diff --git a/backend/drizzle/0045_mcp_operation_replay_tombstones.sql b/backend/drizzle/0045_mcp_operation_replay_tombstones.sql new file mode 100644 index 00000000..08b3d253 --- /dev/null +++ b/backend/drizzle/0045_mcp_operation_replay_tombstones.sql @@ -0,0 +1,13 @@ +BEGIN; +CREATE TABLE public.mcp_operation_replay_tombstones ( + operation_id uuid PRIMARY KEY, + plan_id uuid NOT NULL UNIQUE, + connection_id uuid NOT NULL REFERENCES public.mcp_connections(id), + idempotency_digest text NOT NULL CHECK (idempotency_digest ~ '^[0-9a-f]{64}$'), + action_id text NOT NULL CHECK (length(action_id) BETWEEN 1 AND 200), + target_server_ids bigint[] NOT NULL CHECK (array_position(target_server_ids, NULL) IS NULL), + target_global boolean NOT NULL, + UNIQUE (connection_id,idempotency_digest) +); +INSERT INTO public.schema_migrations(id) VALUES ('0045_mcp_operation_replay_tombstones') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0046_server_poll_operation_retirement.sql b/backend/drizzle/0046_server_poll_operation_retirement.sql new file mode 100644 index 00000000..7cc40b5d --- /dev/null +++ b/backend/drizzle/0046_server_poll_operation_retirement.sql @@ -0,0 +1,186 @@ +-- Inactive poll-history V1-to-V2 retirement. Application owner authenticates +-- both ciphertext versions; SQL independently fences identity and lifetime. +BEGIN; +LOCK TABLE public.server_poll_archives,public.server_poll_storage_admissions IN ACCESS EXCLUSIVE MODE; +ALTER TABLE public.server_poll_archives DROP CONSTRAINT server_poll_archives_version_check; +ALTER TABLE public.server_poll_archives ADD CONSTRAINT server_poll_archives_version_check + CHECK(version=1 OR (version=2 AND decoded_bytes<=24576 AND compressed_bytes<=24576 AND ciphertext_bytes<=33000)); + +CREATE OR REPLACE FUNCTION public.server_poll_archive_immutable() RETURNS trigger +LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +BEGIN + IF TG_OP='DELETE' AND public.server_poll_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF; + IF TG_OP='UPDATE' AND OLD.version=1 AND NEW.version=2 + AND current_setting('mm.poll_retire_receipt',true)=OLD.receipt_id::text + AND NEW.ciphertext_bytes<=OLD.ciphertext_bytes + AND NEW.ciphertext ~ '^[A-Za-z0-9+/]+={0,2}$' + AND replace(encode(decode(NEW.ciphertext,'base64'),'base64'),E'\n','')=NEW.ciphertext + AND octet_length(decode(NEW.ciphertext,'base64'))=NEW.compressed_bytes+28 + AND (to_jsonb(OLD)-ARRAY['version','source_digest','source_sha256','decoded_bytes','compressed_bytes','ciphertext','ciphertext_bytes','archive_proof'])= + (to_jsonb(NEW)-ARRAY['version','source_digest','source_sha256','decoded_bytes','compressed_bytes','ciphertext','ciphertext_bytes','archive_proof']) + AND EXISTS( + SELECT 1 FROM public.server_poll_storage_admissions a JOIN public.mcp_operations o ON o.id=a.execution_id AND o.connection_id=a.connection_id + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0 + JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=o.id AND t.plan_id=p.id AND t.connection_id=o.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id AND c.owner_id=a.owner_id + WHERE a.receipt_id=OLD.receipt_id AND a.identity_kind='mcp' AND a.storage_state='compacted' + AND a.execution_id=OLD.execution_id AND a.connection_id=OLD.connection_id AND a.owner_id=OLD.owner_id + AND a.binding_proof=OLD.binding_proof AND a.context_proof=OLD.context_proof AND a.archive_bytes=OLD.ciphertext_bytes + AND a.original_activity_at=OLD.original_activity_at AND a.authority_expires_at=OLD.original_expires_at + AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL AND isfinite(o.finished_at) + AND o.finished_at0 + ) THEN RETURN NEW; END IF; + RAISE EXCEPTION 'poll archive immutable outside exact retirement or original lifetime GC'; +END $$; + +CREATE FUNCTION public.server_poll_retirement_final_guard() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM public.mcp_operation_replay_tombstones t + WHERE t.operation_id=NEW.execution_id AND t.connection_id=NEW.connection_id AND t.action_id='servers.poll' + AND t.target_global AND NOT EXISTS(SELECT 1 FROM public.mcp_plans p WHERE p.id=t.plan_id)) + OR EXISTS(SELECT 1 FROM public.mcp_operations o WHERE o.id=NEW.execution_id) + OR NOT public.server_poll_storage_accounting_valid() + THEN RAISE EXCEPTION 'poll retirement is incomplete'; END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER server_poll_retirement_final_guard AFTER UPDATE ON public.server_poll_archives + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW WHEN(OLD.version=1 AND NEW.version=2) + EXECUTE FUNCTION public.server_poll_retirement_final_guard(); + +CREATE OR REPLACE FUNCTION server_poll_storage_admission_guard() RETURNS trigger LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +DECLARE retention integer; charged integer; r public.server_poll_archives%ROWTYPE; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + IF TG_OP='DELETE' THEN + IF NOT public.server_poll_storage_gc_allowed(OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_executions WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_intents WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_acks WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_receipts WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_quarantines WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_archives WHERE receipt_id=OLD.receipt_id) THEN RAISE EXCEPTION 'poll lifetime or children prevent admission deletion'; END IF; + UPDATE public.server_poll_storage_usage SET admissions=admissions-CASE WHEN OLD.storage_state='full' THEN 1 ELSE 0 END, + compacted_admissions=compacted_admissions-CASE WHEN OLD.storage_state='compacted' THEN 1 ELSE 0 END, + archive_bytes=archive_bytes-COALESCE(OLD.archive_bytes,0), + reserved_rows=reserved_rows-CASE WHEN OLD.storage_state='full' THEN 4019 ELSE 2 END, + reserved_bytes=reserved_bytes-CASE WHEN OLD.storage_state='full' THEN 8421376 ELSE 32768+OLD.archive_bytes END + WHERE scope IN('global','owner:'||OLD.owner_id); + GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll quota refund mismatch'; END IF; RETURN OLD; + END IF; + IF TG_OP='UPDATE' THEN + IF OLD.storage_state='compacted' AND NEW.storage_state='compacted' + AND current_setting('mm.poll_retire_receipt',true)=OLD.receipt_id::text + AND (to_jsonb(OLD)-'archive_bytes')=(to_jsonb(NEW)-'archive_bytes') THEN + SELECT * INTO r FROM public.server_poll_archives WHERE receipt_id=OLD.receipt_id AND version=2; + IF NOT FOUND OR r.ciphertext_bytes<>NEW.archive_bytes OR NEW.archive_bytes>OLD.archive_bytes + OR r.execution_id<>OLD.execution_id OR r.connection_id<>OLD.connection_id OR r.owner_id<>OLD.owner_id + THEN RAISE EXCEPTION 'retired poll charge source mismatch'; END IF; + UPDATE public.server_poll_storage_usage + SET archive_bytes=archive_bytes-OLD.archive_bytes+NEW.archive_bytes, + reserved_bytes=reserved_bytes-OLD.archive_bytes+NEW.archive_bytes + WHERE scope IN ('global','owner:'||OLD.owner_id) AND compacted_admissions>=1 AND archive_bytes>=OLD.archive_bytes; + GET DIAGNOSTICS charged=ROW_COUNT; + IF charged<>2 THEN RAISE EXCEPTION 'retired poll charge adjustment mismatch'; END IF; + RETURN NEW; + END IF; + -- This durable one-shot marker is written and ACKed in its OWN transaction + -- before archive construction. A committed/unknown claim never resets and + -- the worker never retries destructive work against an already claimed full + -- source. Claim time is maintenance metadata, not renewed execution activity. + IF OLD.storage_state='full' AND NEW.storage_state='full' AND OLD.compaction_claim IS NULL AND OLD.compaction_claimed_at IS NULL AND OLD.compaction_claim_xid IS NULL + AND NEW.compaction_claim IS NOT NULL AND NEW.compaction_claim_xid IS NULL + AND(to_jsonb(OLD)-ARRAY['compaction_claim','compaction_claimed_at','compaction_claim_xid'])=(to_jsonb(NEW)-ARRAY['compaction_claim','compaction_claimed_at','compaction_claim_xid']) + AND public.server_poll_storage_compact_allowed(OLD.receipt_id) + THEN NEW.compaction_claimed_at:=clock_timestamp(); NEW.compaction_claim_xid:=pg_current_xact_id(); RETURN NEW; END IF; + IF(to_jsonb(OLD)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes','storage_state','compact_at','archive_bytes','original_activity_at']) IS DISTINCT FROM + (to_jsonb(NEW)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes','storage_state','compact_at','archive_bytes','original_activity_at']) THEN RAISE EXCEPTION 'poll admission identity is immutable'; END IF; + IF OLD.storage_state<>'full' THEN RAISE EXCEPTION 'compacted poll identity cannot reopen or change'; END IF; + IF NEW.storage_state='compacted' THEN + SELECT * INTO r FROM public.server_poll_archives WHERE receipt_id=OLD.receipt_id AND compaction_xid=pg_current_xact_id(); + IF NOT FOUND OR NEW.used_rows<>OLD.used_rows OR NEW.used_bytes<>OLD.used_bytes OR NEW.claim_token IS DISTINCT FROM OLD.claim_token + OR NEW.claim_attempted_at IS DISTINCT FROM OLD.claim_attempted_at OR NEW.archive_bytes<>r.ciphertext_bytes OR NEW.original_activity_at<>r.original_activity_at + OR EXISTS(SELECT 1 FROM public.server_poll_executions WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_intents WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_acks WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_receipts WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_quarantines WHERE receipt_id=OLD.receipt_id) + THEN RAISE EXCEPTION 'poll compaction requires the complete current-transaction archive'; END IF; + NEW.compact_at:=r.created_at; + UPDATE public.server_poll_storage_usage SET admissions=admissions-1,compacted_admissions=compacted_admissions+1, + archive_bytes=archive_bytes+r.ciphertext_bytes,reserved_rows=reserved_rows-4019+2,reserved_bytes=reserved_bytes-8421376+32768+r.ciphertext_bytes + WHERE scope IN('global','owner:'||OLD.owner_id) AND admissions>=1; + GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll compaction charge mismatch'; END IF; + RETURN NEW; + END IF; + IF NEW.compact_at IS NOT NULL OR NEW.archive_bytes IS NOT NULL OR NEW.original_activity_at IS NOT NULL THEN RAISE EXCEPTION 'full poll cannot carry archive metadata'; END IF; + IF OLD.claim_token IS NOT NULL AND NEW.claim_token IS NULL AND OLD.claim_attempted_at IS NULL AND NEW.claim_attempted_at IS NOT NULL + AND NEW.used_rows=OLD.used_rows AND NEW.used_bytes=OLD.used_bytes AND OLD.authority_expires_at>clock_timestamp() + THEN NEW.claim_attempted_at:=clock_timestamp(); RETURN NEW; END IF; + IF OLD.claim_token IS NOT DISTINCT FROM NEW.claim_token AND OLD.claim_attempted_at IS NOT DISTINCT FROM NEW.claim_attempted_at + AND NEW.used_rows=OLD.used_rows+1 AND NEW.used_bytes>OLD.used_bytes THEN RETURN NEW; END IF; + RAISE EXCEPTION 'poll admission cannot refresh, reopen or refund itself'; + END IF; + IF NEW.storage_state<>'full' OR NEW.compact_at IS NOT NULL OR NEW.archive_bytes IS NOT NULL OR NEW.original_activity_at IS NOT NULL + OR NEW.compaction_claim IS NOT NULL OR NEW.compaction_claimed_at IS NOT NULL OR NEW.compaction_claim_xid IS NOT NULL + OR NOT public.server_poll_storage_accounting_valid() THEN RAISE EXCEPTION 'new poll requires full reservation and exact ledger'; END IF; + SELECT requests_retention_days INTO retention FROM public.mcp_settings WHERE id=1; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 OR NEW.authority_issued_at>clock_timestamp() OR NEW.authority_expires_at<=clock_timestamp() + OR NEW.claim_token IS NULL OR NEW.claim_attempted_at IS NOT NULL OR NEW.used_rows<>0 OR NEW.used_bytes<>0 THEN RAISE EXCEPTION 'new poll admission needs original live authority'; END IF; + IF EXISTS(SELECT 1 FROM public.server_poll_storage_admissions WHERE receipt_id=NEW.receipt_id OR(identity_kind=NEW.identity_kind AND execution_id=NEW.execution_id)) THEN RAISE EXCEPTION 'poll identity already spent'; END IF; + NEW.created_at:=clock_timestamp(); NEW.retention_days:=retention; NEW.safety_ms:=60000; + NEW.gc_not_before:=NEW.authority_expires_at+make_interval(days=>retention)+interval '60 seconds'; + INSERT INTO public.server_poll_storage_usage(scope) VALUES('owner:'||NEW.owner_id) ON CONFLICT DO NOTHING; + IF(SELECT count(*) FROM public.server_poll_storage_usage WHERE scope IN('global','owner:'||NEW.owner_id) + AND admissions2 THEN RAISE EXCEPTION 'poll admission quota exhausted'; END IF; + UPDATE public.server_poll_storage_usage SET admissions=admissions+1,reserved_rows=reserved_rows+4019,reserved_bytes=reserved_bytes+8421376 WHERE scope IN('global','owner:'||NEW.owner_id); + GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll admission quota unavailable'; END IF; + RETURN NEW; +END $$; +CREATE OR REPLACE FUNCTION server_poll_storage_gc_allowed(id uuid) RETURNS boolean LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +DECLARE a public.server_poll_storage_admissions%ROWTYPE; retention integer; activity timestamptz; terminal_at timestamptz; +BEGIN + IF current_setting('mm.poll_storage_gc',true) IS DISTINCT FROM 'on' THEN RETURN false; END IF; + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + SELECT * INTO a FROM public.server_poll_storage_admissions WHERE receipt_id=id FOR UPDATE; + IF NOT FOUND OR a.authority_expires_at>=clock_timestamp() OR a.gc_not_before>=clock_timestamp() THEN RETURN false; END IF; + SELECT requests_retention_days INTO retention FROM public.mcp_settings WHERE mcp_settings.id=1 FOR SHARE; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF; + terminal_at:=a.created_at; + IF a.identity_kind='mcp' THEN + IF NOT pg_try_advisory_xact_lock(hashtextextended('mcp-operation:'||a.execution_id::text,0)) THEN RETURN false; END IF; + BEGIN + SELECT GREATEST(o.finished_at,p.expires_at) INTO terminal_at FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id JOIN public.mcp_connections c ON c.id=o.connection_id + WHERE o.id=a.execution_id AND o.connection_id=a.connection_id AND c.owner_id=a.owner_id AND p.operation_id='servers.poll' + AND o.state IN('applied','failed','unknown','rejected','partial','cancelled') AND o.finished_at IS NOT NULL FOR UPDATE OF o,p NOWAIT; + IF NOT FOUND THEN + IF a.storage_state<>'compacted' OR EXISTS(SELECT 1 FROM public.mcp_operations o WHERE o.id=a.execution_id) OR NOT EXISTS ( + SELECT 1 FROM public.mcp_operation_replay_tombstones t JOIN public.mcp_connections c ON c.id=t.connection_id + WHERE t.operation_id=a.execution_id AND t.connection_id=a.connection_id AND c.owner_id=a.owner_id + AND t.action_id='servers.poll' AND t.target_global AND cardinality(t.target_server_ids)>0 + AND NOT EXISTS(SELECT 1 FROM public.mcp_plans WHERE mcp_plans.id=t.plan_id) + ) OR EXISTS(SELECT 1 FROM public.server_poll_archives r WHERE r.receipt_id=id AND + (r.version<>2 OR r.execution_id<>a.execution_id OR r.connection_id<>a.connection_id OR r.owner_id<>a.owner_id + OR r.original_activity_at IS DISTINCT FROM a.original_activity_at OR r.original_expires_at<>a.authority_expires_at + OR r.binding_proof<>a.binding_proof OR r.context_proof<>a.context_proof OR r.ciphertext_bytes<>a.archive_bytes)) + THEN RETURN false; END IF; + terminal_at:=a.original_activity_at; + END IF; + IF terminal_at IS NULL THEN RETURN false; END IF; + EXCEPTION WHEN lock_not_available THEN RETURN false; END; + END IF; + IF a.storage_state='compacted' THEN activity:=GREATEST(a.original_activity_at,a.authority_expires_at,terminal_at); + ELSE + SELECT GREATEST(a.authority_expires_at,a.created_at,COALESCE(a.claim_attempted_at,a.created_at),terminal_at, + COALESCE((SELECT created_at FROM public.server_poll_executions WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(created_at) FROM public.server_poll_intents WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(created_at) FROM public.server_poll_acks WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM public.server_poll_receipts WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM public.server_poll_quarantines WHERE receipt_id=id),a.created_at)) INTO activity; + END IF; + RETURN isfinite(activity) AND activity+make_interval(days=>GREATEST(a.retention_days,retention))+a.safety_ms*interval '1 millisecond'1 AND NOT EXISTS( + SELECT 1 FROM public.server_poll_storage_admissions a WHERE a.receipt_id=OLD.receipt_id + ) THEN RETURN OLD; END IF; + RAISE EXCEPTION 'poll retirement claim cannot be replaced or reopened'; +END $$; +CREATE TRIGGER server_poll_retirement_claim_immutable + BEFORE INSERT OR UPDATE OR DELETE ON public.server_poll_retirement_claims + FOR EACH ROW EXECUTE FUNCTION public.server_poll_retirement_claim_immutable(); +CREATE TRIGGER server_poll_retirement_claim_no_truncate + BEFORE TRUNCATE ON public.server_poll_retirement_claims + FOR EACH STATEMENT EXECUTE FUNCTION public.server_poll_retirement_claim_immutable(); +CREATE FUNCTION public.server_poll_retirement_requires_claim() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM public.server_poll_retirement_claims c + WHERE c.receipt_id=OLD.receipt_id AND c.old_archive_proof=OLD.archive_proof + AND c.claim_token::text=current_setting('mm.poll_retire_claim',true) + AND c.claim_xid<>pg_current_xact_id()) + THEN RAISE EXCEPTION 'poll retirement requires prior exact claim'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER server_poll_retirement_requires_claim BEFORE UPDATE ON public.server_poll_archives + FOR EACH ROW WHEN(OLD.version=1 AND NEW.version=2) + EXECUTE FUNCTION public.server_poll_retirement_requires_claim(); +INSERT INTO public.schema_migrations(id) VALUES ('0047_server_poll_retirement_claims') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0048_mcp_ordinary_retirement_claims.sql b/backend/drizzle/0048_mcp_ordinary_retirement_claims.sql new file mode 100644 index 00000000..c0107ac5 --- /dev/null +++ b/backend/drizzle/0048_mcp_ordinary_retirement_claims.sql @@ -0,0 +1,27 @@ +BEGIN; +CREATE TABLE public.mcp_ordinary_retirement_claims ( + operation_id uuid PRIMARY KEY, + plan_id uuid NOT NULL UNIQUE, + connection_id uuid NOT NULL REFERENCES public.mcp_connections(id), + source_proof text NOT NULL CHECK(source_proof ~ '^[0-9a-f]{64}$'), + claim_token uuid NOT NULL, + claim_xid xid8 NOT NULL DEFAULT pg_current_xact_id() +); +CREATE FUNCTION public.mcp_ordinary_retirement_claim_immutable() RETURNS trigger +LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +BEGIN + IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS( + SELECT 1 FROM public.mcp_operations o JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + WHERE o.id=NEW.operation_id AND o.plan_id=NEW.plan_id AND o.connection_id=NEW.connection_id + AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL + AND isfinite(o.finished_at) AND o.finished_at>'version') IS NOT DISTINCT FROM '1' + AND (source->>'receiptId') IS NOT DISTINCT FROM receipt_id::text + AND (source->>'executionId') IS NOT DISTINCT FROM operation_id::text) +); +CREATE FUNCTION public.server_probe_retired_source_immutable() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$ +BEGIN + RAISE EXCEPTION 'retired diagnostic source is immutable'; +END $$; +CREATE TRIGGER server_probe_retired_source_immutable + BEFORE UPDATE OR DELETE ON public.server_probe_retired_sources + FOR EACH ROW EXECUTE FUNCTION public.server_probe_retired_source_immutable(); +CREATE TRIGGER server_probe_retired_source_no_truncate BEFORE TRUNCATE ON public.server_probe_retired_sources + FOR EACH STATEMENT EXECUTE FUNCTION public.server_probe_retired_source_immutable(); +INSERT INTO public.schema_migrations(id) VALUES ('0050_server_probe_retired_sources') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0051_server_probe_retirement_claims.sql b/backend/drizzle/0051_server_probe_retirement_claims.sql new file mode 100644 index 00000000..15e52072 --- /dev/null +++ b/backend/drizzle/0051_server_probe_retirement_claims.sql @@ -0,0 +1,52 @@ +BEGIN; +CREATE TABLE public.server_probe_retirement_claims ( + receipt_id uuid PRIMARY KEY, + operation_id uuid NOT NULL UNIQUE, + plan_id uuid NOT NULL UNIQUE, + connection_id uuid NOT NULL, + source_proof text NOT NULL CHECK(source_proof ~ '^[a-f0-9]{64}$'), + claim_token uuid NOT NULL, + claim_xid xid8 NOT NULL DEFAULT pg_current_xact_id() +); +CREATE FUNCTION public.server_probe_retirement_claim_guard() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public SET TimeZone='UTC' AS $$ +BEGIN + IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS( + SELECT 1 FROM public.server_probe_storage_admissions a + JOIN public.mcp_operations o ON o.id=a.execution_id + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id AND c.owner_id=a.owner_id + WHERE a.receipt_id=NEW.receipt_id AND a.identity_kind='mcp' AND a.claim_token IS NULL + AND a.claim_attempted_at IS NOT NULL AND a.cancelled_at IS NULL AND a.purged_at IS NULL + AND o.id=NEW.operation_id AND p.id=NEW.plan_id AND c.id=NEW.connection_id + AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL + AND isfinite(o.finished_at) AND o.finished_atpg_current_xact_id() AND r.claim_token::text=current_setting('mm.probe_retirement_claim',true) + AND NEW.source->>'planId'=r.plan_id::text AND NEW.source->>'connectionId'=r.connection_id::text + AND NEW.source->>'ownerId'=a.owner_id AND NEW.source->>'bindingProof'=a.binding_proof + AND NEW.source->'serverIds'=to_jsonb(t.target_server_ids) + ) THEN RETURN NEW; END IF; + RAISE EXCEPTION 'diagnostic archive requires an exact committed retirement claim'; +END $$; +CREATE TRIGGER server_probe_retired_source_insert_guard BEFORE INSERT ON public.server_probe_retired_sources + FOR EACH ROW EXECUTE FUNCTION public.server_probe_retired_source_insert_guard(); +INSERT INTO public.schema_migrations(id) VALUES ('0051_server_probe_retirement_claims') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0052_server_probe_retired_gc.sql b/backend/drizzle/0052_server_probe_retired_gc.sql new file mode 100644 index 00000000..18fc51ea --- /dev/null +++ b/backend/drizzle/0052_server_probe_retired_gc.sql @@ -0,0 +1,96 @@ +BEGIN; +ALTER TABLE public.server_probe_retired_sources + DROP CONSTRAINT server_probe_retired_sources_receipt_id_fkey, + ADD CONSTRAINT server_probe_retired_sources_receipt_id_fkey + FOREIGN KEY(receipt_id) REFERENCES public.server_probe_storage_admissions(receipt_id) ON DELETE CASCADE; + +CREATE FUNCTION public.server_probe_retired_gc_allowed(target_receipt uuid,current_retention integer) RETURNS boolean +LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +DECLARE activity timestamptz; retained_days integer; safety integer; +BEGIN + IF current_setting('mm.probe_storage_gc',true) IS DISTINCT FROM 'on' + OR current_setting('mm.probe_retired_gc_receipt',true) IS DISTINCT FROM target_receipt::text THEN RETURN false; END IF; + SELECT (r.source->>'activityAt')::timestamptz,a.retention_days,a.safety_ms + INTO activity,retained_days,safety + FROM public.server_probe_retired_sources r + JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.execution_id=r.operation_id + JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id + JOIN public.mcp_connections c ON c.id=t.connection_id AND c.owner_id=a.owner_id + WHERE r.receipt_id=target_receipt AND r.proof=current_setting('mm.probe_retired_gc_proof',true) + AND a.identity_kind='mcp' AND a.cancelled_at IS NULL AND a.claim_token IS NULL AND isfinite(a.claim_attempted_at) + AND t.action_id='servers.probes.run' AND NOT t.target_global + AND r.source->>'planId'=t.plan_id::text AND r.source->>'connectionId'=c.id::text + AND r.source->'serverIds'=to_jsonb(t.target_server_ids) AND r.source->>'ownerId'=a.owner_id + AND r.source->>'bindingProof'=a.binding_proof + AND (r.source->>'authorityExpiresAt')::timestamptz=a.authority_expires_at + AND (r.source->>'gcNotBefore')::timestamptz=a.gc_not_before + AND (r.source->>'retentionDays')::int=a.retention_days AND (r.source->>'safetyMs')::int=a.safety_ms + AND isfinite(a.authority_expires_at) AND isfinite(a.gc_not_before) + AND a.authority_expires_atactivity)) + OR EXISTS(SELECT 1 FROM public.server_probe_execution_stages s WHERE s.receipt_id=target_receipt + AND GREATEST(s.created_at,COALESCE(s.received_at,s.created_at))>activity) + THEN RETURN false; END IF; + RETURN activity+make_interval(days=>GREATEST(retained_days,current_retention)) + +GREATEST(safety,60000)*interval '1 millisecond'=clock_timestamp() OR a.authority_expires_at>=clock_timestamp() THEN RETURN false; END IF; + SELECT settings.requests_retention_days INTO current_retention FROM mcp_settings settings WHERE settings.id=1; + IF current_retention IS NULL OR current_retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF; + IF EXISTS(SELECT 1 FROM public.server_probe_retired_sources WHERE receipt_id=id) THEN + RETURN public.server_probe_retired_gc_allowed(id,current_retention); + END IF; + terminal_at:=a.created_at; + IF a.identity_kind='mcp' THEN + SELECT GREATEST(o.finished_at,p.expires_at,COALESCE(cancellation.created_at,o.finished_at)) INTO terminal_at + FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN mcp_connections owner_connection ON owner_connection.id=o.connection_id + LEFT JOIN mcp_diagnostic_cancellations cancellation ON cancellation.operation_id=o.id + WHERE o.id=a.execution_id AND owner_connection.owner_id=a.owner_id AND p.operation_id='servers.probes.run' + AND o.state IN ('applied','failed','unknown','rejected','partial','cancelled') AND o.finished_at IS NOT NULL + AND (cancellation.operation_id IS NULL OR(cancellation.connection_id=o.connection_id AND cancellation.actor_id=a.owner_id)) + FOR UPDATE OF o,p,owner_connection; + IF NOT FOUND OR terminal_at IS NULL THEN RETURN false; END IF; + END IF; + SELECT GREATEST(a.created_at,a.authority_expires_at,terminal_at,COALESCE(a.cancelled_at,a.created_at), + COALESCE((SELECT finished_at FROM server_probe_executions WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM server_probe_execution_quarantines WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(GREATEST(created_at,COALESCE(received_at,created_at))) FROM server_probe_execution_stages WHERE receipt_id=id),a.created_at)) INTO activity; + RETURN activity+make_interval(days=>GREATEST(a.retention_days,current_retention))+a.safety_ms*interval '1 millisecond'>'version') IS NOT DISTINCT FROM '1' + AND (source->>'executionId') IS NOT DISTINCT FROM operation_id::text + AND (source->>'receiptId') IS NOT DISTINCT FROM receipt_id::text) +); +CREATE FUNCTION public.server_probe_terminal_proof_guard() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$ +BEGIN + IF TG_OP='INSERT' THEN + IF EXISTS( + SELECT 1 FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id + JOIN public.server_probe_storage_admissions a ON a.execution_id=o.id AND a.identity_kind='mcp' AND a.owner_id=c.owner_id + JOIN public.server_probe_executions j ON j.receipt_id=a.receipt_id + JOIN public.server_probe_execution_payloads b ON b.receipt_id=a.receipt_id AND b.receipt_proof=j.receipt_proof + WHERE o.id=NEW.operation_id AND a.receipt_id=NEW.receipt_id AND a.purged_at IS NULL AND a.cancelled_at IS NULL + AND a.claim_token IS NULL AND a.claim_attempted_at IS NOT NULL + AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL + AND p.state='completed' AND p.operation_id='servers.probes.run' AND p.operation_version=1 + AND j.state='completed' AND j.execution_id=o.id AND j.mcp_connection_id=c.id AND j.actor_id=c.owner_id + AND NEW.source->>'connectionId'=c.id::text AND NEW.source->>'planId'=p.id::text + AND NEW.source->>'ownerId'=a.owner_id AND NEW.source->>'bindingProof'=a.binding_proof + AND NEW.source->>'receiptProof'=b.receipt_proof + AND NEW.source->>'ciphertextSha256'=encode(sha256(convert_to(b.receipt_ciphertext,'UTF8')),'hex') + ) THEN RETURN NEW; END IF; + ELSIF TG_OP='DELETE' THEN + IF EXISTS( + SELECT 1 FROM public.server_probe_retirement_claims r + JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id AND t.plan_id=r.plan_id AND t.connection_id=r.connection_id + WHERE r.operation_id=OLD.operation_id AND r.receipt_id=OLD.receipt_id AND r.source_proof=OLD.source_proof + AND r.claim_xid<>pg_current_xact_id() AND r.claim_token::text=current_setting('mm.probe_retirement_claim',true) + AND t.action_id='servers.probes.run' AND NOT t.target_global AND t.idempotency_digest=OLD.idempotency_digest + AND OLD.source->'serverIds'=to_jsonb(t.target_server_ids) + ) THEN RETURN OLD; END IF; + END IF; + RAISE EXCEPTION 'diagnostic terminal proof must match its original source or exact retirement claim'; +END $$; +CREATE TRIGGER server_probe_terminal_proof_guard BEFORE INSERT OR UPDATE OR DELETE + ON public.server_probe_terminal_proofs FOR EACH ROW EXECUTE FUNCTION public.server_probe_terminal_proof_guard(); +CREATE TRIGGER server_probe_terminal_proof_no_truncate BEFORE TRUNCATE + ON public.server_probe_terminal_proofs FOR EACH STATEMENT EXECUTE FUNCTION public.server_probe_terminal_proof_guard(); + +CREATE OR REPLACE FUNCTION public.server_probe_retirement_claim_guard() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public SET TimeZone='UTC' AS $$ +BEGIN + IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS( + SELECT 1 FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id + WHERE o.id=NEW.operation_id AND p.id=NEW.plan_id AND c.id=NEW.connection_id + AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL + AND isfinite(o.finished_at) AND o.finished_at>'ownerId'=c.owner_id AND f.source->>'connectionId'=c.id::text AND f.source->>'planId'=p.id::text) + AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions a + WHERE a.receipt_id=NEW.receipt_id OR(a.identity_kind='mcp' AND a.execution_id=o.id)))) + ) THEN RETURN NEW; END IF; + RAISE EXCEPTION 'diagnostic retirement claim cannot be replaced or reopened'; +END $$; +INSERT INTO public.schema_migrations(id) VALUES ('0053_server_probe_terminal_proofs') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/package.json b/backend/package.json index 4f0bdc37..e785fc4b 100644 --- a/backend/package.json +++ b/backend/package.json @@ -66,7 +66,7 @@ "test:mcp:ipsec-export-journal:pg": "node scripts/test-mcp-postgres.mjs --ipsec-export-journal --restore", "test:mcp:ipsec-export:http:pg": "node scripts/test-mcp-postgres.mjs --ipsec-export-http --restore", "test:mcp:diagnostic-cancel-host:pg": "node scripts/test-mcp-postgres.mjs --diagnostic-cancel-host --restore", - "test:server-probe-management": "tsx src/services/server-probe-authority.test.ts && tsx src/services/server-probe-public-result.test.ts && tsx src/services/server-probe-execution.test.ts && tsx src/services/server-probe-rest-contract.test.ts && tsx src/services/server-probe-rest-execution.test.ts && tsx src/services/server-probe-journal.test.ts && tsx src/services/server-probe-journal-history.test.ts && tsx src/modules/mcp/server-probe-actions.test.ts && tsx src/modules/mcp/diagnostic-cancellation.test.ts && tsx src/modules/mcp/service-diagnostic-progress.test.ts && tsx src/services/mikrotik-get-admission.test.ts && tsx src/services/server-probe-storage-journal-adapter.test.ts && tsx --test src/services/server-probe-storage-history.test.ts && npm run test:server-probe-storage-models", + "test:server-probe-management": "tsx src/services/server-probe-authority.test.ts && tsx src/services/server-probe-public-result.test.ts && tsx src/services/server-probe-execution.test.ts && tsx src/services/server-probe-rest-contract.test.ts && tsx src/services/server-probe-rest-execution.test.ts && tsx src/services/server-probe-journal.test.ts && tsx src/services/server-probe-journal-history.test.ts && tsx src/modules/mcp/server-probe-actions.test.ts && tsx src/modules/mcp/diagnostic-cancellation.test.ts && tsx src/modules/mcp/service-diagnostic-progress.test.ts && tsx src/services/mikrotik-get-admission.test.ts && tsx src/services/server-probe-storage-journal-adapter.test.ts && npm run test:server-probe-history && npm run test:server-probe-storage-models", "test:server-poll-http:pg": "node scripts/test-server-poll-http-owned.mjs", "test:server-poll-approval:pg": "node scripts/test-server-poll-approval-owned.mjs", "test:server-poll-atomic:pg": "node scripts/test-server-poll-atomic-owned.mjs", @@ -155,8 +155,9 @@ "test:server-poll-durable:pg": "node scripts/test-mcp-postgres.mjs --server-poll-durable --restore", "test:server-probe-legacy:pg": "node scripts/test-mcp-postgres.mjs --probe-legacy-orphans", "test:server-probe-storage:pg": "node scripts/test-mcp-postgres.mjs --probe-storage-journal", - "test:server-probe-history": "tsx --test src/services/server-probe-storage-history.test.ts", + "test:server-probe-history": "tsx --test src/services/server-probe-storage-history.test.ts src/services/server-probe-retired-source.test.ts", "test:server-probe-history:pg": "node scripts/test-mcp-postgres.mjs --probe-storage-history", + "test:server-probe-retirement:pg": "node scripts/test-mcp-postgres.mjs --probe-operation-retirement", "test:server-probe-storage-models": "tsx src/services/server-probe-journal-maintenance.test.ts && tsx src/modules/mcp/diagnostic-cancellation-source.test.ts && tsx src/services/server-probe-storage-source.test.ts && tsx src/services/server-probe-storage-connection.test.ts && tsx src/modules/mcp/server-probe-storage-host.test.ts && tsx src/services/server-probe-storage-rest.test.ts && tsx src/services/server-probe-storage-metadata.test.ts && tsx src/services/server-probe-legacy-verifier.test.ts && tsx src/services/server-probe-legacy-snapshot.test.ts && tsx src/services/server-probe-legacy-inspection.test.ts && tsx src/db/database-source-observation.test.ts && tsx src/services/server-probe-retention.test.ts && tsx src/modules/mcp/server-probe-registration.test.ts", "test:mcp:migration-data:pg": "node scripts/test-mcp-postgres.mjs --migration-data-inventory --restore", "test:mcp:poll-memory:pg": "npm run build -w @mmapp/contracts && node scripts/acceptance/build-poll-memory.mjs && node scripts/test-mcp-postgres.mjs --poll-memory", diff --git a/backend/scripts/acceptance/chr-netflow-drain-witness-pg.mjs b/backend/scripts/acceptance/chr-netflow-drain-witness-pg.mjs new file mode 100644 index 00000000..54dbafc0 --- /dev/null +++ b/backend/scripts/acceptance/chr-netflow-drain-witness-pg.mjs @@ -0,0 +1,123 @@ +import assert from 'node:assert/strict' +import { execFileSync } from 'node:child_process' +import { randomBytes } from 'node:crypto' +import { lstat, mkdtemp, rm } from 'node:fs/promises' +import { createServer } from 'node:net' +import { tmpdir } from 'node:os' +import { isAbsolute, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import EmbeddedPostgres from 'embedded-postgres' +import pg from 'pg' +import { applySqlMigrations } from '../../src/db/migrate.ts' +import { MIGRATION_MANIFEST } from '../../src/db/migration-manifest.ts' + +assert.deepEqual(process.argv.slice(2), ['--owned-embedded']) +const bin = process.env.MMAPP_PG_CLIENT_BIN +assert(bin && isAbsolute(bin) && resolve(bin) === bin, 'Set MMAPP_PG_CLIENT_BIN to the exact local PostgreSQL client directory') +const psql = join(bin, process.platform === 'win32' ? 'psql.exe' : 'psql') +const executable = await lstat(psql) +assert(executable.isFile() && !executable.isSymbolicLink(), 'PostgreSQL client must be a regular file') +const clientEnv = Object.fromEntries(Object.entries(process.env).filter(([name]) => !/^PG/i.test(name))) +const run = (command, args, options = {}) => execFileSync(command, args, { + timeout: 10000, maxBuffer: 16384, encoding: 'utf8', windowsHide: true, ...options, +}) +assert.match(run(psql, ['--version'], { env: clientEnv }), /^psql \(PostgreSQL\) (?:1[6-9]|[2-9][0-9])\./) +const witness = fileURLToPath(new URL('../../../tmp/mcp-netflow-linux-topology-20261104/collector-drain-witness.py', import.meta.url)) +const { IDENTITY_SQL, SQL } = JSON.parse(run('python', ['-I', '-B', '-c', + 'import json,runpy,sys; m=runpy.run_path(sys.argv[1]); print(json.dumps({k:m[k] for k in ("IDENTITY_SQL","SQL")}))', witness])) +assert(typeof SQL === 'string' && typeof IDENTITY_SQL === 'string') + +const directory = await mkdtemp(join(tmpdir(), 'mm-netflow-drain-pg-')) +const password = randomBytes(24).toString('hex') +const host = '127.0.0.1', database = 'postgres', user = 'mcp_test' +const listener = createServer() +let embedded, pool, started = false +try { + await new Promise((done, reject) => { listener.once('error', reject); listener.listen(0, host, done) }) + const { port } = listener.address() + await new Promise((done, reject) => listener.close(error => error ? reject(error) : done())) + embedded = new EmbeddedPostgres({ + databaseDir: join(directory, 'pg'), port, user, password, persistent: true, createPostgresUser: false, + initdbFlags: ['--encoding=UTF8'], postgresFlags: ['-h', host, '-c', 'unix_socket_directories='], + onLog: () => {}, onError: () => {}, + }) + await embedded.initialise() + await embedded.start() + started = true + pool = new pg.Pool({ host, port, database, user, password, max: 1, connectionTimeoutMillis: 5000 }) + await applySqlMigrations(pool) + const { rows: [version] } = await pool.query("SELECT current_setting('server_version_num')::int AS number, (SELECT count(*)::int FROM schema_migrations) AS migrations") + assert(version.number >= 180000 && version.number <= 189999) + assert.equal(version.migrations, MIGRATION_MANIFEST.length) + + const runId = 'a012bcde', owner = 'netflow-operator-' + runId, collectorPort = 51000 + const read = (sql, probePort = collectorPort) => JSON.parse(run(psql, ['-X', '-w', '-qAt', + '-h', host, '-p', String(port), '-U', user, '-d', database, '-v', 'ON_ERROR_STOP=1', + '-v', 'database=' + database, '-v', 'socket_dir=/tmp/mcp-netflow-' + runId + '/pg-socket', + '-v', 'run_id=' + runId, '-v', 'port=' + probePort], + { env: { ...clientEnv, PGCONNECT_TIMEOUT: '2', PGPASSWORD: password }, input: sql }).trim()) + const identity = read(IDENTITY_SQL) + assert.deepEqual(Object.keys(identity).sort(), ['oid', 'source']) + assert.match(identity.oid, /^[1-9][0-9]{0,9}$/) + // The production source predicate deliberately rejects this TCP fixture. + assert.equal(identity.source, false) + const expect = (planCount, enabled, verified, ordered = true, sqlPort = collectorPort) => + assert.deepEqual(read(SQL, sqlPort), { ...identity, connectionCount: 1, planCount, enabled, verified, ordered }) + + const uuid = index => 'a012bcde-0000-4000-8000-' + String(index).padStart(12, '0') + const connection = uuid(1) + await pool.query(`INSERT INTO mcp_connections(id,name,owner_id,owner_permissions,policy,generation) + VALUES($1,$2,$3,'[]','{}','inert-witness')`, [connection, 'Owned NetFlow ' + runId, owner]) + await pool.query("UPDATE traffic_flow_settings SET collector_ip='198.18.242.3', flow_listen_port=$1, enabled=false WHERE id=1", [collectorPort]) + expect(0, false, []) + + async function seed(index, enabled) { + const plan = uuid(index), operation = uuid(index + 1), receipt = uuid(index + 2) + const base = Date.UTC(2026, 0, 1) + index * 1000 + const at = offset => new Date(base + offset) + const result = { receiptId: receipt, configurationConfirmed: true, receiptConfirmed: true, + listener: 'request-acknowledged', automaticReplay: false } + await pool.query(`INSERT INTO mcp_plans(id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at,approved_by,approved_at,created_at) + VALUES($1,$2,'netflow.settings.update',3,1,'inert-witness','{}','{}','completed',$3,$4,$5,$6)`, + [plan, connection, at(9000), owner, at(100), at(0)]) + await pool.query(`INSERT INTO mcp_operations(id,plan_id,connection_id,idempotency_key,state,verification,created_at,finished_at) + VALUES($1,$2,$3,$4,'applied','passed',$5,$6)`, [operation, plan, connection, 'inert-' + index, at(200), at(400)]) + await pool.query(`INSERT INTO mcp_operation_steps(operation_id,step_index,state,intended_state,result,created_at,finished_at) + VALUES($1,0,'verified',$2,$3,$4,$5)`, + [operation, { service: 'netflow.settings.update', input: { receiptId: receipt, enabled } }, result, at(300), at(400)]) + await pool.query(`INSERT INTO mcp_audit(actor_id,action,connection_id,payload,created_at) + VALUES($1,'plan.approve',$2,$3,$4)`, [owner, connection, { planId: plan }, at(100)]) + await pool.query(`INSERT INTO flow_settings_intents(receipt_id,phase,request_proof,state,result,created_at) + VALUES($1,'configuration','inert-proof','completed',$2,$3),($1,'listener','inert-proof','completed',$2,$3)`, [receipt, result, at(300)]) + await pool.query('UPDATE traffic_flow_settings SET enabled=$1 WHERE id=1', [enabled]) + return [plan, operation, receipt] + } + const enabled = await seed(10, true) + expect(1, true, [enabled]) + const disabled = await seed(20, false) + expect(2, false, [enabled, disabled]) + expect(2, null, [enabled, disabled], true, collectorPort + 1) + + await pool.query("UPDATE flow_settings_intents SET request_proof='foreign' WHERE receipt_id=$1 AND phase='listener'", [disabled[2]]) + expect(2, false, [enabled]) + await pool.query("UPDATE flow_settings_intents SET request_proof='inert-proof' WHERE receipt_id=$1 AND phase='listener'", [disabled[2]]) + const { rows: [duplicate] } = await pool.query(`INSERT INTO mcp_audit(actor_id,action,connection_id,payload) + VALUES($1,'plan.approve',$2,$3) RETURNING id`, [owner, connection, { planId: disabled[0] }]) + expect(2, false, [enabled]) + await pool.query('DELETE FROM mcp_audit WHERE id=$1', [duplicate.id]) + const { rows: [original] } = await pool.query(`UPDATE mcp_operations SET finished_at=(SELECT created_at + interval '1 second' FROM mcp_plans WHERE id=$2) + WHERE id=$1 RETURNING created_at`, [enabled[1], disabled[0]]) + expect(2, false, [enabled, disabled], false) + await pool.query("UPDATE mcp_operations SET finished_at=$2::timestamptz + interval '200 milliseconds' WHERE id=$1", [enabled[1], original.created_at]) + expect(2, false, [enabled, disabled]) + console.log(JSON.stringify({ migrations: version.migrations, exactPsqlQueries: true, inertEnableDisableAndRejections: true, + linuxSocketSourceAccepted: false, realMcpExecutionClaimed: false })) +} finally { + try { + await pool?.end() + } finally { + if (started) await embedded.stop() + if (listener.listening) await new Promise(done => listener.close(done)) + await rm(directory, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }) + } +} diff --git a/backend/scripts/test-mcp-postgres.mjs b/backend/scripts/test-mcp-postgres.mjs index 7dafc4b3..b8f16b26 100644 --- a/backend/scripts/test-mcp-postgres.mjs +++ b/backend/scripts/test-mcp-postgres.mjs @@ -94,8 +94,11 @@ const operatorPeerMetadata = process.argv.includes('--operator-peer-metadata') if (operatorPeerMetadata && process.argv.slice(2).some(arg => arg !== '--operator-peer-metadata')) throw new Error('Operator peer metadata requires a separate disposable cluster without restore') const externalPeerMetadata = process.argv.includes('--external-peer-metadata') if (externalPeerMetadata && process.argv.slice(2).some(arg => arg !== '--external-peer-metadata')) throw new Error('Guarded external peer metadata requires a separate disposable cluster without restore') -const mcpOperationOrigin = process.argv.includes('--mcp-operation-origin') -if (mcpOperationOrigin && process.argv.slice(2).some(arg => arg !== '--mcp-operation-origin')) throw new Error('MCP operation root requires a separate disposable cluster without restore') +const pollOperationRetirement = process.argv.includes('--poll-operation-retirement') +const ordinaryOperationRetirement = process.argv.includes('--ordinary-operation-retirement') +const probeOperationRetirement = process.argv.includes('--probe-operation-retirement') +const mcpOperationOrigin = process.argv.includes('--mcp-operation-origin') || pollOperationRetirement || ordinaryOperationRetirement || probeOperationRetirement +if (mcpOperationOrigin && process.argv.slice(2).some(arg => arg !== (probeOperationRetirement ? '--probe-operation-retirement' : ordinaryOperationRetirement ? '--ordinary-operation-retirement' : pollOperationRetirement ? '--poll-operation-retirement' : '--mcp-operation-origin'))) throw new Error('MCP operation root requires a separate disposable cluster without restore') const mcpSpeedOriginal = process.argv.includes('--mcp-speed-original') if (mcpSpeedOriginal && process.argv.slice(2).some(arg => arg !== '--mcp-speed-original')) throw new Error('MCP speed component requires a separate disposable cluster without restore') const routerConfigurationGuard = process.argv.includes('--router-configuration-guard') @@ -243,13 +246,13 @@ try { if (regressionOnly) console.log('Selected regression only: primary integration and terminal restore are not run') if (profilesOnly) console.log('Profiles-only regression matrix: primary integration NOT RUN; terminal restore NOT RUN; full acceptance is not claimed') const result = regressionOnly || profilesOnly ? 0 : await new Promise((resolve, reject) => { - const child = spawn(process.execPath, pollMemory ? ['--max-old-space-size=192', '--max-semi-space-size=16', fileURLToPath(new URL('../dist-mcp-memory/services/server-poll-memory.owned.test.js', import.meta.url))] : ['--import', 'tsx', principalSecuritySource ? 'src/modules/mcp/principal-security-source.owned.test.ts' : pollJournalSnapshot ? 'src/services/server-poll-journal-snapshot.owned.test.ts' : pollTerminalArchive ? 'src/services/server-poll-terminal-archive.owned.test.ts' : pollArchiveSchema ? 'src/db/server-poll-terminal-archives.sql.test.ts' : telemetrySettingsConfiguration ? 'src/modules/mcp/telemetry-settings-configuration.owned.test.ts' : routingReservationSchema ? 'src/db/routing-reservation-schema.owned.test.ts' : routingForwardRounds ? 'src/modules/mcp/routing-network-forward-rounds.owned.test.ts' : flowActiveServices ? 'src/services/traffic-flow-active-services.owned.test.ts' : flowActiveRebuild ? 'src/services/traffic-flow-active-rebuild.owned.test.ts' : flowQuality ? 'src/services/traffic-flow-quality.owned.test.ts' : flowWorkerIpc ? 'src/services/traffic-flow-worker-ipc.owned.test.ts' : manualUptime ? 'src/modules/mcp/manual-uptime-http.test.ts' : manualPingGroup ? 'src/modules/mcp/manual-ping-group-http.test.ts' : manualTraffic ? 'src/modules/mcp/manual-traffic-http.test.ts' : manualApiPing ? 'src/modules/mcp/manual-api-ping-http.test.ts' : simpleSpeedChr ? 'src/modules/mcp/simple-speed-chr.test.ts' : simpleSpeed ? 'src/modules/mcp/simple-speed-http.test.ts' : externalPeerMetadata ? 'src/services/bandwidth-external-peer-metadata.sql.test.ts' : operatorPeerMetadata ? 'src/services/bandwidth-operator-metadata.sql.test.ts' : publicRegistryMetadata ? 'src/services/bandwidth-public-registry.sql.test.ts' : operatorDockerSql ? 'src/services/bandwidth-operator-docker-stop.sql.test.ts' : mcpOperationOrigin ? 'src/modules/mcp/operation-execution-origin.sql.test.ts' : mcpSpeedOriginal ? 'src/modules/mcp/speed-bandwidth-original.sql.test.ts' : routerConfigurationGuard ? 'src/services/router-configuration-guard.sql.test.ts' : scheduledSpeedOriginal ? 'src/services/bandwidth-scheduled-speed-original.sql.test.ts' : bandwidthSpeedWorkflow ? 'src/services/bandwidth-speed-workflow.sql.test.ts' : restSpeedOriginal ? 'src/services/bandwidth-rest-speed-original.sql.test.ts' : operationLocks ? 'src/services/operation-config-lock-lease.sql.test.ts' : serviceExecutionOrigin ? 'src/modules/mcp/service-execution-origin.sql.test.ts' : bandwidthNodeSource ? 'src/services/bandwidth-node-source.sql.test.ts' : bandwidthExternalRegistry ? 'src/services/bandwidth-external-node-registry.sql.test.ts' : bandwidthNodeRegistry ? 'src/services/bandwidth-node-registry.sql.test.ts' : bandwidthControlSource ? 'src/services/bandwidth-control-source.sql.test.ts' : bandwidthAdmission ? 'src/services/bandwidth-admission.sql.test.ts' : serverPollStorageFresh ? 'src/services/server-poll-storage-fresh.sql.test.ts' : serverPollStorage ? 'src/services/server-poll-storage.sql.test.ts' : probeStorageFresh ? 'src/services/server-probe-storage-fresh.sql.test.ts' : probeLegacyOrphans ? 'src/services/server-probe-legacy-orphans.sql.test.ts' : diagnosticCancelHost ? 'src/modules/mcp/diagnostic-cancellation-storage.sql.test.ts' : ipsecExportHttp ? 'src/modules/mcp/ipsec-existing-certificate-export-http.sql.test.ts' : ipsecExportJournal ? 'src/services/ipsec-existing-certificate-export-rest-journal.sql.test.ts' : migrationDataInventory ? 'src/db/migration-data-inventory.sql.test.ts' : probeStorageHistory ? 'src/services/server-probe-storage-history.sql.test.ts' : probeStorageJournal ? 'src/services/server-probe-storage-journal.sql.test.ts' : controlledSchemaReference ? 'src/db/controlled-schema-reference.sql.test.ts' : controlledMigrations ? 'src/db/controlled-migrations.sql.test.ts' : databaseSchemaControl ? 'src/db/database-schema-control.sql.test.ts' : migrationSchemaInventory ? 'src/db/migration-schema-inventory.sql.test.ts' : routingNetworkNative ? 'src/modules/mcp/routing-network-native.sql.test.ts' : routingReservationRuntime ? 'src/modules/mcp/routing-reservation-runtime.sql.test.ts' : routingReservations ? 'src/modules/mcp/routing-reservation.sql.test.ts' : collectorPartitionWriters ? 'src/services/collector-partition-writes.sql.test.ts' : serverPollDurable ? 'src/services/server-poll-durable-session.sql.test.ts' : routingRecoveryAuthority ? 'src/modules/mcp/routing-recovery-authority.sql.test.ts' : legacyFlowRebuild ? 'src/test/legacy-flow-rebuild-sql.ts' : probeJournal ? 'src/services/server-probe-journal.sql.test.ts' : flowHostKeys ? 'src/modules/mcp/flow-host-keys-http.test.ts' : flowRebuildMcp ? 'src/modules/mcp/flow-facts-rebuild-http.test.ts' : flowRebuild ? 'src/test/flow-facts-rebuild-sql.ts' : flowWrites ? 'src/services/traffic-flow-write-transaction.pg.test.ts' : nativeConservation ? 'src/modules/mcp/native-conservation-sql.test.ts' : nativeCreateId ? 'src/modules/mcp/native-create-id-http.test.ts' : ipsecAdapters ? "src/modules/mcp/ipsec-adapters-http.test.ts" : recoveryMaintenance ? "src/modules/mcp/recovery-maintenance-http.test.ts" : scopedCheckpointAdapters ? "src/modules/mcp/scoped-checkpoint-adapters-http.test.ts" : wireguardTransfer ? 'src/modules/mcp/wireguard-transfer-http.test.ts' : interfaceEventsCompatibility ? 'src/modules/mcp/interface-events-compatibility-http.test.ts' : scopedRoutingDiagnostics ? 'src/modules/mcp/scoped-routing-diagnostics-http.test.ts' : applicationAdapters ? 'src/modules/mcp/application-adapters-http.test.ts' : greRevision ? 'src/modules/mcp/gre-revision-http.test.ts' : chrGreRevisionRecovery ? 'src/modules/mcp/chr-gre-revision-recovery.test.ts' : chrGreRevision ? 'src/modules/mcp/chr-gre-revision.test.ts' : chrLegacyOspf ? 'src/modules/mcp/chr-ospf-legacy.test.ts' : chrFilters ? 'src/modules/mcp/chr-filter-write.test.ts' : chrGreMutation ? 'src/modules/mcp/chr-gre-mutation.test.ts' : chrFirewallRestore ? 'src/modules/mcp/chr-firewall-restore.test.ts' : chrAddress ? 'src/modules/mcp/chr-address-list.test.ts' : chrFirewallWrite ? 'src/modules/mcp/chr-firewall-write.test.ts' : chrVpn ? 'src/modules/mcp/chr-vpn-catalog.test.ts' : chrFirewallMove ? 'src/modules/mcp/chr-firewall-move.test.ts' : chrRecursive ? 'src/modules/mcp/chr-recursive-restore.test.ts' : chrIdentity ? 'src/modules/mcp/chr-ospf-identity-readonly.test.ts' : chrRead ? 'src/modules/mcp/chr-readonly-acceptance.test.ts' : chr ? 'src/modules/mcp/chr-acceptance.test.ts' : 'src/modules/mcp/integration.test.ts'], { + const child = spawn(process.execPath, pollMemory ? ['--max-old-space-size=192', '--max-semi-space-size=16', fileURLToPath(new URL('../dist-mcp-memory/services/server-poll-memory.owned.test.js', import.meta.url))] : ['--import', 'tsx', principalSecuritySource ? 'src/modules/mcp/principal-security-source.owned.test.ts' : pollJournalSnapshot ? 'src/services/server-poll-journal-snapshot.owned.test.ts' : pollTerminalArchive ? 'src/services/server-poll-terminal-archive.owned.test.ts' : pollArchiveSchema ? 'src/db/server-poll-terminal-archives.sql.test.ts' : telemetrySettingsConfiguration ? 'src/modules/mcp/telemetry-settings-configuration.owned.test.ts' : routingReservationSchema ? 'src/db/routing-reservation-schema.owned.test.ts' : routingForwardRounds ? 'src/modules/mcp/routing-network-forward-rounds.owned.test.ts' : flowActiveServices ? 'src/services/traffic-flow-active-services.owned.test.ts' : flowActiveRebuild ? 'src/services/traffic-flow-active-rebuild.owned.test.ts' : flowQuality ? 'src/services/traffic-flow-quality.owned.test.ts' : flowWorkerIpc ? 'src/services/traffic-flow-worker-ipc.owned.test.ts' : manualUptime ? 'src/modules/mcp/manual-uptime-http.test.ts' : manualPingGroup ? 'src/modules/mcp/manual-ping-group-http.test.ts' : manualTraffic ? 'src/modules/mcp/manual-traffic-http.test.ts' : manualApiPing ? 'src/modules/mcp/manual-api-ping-http.test.ts' : simpleSpeedChr ? 'src/modules/mcp/simple-speed-chr.test.ts' : simpleSpeed ? 'src/modules/mcp/simple-speed-http.test.ts' : externalPeerMetadata ? 'src/services/bandwidth-external-peer-metadata.sql.test.ts' : operatorPeerMetadata ? 'src/services/bandwidth-operator-metadata.sql.test.ts' : publicRegistryMetadata ? 'src/services/bandwidth-public-registry.sql.test.ts' : operatorDockerSql ? 'src/services/bandwidth-operator-docker-stop.sql.test.ts' : probeOperationRetirement ? 'src/services/server-probe-operation-retirement.owned.test.ts' : ordinaryOperationRetirement ? 'src/modules/mcp/ordinary-operation-retirement.owned.test.ts' : pollOperationRetirement ? 'src/services/server-poll-operation-retirement.owned.test.ts' : mcpOperationOrigin ? 'src/modules/mcp/operation-execution-origin.sql.test.ts' : mcpSpeedOriginal ? 'src/modules/mcp/speed-bandwidth-original.sql.test.ts' : routerConfigurationGuard ? 'src/services/router-configuration-guard.sql.test.ts' : scheduledSpeedOriginal ? 'src/services/bandwidth-scheduled-speed-original.sql.test.ts' : bandwidthSpeedWorkflow ? 'src/services/bandwidth-speed-workflow.sql.test.ts' : restSpeedOriginal ? 'src/services/bandwidth-rest-speed-original.sql.test.ts' : operationLocks ? 'src/services/operation-config-lock-lease.sql.test.ts' : serviceExecutionOrigin ? 'src/modules/mcp/service-execution-origin.sql.test.ts' : bandwidthNodeSource ? 'src/services/bandwidth-node-source.sql.test.ts' : bandwidthExternalRegistry ? 'src/services/bandwidth-external-node-registry.sql.test.ts' : bandwidthNodeRegistry ? 'src/services/bandwidth-node-registry.sql.test.ts' : bandwidthControlSource ? 'src/services/bandwidth-control-source.sql.test.ts' : bandwidthAdmission ? 'src/services/bandwidth-admission.sql.test.ts' : serverPollStorageFresh ? 'src/services/server-poll-storage-fresh.sql.test.ts' : serverPollStorage ? 'src/services/server-poll-storage.sql.test.ts' : probeStorageFresh ? 'src/services/server-probe-storage-fresh.sql.test.ts' : probeLegacyOrphans ? 'src/services/server-probe-legacy-orphans.sql.test.ts' : diagnosticCancelHost ? 'src/modules/mcp/diagnostic-cancellation-storage.sql.test.ts' : ipsecExportHttp ? 'src/modules/mcp/ipsec-existing-certificate-export-http.sql.test.ts' : ipsecExportJournal ? 'src/services/ipsec-existing-certificate-export-rest-journal.sql.test.ts' : migrationDataInventory ? 'src/db/migration-data-inventory.sql.test.ts' : probeStorageHistory ? 'src/services/server-probe-storage-history.sql.test.ts' : probeStorageJournal ? 'src/services/server-probe-storage-journal.sql.test.ts' : controlledSchemaReference ? 'src/db/controlled-schema-reference.sql.test.ts' : controlledMigrations ? 'src/db/controlled-migrations.sql.test.ts' : databaseSchemaControl ? 'src/db/database-schema-control.sql.test.ts' : migrationSchemaInventory ? 'src/db/migration-schema-inventory.sql.test.ts' : routingNetworkNative ? 'src/modules/mcp/routing-network-native.sql.test.ts' : routingReservationRuntime ? 'src/modules/mcp/routing-reservation-runtime.sql.test.ts' : routingReservations ? 'src/modules/mcp/routing-reservation.sql.test.ts' : collectorPartitionWriters ? 'src/services/collector-partition-writes.sql.test.ts' : serverPollDurable ? 'src/services/server-poll-durable-session.sql.test.ts' : routingRecoveryAuthority ? 'src/modules/mcp/routing-recovery-authority.sql.test.ts' : legacyFlowRebuild ? 'src/test/legacy-flow-rebuild-sql.ts' : probeJournal ? 'src/services/server-probe-journal.sql.test.ts' : flowHostKeys ? 'src/modules/mcp/flow-host-keys-http.test.ts' : flowRebuildMcp ? 'src/modules/mcp/flow-facts-rebuild-http.test.ts' : flowRebuild ? 'src/test/flow-facts-rebuild-sql.ts' : flowWrites ? 'src/services/traffic-flow-write-transaction.pg.test.ts' : nativeConservation ? 'src/modules/mcp/native-conservation-sql.test.ts' : nativeCreateId ? 'src/modules/mcp/native-create-id-http.test.ts' : ipsecAdapters ? "src/modules/mcp/ipsec-adapters-http.test.ts" : recoveryMaintenance ? "src/modules/mcp/recovery-maintenance-http.test.ts" : scopedCheckpointAdapters ? "src/modules/mcp/scoped-checkpoint-adapters-http.test.ts" : wireguardTransfer ? 'src/modules/mcp/wireguard-transfer-http.test.ts' : interfaceEventsCompatibility ? 'src/modules/mcp/interface-events-compatibility-http.test.ts' : scopedRoutingDiagnostics ? 'src/modules/mcp/scoped-routing-diagnostics-http.test.ts' : applicationAdapters ? 'src/modules/mcp/application-adapters-http.test.ts' : greRevision ? 'src/modules/mcp/gre-revision-http.test.ts' : chrGreRevisionRecovery ? 'src/modules/mcp/chr-gre-revision-recovery.test.ts' : chrGreRevision ? 'src/modules/mcp/chr-gre-revision.test.ts' : chrLegacyOspf ? 'src/modules/mcp/chr-ospf-legacy.test.ts' : chrFilters ? 'src/modules/mcp/chr-filter-write.test.ts' : chrGreMutation ? 'src/modules/mcp/chr-gre-mutation.test.ts' : chrFirewallRestore ? 'src/modules/mcp/chr-firewall-restore.test.ts' : chrAddress ? 'src/modules/mcp/chr-address-list.test.ts' : chrFirewallWrite ? 'src/modules/mcp/chr-firewall-write.test.ts' : chrVpn ? 'src/modules/mcp/chr-vpn-catalog.test.ts' : chrFirewallMove ? 'src/modules/mcp/chr-firewall-move.test.ts' : chrRecursive ? 'src/modules/mcp/chr-recursive-restore.test.ts' : chrIdentity ? 'src/modules/mcp/chr-ospf-identity-readonly.test.ts' : chrRead ? 'src/modules/mcp/chr-readonly-acceptance.test.ts' : chr ? 'src/modules/mcp/chr-acceptance.test.ts' : 'src/modules/mcp/integration.test.ts'], { cwd: pollMemory ? directory : new URL('../', import.meta.url), stdio: 'inherit', env: { ...(pollMemory ? pollMemoryEnvironment : process.env), ...(routingOriginalEffects ? { MMAPP_OWN_ROUTING_ORIGINAL_EFFECTS: 'true' } : {}), DATABASE_URL: `postgresql://mcp_test:${password}@127.0.0.1:${port}/postgres`, DATABASE_PATH: join(directory, 'no-legacy-database.sqlite'), SQLITE_IMPORT_FULL_HISTORY: 'false', MCP_STATE_DIR: join(directory, 'state'), AUTH_REQUIRED: 'true', AUTH_JWT_SECRET: 'mcp-integration-test-only', MMAPP_TEST_WG_EXPORT_ONLY: String(wireguardExport), - MCP_PUBLIC_URL: 'http://localhost:8000/mcp', NODE_ENV: 'test', PATH: testPath, NODE_OPTIONS: pollMemory ? '' : process.env.NODE_OPTIONS, MMAPP_OWN_PRINCIPAL_SECURITY_SOURCE: String(principalSecuritySource), MMAPP_OWN_POLL_JOURNAL_SNAPSHOT: String(pollJournalSnapshot), MMAPP_OWN_POLL_MEMORY: String(pollMemory), MMAPP_OWN_POLL_BOOTSTRAP: String(pollMemoryBootstrap), MMAPP_OWN_POLL_COST_DIAGNOSTICS: String(pollMemoryCost), MMAPP_OWN_POLL_ARCHIVE_CLAIM_LOSS: String(pollArchiveClaimLoss), MMAPP_OWN_POLL_ARCHIVE_HISTORY: String(pollArchiveHistory), MMAPP_OWN_POLL_ARCHIVE_COLD: String(pollArchiveCold), MMAPP_OWN_POLL_ARCHIVE_LOST_ACK: String(pollArchiveLostAck), MMAPP_OWN_POLL_TERMINAL_ARCHIVE: String(pollTerminalArchive), MMAPP_OWN_POLL_ARCHIVE_SCHEMA: String(pollArchiveSchema), MMAPP_OWN_TELEMETRY_SETTINGS_CONFIGURATION: String(telemetrySettingsConfiguration), MMAPP_OWN_ROUTING_RESERVATION_SCHEMA: String(routingReservationSchema), MMAPP_OWN_ROUTING_FORWARD_ROUNDS: String(routingForwardRounds), MMAPP_OWN_FLOW_ACTIVE_SERVICES: String(flowActiveServices), MMAPP_OWN_FLOW_ACTIVE_REBUILD: String(flowActiveRebuild), MMAPP_OWN_FLOW_QUALITY: String(flowQuality), MMAPP_OWN_FLOW_WORKER_IPC: String(flowWorkerIpc), MMAPP_OWN_MANUAL_UPTIME: String(manualUptime), MMAPP_OWN_MANUAL_PING_GROUP: String(manualPingGroup), MMAPP_OWN_MANUAL_TRAFFIC: String(manualTraffic), MMAPP_OWN_MANUAL_API_PING: String(manualApiPing), MMAPP_OWN_SIMPLE_SPEED: String(simpleSpeed), MMAPP_OWN_SIMPLE_SPEED_CHR: String(simpleSpeedChr), MMAPP_OWN_DIAGNOSTIC_CANCEL_HOST: String(diagnosticCancelHost), MMAPP_OWN_OPERATOR_DOCKER_SQL: String(operatorDockerSql), MMAPP_OWN_PUBLIC_REGISTRY_METADATA: String(publicRegistryMetadata), MMAPP_OWN_OPERATOR_PEER_METADATA: String(operatorPeerMetadata), MMAPP_OWN_EXTERNAL_PEER_METADATA: String(externalPeerMetadata), MMAPP_OWN_BANDWIDTH_EXTERNAL_REGISTRY: String(externalPeerMetadata || operatorPeerMetadata || publicRegistryMetadata || operatorDockerSql || mcpSpeedOriginal || bandwidthSpeedWorkflow || bandwidthExternalRegistry), MMAPP_OWN_BANDWIDTH_SPEED_WORKFLOW: String(bandwidthSpeedWorkflow), MMAPP_OWN_REST_SPEED_ORIGINAL: String(restSpeedOriginal), MMAPP_OWN_MCP_OPERATION_ORIGIN: String(mcpOperationOrigin), MMAPP_OWN_MCP_SPEED_ORIGINAL: String(mcpSpeedOriginal), MMAPP_OWN_SCHEDULED_SPEED_ORIGINAL: String(scheduledSpeedOriginal), MMAPP_TEST_ROUTER_GUARD_OWNED: String(routerConfigurationGuard), MMAPP_TEST_ROUTER_GUARD_CLUSTER_DISPOSABLE: String(routerConfigurationGuard), MMAPP_TEST_ROUTER_GUARD_ADMIN_URL: routerConfigurationGuard ? `postgresql://mcp_test:${password}@127.0.0.1:${port}/postgres` : '', MMAPP_OWN_BANDWIDTH_ADMISSION: String(externalPeerMetadata || operatorPeerMetadata || publicRegistryMetadata || operatorDockerSql || mcpOperationOrigin || mcpSpeedOriginal || scheduledSpeedOriginal || bandwidthSpeedWorkflow || restSpeedOriginal || bandwidthExternalRegistry || operationLocks || serviceExecutionOrigin || bandwidthAdmission || bandwidthControlSource || bandwidthNodeRegistry || bandwidthNodeSource), MMAPP_OWN_BANDWIDTH_RESTORE: String((bandwidthAdmission || bandwidthControlSource || bandwidthNodeRegistry || bandwidthNodeSource) && process.argv.includes('--restore')), MMAPP_OWN_IPSEC_EXPORT_HTTP: String(ipsecExportHttp), MMAPP_OWN_IPSEC_EXPORT_JOURNAL: String(ipsecExportJournal), MMAPP_OWN_SQL_DB: 'postgres', MMAPP_OWN_PROBE_LEGACY_ORPHANS: String(probeLegacyOrphans), MMAPP_OWN_PROBE_STORAGE_HISTORY: String(probeStorageHistory), MMAPP_OWN_PROBE_STORAGE_FRESH: String(probeStorageFresh), MMAPP_OWN_PROBE_STORAGE_JOURNAL: String(probeStorageJournal || probeStorageFresh), MMAPP_OWN_CONTROLLED_SCHEMA_REFERENCE: String(controlledSchemaReference), MMAPP_OWN_CONTROLLED_MIGRATIONS: String(controlledMigrations), MMAPP_OWN_DATABASE_SCHEMA_CONTROL: String(databaseSchemaControl), MMAPP_OWN_MIGRATION_DATA_INVENTORY: String(migrationDataInventory), MMAPP_OWN_MIGRATION_SCHEMA_INVENTORY: String(migrationSchemaInventory), MMAPP_OWN_ROUTING_NETWORK_NATIVE: String(routingNetworkNative), MMAPP_OWN_ROUTING_RESERVATIONS: String(routingReservations || routingReservationRuntime), MMAPP_OWN_COLLECTOR_PARTITION_WRITERS: String(collectorPartitionWriters), MMAPP_OWN_SERVER_POLL_STORAGE_FRESH: String(serverPollStorageFresh), MMAPP_OWN_SERVER_POLL_STORAGE: String(serverPollStorage), MMAPP_OWN_SERVER_POLL_DURABLE: String(serverPollDurable), MMAPP_OWN_ROUTING_RECOVERY_AUTHORITY: String(routingRecoveryAuthority), MMAPP_OWN_LEGACY_FLOW_REBUILD: String(legacyFlowRebuild), MMAPP_OWN_PROBE_JOURNAL: String(probeJournal), MMAPP_MODULE_COMPLETION_ONLY: String(moduleCompletion), MMAPP_MODULE_COMPLETION_TARGET: completionTarget, MMAPP_OWN_FLOW_HOST_KEYS: String(flowHostKeys), MMAPP_OWN_MCP_FLOW_REBUILD: String(flowRebuildMcp), MMAPP_OWN_FLOW_WRITES: String(flowWrites), MMAPP_OWN_FLOW_REBUILD: String(flowRebuild) }, + MCP_PUBLIC_URL: 'http://localhost:8000/mcp', NODE_ENV: 'test', PATH: testPath, NODE_OPTIONS: pollMemory ? '' : process.env.NODE_OPTIONS, MMAPP_OWN_PRINCIPAL_SECURITY_SOURCE: String(principalSecuritySource), MMAPP_OWN_POLL_JOURNAL_SNAPSHOT: String(pollJournalSnapshot), MMAPP_OWN_POLL_MEMORY: String(pollMemory), MMAPP_OWN_POLL_BOOTSTRAP: String(pollMemoryBootstrap), MMAPP_OWN_POLL_COST_DIAGNOSTICS: String(pollMemoryCost), MMAPP_OWN_POLL_ARCHIVE_CLAIM_LOSS: String(pollArchiveClaimLoss), MMAPP_OWN_POLL_ARCHIVE_HISTORY: String(pollArchiveHistory), MMAPP_OWN_POLL_ARCHIVE_COLD: String(pollArchiveCold), MMAPP_OWN_POLL_ARCHIVE_LOST_ACK: String(pollArchiveLostAck), MMAPP_OWN_POLL_TERMINAL_ARCHIVE: String(pollTerminalArchive), MMAPP_OWN_POLL_ARCHIVE_SCHEMA: String(pollArchiveSchema), MMAPP_OWN_TELEMETRY_SETTINGS_CONFIGURATION: String(telemetrySettingsConfiguration), MMAPP_OWN_ROUTING_RESERVATION_SCHEMA: String(routingReservationSchema), MMAPP_OWN_ROUTING_FORWARD_ROUNDS: String(routingForwardRounds), MMAPP_OWN_FLOW_ACTIVE_SERVICES: String(flowActiveServices), MMAPP_OWN_FLOW_ACTIVE_REBUILD: String(flowActiveRebuild), MMAPP_OWN_FLOW_QUALITY: String(flowQuality), MMAPP_OWN_FLOW_WORKER_IPC: String(flowWorkerIpc), MMAPP_OWN_MANUAL_UPTIME: String(manualUptime), MMAPP_OWN_MANUAL_PING_GROUP: String(manualPingGroup), MMAPP_OWN_MANUAL_TRAFFIC: String(manualTraffic), MMAPP_OWN_MANUAL_API_PING: String(manualApiPing), MMAPP_OWN_SIMPLE_SPEED: String(simpleSpeed), MMAPP_OWN_SIMPLE_SPEED_CHR: String(simpleSpeedChr), MMAPP_OWN_DIAGNOSTIC_CANCEL_HOST: String(diagnosticCancelHost), MMAPP_OWN_OPERATOR_DOCKER_SQL: String(operatorDockerSql), MMAPP_OWN_PUBLIC_REGISTRY_METADATA: String(publicRegistryMetadata), MMAPP_OWN_OPERATOR_PEER_METADATA: String(operatorPeerMetadata), MMAPP_OWN_EXTERNAL_PEER_METADATA: String(externalPeerMetadata), MMAPP_OWN_BANDWIDTH_EXTERNAL_REGISTRY: String(externalPeerMetadata || operatorPeerMetadata || publicRegistryMetadata || operatorDockerSql || mcpSpeedOriginal || bandwidthSpeedWorkflow || bandwidthExternalRegistry), MMAPP_OWN_BANDWIDTH_SPEED_WORKFLOW: String(bandwidthSpeedWorkflow), MMAPP_OWN_REST_SPEED_ORIGINAL: String(restSpeedOriginal), MMAPP_OWN_MCP_OPERATION_ORIGIN: String(mcpOperationOrigin), MMAPP_OWN_PROBE_OPERATION_RETIREMENT: String(probeOperationRetirement), MMAPP_OWN_MCP_SPEED_ORIGINAL: String(mcpSpeedOriginal), MMAPP_OWN_SCHEDULED_SPEED_ORIGINAL: String(scheduledSpeedOriginal), MMAPP_TEST_ROUTER_GUARD_OWNED: String(routerConfigurationGuard), MMAPP_TEST_ROUTER_GUARD_CLUSTER_DISPOSABLE: String(routerConfigurationGuard), MMAPP_TEST_ROUTER_GUARD_ADMIN_URL: routerConfigurationGuard ? `postgresql://mcp_test:${password}@127.0.0.1:${port}/postgres` : '', MMAPP_OWN_BANDWIDTH_ADMISSION: String(externalPeerMetadata || operatorPeerMetadata || publicRegistryMetadata || operatorDockerSql || mcpOperationOrigin || mcpSpeedOriginal || scheduledSpeedOriginal || bandwidthSpeedWorkflow || restSpeedOriginal || bandwidthExternalRegistry || operationLocks || serviceExecutionOrigin || bandwidthAdmission || bandwidthControlSource || bandwidthNodeRegistry || bandwidthNodeSource), MMAPP_OWN_BANDWIDTH_RESTORE: String((bandwidthAdmission || bandwidthControlSource || bandwidthNodeRegistry || bandwidthNodeSource) && process.argv.includes('--restore')), MMAPP_OWN_IPSEC_EXPORT_HTTP: String(ipsecExportHttp), MMAPP_OWN_IPSEC_EXPORT_JOURNAL: String(ipsecExportJournal), MMAPP_OWN_SQL_DB: 'postgres', MMAPP_OWN_PROBE_LEGACY_ORPHANS: String(probeLegacyOrphans), MMAPP_OWN_PROBE_STORAGE_HISTORY: String(probeStorageHistory), MMAPP_OWN_PROBE_STORAGE_FRESH: String(probeStorageFresh), MMAPP_OWN_PROBE_STORAGE_JOURNAL: String(probeStorageJournal || probeStorageFresh), MMAPP_OWN_CONTROLLED_SCHEMA_REFERENCE: String(controlledSchemaReference), MMAPP_OWN_CONTROLLED_MIGRATIONS: String(controlledMigrations), MMAPP_OWN_DATABASE_SCHEMA_CONTROL: String(databaseSchemaControl), MMAPP_OWN_MIGRATION_DATA_INVENTORY: String(migrationDataInventory), MMAPP_OWN_MIGRATION_SCHEMA_INVENTORY: String(migrationSchemaInventory), MMAPP_OWN_ROUTING_NETWORK_NATIVE: String(routingNetworkNative), MMAPP_OWN_ROUTING_RESERVATIONS: String(routingReservations || routingReservationRuntime), MMAPP_OWN_COLLECTOR_PARTITION_WRITERS: String(collectorPartitionWriters), MMAPP_OWN_SERVER_POLL_STORAGE_FRESH: String(serverPollStorageFresh), MMAPP_OWN_SERVER_POLL_STORAGE: String(serverPollStorage), MMAPP_OWN_SERVER_POLL_DURABLE: String(serverPollDurable), MMAPP_OWN_ROUTING_RECOVERY_AUTHORITY: String(routingRecoveryAuthority), MMAPP_OWN_LEGACY_FLOW_REBUILD: String(legacyFlowRebuild), MMAPP_OWN_PROBE_JOURNAL: String(probeJournal), MMAPP_MODULE_COMPLETION_ONLY: String(moduleCompletion), MMAPP_MODULE_COMPLETION_TARGET: completionTarget, MMAPP_OWN_FLOW_HOST_KEYS: String(flowHostKeys), MMAPP_OWN_MCP_FLOW_REBUILD: String(flowRebuildMcp), MMAPP_OWN_FLOW_WRITES: String(flowWrites), MMAPP_OWN_FLOW_REBUILD: String(flowRebuild) }, }) child.once('error', reject); child.once('exit', code => resolve(code ?? 1)) }) @@ -286,6 +289,9 @@ try { else if (result === 0 && operatorPeerMetadata) console.log('Operator peer metadata: actual managed endpoint SQL and genuine localhost license origin; synthetic Q, external metadata and full operator window remain open') else if (result === 0 && publicRegistryMetadata) console.log('Public registry metadata: actual owned PostgreSQL and same physical lease; no secret/Q/native/write authority or backend-window acceptance') else if (result === 0 && operatorDockerSql) console.log('Operator Docker SQL: actual owned PostgreSQL with mocked Docker HTTP only; not a real Docker/backend-window acceptance') + else if (result === 0 && ordinaryOperationRetirement) console.log('Ordinary operation retirement: owned actual logs-settings apply and worker') + else if (result === 0 && pollOperationRetirement) console.log('Poll operation retirement: owned authenticated historical fixtures and current reader/GC') + else if (result === 0 && probeOperationRetirement) console.log('Diagnostic retirement: actual owned worker tick, PostgreSQL and product read/replay; no live scheduling') else if (result === 0 && mcpOperationOrigin) console.log('MCP operation root: owned PostgreSQL ordinary runner provenance; no restore/recovery, global writer activation, CHR or deployment') else if (result === 0 && mcpSpeedOriginal) console.log('MCP speed: genuine trusted service context with real connection/grant/SQL, host receipt and localhost native; synthetic Q/mapping; not a public MCP runner/action acceptance') else if (result === 0 && routerConfigurationGuard) console.log('Router configuration guard: dedicated owned databases/roles and localhost native fixture; no application migration, production activation or real CHR') diff --git a/backend/src/db/migration-manifest.test.ts b/backend/src/db/migration-manifest.test.ts index 9a3cac6e..e5757324 100644 --- a/backend/src/db/migration-manifest.test.ts +++ b/backend/src/db/migration-manifest.test.ts @@ -12,10 +12,10 @@ const reviewed = (source: string, extra: Partial = {}) => { const bytes = Buffer.from(source), file = "9000_fixture.sql" return () => analyzeMigration(file, bytes, { file, sha256: createHash("sha256").update(bytes).digest("hex"), transaction: "owner-transaction", selfMarker: false, opaqueBodies: 0, bodyReview: [], effects: ["Explicit inert fixture review"], ...extra }) } -await test("all44 exact file pins and every top-level schema operation are inventoried", () => { - assert.equal(inventory.length, 44); assert.equal(MIGRATION_MANIFEST.length, 44) +await test("all 53 exact file pins and every top-level schema operation are inventoried", () => { + assert.equal(inventory.length, 53); assert.equal(MIGRATION_MANIFEST.length, 53) assert.equal(inventory.filter(file => file.transaction === "owner-transaction").length, 17) - assert.equal(inventory.filter(file => file.transaction === "self-managed-single-transaction").length, 27) + assert.equal(inventory.filter(file => file.transaction === "self-managed-single-transaction").length, 36) assert(inventory.every(file => file.operations.length === file.statementCount && file.effects.length > 0)) let tables = new Set() for (const file of inventory) { @@ -32,8 +32,8 @@ await test("DO BEGIN and stored function BEGIN are never transaction boundaries" assert.equal(endpoint.topLevelTransactionControls.length, 2); assert.equal(endpoint.opaqueBodies.length, 2) assert.deepEqual(endpoint.opaqueBodies.map(body => body.kind), ["stored-function-body", "executed-do-body"]) }) -await test("0035 through0044 have exact atomic file-owned terminal markers", () => { - assert.deepEqual(inventory.filter(file => file.selfMarker).map(file => file.id), ["0035_server_probe_journal", "0036_diagnostic_cancellations", "0037_server_probe_storage_v2", "0038_server_poll_journal_and_writer_guards", "0039_server_poll_storage", "0040_mcp_routing_reservations", "0041_server_poll_terminal_archives", "0042_server_poll_settings_epoch", "0043_mcp_routing_forward_charges", "0044_mcp_routing_manual_inverse_charges"]) +await test("0035 through 0053 have exact atomic file-owned terminal markers", () => { + assert.deepEqual(inventory.filter(file => file.selfMarker).map(file => file.id), ["0035_server_probe_journal", "0036_diagnostic_cancellations", "0037_server_probe_storage_v2", "0038_server_poll_journal_and_writer_guards", "0039_server_poll_storage", "0040_mcp_routing_reservations", "0041_server_poll_terminal_archives", "0042_server_poll_settings_epoch", "0043_mcp_routing_forward_charges", "0044_mcp_routing_manual_inverse_charges", "0045_mcp_operation_replay_tombstones", "0046_server_poll_operation_retirement", "0047_server_poll_retirement_claims", "0048_mcp_ordinary_retirement_claims", "0049_mcp_ordinary_telemetry_retirement", "0050_server_probe_retired_sources", "0051_server_probe_retirement_claims", "0052_server_probe_retired_gc", "0053_server_probe_terminal_proofs"]) for (const file of inventory.filter(file => file.selfMarker)) { assert.equal(file.selfMarker!.atomicWithAllFileEffects, true); assert.equal(file.selfMarker!.statement, file.statementCount - 2); assert.equal(file.currentOuterMarkerGap, false) } assert.equal(inventory.filter(file => file.currentOuterMarkerGap).length, 34) }) diff --git a/backend/src/db/migration-manifest.ts b/backend/src/db/migration-manifest.ts index 31c0fc37..e9ccc324 100644 --- a/backend/src/db/migration-manifest.ts +++ b/backend/src/db/migration-manifest.ts @@ -702,6 +702,132 @@ const classifiedMigrationPins: readonly MigrationReview[] = [ "Adds one one-unit immutable manual inverse charge table, two charge indexes, one manual-history page index, and four triggers. Existing43 bytes, reserved2 and generic debit ownership remain unchanged.", "No inferred/backfilled identity, legacy timestamp debit, permission change, source enrollment, public controller activation or READY. Full RT3/all-writer/family-deletion proof remains open." ] + }, + { + "file": "0045_mcp_operation_replay_tombstones.sql", + "sha256": "8edf5b074c7eef49dad7835c44ac2539df4347f19eb9dd9b2aa7005bafaadbc0", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds one empty replay identity table with plan/operation uniqueness and connection-scoped idempotency digest uniqueness.", + "Table creation refuses an unmarked collision; the exact marker commits with the schema. No backfill, operation deletion, or worker activation." + ] + }, + { + "file": "0046_server_poll_operation_retirement.sql", + "sha256": "75d7956ea8134b01bcc88da083b77eb52f6e1ad70ad854918ed3380cb0e8cf88", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 4, + "bodyReview": [ + "Archive guard permits only fixed receipt-scoped V1 to V2 updates with matching approved terminal operation, tombstone and SQL 180-day cutoff.", + "Deferred guard independently requires the tombstone, physically absent operation and plan, and exact ledger accounting.", + "Admission guard retains existing paths and applies exact V2 archive-byte delta to both owners without changing row charges or lifetime fields.", + "Existing GC retains its authority/activity cutoff and admits retired identity only with exact poll tombstone and compatible V2 archive when present." + ], + "effects": [ + "Extends the same archive table to a bounded version 2, changes three guards and adds one deferred retirement guard.", + "No automatic conversion, worker activation, operation deletion or retention-window adjustment in migration." + ] + }, + { + "file": "0047_server_poll_retirement_claims.sql", + "sha256": "8c01ab14d241f715af021e8a7d7f5bb273b450b32f566140359260282524e01d", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 2, + "bodyReview": [ + "Protects exact current-transaction V1 claim insertion and permits deletion only as the owning admission is cascaded by GC.", + "Requires exact receipt, V1 archive proof and claim token already committed in another transaction before V2 update." + ], + "effects": [ + "Adds one durable one-shot poll retirement claim per admission, cascading only with that admission's existing guarded GC.", + "No migration-time conversion, backfill, dispatch, or quota change." + ] + }, + { + "file": "0048_mcp_ordinary_retirement_claims.sql", + "sha256": "5bc3598c9f5d09611e98d05345accda6d39b199a583bba789b97a24140fafbfd", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 1, + "bodyReview": [ + "Makes ordinary claims immutable and allows insert only for the exact linked settled and aged logs.settings.update v1 operation." + ], + "effects": [ + "Adds one durable ordinary-operation claim with an authenticated source digest. No migration-time data purge or backfill." + ] + }, + { + "file": "0049_mcp_ordinary_telemetry_retirement.sql", + "sha256": "68993e924d20db6b08973846a0da367fe89fe2948c414c810e9b171dbbae6761", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 1, + "bodyReview": [ + "Extends the same immutable ordinary claim guard to four explicitly versioned settled telemetry settings families." + ], + "effects": [ + "Replaces only the ordinary claim guard function. No backfill, data purge, or changes to poll or diagnostic storage." + ] + }, + { + "file": "0050_server_probe_retired_sources.sql", + "sha256": "0468ac7e53638b532557273f55d065ff4cde092d8b08073f5860bab35263aead", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 1, + "bodyReview": [ + "Refuses update, delete, and truncate of a signed retired diagnostic source pending a separately reviewed lifecycle." + ], + "effects": [ + "Creates empty bounded storage linking exact diagnostic admission and replay tombstone IDs. No backfill, retirement, or GC activation." + ] + }, + { + "file": "0051_server_probe_retirement_claims.sql", + "sha256": "b38876b5da44cf14e20a9df0baaa233ea4de81e471b38dddd370e149c8116243", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 2, + "bodyReview": [ + "Restricts diagnostic claims to their original settled 180-day source and makes each claim permanent.", + "Restricts archival insertion to the exact committed claim and current original source and tombstone identity." + ], + "effects": [ + "Creates empty diagnostic retirement claims and guards the archive insert. No conversion, backfill, or GC activation." + ] + }, + { + "file": "0052_server_probe_retired_gc.sql", + "sha256": "ea56a1f1bdabfce606bdfec56b6a7b31d45b0187fbea4c900bc656ae320f2fdd", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 3, + "bodyReview": [ + "Validates the exact retired diagnostic identity and the later of current and persisted retention plus the original safety window.", + "Only permits archive removal during an exact guarded admission cascade after journal removal.", + "Adds the exact retired GC branch while retaining the live admission eligibility predicate." + ], + "effects": [ + "Changes only the retired source admission FK to guarded cascade and adds an explicit exact GC path. No collection runs during migration." + ] + }, + { + "file": "0053_server_probe_terminal_proofs.sql", + "sha256": "759ec8c349fcb24c43af410ad6ade714432678b3a021c94bd982080d6a8986be", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 2, + "bodyReview": [ + "Pins terminal proof to the current retained original on insert and an exact committed retirement claim on delete.", + "Extends claims only to the identical signed terminal proof when the original diagnostic admission is gone." + ], + "effects": [ + "Creates empty bounded private terminal proof storage. Existing payload retention, scheduler, and archive GC remain unchanged." + ] } ] @@ -724,7 +850,7 @@ export const migrationManifestDigest = () => hash(JSON.stringify(MIGRATION_MANIF // resetting the accumulated public table set at the reviewed destructive0002. // This is a necessary presence check, not a substitute for exact live column, // constraint/index/function/trigger comparison by the control protocol. -export const CURRENT_REQUIRED_TABLES: readonly string[] = Object.freeze(["acme_settings","alert_engine_confirm_pending","alert_engine_cursor","alert_engine_prev_live","alert_engine_state","alert_groups","alert_history","alert_outbox","alert_rule_conditions","alert_rule_targets","alert_rules","alert_telegram_settings","app_users","backup_entries","backup_schedule_settings","backup_storage_settings","certificate_issue_jobs","certificate_renew_settings","config_revisions","data_migration","events","evobgp_settings","filter_rules","flow_asn_meta","flow_buckets","flow_daily_dims","flow_daily_facts","flow_hour_facts","flow_ip_meta","flow_minute_dims","flow_minute_stats","flow_settings_intents","flow_write_receipts","geoip_settings","internet_path_settings","internet_path_snapshots","mcp_artifacts","mcp_audit","mcp_budget_buckets","mcp_connections","mcp_credentials","mcp_diagnostic_cancellations","mcp_drift","mcp_grants","mcp_management_channels","mcp_oauth_clients","mcp_oauth_requests","mcp_operation_checkpoints","mcp_operation_steps","mcp_operations","mcp_owner_revocations","mcp_path_checks","mcp_plans","mcp_portal_events","mcp_protections","mcp_read_leases","mcp_recovery_intents","mcp_requests","mcp_routing_forward_charges","mcp_routing_manual_inverse_charges","mcp_routing_observations","mcp_routing_profiles","mcp_routing_reservation_slots","mcp_routing_reservations","mcp_scoped_recovery_intents","mcp_secrets","mcp_settings","mcp_unauthorized_buckets","recursive_routes","router_log_collection_state","router_log_entries","router_log_settings","scheduler_runs","schema_migrations","server_poll_acks","server_poll_archives","server_poll_executions","server_poll_intents","server_poll_quarantines","server_poll_receipts","server_poll_storage_admissions","server_poll_storage_usage","server_probe_execution_payloads","server_probe_execution_quarantines","server_probe_execution_stages","server_probe_executions","server_probe_storage_admissions","server_probe_storage_usage","server_snapshots","servers","servers_api_ping_settings","servers_rest_ping_samples","traffic_collection_state","traffic_flow_settings","traffic_samples","traffic_settings","uptime_probe_samples","uptime_probes","uptime_resource_samples","uptime_settings","uptime_speed_probes","uptime_speed_test_runs","user_interface_bindings","vrf_production_tickets"]) +export const CURRENT_REQUIRED_TABLES: readonly string[] = Object.freeze(["acme_settings","alert_engine_confirm_pending","alert_engine_cursor","alert_engine_prev_live","alert_engine_state","alert_groups","alert_history","alert_outbox","alert_rule_conditions","alert_rule_targets","alert_rules","alert_telegram_settings","app_users","backup_entries","backup_schedule_settings","backup_storage_settings","certificate_issue_jobs","certificate_renew_settings","config_revisions","data_migration","events","evobgp_settings","filter_rules","flow_asn_meta","flow_buckets","flow_daily_dims","flow_daily_facts","flow_hour_facts","flow_ip_meta","flow_minute_dims","flow_minute_stats","flow_settings_intents","flow_write_receipts","geoip_settings","internet_path_settings","internet_path_snapshots","mcp_artifacts","mcp_audit","mcp_budget_buckets","mcp_connections","mcp_credentials","mcp_diagnostic_cancellations","mcp_drift","mcp_grants","mcp_management_channels","mcp_oauth_clients","mcp_oauth_requests","mcp_operation_checkpoints","mcp_operation_replay_tombstones","mcp_operation_steps","mcp_operations","mcp_ordinary_retirement_claims","mcp_owner_revocations","mcp_path_checks","mcp_plans","mcp_portal_events","mcp_protections","mcp_read_leases","mcp_recovery_intents","mcp_requests","mcp_routing_forward_charges","mcp_routing_manual_inverse_charges","mcp_routing_observations","mcp_routing_profiles","mcp_routing_reservation_slots","mcp_routing_reservations","mcp_scoped_recovery_intents","mcp_secrets","mcp_settings","mcp_unauthorized_buckets","recursive_routes","router_log_collection_state","router_log_entries","router_log_settings","scheduler_runs","schema_migrations","server_poll_acks","server_poll_archives","server_poll_executions","server_poll_intents","server_poll_quarantines","server_poll_receipts","server_poll_retirement_claims","server_poll_storage_admissions","server_poll_storage_usage","server_probe_execution_payloads","server_probe_execution_quarantines","server_probe_execution_stages","server_probe_executions","server_probe_retired_sources","server_probe_retirement_claims","server_probe_storage_admissions","server_probe_storage_usage","server_probe_terminal_proofs","server_snapshots","servers","servers_api_ping_settings","servers_rest_ping_samples","traffic_collection_state","traffic_flow_settings","traffic_samples","traffic_settings","uptime_probe_samples","uptime_probes","uptime_resource_samples","uptime_settings","uptime_speed_probes","uptime_speed_test_runs","user_interface_bindings","vrf_production_tickets"]) export interface LegacyBaselineEvidence { inventoryComplete: boolean; databaseGeneration: string; unresolvedAttemptIds: readonly string[] diff --git a/backend/src/modules/mcp/actions.ts b/backend/src/modules/mcp/actions.ts index 16bcc7e7..0556ae93 100644 --- a/backend/src/modules/mcp/actions.ts +++ b/backend/src/modules/mcp/actions.ts @@ -14,6 +14,7 @@ import { logFiltersSchema, logSettingsPatchSchema } from "@mmapp/contracts" import { appUserCreateSchema, appUserUpdateSchema, appUserRoleSchema, sectionPermSchema, serverPermSchema, userBindingCreateSchema } from "@mmapp/contracts/users" import type { McpOperationDescriptor } from "@mmapp/contracts/mcp" import { hasPermission } from "../../lib/permissions.js" +import { assertOperationReplayScope, findOperationReplayTombstone, operationHistoryExpired } from "./operation-replay-tombstones.js" import { pool } from "../../db/index.js" import { getServerReadById } from "../servers/service/servers-service.js" import { listUsers, getUserById, listInterfaceCatalog, getInterfaceCatalogState } from "../users/service/users-service.js" @@ -473,7 +474,12 @@ add("changes.cancel", "mm_change_cancel", "Отменить свой незап add("operations.read", "mm_operation_get", "Результат своей операции", "mcp", null, "read", "connection", z.object({ operationId: z.string().uuid() }).strict(), { read: async (input, context) => { const { rows: [operation] } = await pool.query("SELECT * FROM mcp_operations WHERE id=$1 AND connection_id=$2", [input.operationId, context.connection.id]) - if (!operation) throw new McpError("NOT_FOUND", "Операция не найдена", 404) + if (!operation) { + const tombstone = await findOperationReplayTombstone(pool, context.connection.id, String(input.operationId)) + if (!tombstone) throw new McpError("NOT_FOUND", "Операция не найдена", 404) + assertOperationReplayScope(tombstone, context.connection) + operationHistoryExpired() + } await assertPlanReadScope(await (await import("./runtime.js")).loadPlan(operation.plan_id), context) const { rows: steps } = await pool.query("SELECT * FROM mcp_operation_steps WHERE operation_id=$1 ORDER BY step_index", [input.operationId]) return { ...operation, steps } diff --git a/backend/src/modules/mcp/diagnostic-cancellation-source.ts b/backend/src/modules/mcp/diagnostic-cancellation-source.ts index 2946b8a9..8feb2d70 100644 --- a/backend/src/modules/mcp/diagnostic-cancellation-source.ts +++ b/backend/src/modules/mcp/diagnostic-cancellation-source.ts @@ -115,3 +115,9 @@ export const createDiagnosticCancellationSourceForTests = sourceLoader /** Historical authentication only; confers no execution/cancellation grant. */ export const readServerProbeExecutionSource = readSourceLoader(productionCrypto) export const createReadServerProbeExecutionSourceForTests = readSourceLoader +/** Exact previously locked source projection for private history conversion. */ +export function authenticateServerProbeExecutionSourceRow(row: Record, operationId: string, + owner: Readonly) { + checkLookup(operationId, owner) + return validateSourceRow(row, operationId, owner, productionCrypto) +} diff --git a/backend/src/modules/mcp/global-reconciliation-actions.test.ts b/backend/src/modules/mcp/global-reconciliation-actions.test.ts new file mode 100644 index 00000000..14a8e295 --- /dev/null +++ b/backend/src/modules/mcp/global-reconciliation-actions.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { test } from "node:test" +import type { McpAction, ActionContext } from "./action-types.js" +import { registerGlobalReconciliationActionsForTests } from "./global-reconciliation-actions.js" + +type Register = Parameters[0] +type Dependencies = Parameters[1] + +function fixture() { + const connectionId = randomUUID(), operationId = randomUUID(), ownerId = "current-retired-owner" + const context = { principal: { connectionId }, connection: { id: connectionId, owner_id: ownerId, owner_permissions: ["mm:servers:write"], + policy: { permissions: ["mm:servers:write"], scope: { allowGlobal: true, operationIds: ["servers.poll", "operations.global.reconcile"], serverIds: [1, 2] } } } } as ActionContext + let action!: Pick + let history: { affectedServerIds: number[] } | undefined = { affectedServerIds: [1, 2] } + let revoke = false, retiredCalls = 0, live = false + const deps = { + connection: async () => { + if (revoke) throw Error("revoked-current-grant") + return structuredClone(context.connection) + }, + query: async (sql: string) => { + if (sql.startsWith("SELECT id FROM servers")) return [{ id: 1 }] + if (live) return [{ id: operationId, state: "applied", operation_id: "servers.poll", evidence: { sealed: "live" } }] + return [] + }, + operation: (id: string) => ({ id, scope: "global", requiredPermissions: ["mm:servers:write"] }), + service: () => ({ reconcile: async () => ({ matchesDesiredState: true, matchesOriginalState: false }) }), + decrypt: async () => JSON.stringify({ target: { scope: "global" }, objects: [{ serverId: 1 }], steps: [{ kind: "service", service: "servers.poll", input: {} }] }), + retired: async (identity, authorize) => { + retiredCalls++ + assert.deepEqual(identity, { operationId, connectionId, ownerId }) + await authorize() + return history + }, + } satisfies Omit & Partial> + registerGlobalReconciliationActionsForTests(((...args: Parameters) => { action = args[8] }) as Register, deps as unknown as Dependencies) + return { context, operationId, history: (value: typeof history) => { history = value }, live: () => { live = true }, + revoke: () => { revoke = true }, calls: () => retiredCalls, read: () => action.read!({ operationId }, context), + target: () => action.target({ operationId }, context) } +} + +test("global action reconciles retired poll only under current ownership and returns no archived material", async () => { + const f = fixture() + assert.deepEqual(await f.target(), { scope: "global" }) + const result = await f.read() as { observedAt: string } + const { observedAt, ...rest } = result + assert(Number.isFinite(Date.parse(observedAt))) + assert.deepEqual(rest, { operationId: f.operationId, state: "applied", automaticRetry: false, + results: [{ stepIndex: 0, service: "servers.poll", matchesDesiredState: true, matchesOriginalState: false }] }) + assert.equal(f.calls(), 2) +}) + +test("global action refuses missing or inaccessible retired history", async () => { + const f = fixture() + f.history(undefined) + await assert.rejects(f.read(), { code: "NOT_FOUND" }) + f.history({ affectedServerIds: [1, 2] }) + f.context.connection.policy.scope.serverIds = [1] + await assert.rejects(f.read(), { code: "SERVER_DENIED" }) + f.context.connection.policy.scope.serverIds = [1, 2] + f.context.connection.policy.scope.operationIds = ["operations.global.reconcile"] + await assert.rejects(f.read(), { code: "PERMISSION_DENIED" }) + f.revoke() + await assert.rejects(f.read(), /revoked-current-grant/) +}) + +test("global action uses the live reconciliation path while original source exists", async () => { + const f = fixture() + f.live() + await f.read() + assert.equal(f.calls(), 0) +}) diff --git a/backend/src/modules/mcp/global-reconciliation-actions.ts b/backend/src/modules/mcp/global-reconciliation-actions.ts index 7de41027..74f89878 100644 --- a/backend/src/modules/mcp/global-reconciliation-actions.ts +++ b/backend/src/modules/mcp/global-reconciliation-actions.ts @@ -1,5 +1,6 @@ import { z } from "zod" import { pool } from "../../db/index.js" +import { readRetiredServerPollHistory } from "../../services/server-poll-history-reader.js" import { hasPermission } from "../../lib/permissions.js" import { effectiveMcpConnection } from "./store.js" import { decryptSecret, fingerprint, McpError } from "./security.js" @@ -7,40 +8,84 @@ import { findMcpOperation } from "./registry.js" import { findMcpService } from "./service-registry.js" import type { add } from "./actions.js" import type { ActionContext, ChangeEvidence } from "./action-types.js" +import type { ConnectionRow } from "./store.js" -async function authorizedSource(operationId: string, context: ActionContext) { +const production = { + connection: effectiveMcpConnection, + query: async (sql: string, values: unknown[] = []) => (await pool.query(sql, values)).rows, + operation: findMcpOperation, + service: findMcpService, + decrypt: decryptSecret, + retired: readRetiredServerPollHistory, +} +type Dependencies = typeof production +const notFound = () => new McpError("NOT_FOUND", "Операция не найдена", 404) + +async function currentAccess(context: ActionContext, deps: Dependencies) { context.signal?.throwIfAborted() - const current = await effectiveMcpConnection(context.principal) + const current = await deps.connection(context.principal) if (!current.policy.scope.allowGlobal || fingerprint(current.policy) !== fingerprint(context.connection.policy) || fingerprint(current.owner_permissions) !== fingerprint(context.connection.owner_permissions)) throw new McpError("PERMISSION_DENIED", "Глобальный допуск сверки изменился", 403) - const ids = (await pool.query("SELECT id FROM servers ORDER BY id LIMIT 1001")).rows.map(row => Number(row.id)) + const ids = (await deps.query("SELECT id FROM servers ORDER BY id LIMIT 1001")).map(row => Number(row.id)) if (ids.length > 1000) throw new McpError("LIMIT_EXCEEDED", "Каталог глобальной сверки превышает лимит", 409) if (ids.some(id => !current.policy.scope.serverIds.includes(id))) throw new McpError("SERVER_DENIED", "Глобальная сверка требует доступа ко всему каталогу серверов", 403) - const row = (await pool.query("SELECT o.id,o.state,p.operation_id,p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2", [operationId, current.id])).rows[0] - if (!row) throw new McpError("NOT_FOUND", "Операция не найдена", 404) - const original = findMcpOperation(row.operation_id) + return current +} + +function requireOriginal(actionId: string, current: ConnectionRow, deps: Dependencies) { + const original = deps.operation(actionId) if (!original || original.scope !== "global" || !current.policy.scope.operationIds.includes(original.id) || original.requiredPermissions.some(permission => !hasPermission(current.policy.permissions, permission) || !hasPermission(current.owner_permissions, permission))) throw new McpError("PERMISSION_DENIED", "Утрачено право на исходную глобальную операцию", 403) - const evidence = JSON.parse(await decryptSecret(row.evidence.sealed)) as ChangeEvidence - if (evidence.target.scope !== "global" || !evidence.steps.length || evidence.steps.some(step => step.kind !== "service" || !findMcpService(step.service)?.reconcile)) throw new McpError("UNSUPPORTED_CAPABILITY", "Для глобальной операции нет безопасной read-only сверки", 409) - const declared = evidence.objects.flatMap(object => object.serverId === undefined ? [] : [object.serverId]) +} + +function requireServerScope(declared: number[], current: ConnectionRow) { if (declared.some(id => !current.policy.scope.serverIds.includes(id))) throw new McpError("SERVER_DENIED", "Зависимость операции вне текущего глобального допуска", 403) +} + +async function authorizedSource(operationId: string, context: ActionContext, deps: Dependencies) { + const current = await currentAccess(context, deps) + const row = (await deps.query("SELECT o.id,o.state,p.operation_id,p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2", [operationId, current.id]))[0] + if (!row) { + const authorize = async () => { + const fresh = await currentAccess(context, deps) + if (fresh.id !== current.id || fresh.owner_id !== current.owner_id) throw notFound() + } + const archived = await deps.retired({ operationId, connectionId: current.id, ownerId: current.owner_id }, authorize, + { deadlineAt: Date.now() + 10000, signal: context.signal }) + context.signal?.throwIfAborted() + if (!archived) throw notFound() + requireOriginal("servers.poll", current, deps) + requireServerScope(archived.affectedServerIds, current) + return { kind: "retired-poll" as const } + } + requireOriginal(row.operation_id, current, deps) + const evidence = JSON.parse(await deps.decrypt(row.evidence.sealed)) as ChangeEvidence + if (evidence.target.scope !== "global" || !evidence.steps.length || evidence.steps.some(step => step.kind !== "service" || !deps.service(step.service)?.reconcile)) throw new McpError("UNSUPPORTED_CAPABILITY", "Для глобальной операции нет безопасной read-only сверки", 409) + requireServerScope(evidence.objects.flatMap(object => object.serverId === undefined ? [] : [object.serverId]), current) if (["pending", "running"].includes(row.state)) throw new McpError("STATE_CONFLICT", "Операция ещё выполняется", 409) context.signal?.throwIfAborted() - return { row, evidence } + return { kind: "live" as const, row, evidence } } -export function registerGlobalReconciliationActions(register: typeof add) { + +function registerActions(register: typeof add, deps: Dependencies) { register("operations.global.reconcile", "mm_global_operation_reconcile", "Сверить глобальную сервисную операцию без повторной записи", "mcp", null, "read", "global", z.object({ operationId: z.string().uuid() }).strict(), { - target: async (input, context) => { await authorizedSource(String(input.operationId), context); return { scope: "global" } }, + target: async (input, context) => { await authorizedSource(String(input.operationId), context, deps); return { scope: "global" } }, read: async (input, context) => { - const { row, evidence } = await authorizedSource(String(input.operationId), context) - const authorize = async () => { await authorizedSource(String(input.operationId), context) } + const operationId = String(input.operationId) + const source = await authorizedSource(operationId, context, deps) + if (source.kind === "retired-poll") return { operationId, state: "applied", observedAt: new Date().toISOString(), + results: [{ stepIndex: 0, service: "servers.poll", matchesDesiredState: true, matchesOriginalState: false }], automaticRetry: false } + const { row, evidence } = source + const authorize = async () => { if ((await authorizedSource(operationId, context, deps)).kind !== "live") throw notFound() } const results = [] for (const [index, step] of evidence.steps.entries()) { if (step.kind !== "service") throw new McpError("UNSUPPORTED_CAPABILITY", "Нативная глобальная сверка недоступна", 409) await authorize() - results.push({ stepIndex: index, service: step.service, ...await findMcpService(step.service)!.reconcile!(step.input, authorize) }) + results.push({ stepIndex: index, service: step.service, ...await deps.service(step.service)!.reconcile!(step.input, authorize) }) } await authorize() return { operationId: row.id, state: row.state, observedAt: new Date().toISOString(), results, automaticRetry: false } }, }) } + +export const registerGlobalReconciliationActions = (register: typeof add) => registerActions(register, production) +export const registerGlobalReconciliationActionsForTests = registerActions diff --git a/backend/src/modules/mcp/operation-execution-origin.sql.test.ts b/backend/src/modules/mcp/operation-execution-origin.sql.test.ts index 8739755b..63ee3e5e 100644 --- a/backend/src/modules/mcp/operation-execution-origin.sql.test.ts +++ b/backend/src/modules/mcp/operation-execution-origin.sql.test.ts @@ -5,6 +5,8 @@ await prepareOwnedBandwidthState() const { pool, closePool } = await import("../../db/index.js") try { const { applySqlMigrations } = await import("../../db/migrate.js"); await applySqlMigrations(pool) + const { acceptOperationReplayTombstones } = await import("./operation-replay-tombstones.sql-helper.js") + await acceptOperationReplayTombstones() const { acceptOwnedMcpOperationRoot } = await import("./operation-execution-origin.sql-helper.js") console.log("Owned actual MCP operation root:", JSON.stringify(await acceptOwnedMcpOperationRoot())) } finally { await closePool() } diff --git a/backend/src/modules/mcp/operation-replay-tombstones.sql-helper.ts b/backend/src/modules/mcp/operation-replay-tombstones.sql-helper.ts new file mode 100644 index 00000000..e6dddc2f --- /dev/null +++ b/backend/src/modules/mcp/operation-replay-tombstones.sql-helper.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { assertOwnedBandwidthSql } from "../../services/bandwidth-admission-process.sql-helper.js" + +export async function acceptOperationReplayTombstones() { + assertOwnedBandwidthSql() + assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN, "true") + const { pool } = await import("../../db/index.js") + const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp") + const { createMcpConnection, authenticateMcpToken } = await import("./store.js") + const { applyPlan, invokeMcpTool } = await import("./runtime.js") + const { operationIdempotencyDigest } = await import("./operation-replay-tombstones.js") + const { encryptSecret, McpError } = await import("./security.js") + const { default: Fastify } = await import("fastify") + const { default: authPlugin } = await import("../../plugins/auth.js") + const { createMcpRoutes } = await import("../../routes/mcp.js") + const { env } = await import("../../config.js") + const { SignJWT } = await import("jose") + const makeConnection = async () => { + const actor = { id: randomUUID(), email: "owned-tombstone@example.test", name: "Owned replay fixture", apps: ["mm"], + permissions: ["mm:users:write", "mm:mcp:read"], issuedAt: Date.now()-1000, expiresAt: Date.now()+300000 } + const issued = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: actor.name, mode: "approval", + permissions: actor.permissions, expiresAt: new Date(actor.expiresAt).toISOString(), + scope: { serverIds: [], operationIds: ["users.create", "operations.read", "changes.apply"], allowGlobal: true }, + limits: { callsPerMinute: 100, mutationsPerHour: 10 } }), actor) + return { actor, principal: await authenticateMcpToken(issued.token) } + } + const app = Fastify() + const ids: string[] = [] + const matches = (code: string, status: number) => (error: unknown) => + error instanceof McpError && error.code === code && error.statusCode === status + await pool.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1") + try { + const owner = await makeConnection(), foreign = await makeConnection() + // A real completed plan has already initialized the owned encryption state. + // The fixture seeds its tombstone directly, so exercise the same initializer. + await encryptSecret("owned replay fixture state") + const connectionId = owner.principal.connectionId + ids.push(connectionId, foreign.principal.connectionId) + const operationId = randomUUID(), planId = randomUUID(), key = "owned-replay-" + randomUUID() + await pool.query(`INSERT INTO public.mcp_operation_replay_tombstones + (operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global) + VALUES($1,$2,$3,$4,'users.create','{}',true)`, [operationId, planId, connectionId, operationIdempotencyDigest(connectionId,key)]) + await assert.rejects(applyPlan(owner.principal, planId, key), matches("OPERATION_HISTORY_EXPIRED",410)) + await assert.rejects(applyPlan(owner.principal, planId, randomUUID()), matches("OPERATION_HISTORY_EXPIRED",410)) + await assert.rejects(applyPlan(owner.principal, randomUUID(), key), matches("STATE_CONFLICT",409)) + await assert.rejects(applyPlan(foreign.principal, planId, key), matches("NOT_FOUND",404)) + await assert.rejects(invokeMcpTool(owner.principal,"mm_change_apply",{ planId,idempotencyKey:key }), matches("OPERATION_HISTORY_EXPIRED",410)) + await assert.rejects(invokeMcpTool(owner.principal,"mm_operation_get",{ operationId }), matches("OPERATION_HISTORY_EXPIRED",410)) + await assert.rejects(invokeMcpTool(foreign.principal,"mm_operation_get",{ operationId }), matches("NOT_FOUND",404)) + await app.register(authPlugin) + await app.register(createMcpRoutes(async () => {})) + for (const subject of [owner, foreign]) { + const token = await new SignJWT({ sub: subject.actor.id, email: subject.actor.email, name: subject.actor.name, + apps: subject.actor.apps, permissions: subject.actor.permissions }) + .setProtectedHeader({ alg:"HS256" }).setIssuer(env.authIssuer).setIssuedAt().setExpirationTime("4m") + .sign(new TextEncoder().encode(env.jwtSecret)) + const reply = await app.inject({ method: "POST", url: "/mcp/plans/"+planId+"/apply", headers: { authorization:"Bearer "+token } }) + assert.equal(reply.statusCode, subject === owner ? 410 : 404) + assert.equal(reply.json().code, subject === owner ? "OPERATION_HISTORY_EXPIRED" : "NOT_FOUND") + } + await pool.query("UPDATE public.mcp_connections SET policy=jsonb_set(policy,'{scope,allowGlobal}','false') WHERE id=$1", [connectionId]) + await assert.rejects(applyPlan(owner.principal, planId, key), matches("NOT_FOUND",404)) + await assert.rejects(invokeMcpTool(owner.principal,"mm_operation_get",{ operationId }), matches("NOT_FOUND",404)) + const { rows:[proof] } = await pool.query(`SELECT + NOT EXISTS(SELECT 1 FROM public.mcp_operations WHERE connection_id=ANY($1::uuid[])) AS no_operation, + NOT EXISTS(SELECT 1 FROM public.mcp_budget_buckets WHERE connection_id=ANY($1::uuid[]) AND kind='mutations') AS no_write_budget`, [ids]) + assert.deepEqual(proof,{no_operation:true,no_write_budget:true}) + console.log("Owned operation tombstones: scoped MCP and Web 410, replay conflict and no dispatch") + } finally { + await app.close() + await pool.query("DELETE FROM public.mcp_operation_replay_tombstones WHERE connection_id=ANY($1::uuid[])", [ids]) + } +} diff --git a/backend/src/modules/mcp/operation-replay-tombstones.ts b/backend/src/modules/mcp/operation-replay-tombstones.ts new file mode 100644 index 00000000..651ab64b --- /dev/null +++ b/backend/src/modules/mcp/operation-replay-tombstones.ts @@ -0,0 +1,64 @@ +import { createHash } from "node:crypto" +import type { PoolClient } from "pg" +import { hasPermission } from "../../lib/permissions.js" +import { listActions } from "./actions.js" +import { McpError } from "./security.js" +import { effectiveMcpConnection, type ConnectionRow, type McpPrincipal } from "./store.js" + +type Query = Pick +export interface OperationReplayTombstone { + operation_id: string + plan_id: string + connection_id: string + action_id: string + target_server_ids: string[] + target_global: boolean +} + +// Idempotency tokens are caller input and already stored in plaintext on live +// operations. This domain-separated digest avoids copying that plaintext and +// survives DB restore and access-key rotation. It is not a keyed secret proof. +export function operationIdempotencyDigest(connectionId: string, key: string): string { + return createHash("sha256").update("mcp-operation-replay-v1\0").update(connectionId).update("\0").update(key).digest("hex") +} + +export function operationHistoryExpired(): never { + throw new McpError("OPERATION_HISTORY_EXPIRED", "Срок хранения завершённой операции истёк. Повторное применение запрещено.", 410) +} + +export function assertOperationReplayScope(row: OperationReplayTombstone, connection: ConnectionRow): void { + const scope = connection.policy.scope + if (row.connection_id !== connection.id || !scope.operationIds.includes(row.action_id)) throw operationReplayNotFound() + const action = listActions().find(candidate => candidate.operation.id === row.action_id) + if (!action || action.operation.requiredPermissions.some(permission => + !hasPermission(connection.policy.permissions, permission) || !hasPermission(connection.owner_permissions, permission))) throw operationReplayNotFound() + if (row.target_global && !scope.allowGlobal || row.target_server_ids.some(id => !scope.serverIds.some(server => String(server) === id))) throw operationReplayNotFound() +} + +function operationReplayNotFound(): McpError { + return new McpError("NOT_FOUND", "Операция не найдена", 404) +} + +const projection = "t.operation_id,t.plan_id,t.connection_id,t.action_id,t.target_server_ids::text[] AS target_server_ids,t.target_global" + +export async function findOperationReplayTombstone(db: Query, connectionId: string, operationId: string) { + return (await db.query(`SELECT ${projection} FROM mcp_operation_replay_tombstones t + WHERE t.connection_id=$1 AND t.operation_id=$2`, [connectionId, operationId])).rows[0] +} + +export async function findOwnerPlanReplayTombstone(db: Query, ownerId: string, planId: string) { + return (await db.query(`SELECT ${projection} FROM mcp_operation_replay_tombstones t + JOIN mcp_connections c ON c.id=t.connection_id WHERE c.owner_id=$1 AND t.plan_id=$2`, [ownerId, planId])).rows[0] +} + +export async function refuseExpiredOperationReplay(db: Query, principal: McpPrincipal, connectionId: string, planId: string, key: string): Promise { + const rows = (await db.query(`SELECT ${projection} FROM mcp_operation_replay_tombstones t + WHERE t.connection_id=$1 AND (t.plan_id=$2 OR t.idempotency_digest=$3)`, + [connectionId, planId, operationIdempotencyDigest(connectionId, key)])).rows + if (!rows.length) return + const conflict = rows.find(row => row.plan_id !== planId.toLowerCase()) + const current = await effectiveMcpConnection(principal, db) + assertOperationReplayScope(conflict ?? rows[0], current) + if (conflict) throw new McpError("STATE_CONFLICT", "Ключ уже используется другим планом", 409) + operationHistoryExpired() +} diff --git a/backend/src/modules/mcp/ordinary-operation-retirement.owned.test.ts b/backend/src/modules/mcp/ordinary-operation-retirement.owned.test.ts new file mode 100644 index 00000000..c295ef6d --- /dev/null +++ b/backend/src/modules/mcp/ordinary-operation-retirement.owned.test.ts @@ -0,0 +1,148 @@ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { setTimeout as delay } from "node:timers/promises" +import { assertOwnedBandwidthSql,prepareOwnedBandwidthState } from "../../services/bandwidth-admission-process.sql-helper.js" + +assertOwnedBandwidthSql() +assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN,"true") +await prepareOwnedBandwidthState() +const {pool,closePool}=await import("../../db/index.js") +const {applySqlMigrations}=await import("../../db/migrate.js") +const {mcpConnectionCreateSchema}=await import("@mmapp/contracts/mcp") +const {createMcpConnection,authenticateMcpToken}=await import("./store.js") +const {invokeMcpTool,approvePlan,applyPlan}=await import("./runtime.js") +const {OrdinaryRetirementCommitUnknown,retireOrdinaryMcpOperations}=await import("./ordinary-operation-retirement.js") +const {tickMcpWorkers}=await import("./workers.js") +const {pauseSchedulerJobsAndWait,resumeSchedulerJobs}=await import("../../services/scheduler-running.js") +const {Client}=await import("pg") +const {McpError}=await import("./security.js") +const errorIs=(code:string,status:number)=>(error:unknown)=>error instanceof McpError&&error.code===code&&error.statusCode===status + +try { + await applySqlMigrations(pool) + await pauseSchedulerJobsAndWait() + await pool.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1") + const serverId=Number((await pool.query("INSERT INTO servers(name,host,enabled) VALUES('owned logs retention','127.0.0.1',false) RETURNING id")).rows[0].id) + async function access(){ + const actor={id:randomUUID(),email:"owned-logs-retirement@example.test",name:"Owned settings retention",apps:["mm"], + permissions:["mm:settings:admin","mm:mcp:approve","mm:traffic:write","mm:uptime:write","mm:servers:write","mm:network:write"], + issuedAt:Date.now()-1000,expiresAt:Date.now()+300000} + const issued=await createMcpConnection(mcpConnectionCreateSchema.parse({name:actor.name,mode:"approval", + permissions:actor.permissions,expiresAt:new Date(actor.expiresAt).toISOString(), + scope:{serverIds:[serverId],operationIds:["logs.settings.update","traffic.settings.update","uptime.settings.update", + "servers.api-ping.settings.update","internet-path.settings.update","operations.read","changes.apply"],allowGlobal:true}, + limits:{callsPerMinute:100,mutationsPerHour:10}}),actor) + return {actor,principal:await authenticateMcpToken(issued.token)} + } + const {actor,principal}=await access(),foreign=(await access()).principal + async function operation(retentionDays:number,ageMs:number,tool="mm_logs_settings_update_plan", + settings:Record={retentionDays}){ + const plan=await invokeMcpTool(principal,tool,{settings}) as {planId:string;state:string} + if(plan.state==="awaiting-approval")await approvePlan(plan.planId,actor,true) + const key=randomUUID(),applied=await applyPlan(principal,plan.planId,key) + let terminal + for(const deadline=Date.now()+15000;Date.now()(await pool.query(`SELECT i.id,o.state,p.state AS plan_state,s.state AS step_state, + k.operation_id AS checkpoint,t.operation_id AS tombstone,c.claim_token + FROM unnest($1::uuid[]) i(id) + LEFT JOIN mcp_operations o ON o.id=i.id LEFT JOIN mcp_plans p ON p.id=o.plan_id + LEFT JOIN mcp_operation_steps s ON s.operation_id=i.id LEFT JOIN mcp_operation_checkpoints k ON k.operation_id=i.id + LEFT JOIN mcp_operation_replay_tombstones t ON t.operation_id=i.id + LEFT JOIN mcp_ordinary_retirement_claims c ON c.operation_id=i.id ORDER BY i.id`,[ids])).rows + const originalQuery=Client.prototype.query + let commits=0 + try{ + Client.prototype.query=function(this:InstanceType,...args:unknown[]){ + if((args[0] as {text?:string})?.text==="COMMIT"&&++commits===3) + return Promise.reject(Error("owned ordinary destructive COMMIT transport loss")) + return Reflect.apply(originalQuery,this,args) + } as typeof Client.prototype.query + await assert.rejects(retireOrdinaryMcpOperations(), + (error:unknown)=>error instanceof OrdinaryRetirementCommitUnknown&&error.automaticReplay===false) + }finally{Client.prototype.query=originalQuery} + assert.equal(commits,3) + const before=await projection(),spentBefore=before.find(row=>row.id===spent.id) + assert.equal(spentBefore?.state,"applied");assert(spentBefore?.claim_token);assert.equal(spentBefore?.tombstone,null) + await tickMcpWorkers() + const after=await projection() + assert.deepEqual(after.filter(row=>row.id!==due.id),before.filter(row=>row.id!==due.id)) + const retired=after.find(row=>row.id===due.id) + assert(retired?.claim_token);assert.equal(retired?.tombstone,due.id) + for(const field of ["state","plan_state","step_state","checkpoint"])assert.equal(retired[field],null) + assert.equal((await pool.query("SELECT 1 FROM mcp_plans WHERE id=$1",[due.planId])).rows.length,0) + assert.equal((await pool.query("SELECT retention_days FROM router_log_settings WHERE id=1")).rows[0].retention_days,54) + await tickMcpWorkers() + assert.deepEqual(await projection(),after) + await assert.rejects(applyPlan(principal,due.planId,due.key),errorIs("OPERATION_HISTORY_EXPIRED",410)) + await assert.rejects(applyPlan(principal,randomUUID(),due.key),errorIs("STATE_CONFLICT",409)) + await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:due.id}),errorIs("NOT_FOUND",404)) + const telemetry=[ + {action:"traffic.settings.update",version:2,record:await operation(55,age+60000,"mm_traffic_settings_update_plan")}, + {action:"uptime.settings.update",version:2,record:await operation(56,age+60000,"mm_uptime_settings_update_plan",{enabled:false,retentionDays:56})}, + {action:"servers.api-ping.settings.update",version:2,record:await operation(0,age+60000,"mm_servers_api_ping_settings_update_plan",{intervalSec:123})}, + {action:"internet-path.settings.update",version:1,record:await operation(57,age+60000,"mm_internet_path_settings_update_plan")}, + ] + const telemetryRows=async()=>(await pool.query(`SELECT p.operation_id,p.operation_version,o.id + FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id WHERE o.id=ANY($1::uuid[]) ORDER BY p.operation_id`, + [telemetry.map(item=>item.record.id)])).rows + assert.deepEqual((await telemetryRows()).map(row=>[row.operation_id,row.operation_version]), + telemetry.map(item=>[item.action,item.version]).sort((a,b)=>String(a[0]).localeCompare(String(b[0])))) + const telemetrySettings=async()=>(await pool.query(`SELECT to_jsonb(t) AS traffic,to_jsonb(u) AS uptime, + to_jsonb(a) AS api_ping,to_jsonb(i) AS internet FROM traffic_settings t,uptime_settings u, + servers_api_ping_settings a,internet_path_settings i WHERE t.id=1 AND u.id=1 AND a.id=1 AND i.id=1`)).rows[0] + const settingsBefore=await telemetrySettings() + const traffic=telemetry[0].record + const {rows:[originalStep]}=await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1",[traffic.id]) + await pool.query("UPDATE mcp_operation_steps SET result=jsonb_set(result,'{retentionDays}','999') WHERE operation_id=$1",[traffic.id]) + await tickMcpWorkers() + assert.equal((await pool.query("SELECT 1 FROM mcp_ordinary_retirement_claims WHERE operation_id=$1",[traffic.id])).rows.length,0) + assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[traffic.id])).rows.length,1) + await pool.query("UPDATE mcp_operation_steps SET result=$2 WHERE operation_id=$1",[traffic.id,originalStep.result]) + await tickMcpWorkers() + await tickMcpWorkers() + // The keyset owner may need one tick to wrap after the rejected oldest row. + await tickMcpWorkers() + assert.deepEqual(await telemetryRows(),[]) + assert.deepEqual(await telemetrySettings(),settingsBefore) + for(const {action,record} of telemetry){ + const {rows:[proof]}=await pool.query(`SELECT t.action_id,t.target_global,t.target_server_ids::text[] AS server_ids, + NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan_gone, + NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step_gone, + NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint_gone + FROM mcp_operation_replay_tombstones t WHERE t.operation_id=$1 AND t.plan_id=$2`,[record.id,record.planId]) + assert.deepEqual(proof,{action_id:action,target_global:true,server_ids:[String(serverId)],plan_gone:true,step_gone:true,checkpoint_gone:true}) + await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("OPERATION_HISTORY_EXPIRED",410)) + await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:record.id}),errorIs("NOT_FOUND",404)) + } + await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{scope,serverIds}','[]') WHERE id=$1",[principal.connectionId]) + await assert.rejects(applyPlan(principal,due.planId,due.key),errorIs("NOT_FOUND",404)) + for(const {record} of telemetry)await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("NOT_FOUND",404)) + console.log("Owned settings operations: five exact families, authenticated apply, physical 180-day purge, preserved settings and scoped replay") +}finally{resumeSchedulerJobs();await closePool()} diff --git a/backend/src/modules/mcp/ordinary-operation-retirement.ts b/backend/src/modules/mcp/ordinary-operation-retirement.ts new file mode 100644 index 00000000..4bfaf74e --- /dev/null +++ b/backend/src/modules/mcp/ordinary-operation-retirement.ts @@ -0,0 +1,199 @@ +import { randomUUID } from "node:crypto" +import { logSettingsPatchSchema, logSettingsSchema } from "@mmapp/contracts" +import { z } from "zod" +import { connectProbeStorage } from "../../services/server-probe-storage-connection.js" +import type { ProbeStorageSql } from "../../services/server-probe-journal-maintenance.js" +import { sharedDatabaseGate } from "../../services/shared-database-gate.js" +import { operationIdempotencyDigest } from "./operation-replay-tombstones.js" +import { decryptSecretExisting, fingerprint, privateFingerprinterExisting, redact } from "./security.js" + +const uuid = z.string().uuid() +const positive = z.number().int().positive().safe() +const object = z.record(z.string(),z.unknown()) +const sealed = z.object({sealed:z.string().min(1).max(262144)}).strict() +const ids = z.array(positive).max(1000) +const retention = z.number().int().min(1).max(365) +const bool = z.boolean() +const interval = (min:number,max:number)=>z.number().int().min(min).max(max) +// Drizzle's string-mode timestamptz keeps PostgreSQL's space and hour-only +// offset. Accept that exact wire form as well as an ISO timestamptz. +const timestamp = z.string().regex(/^\d{4}-\d\d-\d\d[ T]\d\d:\d\d:\d\d(?:\.\d{1,6})?(?:Z|[+-]\d\d(?::\d\d)?)$/) + .refine(value=>Number.isFinite(Date.parse(value.replace(" ","T").replace(/([+-]\d\d)$/,"$1:00")))) +const metadata = { + id:z.literal(1),createdAt:timestamp,updatedAt:timestamp, + lastCollectedAt:timestamp.nullable(),lastDurationMs:z.number().int().safe().nullable(), + lastError:z.string().max(16384).nullable(), +} +function telemetryFamily(version:number,domain:string,settings:z.ZodObject,hasEpoch:boolean,hasConfigurationProof:boolean){ + const originalSettings=hasConfigurationProof?settings.extend({configurationProof:z.string().regex(/^[0-9a-f]{64}$/)}):settings + const effect=settings.extend({...metadata,...(hasEpoch?{pollConfigurationEpoch:uuid}:{})}).strict() + return {version,domain,patch:settings.partial().strict().refine(value=>Object.keys(value).length>0), + before:z.object({value:originalSettings.strict(),serverIds:ids}).strict(),effect} +} +const families={ + "logs.settings.update":{version:1,domain:null,patch:logSettingsPatchSchema,before:logSettingsSchema.strict(),effect:logSettingsSchema.strict()}, + "traffic.settings.update":telemetryFamily(2,"traffic-settings",z.object({enabled:bool,intervalSec:interval(5,3600),retentionDays:retention}),true,true), + "uptime.settings.update":telemetryFamily(2,"uptime-settings",z.object({enabled:bool,resourcesEnabled:bool,pingEnabled:bool, + speedEnabled:bool,intervalSec:interval(5,86400),probeIntervalSec:interval(5,86400),speedIntervalSec:interval(10,86400),retentionDays:retention}),true,true), + "servers.api-ping.settings.update":telemetryFamily(2,"servers-api-ping-settings",z.object({enabled:bool,intervalSec:interval(10,86400)}),false,true), + "internet-path.settings.update":telemetryFamily(1,"internet-path-settings",z.object({enabled:bool,intervalSec:interval(30,86400),retentionDays:retention}),true,false), +} as const +const planEvidence = z.object({ + target:z.object({scope:z.literal("global")}).strict(), + objects:z.array(z.object({kind:z.string(),id:z.string(),serverId:positive.optional()}).strict()).max(1001), + steps:z.tuple([z.object({kind:z.literal("service"),service:z.string(),input:z.object({settings:object}).strict()}).strict()]), + snapshots:z.tuple([]),before:object,warnings:z.array(z.string()).max(8), + verificationScope:z.literal("configuration").optional(), + requiresHuman:z.literal(true).optional(), +}).strict() + +class RetirementRefused extends Error {} +export class OrdinaryRetirementCommitUnknown extends Error { + readonly automaticReplay = false + constructor() { super("Ordinary operation retirement outcome unknown; automatic replay forbidden") } +} +const requireThat: (value: unknown) => asserts value = value => { + if (!value) throw new RetirementRefused() +} + +// Fixed reviewed settings families without artifact, historic receipt or +// inverse-plan consumers. Other action families need separate review. +const ELIGIBLE = `FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0 + JOIN public.mcp_operation_checkpoints k ON k.operation_id=o.id AND k.step_index=0 + WHERE (p.operation_id,p.operation_version) IN ( + ('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2), + ('servers.api-ping.settings.update',2),('internet-path.settings.update',1) + ) AND p.state='completed' + AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL + AND isfinite(o.finished_at) AND o.finished_at($1::timestamptz,$2::uuid)) + ORDER BY o.finished_at,o.id LIMIT 8` +const SOURCE = `SELECT to_jsonb(o) AS operation,to_jsonb(p) AS plan,to_jsonb(s) AS step,to_jsonb(k) AS checkpoint ${ELIGIBLE} + AND o.id=$1 AND o.connection_id=$2 + AND octet_length(to_jsonb(o)::text)+octet_length(to_jsonb(p)::text)+octet_length(to_jsonb(s)::text)+octet_length(to_jsonb(k)::text)<=1048576 + FOR UPDATE OF o,p,s,k NOWAIT` + +async function transaction(deadlineAt:number, work:(sql:ProbeStorageSql)=>Promise):Promise { + const sql = await connectProbeStorage({deadlineAt}) + let committing=false,committed=false + try { + await sql.query("BEGIN") + await sql.query("SELECT set_config('TimeZone','UTC',true),set_config('statement_timeout','5000ms',true),set_config('lock_timeout','1000ms',true)") + const result=await work(sql) + committing=true + await sql.query("COMMIT") + committed=true + return result + } catch(error) { + if(committing)throw new OrdinaryRetirementCommitUnknown() + try{await sql.query("ROLLBACK")}catch{/* Destroy this checkout. */} + throw error + } finally { + sql.release(!committed) + } +} + +async function prepare(sql:ProbeStorageSql, operationId:string, connectionId:string) { + requireThat((await sql.query("SELECT id FROM public.mcp_connections WHERE id=$1 FOR UPDATE NOWAIT",[connectionId])).rows.length===1) + requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked",["mcp-operation:"+operationId])).rows[0]?.locked===true) + const rows=(await sql.query(SOURCE,[operationId,connectionId])).rows + requireThat(rows.length===1) + const raw=rows[0], operation=object.parse(raw.operation), plan=object.parse(raw.plan) + const step=object.parse(raw.step),checkpoint=object.parse(raw.checkpoint) + const open=async(value:unknown)=>JSON.parse(await decryptSecretExisting(sealed.parse(value).sealed)) + const original=planEvidence.parse(await open(plan.evidence)) + const action=z.enum(Object.keys(families) as [keyof typeof families,...(keyof typeof families)[]]).parse(plan.operation_id) + const family=families[action],service=original.steps[0] + requireThat(plan.operation_version===family.version&&service.service===action) + requireThat(fingerprint(await open(plan.payload))===fingerprint(service.input)) + requireThat(fingerprint(step.intended_state)===fingerprint(redact(service))) + const patch=object.parse(family.patch.parse(service.input.settings)) + const effect=object.parse(family.effect.parse(step.result)) + requireThat(Object.entries(patch).every(([key,value])=>effect[key]===value)) + requireThat(fingerprint(operation.result)===fingerprint({confirmedWrites:1,verificationScope:"configuration"})) + for(const value of [checkpoint.before_sealed,checkpoint.after_sealed]){ + requireThat(typeof value==="string"&&value.length<=1024&&fingerprint(await open({sealed:value}))===fingerprint([])) + } + const serverObjects=original.objects.filter(item=>item.kind==="server") + const serverIds=ids.parse(serverObjects.map(item=>item.serverId)) + requireThat(new Set(serverIds).size===serverIds.length&&serverObjects.every(item=>item.id===String(item.serverId))) + const domainObjects=original.objects.filter(item=>item.kind!=="server") + const before=family.before.parse(original.before) + if(family.domain===null){ + requireThat(domainObjects.length===0&&original.requiresHuman===undefined) + }else{ + requireThat(original.requiresHuman===true&&original.verificationScope==="configuration" + &&domainObjects.length===1&&domainObjects[0].kind===family.domain&&domainObjects[0].id==="1"&&domainObjects[0].serverId===undefined) + requireThat("serverIds" in before&&fingerprint(before.serverIds)===fingerprint(serverIds)) + } + const signer=await privateFingerprinterExisting("mcp-ordinary-retirement-source-v1") + return {planId:uuid.parse(plan.id),action,serverIds,key:z.string().min(1).max(200).parse(operation.idempotency_key),proof:signer(fingerprint(raw))} +} + +async function retire(operationId:string,connectionId:string,deadlineAt:number) { + const token=randomUUID() + await transaction(deadlineAt,async sql=>{ + const source=await prepare(sql,operationId,connectionId) + const claim=await sql.query(`INSERT INTO public.mcp_ordinary_retirement_claims(operation_id,plan_id,connection_id,source_proof,claim_token) + VALUES($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING RETURNING operation_id`,[operationId,source.planId,connectionId,source.proof,token]) + requireThat(claim.rows.length===1&&claim.rows[0].operation_id===operationId) + }) + await transaction(deadlineAt,async sql=>{ + const claim=(await sql.query(`SELECT source_proof,plan_id FROM public.mcp_ordinary_retirement_claims + WHERE operation_id=$1 AND connection_id=$2 AND claim_token=$3 AND claim_xid<>pg_current_xact_id()`,[operationId,connectionId,token])).rows + requireThat(claim.length===1) + const source=await prepare(sql,operationId,connectionId) + requireThat(claim[0].source_proof===source.proof&&claim[0].plan_id===source.planId) + const tombstone=await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones + (operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global) + VALUES($1,$2,$3,$4,$5,$6,true) RETURNING operation_id`, + [operationId,source.planId,connectionId,operationIdempotencyDigest(connectionId,source.key),source.action,source.serverIds]) + requireThat(tombstone.rows.length===1&&tombstone.rows[0].operation_id===operationId) + const checkpoint=await sql.query("DELETE FROM public.mcp_operation_checkpoints WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId]) + const step=await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId]) + const operation=await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id AS operation_id",[operationId,connectionId,source.planId]) + const plan=await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id AS plan_id",[source.planId,connectionId]) + requireThat([checkpoint,step,operation].every(result=>result.rows.length===1&&result.rows[0].operation_id===operationId) + &&plan.rows.length===1&&plan.rows[0].plan_id===source.planId) + }) +} + +let cursor:{finishedAt:string;operationId:string}|undefined +export function retireOrdinaryMcpOperations() { + return sharedDatabaseGate.request(async()=>{ + const deadlineAt=Date.now()+15000 + const rows=await transaction(deadlineAt,async sql=>(await sql.query(CANDIDATES,[cursor?.finishedAt??null,cursor?.operationId??null])).rows) + requireThat(rows.length<=8) + let scanned=0,retired=0 + for(const row of rows){ + if(Date.now()>=deadlineAt-2000||retired>=2)break + const id=uuid.parse(row.id),connectionId=uuid.parse(row.connection_id) + requireThat(typeof row.finished_at==="string"&&Number.isFinite(Date.parse(row.finished_at))) + cursor={finishedAt:row.finished_at,operationId:id} + scanned++ + try{await retire(id,connectionId,deadlineAt);retired++} + catch(error){if(error instanceof OrdinaryRetirementCommitUnknown)throw error} + } + if(scanned===rows.length&&rows.length<8)cursor=undefined + return {scanned,retired,automaticReplay:false as const} + }) +} diff --git a/backend/src/modules/mcp/routing-community-reserved-source.test.ts b/backend/src/modules/mcp/routing-community-reserved-source.test.ts index 0c770035..3fa30147 100644 --- a/backend/src/modules/mcp/routing-community-reserved-source.test.ts +++ b/backend/src/modules/mcp/routing-community-reserved-source.test.ts @@ -1,5 +1,8 @@ import assert from "node:assert/strict" +import { readFile } from "node:fs/promises" import { test } from "node:test" +import { runInNewContext } from "node:vm" +import ts from "typescript" import type { ActionContext } from "./action-types.js" import { loadReservedScopedRecoverySource } from "./routing-community-scoped-recovery-source.js" import { executeReservedScopedRecovery, scopedRecoveryIntentFromCheckpoint, validateReservedInverseVerified } from "./routing-community-scoped-recovery.js" @@ -16,6 +19,67 @@ test("a serialized allocation cannot enter the shared inverse engine", async () await assert.rejects(executeReservedScopedRecovery(forged, {} as Parameters[1]), /capability/) }) +test("confirmed inverse spend quarantines the exact slot on post-spend refusal without replay", async () => { + // Execute the actual boundary body with only its I/O dependencies substituted. + // This does not mint a production capability or claim native/SQL acceptance. + const source = ts.createSourceFile("routing-community-scoped-recovery.ts", + await readFile(new URL("./routing-community-scoped-recovery.ts", import.meta.url), "utf8"), ts.ScriptTarget.ES2022) + const declaration = source.statements.find(node => ts.isFunctionDeclaration(node) && node.name?.text === "executeReservedScopedRecovery") + assert(declaration) + const compiled = ts.transpileModule(declaration.getText(source), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }).outputText + const allocation = Object.freeze({ kind: "routing-reserved-inverse-allocation" }) + const execution = Object.freeze({ kind: "routing-reserved-inverse-execution" }) + const principal = { connectionId: "owned", credentialId: "original" } + const original = new Error("original authority failure"), result = Object.freeze({ outcome: "recovered" }) + const scenarios = { + spend: ["spend"], + inspect: ["spend", "inspect", "quarantine"], + authority: ["spend", "inspect", "authority", "quarantine"], + source: ["spend", "inspect", "authority", "source", "quarantine"], + proof: ["spend", "inspect", "authority", "source", "proof", "quarantine"], + engine: ["spend", "inspect", "authority", "source", "proof", "engine", "quarantine"], + success: ["spend", "inspect", "authority", "source", "proof", "engine"], + } as const + for (const [scenario, expected] of Object.entries(scenarios)) { + const events: string[] = [], exports: Record = {}, capabilities = new WeakMap() + let request: object | undefined + const at = (phase: string) => { events.push(phase); if (phase === scenario) throw original } + runInNewContext(compiled, { + exports, reservedExecutions: capabilities, fingerprint: JSON.stringify, + createRoutingReservationStore: () => ({ + async beginInverseAttempt(value: unknown) { + assert.equal(value, allocation); at("spend"); return { reservationId: "spent-slot", execution } + }, + async inspectInverseExecution(value: unknown) { + assert.equal(value, execution); at("inspect") + return { principal, sourceOperationId: "source", deadline: "2026-10-29T01:00:00.000Z" } + }, + async markUnknown(id: string, lane: string) { + assert.equal(id, "spent-slot"); assert.equal(lane, "inverse"); at("quarantine") + throw new Error("quarantine persistence unavailable") + }, + }), + effectiveMcpConnection: async () => { at("authority"); return { owner_id: "owner", policy: {}, owner_permissions: [] } }, + loadReservedScopedRecoverySource: async () => { + at("source"); return { profile: { serverId: 1 }, input: {}, configuration: { packet: {} } } + }, + accessGeneration: async () => "generation", + deadline: () => Date.parse("2026-10-29T01:00:00.000Z"), + sealedProof: async () => { at("proof"); return "proof" }, + executeScopedRecovery: async (value: object) => { + request = value; assert.equal(capabilities.get(value), execution); at("engine"); return result + }, + }) + const execute = exports.executeReservedScopedRecovery as + (allocation: unknown, expected: { sourceOperationId: string; principal: typeof principal }) => Promise + const outcome = execute(allocation, { sourceOperationId: "source", principal }) + if (scenario === "success") assert.equal(await outcome, result) + else await assert.rejects(outcome, error => error === original) + assert.deepEqual(events, expected, scenario) + if (request) assert.equal(capabilities.has(request), false) + } +}) + test("configuration equality or generic VERIFIED cannot replace reserved signed receipts", async () => { const expected = {} as Parameters[1] for (const row of [{}, { state: "verified", phase: "RECOVERY_VERIFIED", configurationVerified: true }, { inverse_intent_sealed: "x".repeat(524289) }]) { diff --git a/backend/src/modules/mcp/routing-community-scoped-acceptance.ts b/backend/src/modules/mcp/routing-community-scoped-acceptance.ts index 2ff02082..da0dd0f1 100644 --- a/backend/src/modules/mcp/routing-community-scoped-acceptance.ts +++ b/backend/src/modules/mcp/routing-community-scoped-acceptance.ts @@ -35,8 +35,9 @@ import { registerManagementChannel } from "./management.js" import { isSchedulerJobsPaused, pauseSchedulerJobsAndWait, resumeSchedulerJobs } from "../../services/scheduler-running.js" /** Disposable SQL + loopback HTTP only. Does not access or configure CHR. */ -export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options: { originalEffectsOnly?: boolean; originalCredentialRevocationOnly?: boolean; forwardRoundsOnly?: boolean; manualInverseChargeOnly?: boolean; manualInverseChargeCase?: "positive" | "quota" | "concurrent" | "unknown" | "revoked" | "nonpersistent" | "mixed-forward" } = {}) { - const sid = 131, writer: AuthUser = { ...actor, permissions: [...new Set([...actor.permissions, "mm:filters:write"])] } +export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options: { originalEffectsOnly?: boolean; originalCredentialRevocationOnly?: boolean; forwardRoundsOnly?: boolean; manualInverseChargeOnly?: boolean; manualInverseChargeCase?: "positive" | "quota" | "concurrent" | "unknown" | "revoked" | "nonpersistent" | "mixed-forward"; lateInverseAuthorityChange?: "edit" | "disable" | "grant" } = {}) { + const sid = options.lateInverseAuthorityChange ? { edit: 131, disable: 132, grant: 133 }[options.lateInverseAuthorityChange] : 131 + const writer: AuthUser = { ...actor, permissions: [...new Set([...actor.permissions, "mm:filters:write"])] } const initial = compileManagedFilterRules([{ community: "65000:10", gateway: "192.0.2.1", outIface: "", action: "route", description: "" }, { community: "65000:20", gateway: "192.0.2.5", outIface: "", action: "route", description: "" }]) const filters = ["a", "b"].map(key => ({ ".id": "*f" + key, chain: "custom-" + key, rule: initial, disabled: "false", comment: managedComment("NEVER_SCOPED_COMMENT_" + key), opaque: "NEVER_SCOPED_OPAQUE" })) let mode: "normal" | "ignored" | "lost" | "collateral" | "post-failed" | "post-unknown" = "normal", writes = 0, requests = 0 @@ -423,8 +424,9 @@ export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options: await assertOriginalCredentialRevocation() return {accepted:true,originalCredentialRevocation:true,unauthorized401:true,fullAdmissionRollback:true,nativeReads:0,nativeWrites:0,realSql:true,allWriterLedgerComplete:false} } - if (options.manualInverseChargeOnly) { + if (options.manualInverseChargeOnly || options.lateInverseAuthorityChange) { const chargeCase = options.manualInverseChargeCase ?? "positive" + if (options.lateInverseAuthorityChange) assert.equal(chargeCase, "positive") const persistent = chargeCase !== "nonpersistent" // Same genuine fixed-reader baseline/CAS used by the existing persistent // acceptance below. No hand-written baseline, sealed plan or authority. @@ -573,6 +575,48 @@ export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options: realSql: true, allWriterLedgerComplete: false, reservedDispatchEnabled: false, autoRecoveryEnabled: false } } console.log(JSON.stringify({ stage: "manual-inverse:human-recovery-apply" })) + const lateChange = options.lateInverseAuthorityChange + if (lateChange) { + const priorIntent = (await pool.query("SELECT coalesce(max(id),0)::text AS id FROM mcp_audit WHERE connection_id=$1 AND action='configuration.intent'", [connectionId])).rows[0].id + let writeId: string | undefined + afterIntent = async () => { + const intents = (await pool.query(`SELECT payload FROM mcp_audit WHERE connection_id=$1 AND id>$2::bigint + AND action='configuration.intent' AND payload->>'path'=$3`, + [connectionId, priorIntent, "/routing/filter/rule/" + loaded.configuration.packet.nativeId])).rows + assert.equal(intents.length, 1) + assert.equal(intents[0].payload.method, "PATCH") + assert.equal(intents[0].payload.serverId, sid) + writeId = intents[0].payload.writeId + assert.match(writeId ?? "", /^[0-9a-f-]{36}$/) + const sql = { + edit: "UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late inverse edit\"') WHERE id=$1", + disable: "UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", + grant: "UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1", + }[lateChange] + assert.equal((await pool.query(sql, [id])).rowCount, 1) + } + const inverse = await execute(recovery.planId) + assert.equal(afterIntent, undefined) + assert.ok(writeId) + assert.notEqual(inverse.state, "applied") + assert.equal(writes, afterForward.writes) + assert.equal(inversePostChecks, 0) + const auditRows = (await pool.query(`SELECT action FROM mcp_audit WHERE connection_id=$1 AND payload->>'writeId'=$2 + ORDER BY id`, [connectionId, writeId])).rows + assert.deepEqual(auditRows.map(row => row.action), ["configuration.intent", "configuration.unconfirmed"]) + const refusal = (await pool.query(`SELECT payload->>'phase' AS phase, payload->>'code' AS code + FROM mcp_audit WHERE connection_id=$1 AND action='routing.scoped.recovery-unconfirmed'`, [connectionId])).rows + assert.deepEqual(refusal, [{ phase: "before-dispatch-context", code: "STATE_CONFLICT" }]) + assert.deepEqual((await pool.query(`SELECT state,verified_sealed FROM mcp_scoped_recovery_intents + WHERE source_operation_id=$1 AND recovery_operation_id=$2`, [source.id, inverse.id])).rows, + [{ state: "unknown", verified_sealed: null }]) + assert.equal((await pool.query(`SELECT count(*)::int AS n FROM mcp_operation_checkpoints + WHERE operation_id=$1 AND configuration_sealed IS NOT NULL`, [inverse.id])).rows[0].n, 0) + assert.equal((await pool.query("SELECT error_code FROM mcp_operations WHERE id=$1", [inverse.id])).rows[0].error_code, + "MUTATION_OUTCOME_UNKNOWN") + return { accepted: true, lateInverseAuthorityChange: lateChange, durableIntents: 1, inverseNativeWrites: 0, + realSql: true, realMcpDispatcher: true, fullInverseLifecycleRepeated: false } + } if (chargeCase === "quota" || chargeCase === "revoked") { if (chargeCase === "revoked") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [id]) const beforeOperations = (await pool.query("SELECT count(*)::int AS n FROM mcp_operations WHERE connection_id=$1", [connectionId])).rows[0].n diff --git a/backend/src/modules/mcp/routing-community-scoped-recovery.ts b/backend/src/modules/mcp/routing-community-scoped-recovery.ts index 11bb5e20..1480d6e9 100644 --- a/backend/src/modules/mcp/routing-community-scoped-recovery.ts +++ b/backend/src/modules/mcp/routing-community-scoped-recovery.ts @@ -257,24 +257,30 @@ async function checkpoint(value: Session, intent: Intent, phase: Checkpoint["pha export async function executeReservedScopedRecovery(allocation: RoutingReservedInverseAllocation, expected: { sourceOperationId: string; principal: McpPrincipal }) { const store = createRoutingReservationStore(), spent = await store.beginInverseAttempt(allocation) const execution = spent.execution - const binding = await store.inspectInverseExecution(execution) - if (binding.sourceOperationId !== expected.sourceOperationId || fingerprint(binding.principal) !== fingerprint(expected.principal)) throw fail("Reserved runner identity differs") - const current = await effectiveMcpConnection(binding.principal) - const context: ActionContext = { principal: binding.principal, connection: current } - const source = await loadReservedScopedRecoverySource(execution, context) - const channel: ManagementTransportBinding | undefined = source.input.managementBindingCiphertext == null ? undefined - : JSON.parse(await decryptSecret(String(source.input.managementBindingCiphertext))) - const input: Record = { mode: "reserved", serverId: source.profile.serverId, sourceOperationId: binding.sourceOperationId, - sourceProof: source.sourceProof, configurationProof: binding.configurationProof, sourceReceiptId: binding.sourceReceiptId, - profile: source.profile, candidateId: source.configuration.packet.originalCandidateId, rules: source.input.beforeRules, beforeRules: source.input.rules, - principal: binding.principal, ownerId: current.owner_id, ownerProof: fingerprint(current.owner_permissions), policyProof: fingerprint(current.policy), connectionGeneration: current.generation, - generation: await accessGeneration(), receiptId: binding.inverseReceiptId, proofRevision: 1, - validUntil: new Date(Math.min(Date.parse(binding.deadline), deadline(source, context, channel?.expiresAt))).toISOString(), managementBindingCiphertext: source.input.managementBindingCiphertext ?? null } - input.requestProof = await sealedProof(input, "request") - reservedExecutions.set(input, execution) - try { return await executeScopedRecovery(input) } - catch (error) { await store.markUnknown(binding.reservationId, "inverse").catch(() => {}); throw error } - finally { reservedExecutions.delete(input) } + try { + const binding = await store.inspectInverseExecution(execution) + if (binding.sourceOperationId !== expected.sourceOperationId || fingerprint(binding.principal) !== fingerprint(expected.principal)) throw fail("Reserved runner identity differs") + const current = await effectiveMcpConnection(binding.principal) + const context: ActionContext = { principal: binding.principal, connection: current } + const source = await loadReservedScopedRecoverySource(execution, context) + const channel: ManagementTransportBinding | undefined = source.input.managementBindingCiphertext == null ? undefined + : JSON.parse(await decryptSecret(String(source.input.managementBindingCiphertext))) + const input: Record = { mode: "reserved", serverId: source.profile.serverId, sourceOperationId: binding.sourceOperationId, + sourceProof: source.sourceProof, configurationProof: binding.configurationProof, sourceReceiptId: binding.sourceReceiptId, + profile: source.profile, candidateId: source.configuration.packet.originalCandidateId, rules: source.input.beforeRules, beforeRules: source.input.rules, + principal: binding.principal, ownerId: current.owner_id, ownerProof: fingerprint(current.owner_permissions), policyProof: fingerprint(current.policy), connectionGeneration: current.generation, + generation: await accessGeneration(), receiptId: binding.inverseReceiptId, proofRevision: 1, + validUntil: new Date(Math.min(Date.parse(binding.deadline), deadline(source, context, channel?.expiresAt))).toISOString(), managementBindingCiphertext: source.input.managementBindingCiphertext ?? null } + input.requestProof = await sealedProof(input, "request") + reservedExecutions.set(input, execution) + try { return await executeScopedRecovery(input) } + finally { reservedExecutions.delete(input) } + } catch (error) { + // Acknowledged spend owns this exact slot even when its authority can no + // longer be inspected. A refused or uncertain spend never enters here. + await store.markUnknown(spent.reservationId, "inverse").catch(() => {}) + throw error + } } /** Fixed-domain receipt verifier used by the reservation store. It recognizes diff --git a/backend/src/modules/mcp/routing-network-scoped-context.test.ts b/backend/src/modules/mcp/routing-network-scoped-context.test.ts index ae1fe43d..a1d645ce 100644 --- a/backend/src/modules/mcp/routing-network-scoped-context.test.ts +++ b/backend/src/modules/mcp/routing-network-scoped-context.test.ts @@ -14,7 +14,9 @@ import { nativeBgpRows, bgpTemplateReferences } from "../../services/bgp-export- import { bgpFilterName, effectiveBgpFilter, stableBgpPolicyRow } from "../../services/bgp-filter-binding-model.js" import { canonicalVrfMembers } from "../../services/vrf-members.js" import { readOspfEffectiveDisabled } from "../../services/ospf-effective-disabled.js" +import { isConfirmedRoutingFibTable } from "../../services/routing-table-fib.js" import * as contextModel from "./routing-community-context-model.js" +import { prepareOriginalEffectSelection } from "./routing-original-effects.js" import { readMeasurementAddress } from "./routing-measurement-address.js" import { routeContextClients } from "./routing-community-context-native.js" import { pathProofTag } from "./path-proof-purpose.js" @@ -65,9 +67,8 @@ async function fixture(autoRecovery = false) { afterHttp?.(); afterHttp = undefined if (afterPath?.path === path) { const run = afterPath.run; afterPath = undefined; run() } }) - await new Promise(resolve => router.listen(0, "127.0.0.1", resolve)) const now = Date.now(), fingerprint = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex") - const endpoint = { id: 1, host: "127.0.0.1", port: (router.address() as { port: number }).port, username: "fixture", password: "private", useSsl: false, verifySsl: true, enabled: true, routingEndpointEpoch: randomUUID() } + const endpoint = { id: 1, host: "127.0.0.1", port: 0, username: "fixture", password: "private", useSsl: false, verifySsl: true, enabled: true, routingEndpointEpoch: randomUUID() } const principal = { connectionId: randomUUID(), credentialId: randomUUID(), credentialExpiresAt: new Date(now + 600000).toISOString() } const connection = { id: principal.connectionId, owner_id: "owner", generation: "generation", owner_permissions: ["mm:network:read"], policy: { permissions: ["mm:network:read"], scope: { serverIds: [1] }, expiresAt: new Date(now + 600000).toISOString() }, owner_valid_until: new Date(now + 600000) } @@ -92,6 +93,7 @@ async function fixture(autoRecovery = false) { "./routing-community-context-native.js": { routeContextClients }, "../../services/vrf-members.js": { canonicalVrfMembers }, "../../services/ospf-effective-disabled.js": { readOspfEffectiveDisabled }, "./routing-measurement-address.js": { readMeasurementAddress } } await loadSource("./routing-community-context.ts", { ...common, "./routing-network-native-session.js": native, + "../../services/routing-table-fib.js": { isConfirmedRoutingFibTable }, "./routing-original-effects.js": { prepareOriginalEffectSelection }, "../../services/bgp-export-model.js": { nativeBgpRows, bgpTemplateReferences }, "../../services/bgp-filter-binding-model.js": { bgpFilterName, effectiveBgpFilter, stableBgpPolicyRow }, "./routing-community-context-model.js": contextModel, "./path-proof-purpose.js": { pathProofTag }, "./management-transport.js": { assertManagementTransport: async () => { throw Error("No alternate management fixture") } } }, context) @@ -103,14 +105,23 @@ async function fixture(autoRecovery = false) { await loadSource("./routing-network-scoped-context.ts", { "./routing-network-native-session.js": native }, privateApi) const create = native.createRoutingNetworkNativeSession as (options: unknown) => Promise const abort = new AbortController() - const session = await create({ sourceOperationId: row.id, context: { principal, connection, signal: abort.signal }, authorize: authority, deadline: now + 119000, manifest: { servers: [1], probes: [] } }) + await new Promise(resolve => router.listen(0, "127.0.0.1", resolve)) + endpoint.port = (router.address() as { port: number }).port + const closeRouter = () => { + router.closeAllConnections() + return new Promise(resolve => router.close(() => resolve())) + } + const session = await create({ sourceOperationId: row.id, context: { principal, connection, signal: abort.signal }, authorize: authority, deadline: now + 119000, manifest: { servers: [1], probes: [] } }).catch(async error => { + await closeRouter() + throw error + }) const readContext = privateApi.readRoutingNetworkScopedCommunityContext as (token: Token) => Promise return { session, readContext: () => readContext(session.sharedContext), forged: () => readContext(JSON.parse(JSON.stringify(session.sharedContext))), readResource: native.readRoutingNetworkSharedResource as (token: Token, server: number, path: string, body?: Record) => Promise, bypassTail: context.readRouteCommunityContextWithSharedNativeSession as (token: Token) => Promise, counters: () => ({ requests, wireBytes, wireRows, rawGetObjects }), bulk: (value: number) => { bulk = value }, padding: (value: number) => { padding = value }, revoke: () => { revoked = true }, abort: () => abort.abort(), afterHttp: (hook: () => void) => { afterHttp = hook }, afterPath: (path: string, hook: () => void) => { afterPath = { path, run: hook } }, - cleanup: async () => { session.close(); router.closeAllConnections(); await new Promise(resolve => router.close(() => resolve())) } } + cleanup: async () => { session.close(); await closeRouter() } } } test("actual source context and snapshots share one pinned HTTP request/byte/row budget including object /get", async () => { diff --git a/backend/src/modules/mcp/routing-recovery-authority.sql-helper.ts b/backend/src/modules/mcp/routing-recovery-authority.sql-helper.ts index 6f1bfe53..9621b123 100644 --- a/backend/src/modules/mcp/routing-recovery-authority.sql-helper.ts +++ b/backend/src/modules/mcp/routing-recovery-authority.sql-helper.ts @@ -19,7 +19,7 @@ export async function acceptRoutingRecoveryAuthoritySql() { const { pool } = await import("../../db/index.js") const { mcpConnectionPolicySchema } = await import("@mmapp/contracts/mcp") const { routeAiProfileSchema } = await import("@mmapp/contracts/mcp-routing") - const { accessGeneration } = await import("./security.js") + const { accessGeneration, McpError } = await import("./security.js") const { readRoutingEndpoints, createRoutingSourceContext } = await import("./routing-observation-source.js") const { resolveRoutingProfileAuthority } = await import("./routing-profile-authority.js") const { captureScopedRecoveryAuthority, scopedRecoveryDeadlineCeiling } = await import("./routing-community-scoped-recovery.js") @@ -27,7 +27,7 @@ export async function acceptRoutingRecoveryAuthoritySql() { assert.equal((await pool.query("SELECT current_database() AS name")).rows[0].name, "postgres") const previousEnabled = (await pool.query("SELECT enabled FROM mcp_settings WHERE id=1")).rows[0].enabled as boolean const profileIds: string[] = [], ids: number[] = [] - let nativeHttpRequests = 0, callbackDispatches = 0, durableIntents = 0 + let nativeHttpRequests = 0, callbackDispatches = 0, dispatchFences = 0 const listener = createServer((req, res) => { nativeHttpRequests++; req.resume(); res.setHeader("content-type", "application/json"); res.end("{}") }) try { await new Promise((resolve, reject) => { listener.once("error", reject); listener.listen(0, "127.0.0.1", () => { listener.off("error", reject); resolve() }) }) @@ -65,12 +65,7 @@ export async function acceptRoutingRecoveryAuthoritySql() { const path = "/routing/filter/rule/*1" await assert.rejects(withVerifiedConfigurationChange({ actorId: ownerId, connectionId, approved: true, serverIds: ids, serverWide: true, objects: [], authorize: async () => { await captured.assert() }, beforeDispatch: async () => { - assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action='configuration.intent' AND payload->>'path'=$2", [connectionId, path])).rows[0].n, 1) - durableIntents++ - if (change === "edit") await pool.query("UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late edit\"') WHERE id=$1", [profileId]) - if (change === "disable") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [profileId]) - if (change === "grant") await pool.query("UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1", [profileId]) - await captured.assert() + dispatchFences++ }, }, () => executeGuardedConfigurationWrite(ids[0], "PATCH", path, async () => { callbackDispatches++ @@ -78,12 +73,17 @@ export async function acceptRoutingRecoveryAuthoritySql() { const req = request({ hostname: "127.0.0.1", port, path: "/rest" + path, method: "PATCH", timeout: 1000 }, res => { res.resume(); res.once("end", resolve) }) req.once("error", reject); req.once("timeout", () => req.destroy(new Error("Synthetic loopback timeout"))); req.end("{}") }) - })), /Постоянный допуск/) - assert.equal(callbackDispatches, 0); assert.equal(nativeHttpRequests, 0) - assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action='configuration.unconfirmed'", [connectionId])).rows[0].n, 1) + })), (error: unknown) => error instanceof McpError && error.code === "STATE_CONFLICT" && /Активный routing profile/.test(error.message)) + assert.equal(callbackDispatches, 0); assert.equal(nativeHttpRequests, 0); assert.equal(dispatchFences, 0) + assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action IN ('configuration.intent','configuration.unconfirmed','configuration.acknowledged')", [connectionId])).rows[0].n, 0) + + if (change === "edit") await pool.query("UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late edit\"') WHERE id=$1", [profileId]) + if (change === "disable") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [profileId]) + if (change === "grant") await pool.query("UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1", [profileId]) + await assert.rejects(captured.assert(), /Постоянный допуск/) } - return { cases: 3, lateMutations: ["edit", "disable", "grant"], deepCaptureCases: 3, durableIntents, callbackDispatches, nativeHttpRequests, - productionSessionAuthorityFence: true, realSql: true, productionHmac: true, fullInverseLifecycleRepeated: false, controllerEnabled: false, externalNetwork: false } + return { cases: 3, deepCaptureCases: 3, directGuardBeforeIntent: true, dispatchFences, callbackDispatches, nativeHttpRequests, + capturedAuthorityChanges: ["edit", "disable", "grant"], realSql: true, productionHmac: true, controllerEnabled: false, externalNetwork: false } } finally { if (profileIds.length) await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=ANY($1::uuid[])", [profileIds]) if (ids.length) await pool.query("UPDATE servers SET enabled=false WHERE id=ANY($1::int[])", [ids]) diff --git a/backend/src/modules/mcp/routing-recovery-authority.sql.test.ts b/backend/src/modules/mcp/routing-recovery-authority.sql.test.ts index fec1bb1c..7a565467 100644 --- a/backend/src/modules/mcp/routing-recovery-authority.sql.test.ts +++ b/backend/src/modules/mcp/routing-recovery-authority.sql.test.ts @@ -2,9 +2,26 @@ import { assertOwnedRoutingRecoveryAuthoritySql, acceptRoutingRecoveryAuthorityS // Guard precedes imports that configure a pool or load private MCP state. assertOwnedRoutingRecoveryAuthoritySql() +process.env.AUTH_REQUIRED = "true" +process.env.AUTH_JWT_SECRET = "owned-routing-recovery-authority-acceptance-secret" +process.env.AUTH_ISSUER = "https://owned-routing-recovery-authority.test" const { pool, closePool } = await import("../../db/index.js") try { const { applySqlMigrations } = await import("../../db/migrate.js") await applySqlMigrations(pool) console.log("captured recovery authority SQL acceptance:", JSON.stringify(await acceptRoutingRecoveryAuthoritySql())) -} finally { await closePool() } + const { updateMcpSettings } = await import("./store.js") + await import("./actions.js") + const { acceptScopedRouteCommunitySwitch } = await import("./routing-community-scoped-acceptance.js") + const actor = { id: "owned-routing-recovery-authority", email: "routing-recovery@example.invalid", name: "Owned recovery authority", + apps: ["mm"], permissions: ["mm:settings:admin", "mm:mcp:admin", "mm:mcp:approve", "mm:network:read", "mm:network:write", "mm:filters:write", "mm:servers:admin"], + issuedAt: Date.now() - 1000, expiresAt: Date.now() + 900000 } + await updateMcpSettings({ enabled: true }, actor) + for (const lateInverseAuthorityChange of ["edit", "disable", "grant"] as const) { + const evidence = await acceptScopedRouteCommunitySwitch(actor, { lateInverseAuthorityChange }) + console.log("genuine late inverse authority SQL acceptance:", JSON.stringify(evidence)) + } +} finally { + await (await import("../../services/scheduler.js")).stopSchedulerForDatabaseOperation() + await closePool() +} diff --git a/backend/src/modules/mcp/routing-reservation.sql-helper.ts b/backend/src/modules/mcp/routing-reservation.sql-helper.ts index fe469807..73d5daaf 100644 --- a/backend/src/modules/mcp/routing-reservation.sql-helper.ts +++ b/backend/src/modules/mcp/routing-reservation.sql-helper.ts @@ -352,6 +352,47 @@ export async function acceptRoutingReservationSql(options: { runtimeAdmission?: await assert.rejects(store.beginForwardAttempt(reservation.forwardAllocation), /capability/) names.push("spent forward/no genuine checkpoint and preterminal unused-proof refuse; unknown never replays or refunds") } + { + const f = await fixture(), source = await f.makeSource(), reservation = await store.reserve(source.operationId) + const { saveScopedConfigurationCheckpoint, saveScopedNetworkCheckpoint } = await import("./routing-community-scoped-checkpoint.js") + const { executeReservedScopedRecovery } = await import("./routing-community-scoped-recovery.js") + await pool.query("UPDATE mcp_operations SET state='running' WHERE id=$1", [source.operationId]) + const step = { kind: "service", service: "routing.community.scoped", input: source.input } + await pool.query("INSERT INTO mcp_operation_steps(operation_id,step_index,state,intended_state) VALUES($1,0,'intent',$2::jsonb)", + [source.operationId, JSON.stringify(security.redact(step))]) + await pool.query("INSERT INTO mcp_operation_checkpoints(operation_id,step_index,before_sealed) VALUES($1,0,$2)", + [source.operationId, await security.encryptSecret(JSON.stringify(source.input.beforeRules))]) + assert(reservation.forwardAllocation) + await store.beginForwardAttempt(reservation.forwardAllocation) + // Use the actual checkpoint writers for production crypto and SQL, but + // deliberately seed a synthetic failed-forward observation. No PATCH + // or genuine network observation is part of this owned component case. + const requestSeal = String(source.input.requestProof) + const configuration = await saveScopedConfigurationCheckpoint(source.input, requestSeal) + await saveScopedNetworkCheckpoint(source.input, requestSeal, { id: randomUUID(), candidateId: "b", + status: "failed", observedAt: configuration.observedAt }) + const published = await store.recordForwardOutcome(reservation.reservationId) + assert(published.allocation) + assert.equal(published.receipt.state.inverse, "held") + const refused = { sourceOperationId: randomUUID(), principal: f.principal } + await assert.rejects(executeReservedScopedRecovery(published.allocation, refused), /Reserved runner identity differs/) + const unknown = await store.read(reservation.reservationId, f.principal) + assert.deepEqual(unknown.state, { ...published.receipt.state, revision: published.receipt.state.revision + 2, inverse: "unknown" }) + assert.equal(unknown.chargedMutationUnits, 2) + assert.equal(unknown.chargedProfileSlots, 2) + assert.equal(unknown.refundsAvailable, false) + const shared = (await pool.query(`SELECT state,slot_phase,inverse_intent_sealed,inverse_configuration_sealed,verified_sealed + FROM mcp_scoped_recovery_intents WHERE reservation_id=$1`, [reservation.reservationId])).rows + assert.deepEqual(shared, [{ state: "unknown", slot_phase: "reserved-attempt", inverse_intent_sealed: null, + inverse_configuration_sealed: null, verified_sealed: null }]) + await assert.rejects(executeReservedScopedRecovery(published.allocation, refused), /capability/) + await assert.rejects(store.recordForwardOutcome(reservation.reservationId), /republished/) + await assert.rejects(store.proveUnusedInverse(reservation.reservationId), /terminal applied/) + assert.equal(await used(f.connectionId), 2) + assert.equal((await store.readHourlyCharges(f.profileId, f.principal)).reservedCharges, 2) + assert.deepEqual((await store.read(reservation.reservationId, f.principal)).state, unknown.state) + names.push("genuine inverse capability on synthetic failed-forward checkpoints: acknowledged spend, late identity refusal, signed same-slot unknown and both charges conserved without replay or refund") + } { const f = await fixture(), source = await f.makeSource(), reservation = await store.reserve(source.operationId) const saved = (await pool.query("SELECT state_proof FROM mcp_routing_reservations WHERE id=$1", [reservation.reservationId])).rows[0].state_proof diff --git a/backend/src/modules/mcp/runtime.ts b/backend/src/modules/mcp/runtime.ts index a2160f65..ea37e174 100644 --- a/backend/src/modules/mcp/runtime.ts +++ b/backend/src/modules/mcp/runtime.ts @@ -1,4 +1,6 @@ import { pollErrorLog } from "../../services/server-poll-error-provenance.js" +import { capturePublishedServerProbeTerminalProof } from "../../services/server-probe-operation-retirement.js" +import { SERVER_PROBE_OPERATION } from "../../services/server-probe-authority.js" import { preparedPlanExpiry } from "./plan-expiry.js" import { acknowledgedCreateId, assertNoCallerCreateId } from "./native-create-identity.js" import { randomUUID, createHash } from "node:crypto" @@ -23,6 +25,7 @@ import { beginMcpOperationAdmission, insertMcpOperationAdmission, retireMcpOpera import { createOspfOwnerSourceGuard, verifyOspfOwnerGraph, verifyOspfOwnerGraphAtPhase } from "./ospf-owner-graph.js" import { evaluateMcpPolicy } from "./policy.js" import { findAction, findActionByOperation } from "./actions.js" +import { refuseExpiredOperationReplay } from "./operation-replay-tombstones.js" import { accessGeneration, decryptSecret, encryptSecret, fingerprint, privateSecretFingerprint, McpError, redact } from "./security.js" import { audit, getConnection, transaction, validatePrincipal, effectiveMcpConnection, assertConnectionVisible, type McpPrincipal, type ConnectionRow, type StoredPlan, type StoredOperation } from "./store.js" import { executeRosStep, readRosSnapshot, resourceById, routerClient, type RosSnapshot } from "./resources.js" @@ -291,6 +294,7 @@ export async function applyPlan(principal: McpPrincipal, planId: string, idempot reservedRouting = !!await routingReservationEvidence(original, originalEvidence) return existing } + await refuseExpiredOperationReplay(client, principal, connection.id, planId, idempotencyKey) await client.query("SELECT pg_advisory_xact_lock(hashtextextended('mcp-runner-admission',0))") const pending = await client.query<{ count: number }>("SELECT count(*)::int AS count FROM mcp_operations WHERE state IN ('pending','running')") if (pending.rows[0].count >= 100) throw new McpError("BUDGET_EXCEEDED", "Очередь исполнителя заполнена; повторите подготовленное применение позже", 429) @@ -359,6 +363,7 @@ export async function applyPlan(principal: McpPrincipal, planId: string, idempot async function executeOperation(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant, operationAdmission?: McpOperationAdmission): Promise { return withMcpOperationExecutionOrigin(operationAdmission, async () => { const lease = await pool.connect() + let terminalProbe: Awaited> let locked = false, released = false const ownerLifetime = new AbortController() // This closure is the SINGLE owner of physical destruction/release. @@ -380,7 +385,7 @@ async function executeOperation(operationId: string, principal: McpPrincipal, ro throw error } if (released || !locked) return - await withOperationConfigurationLockClient(lease, retire, ownerLifetime.signal, + terminalProbe = await withOperationConfigurationLockClient(lease, retire, ownerLifetime.signal, () => executeOperationWithLease(operationId, principal, routingReservationForward, routingReservationId, originalServiceGrant)) } finally { if (!released && locked) { @@ -395,9 +400,14 @@ async function executeOperation(operationId: string, principal: McpPrincipal, ro lease.release() } } + // Capture has its own bounded checkout and may fail without rewriting the + // acknowledged terminal outcome or redispatching the diagnostic. + if (terminalProbe) { + try { await capturePublishedServerProbeTerminalProof(terminalProbe) } catch { /* Existing retention remains authoritative. */ } + } }) } -async function executeOperationWithLease(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant): Promise { +async function executeOperationWithLease(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant) { const { rows: [operation] } = await pool.query("SELECT * FROM mcp_operations WHERE id=$1 AND state='running'", [operationId]) if (!operation) return const plan = await loadPlan(operation.plan_id), evidence = JSON.parse(await decryptSecret(plan.evidence.sealed)) as ChangeEvidence @@ -576,14 +586,18 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin }) await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks) : applyWithLocks()) assertOperationConfigurationLockCheckpoint() - await transaction(async client => { + const receiptId = plan.operation_id === SERVER_PROBE_OPERATION ? diagnosticReceipt?.receiptId : undefined + const finishedAt = await transaction(async client => { assertOperationConfigurationLockCheckpoint() - await client.query("UPDATE mcp_operations SET state='applied',verification=$3,result=$2::jsonb,finished_at=now() WHERE id=$1", [operationId, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, verificationScope: evidence.verificationScope ?? "configuration", ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }), evidence.verificationScope === "acknowledgement" ? "inconclusive" : "passed"]) + const terminal = await client.query<{ finished_at: string }>("UPDATE mcp_operations SET state='applied',verification=$3,result=$2::jsonb,finished_at=now() WHERE id=$1" + + (receiptId ? ` RETURNING to_char(finished_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS finished_at` : ""), + [operationId, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, verificationScope: evidence.verificationScope ?? "configuration", ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }), evidence.verificationScope === "acknowledgement" ? "inconclusive" : "passed"]) assertOperationConfigurationLockCheckpoint() await client.query("UPDATE mcp_plans SET state='completed' WHERE id=$1", [plan.id]) assertOperationConfigurationLockCheckpoint() await audit(client, "mcp-runner", "operation.complete", operation.connection_id, { operationId, confirmedWrites: diagnosticOnly ? 0 : confirmed, ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }) assertOperationConfigurationLockCheckpoint() + return receiptId ? terminal.rows[0]?.finished_at : undefined }) assertOperationConfigurationLockCheckpoint() if (routingReservationId && reservedForwardPassed) { @@ -592,6 +606,7 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin await createRoutingReservationStore().proveUnusedInverse(routingReservationId).catch(() => {}) } assertOperationConfigurationLockCheckpoint() + if (receiptId && finishedAt) return { receiptId, operationId, connectionId:operation.connection_id, finishedAt } } catch (error) { if (routingReservationId && hasConfirmedReservedRoutingForward(error)) confirmed = Math.max(confirmed, 1) if (error instanceof VrfProductionAcknowledgedFailure) confirmed = Math.max(confirmed, error.acknowledgedWrites) diff --git a/backend/src/modules/mcp/server-probe-actions.test.ts b/backend/src/modules/mcp/server-probe-actions.test.ts index ab01c1b2..3d07a16b 100644 --- a/backend/src/modules/mcp/server-probe-actions.test.ts +++ b/backend/src/modules/mcp/server-probe-actions.test.ts @@ -12,9 +12,9 @@ const principal = { connectionId, credentialId, credentialExpiresAt: new Date(no const proof = async (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex") const context = { principal, connection: { id: connectionId, owner_id: "owner", owner_claims_issued_at: new Date(now - 1000) } } as ActionContext let assertions = 0 -function fixture() { +function fixture(retiredSource?: ServerProbeMcpDependencies["retiredSource"]) { const events: string[] = [], receipts = new Map(), contexts: ServerProbeJournalContext[] = [], sealed = new Map() - let evidence: ChangeEvidence, result: unknown = null, stepState = "intent", operationState = "running", scope = [2, 9], revoked = false, sqlTamper: string | undefined, dispatchFailure: unknown, preparedTool: PreparedServerProbe["tool"] = "ping", mismatchDestination = false, nextCipher = 0, revokeOnQuery = false, expiredPrepared = false, clock = now + let evidence: ChangeEvidence, result: unknown = null, stepState = "intent", operationState = "running", scope = [2, 9], revoked = false, sqlTamper: string | undefined, dispatchFailure: unknown, preparedTool: PreparedServerProbe["tool"] = "ping", mismatchDestination = false, nextCipher = 0, revokeOnQuery = false, expiredPrepared = false, clock = now, liveHistory = true const access: ServerProbeAuthority = { transport: "mcp", ownerId: "owner", connectionId, credentialId, expiresAt: now + 600000, generation: "generation", policyProof: "1".repeat(64), ownerProof: "2".repeat(64), active: true, operationAllowed: true, scopeServerIds: scope, ownerPermissions: ["mm:servers:write"], policyPermissions: ["mm:servers:write"] } const readAuthority = async () => { events.push("authority"); if (revoked) throw new ServerProbeExecutionError("PERMISSION_DENIED", "PRIVATE_PROVIDER_SECRET", 403); return { ...access, expiresAt: clock + 600000, scopeServerIds: [...scope] } } const assertProtections = async (ids: number[]) => { events.push("protections"); if (ids.some(id => !scope.includes(id))) throw new Error("out of scope") } @@ -28,6 +28,7 @@ function fixture() { read: async id => { events.push("journal.read"); return receipts.get(id) }, } const deps: ServerProbeMcpDependencies = { + retiredSource, authority: (() => ({ readAuthority, assertProtections })) as ServerProbeMcpDependencies["authority"], historicalAuthority: ((currentPrincipal: typeof principal) => ({ readAuthority: async () => { events.push("read-authority"); return { ...await readAuthority(), connectionId: currentPrincipal.connectionId, credentialId: currentPrincipal.credentialId } }, assertProtections })) as ServerProbeMcpDependencies["historicalAuthority"], history: dependencies => createServerProbeJournalHistoryReader({ ...dependencies, loadArchive: async binding => { @@ -51,7 +52,7 @@ function fixture() { journal: (ctx, dependencies, execution) => { assert.equal(execution?.deadlineAt, Date.parse(prepared().expiresAt)); contexts.push(ctx); return { ...journal, read: async (...args) => { await dependencies.assertAuthority(ctx, args[1]); return journal.read(...args) }, claim: async binding => { await dependencies.assertAuthority(ctx, binding.bindingProof); return journal.claim(binding) } } }, query: async (sql, values) => { events.push("sql"); if (revokeOnQuery) revoked = true - if (sql.includes("probe-history:scope")) { assert.equal(values[2], connectionId); assert.equal(values[3], "owner"); return [{ source_server_id: 9, destination_server_id: preparedTool === "bandwidth" ? 2 : null }] } + if (sql.includes("probe-history:scope")) { assert.equal(values[2], connectionId); assert.equal(values[3], "owner"); return liveHistory ? [{ source_server_id: 9, destination_server_id: preparedTool === "bandwidth" ? 2 : null }] : [] } if (sql.includes("FROM servers")) { assert.ok(sql.includes("ANY($1::int[])")); assert.deepEqual(values[0], scope); return sql.includes("AND id=$2") ? [] : [{ id: 2, host: "peer", enabled: true }] } if (!sql.includes("mcp_operation_steps")) return [{ evidence: { sealed: await seal(JSON.stringify(evidence)) } }] assert.equal(values[0], connectionId); assert.equal(values[1], "owner"); assert.equal(values[2], "servers.probes.run") @@ -68,8 +69,9 @@ function fixture() { }, seal, open, proof, now: () => clock, } const adapter = createServerProbeMcpAdapter(deps) + const retireHistory = () => { liveHistory = false } const prepare = async (tool: PreparedServerProbe["tool"] = "ping") => { evidence = await adapter.prepare({ serverId: 9, request: tool === "bandwidth" ? { tool, bwRemoteAddress: "peer", dstServerId: 8 } : { tool, target: "192.0.2.1" } }, context); const step = evidence.steps[0]; assert.equal(step.kind, "service"); return step.kind === "service" ? step.input : assert.fail() } - return { adapter, events, receipts, contexts, prepare, setResult: (value: unknown) => { result = value; stepState = "applied" }, finishOperation: () => { operationState = "applied"; stepState = "acknowledged" }, setTamper: (value?: string) => { sqlTamper = value }, revoke: () => { revoked = true }, setScope: (value: number[]) => { scope = value }, fail: (value: unknown) => { dispatchFailure = value }, revokeOnQuery: () => { revokeOnQuery = true }, evidence: () => evidence, mismatchDestination: () => { mismatchDestination = true }, expirePrepared: () => { expiredPrepared = true; clock = now + 3600000 } } + return { adapter, events, receipts, contexts, prepare, retireHistory, setResult: (value: unknown) => { result = value; stepState = "applied" }, finishOperation: () => { operationState = "applied"; stepState = "acknowledged" }, setTamper: (value?: string) => { sqlTamper = value }, revoke: () => { revoked = true }, setScope: (value: number[]) => { scope = value }, fail: (value: unknown) => { dispatchFailure = value }, revokeOnQuery: () => { revokeOnQuery = true }, evidence: () => evidence, mismatchDestination: () => { mismatchDestination = true }, expirePrepared: () => { expiredPrepared = true; clock = now + 3600000 } } } for (const tool of ["ping", "traceroute", "bandwidth", "dns", "route", "mtu"] as const) { @@ -128,6 +130,38 @@ for (const input of [{ serverId: 9, request: { tool: "ping", target: "x", creden await assert.rejects(f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0, signal: controller.signal })); assert.equal(f.events.filter(event => event === "execute").length, attempted); assertions += 3 } assert.ok(!("diagnosticProgress" in serverProbeSafeMcpError(new ServerProbeExecutionError("X", "secret", 500, "unknown", 99), { receiptId, operationId, stageBudget: 1 }))); assertions++ +{ + const hash = "3".repeat(64), date = (value: number) => new Date(value).toISOString().replace("Z", "000Z") + const retired = { source: { version: 1 as const, operation: "servers.probes.run" as const, operationVersion: 1 as const, tool: "ping" as const, + receiptId, executionId: operationId, connectionId, ownerId: "owner", planId: randomUUID(), originalCredentialId: credentialId, + sourceServerId: 9, serverIds: [9], issuedAt: now - 1000, authorityExpiresAt: date(now + 600000), terminalAt: date(now + 600000), + activityAt: date(now + 600000), gcNotBefore: date(now + 87060000), retentionDays: 1, safetyMs: 60000, + bindingProof: hash, sourceDigest: hash, terminalDigest: hash, stageSnapshotDigest: hash, receiptProof: hash, ciphertextSha256: hash, + }, payloadPurged: false } + let lookups = 0, changeOnRecheck = false + const f = fixture(async lookup => { + lookups++ + assert.deepEqual(lookup, { receiptId, operationId, connectionId, ownerId: "owner", scopeServerIds: [2, 9] }) + return changeOnRecheck && lookups % 2 === 0 ? undefined : retired + }) + const input = await f.prepare(), output = await f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0 }) + f.setResult(output); f.finishOperation() + const request = { operationId, receiptId, page: 0 } + const live = await f.adapter.readPage(request, context) + assert.equal(lookups, 0) + f.retireHistory() + const rotated = { ...context, principal: { ...principal, credentialId: randomUUID() } } + assert.deepEqual(await f.adapter.readPage(request, rotated), live) + assert.deepEqual(await f.adapter.receiptTarget(request, rotated), { scope: "servers", serverIds: [9] }) + const beforeForeign = lookups + await assert.rejects(f.adapter.readPage(request, { ...rotated, connection: { ...context.connection, owner_id: "foreign" } })) + assert.equal(lookups, beforeForeign) + lookups = 0; changeOnRecheck = true + const privateReads = f.events.filter(event => event === "history.read").length + await assert.rejects(f.adapter.readPage(request, rotated)) + assert.equal(f.events.filter(event => event === "history.read").length, privateReads) + assertions += 7 +} { const f = fixture(), input = await f.prepare("bandwidth"), output = await f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0 }); f.setResult(output); f.finishOperation(); f.expirePrepared() await assert.rejects(f.adapter.service.current(input)) diff --git a/backend/src/modules/mcp/server-probe-actions.ts b/backend/src/modules/mcp/server-probe-actions.ts index 15aadd57..d5c2d3aa 100644 --- a/backend/src/modules/mcp/server-probe-actions.ts +++ b/backend/src/modules/mcp/server-probe-actions.ts @@ -10,6 +10,7 @@ import { serverProbeReceiptPage } from "../../services/server-probe-public-resul import { decryptSecret, encryptSecret, fingerprint, privateSecretFingerprint, McpError, redact } from "./security.js" import type { ActionContext, ChangeEvidence } from "./action-types.js" import type { McpServiceHandler } from "./service-registry.js" +import type { findRetiredServerProbeHistory, ServerProbeRetiredLookup } from "../../services/server-probe-retired-history.js" /** Foundations only: importing this module registers no tools or services. */ export const serverProbeActionSchema = z.object({ serverId: z.number().int().positive().safe(), request: serverProbeExecutionRequest }).strict() @@ -32,6 +33,7 @@ export interface ServerProbeMcpDependencies { executor: typeof createProductionServerProbeExecutor journal: (context: ServerProbeJournalContext, dependencies: ServerProbeJournalDependencies, execution?: { signal?: AbortSignal; deadlineAt?: number }) => ServerProbeJournal query: Query + retiredSource?: (lookup: ServerProbeRetiredLookup) => ReturnType seal: (value: string) => Promise open: (value: string) => Promise proof: (value: unknown) => Promise @@ -52,7 +54,7 @@ export async function openServerProbeMcpBinding(raw: Record, cr return { input, binding } } const ids = (prepared: PreparedServerProbe) => [...new Set([prepared.serverId, ...(prepared.destinationServerId ? [prepared.destinationServerId] : [])])].sort((a, b) => a - b) -function metadata(receipt: ServerProbeReceipt, operationId: string, stageBudget: number) { +export function serverProbeReceiptMetadata(receipt: ServerProbeReceipt, operationId: string, stageBudget: number) { const page = serverProbeReceiptPage(receipt, 0) if (!Number.isSafeInteger(receipt.attemptedStages) || !Number.isSafeInteger(receipt.completedStages) || receipt.completedStages < 0 || receipt.completedStages > receipt.attemptedStages || receipt.attemptedStages > stageBudget) return denied() return { operationId, receiptId: receipt.receiptId, tool: receipt.tool, status: receipt.state, httpStatus: receipt.httpStatus, stageBudget, attemptedStages: receipt.attemptedStages, completedStages: receipt.completedStages, sha256: page.sha256, totalPages: page.totalPages, bytes: page.bytes, automaticReplay: false, forwardingVerified: false } @@ -119,7 +121,7 @@ export function createServerProbeMcpAdapter(overrides: Partial, context: ActionContext): Promise => { const input = serverProbeActionSchema.parse(raw), access = deps.authority(context.principal, { signal: context.signal }) @@ -177,7 +179,7 @@ export function createServerProbeMcpAdapter(overrides: Partial { @@ -200,19 +202,47 @@ export function createServerProbeMcpAdapter(overrides: Partial a - b) + const loadRetired = deps.retiredSource + const readRetired = rows.length === 0 && loadRetired ? () => loadRetired({ + receiptId: request.receiptId, operationId: request.operationId, connectionId: context.principal.connectionId, + ownerId: initial.ownerId, scopeServerIds: initial.scopeServerIds, + }) : undefined + const retired = readRetired ? await readRetired() : undefined + if (readRetired) await fresh() + if (rows.length !== 1 && !retired) return denied() + const row = rows[0] + const serverIds = retired ? retired.source.serverIds : [...new Set([z.number().int().positive().safe().parse(row.source_server_id), ...(row.destination_server_id === null ? [] : [z.number().int().positive().safe().parse(row.destination_server_id)])])].sort((a, b) => a - b) const fence = async () => { const current = await fresh(); if (serverIds.some(id => !current.scopeServerIds.includes(id))) return denied(); await access.assertProtections(serverIds, current); await fresh() } await fence() - return { initial, serverIds, fence } + const retiredFence = retired && readRetired ? async () => { + await fence() + const current = await readRetired() + await fence() + if (!current || fingerprint(current) !== fingerprint(retired)) return denied() + } : undefined + return { initial, serverIds, fence, retired, retiredFence } } const receiptTarget = async (raw: Record, context: ActionContext): Promise => { - const request = serverProbeReceiptReadSchema.parse(raw), { serverIds, fence } = await historicalAccess(request, context) - await fence(); return { scope: "servers", serverIds } + const request = serverProbeReceiptReadSchema.parse(raw), { serverIds, fence, retiredFence } = await historicalAccess(request, context) + await (retiredFence ?? fence)(); return { scope: "servers", serverIds } } const readPage = async (raw: Record, context: ActionContext) => { try { - const request = serverProbeReceiptReadSchema.parse(raw), { initial, serverIds, fence } = await historicalAccess(request, context) + const request = serverProbeReceiptReadSchema.parse(raw), { initial, serverIds, fence, retired, retiredFence } = await historicalAccess(request, context) + if (retired && retiredFence) { + const source = retired.source + const originalContext: ServerProbeJournalContext = { + identity: { kind: "mcp", ownerId: source.ownerId, executionId: source.executionId, connectionId: source.connectionId, credentialId: source.originalCredentialId, + issuedAt: source.issuedAt, expiresAt: Date.parse(source.authorityExpiresAt), operationName: SERVER_PROBE_OPERATION }, + sourceServerId: source.sourceServerId, ...(source.destinationServerId !== undefined ? { destinationServerId: source.destinationServerId } : {}), tool: source.tool, + } + await retiredFence() + const reader = deps.history({ assertReadAuthority: retiredFence }, { signal: context.signal }) + const receipt = await reader.read({ context: originalContext, receiptId: source.receiptId, bindingProof: source.bindingProof }) + const page = serverProbeReceiptPage(receipt, request.page) + await retiredFence() + return page + } await fence() const rows = await deps.query(`SELECT p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id JOIN mcp_connections c ON c.id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2 AND c.owner_id=$3 AND p.operation_id=$4 AND p.operation_version=1 AND octet_length(p.evidence::text)<=2097152 LIMIT 2`, [request.operationId, context.principal.connectionId, initial.ownerId, SERVER_PROBE_OPERATION]) await fence(); if (rows.length !== 1) return denied() diff --git a/backend/src/modules/mcp/server-probe-all-tools.sql-helper.ts b/backend/src/modules/mcp/server-probe-all-tools.sql-helper.ts index f8da91ce..49055095 100644 --- a/backend/src/modules/mcp/server-probe-all-tools.sql-helper.ts +++ b/backend/src/modules/mcp/server-probe-all-tools.sql-helper.ts @@ -171,6 +171,16 @@ export async function acceptServerProbeAllToolsSql() { assert.equal(done.result.confirmedWrites, 0); assert.equal(done.result.completedServiceSteps, 1); assert.equal(done.result.verificationScope, "acknowledgement") const receipt = done.result.diagnosticReceipt as { receiptId: string; stageBudget: number; attemptedStages: number; completedStages: number; totalPages: number; sha256: string } assert(receipt); assert.equal(receipt.attemptedStages, receipt.completedStages); assert(receipt.attemptedStages > 0 && receipt.attemptedStages <= receipt.stageBudget) + let captured = false + for (const until = Date.now()+2000; Date.now()>'terminalAt'=to_char(GREATEST(o.finished_at,p.expires_at) AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS terminal_matches FROM server_probe_terminal_proofs f + JOIN mcp_operations o ON o.id=f.operation_id JOIN mcp_plans p ON p.id=o.plan_id + WHERE f.operation_id=$1 AND f.receipt_id=$2`, [done.id,receipt.receiptId])).rows[0] + if (proof) { assert.equal(proof.terminal_matches,true); captured = true; break } + await delay(20) + } + assert.equal(captured,true,"Owned successful diagnostic did not publish its terminal proof") const step = (await pool.query("SELECT state,result FROM mcp_operation_steps WHERE operation_id=$1", [done.id])).rows assert.equal(step.length, 1); assert.equal(step[0].state, "acknowledged") for (const field of ["output", "data", "contentB64", "bindingCiphertext"]) assert(!Object.hasOwn(step[0].result, field)) @@ -201,6 +211,49 @@ export async function acceptServerProbeAllToolsSql() { cases.push(tool + ": actual public plan/approval/apply, receipt metadata, full stage ledger and checksum pages without redispatch") } + // Force a genuine capture SQL failure after the real diagnostic has + // committed. A sequence records the rejected attempt despite rollback. + const captureFault = await fixture() + const faultPlan = await prepare(captureFault, { tool: "route", target: "127.0.0.1" }) + await pool.query("CREATE SEQUENCE public.owned_probe_capture_failure") + try { + await pool.query(`CREATE FUNCTION public.owned_probe_capture_failure() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.source->>'connectionId'=TG_ARGV[0] THEN + PERFORM nextval('public.owned_probe_capture_failure'); + RAISE EXCEPTION 'owned terminal proof insertion refused'; + END IF; + RETURN NEW; + END $$`) + try { + assert.match(captureFault.principal.connectionId,/^[a-f0-9-]{36}$/) + await pool.query(`CREATE TRIGGER owned_probe_capture_failure BEFORE INSERT ON public.server_probe_terminal_proofs + FOR EACH ROW EXECUTE FUNCTION public.owned_probe_capture_failure('${captureFault.principal.connectionId}')`) + try { + const beforeFault = requests+dnsUdp+dnsTcp, faultKey=randomUUID() + const applied = await applyPlan(captureFault.principal,faultPlan,faultKey), done=await terminal(applied.id) + let attempted = false + for (const until=Date.now()+2500;Date.now()running SQL BEFORE sending it. This // leaves a real pending approved source, allowing genuine durable cancel. // A server-table lock after 'running' would have no intent step yet and the @@ -209,8 +262,9 @@ export async function acceptServerProbeAllToolsSql() { const entered = gate(), resume = gate(); releasePending = resume.resolve let heldOperation = "", held = false pool.query = (async function (...args: unknown[]) { - if (!held && typeof args[0] === "string" && args[0].startsWith("UPDATE mcp_operations SET state='running'")) { - held = true; heldOperation = String((args[1] as unknown[])[0]); entered.resolve(); await resume.promise + const query = args[0] as { text?: string; values?: unknown[] } | undefined + if (!held && query?.text?.startsWith("/* mcp-operation-root:claim */") && typeof query.values?.[0] === "string") { + held = true; heldOperation = query.values[0]; entered.resolve(); await resume.promise } return Reflect.apply(originalQuery, pool, args) }) as typeof pool.query diff --git a/backend/src/modules/mcp/server-probe-storage-host.ts b/backend/src/modules/mcp/server-probe-storage-host.ts index 9b8a1c82..9cb78f0a 100644 --- a/backend/src/modules/mcp/server-probe-storage-host.ts +++ b/backend/src/modules/mcp/server-probe-storage-host.ts @@ -2,6 +2,7 @@ import { createMcpServerProbeAuthority, createMcpServerProbeReadAuthority } from "../../services/server-probe-authority.js" import { createProductionServerProbeExecutor } from "../../services/server-probe-execution.js" import { createServerProbeStorageHistoryReader } from "../../services/server-probe-storage-history.js" +import { findRetiredServerProbeHistory } from "../../services/server-probe-retired-history.js" import { createSqlServerProbeSourceReaders } from "../../services/server-probe-storage-source.js" import { createProbeStorageHostIo } from "../../services/server-probe-storage-host-io.js" import { assertVerifiedConfigurationAccess } from "../../services/config-mutation-policy.js" @@ -28,6 +29,7 @@ export function createStorageServerProbeMcpAdapter(limits: ProbeStorageLimits = }, } const adapter = createServerProbeMcpAdapter({ query: io.rows, journal: io.journal, + retiredSource: lookup => findRetiredServerProbeHistory(io.rows, lookup), authority: (principal, options) => createMcpServerProbeAuthority(principal, options, authorityDependencies), historicalAuthority: (principal, options) => createMcpServerProbeReadAuthority(principal, options, authorityDependencies), history: (dependencies, execution) => createServerProbeStorageHistoryReader({ assertReadAuthority: dependencies.assertReadAuthority, diff --git a/backend/src/modules/mcp/workers.ts b/backend/src/modules/mcp/workers.ts index a34e65ab..10c51fca 100644 --- a/backend/src/modules/mcp/workers.ts +++ b/backend/src/modules/mcp/workers.ts @@ -1,10 +1,13 @@ import { randomUUID } from "node:crypto" import { retainServerProbeStorage } from "../../services/server-probe-retention.js" +import { maintainServerProbeOperationRetirement } from "../../services/server-probe-operation-retirement.js" import { collectExpiredServerPollStorage } from "../../services/server-poll-storage-retention.js" import { compactCompletedServerPollStorage } from "../../services/server-poll-terminal-compaction.js" +import { retireCompletedServerPollOperations } from "../../services/server-poll-operation-retirement.js" import { pool } from "../../db/index.js" import { sharedDatabaseGate } from "../../services/shared-database-gate.js" import { recoverMcpOperations, verifyStep } from "./runtime.js" +import { retireOrdinaryMcpOperations } from "./ordinary-operation-retirement.js" import { audit, getMcpSettings } from "./store.js" import { accessGeneration, decryptSecret, redact } from "./security.js" import { tickRoutingProfiles } from "./routing-profiles.js" @@ -48,11 +51,22 @@ export async function detectMcpDrift() { } } +const mcpRetentionBatchSize = 1000 + async function retainMcpData() { const settings = await getMcpSettings() - await pool.query("DELETE FROM mcp_requests WHERE created_at$1) AND status NOT IN ('running')", [settings.requestsRetentionDays]) - await pool.query("DELETE FROM mcp_audit WHERE created_at$1)", [settings.auditRetentionDays]) - await pool.query("DELETE FROM mcp_artifacts WHERE expires_at$1) AND status NOT IN ('running') + ORDER BY created_at,id LIMIT $2 FOR UPDATE + ) DELETE FROM mcp_requests USING expired WHERE mcp_requests.id=expired.id`, [settings.requestsRetentionDays, mcpRetentionBatchSize]) + await pool.query(`WITH expired AS ( + SELECT id FROM mcp_audit WHERE created_at$1) + ORDER BY created_at,id LIMIT $2 FOR UPDATE + ) DELETE FROM mcp_audit USING expired WHERE mcp_audit.id=expired.id`, [settings.auditRetentionDays, mcpRetentionBatchSize]) + await pool.query(`WITH expired AS ( + SELECT id FROM mcp_artifacts WHERE expires_at retainManualInverseChargesWithinOwner(db, settings.requestsRetentionDays)) @@ -79,8 +93,11 @@ export async function tickMcpWorkers() { await retainMcpData() await retainServerProbeStorage() await compactCompletedServerPollStorage() + await retireCompletedServerPollOperations() await collectExpiredServerPollStorage() + await retireOrdinaryMcpOperations() if ((await getMcpSettings()).enabled) { await detectMcpDrift(); await tickRoutingProfiles() } + await maintainServerProbeOperationRetirement() } finally { if (locked) try { await lease.query("SELECT pg_advisory_unlock(hashtextextended('mcp-workers',0))") } catch { unsafe = true } lease.release(unsafe) diff --git a/backend/src/routes/mcp.ts b/backend/src/routes/mcp.ts index a42df6b4..6191a85d 100644 --- a/backend/src/routes/mcp.ts +++ b/backend/src/routes/mcp.ts @@ -18,6 +18,7 @@ import { accessGeneration, decryptSecret, encryptSecret, McpError, randomToken, import { createMcpConnection, listMcpConnections, getMcpSettings, updateMcpSettings, getConnection, assertConnectionVisible, rotateConnectionToken, setConnectionStatus, updateMcpConnection, transaction, audit } from "../modules/mcp/store.js" import { approvePlan, loadPlan, publicPlan } from "../modules/mcp/runtime.js" import { applyPlan, invokeMcpTool } from "../modules/mcp/runtime.js" +import { findOwnerPlanReplayTombstone } from "../modules/mcp/operation-replay-tombstones.js" import { routeAiProfileSchema } from "@mmapp/contracts/mcp-routing" import { revalidateMcpOwner } from "../modules/mcp/store.js" import { resourceById } from "../modules/mcp/resources.js" @@ -243,7 +244,16 @@ export function createMcpRoutes(checkClaims = assertPortalClaimsCurrent, options return ownerDelegate(async () => invokeMcpTool(principal, action === "recover" ? "mm_operation_recovery_plan" : globalReconcile ? "mm_global_operation_reconcile" : "mm_operation_reconcile", { operationId: id })) }) app.post("/mcp/plans/:id/apply", async request => { - const plan = await loadPlan(idOf(request.params)), actor = actorOf(request.authUser) + const id = idOf(request.params), actor = actorOf(request.authUser) + let plan: { id: string; connection_id: string; operation_id: string } + try { plan = await loadPlan(id) } + catch (error) { + if (!(error instanceof McpError) || error.code !== "NOT_FOUND") throw error + const tombstone = await findOwnerPlanReplayTombstone(pool, actor.id, id) + if (!tombstone) throw error + const principal = await ownerPrincipal(tombstone.connection_id, actor, tombstone.action_id) + return ownerDelegate(async () => applyPlan(principal, tombstone.plan_id, `web:${tombstone.plan_id}`)) + } const principal = await ownerPrincipal(plan.connection_id, actor, plan.operation_id) await audit(pool, actor.id, "plan.web.apply", plan.connection_id, { planId: plan.id }) return ownerDelegate(async () => applyPlan(principal, plan.id, `web:${plan.id}`)) diff --git a/backend/src/services/gre-mutation-state.ts b/backend/src/services/gre-mutation-state.ts index 6458ffae..0b0b8e70 100644 --- a/backend/src/services/gre-mutation-state.ts +++ b/backend/src/services/gre-mutation-state.ts @@ -6,10 +6,11 @@ import { observedBoolean, observedDurationMs, observedInteger } from "./ospf-obs import { readGreNativeRows } from "./gre-observed-state.js" import { greAddressKey } from "./entity-snapshots.js" import type { GreMutation } from "./gre-mutation-input.js" +import { greDependencyPaths } from "./routeros-native-paths.js" +export { greDependencyPaths } from "./routeros-native-paths.js" export const greMutationProblem = (message: string) => Object.assign(new Error(message), { statusCode: 409 }) export type GreNativeRow = Record -export const greDependencyPaths = ["/interface", "/interface/bridge/port", "/interface/vlan", "/interface/list/member", "/ip/vrf", "/ip/route", "/ipv6/route", "/routing/filter/rule", "/routing/ospf/interface-template", "/routing/bfd/configuration", "/routing/bgp/connection", "/routing/rule", ...["ip","ipv6"].flatMap(family=>["filter","nat","mangle","raw"].map(table=>`/${family}/firewall/${table}`))] as const export interface GreMutationState { serverConfiguration: string; gre: GreNativeRow[]; ipv4: GreNativeRow[]; ipv6: GreNativeRow[]; restoreProtectionProof?:string; diff --git a/backend/src/services/mikrotik.ts b/backend/src/services/mikrotik.ts index be733137..243bf8d5 100644 --- a/backend/src/services/mikrotik.ts +++ b/backend/src/services/mikrotik.ts @@ -617,10 +617,7 @@ export function firewallRestPath( return `${root}/${table}` } -export function encodeRosId(rosId: string): string { - const id = rosId.startsWith("*") ? rosId : `*${rosId.replace(/^\*/, "")}` - return encodeURIComponent(id) -} +export { encodeRosId } from "./routeros-native-paths.js" // ── MikrotikClient ───────────────────────────────────────────────────────────── diff --git a/backend/src/services/routeros-native-paths.ts b/backend/src/services/routeros-native-paths.ts new file mode 100644 index 00000000..445c45cf --- /dev/null +++ b/backend/src/services/routeros-native-paths.ts @@ -0,0 +1,17 @@ +export function encodeRosId(rosId: string): string { + const id = rosId.startsWith("*") ? rosId : `*${rosId.replace(/^\*/, "")}` + return encodeURIComponent(id) +} + +export const greDependencyPaths = ["/interface", "/interface/bridge/port", "/interface/vlan", + "/interface/list/member", "/ip/vrf", "/ip/route", "/ipv6/route", "/routing/filter/rule", + "/routing/ospf/interface-template", "/routing/bfd/configuration", "/routing/bgp/connection", "/routing/rule", + ...["ip", "ipv6"].flatMap(family => ["filter", "nat", "mangle", "raw"].map(table => `/${family}/firewall/${table}`))] as const + +export const wireguardInterfaceRuntimeDependencyPaths = ["/routing/bgp/session", "/routing/ospf/neighbor", + "/routing/ospf/interface", "/routing/bfd/session"] as const + +export const wireguardInterfaceDependencyPaths = [...new Set([...greDependencyPaths, "/interface/vxlan", + "/interface/vrrp", "/interface/bonding", "/interface/eoip", "/interface/gre", "/interface/ipip", "/ip/dhcp-server", + "/ip/dhcp-client", "/ipv6/dhcp-server", "/ipv6/dhcp-client", "/ip/ipsec/peer", "/ip/ipsec/policy", + ...wireguardInterfaceRuntimeDependencyPaths])] diff --git a/backend/src/services/server-poll-archive-accounting.test.ts b/backend/src/services/server-poll-archive-accounting.test.ts index 6195f001..ac581ef8 100644 --- a/backend/src/services/server-poll-archive-accounting.test.ts +++ b/backend/src/services/server-poll-archive-accounting.test.ts @@ -4,6 +4,7 @@ import assert from "node:assert/strict" import { createCipheriv, createDecipheriv, createHmac, createHash, randomBytes, randomUUID } from "node:crypto" import { test } from "node:test" import { assertServerPollArchiveAccountingForTests } from "./server-poll-archive-accounting.js" +import type { ServerPollRetiredArchive } from "./server-poll-retired-archive.js" import { createServerPollTerminalArchiveForTests, type ServerPollTerminalArchive, type ServerPollArchiveSourceMetadata } from "./server-poll-terminal-archive.js" import { PollStorageRefused, SERVER_POLL_STORAGE_POLICY as policy } from "./server-poll-storage-model.js" import type { ProbeStorageSql } from "./server-probe-journal-maintenance.js" @@ -19,7 +20,7 @@ const codec = createServerPollTerminalArchiveForTests({ verifySource: async source => source.admission.metadata as unknown as ServerPollArchiveSourceMetadata, }) const id = (n: number) => "00000000-0000-4000-8000-" + n.toString(16).padStart(12, "0") -interface Entry { row: Record; archive?: ServerPollTerminalArchive } +interface Entry { row: Record; archive?: ServerPollTerminalArchive | ServerPollRetiredArchive } const full = (n: number, owner: string): Entry => ({ row: { admission_receipt_id: id(n), identity_kind: "rest", admission_execution_id: randomUUID(), admission_connection_id: null, admission_owner_id: owner, admission_binding_proof: "a".repeat(64), admission_context_proof: "b".repeat(64), storage_state: "full", compact_at: null, admission_archive_bytes: null, admission_activity_at: null, receipt_id: null, has_children: true } }) @@ -34,7 +35,7 @@ async function compact(n: number, owner: string): Promise { storage_state: "compacted", compact_at: new Date("2026-10-29T02:00:00Z"), admission_archive_bytes: archive.ciphertext_bytes, admission_activity_at: archive.original_activity_at, admission_expires_at: archive.original_expires_at, asserted_ciphertext_bytes: archive.ciphertext_bytes, has_children: false } } } -function fixture(entries: Entry[]) { +function fixture(entries: Entry[], retiredCharge?: Parameters[2]) { const totals = new Map() for (const entry of entries) { for (const scope of ["global", "owner:" + entry.row.admission_owner_id]) { @@ -73,10 +74,35 @@ function fixture(entries: Entry[]) { } } return { entries, usage, state, queries, batches, usageBatches, counts: () => ({ factories, charges }), run: () => assertServerPollArchiveAccountingForTests(sql, async () => { factories++; return async row => { charges++; return codec.verifyCharge(row) } - }) } + }, retiredCharge) } } const refused = (work: Promise) => assert.rejects(work, PollStorageRefused) +test("mixed V1/V2 allocations authenticate actual stored ciphertext and never fall back across formats", async () => { + const { createServerPollRetiredArchiveForTests } = await import("./server-poll-retired-archive.js") + const unexpected = async () => { throw new Error("Metadata accounting must not open or write archives") } + const v2 = createServerPollRetiredArchiveForTests({ + proof, decrypt: unexpected, encrypt: unexpected, openV1: unexpected, verifyHistory: unexpected, + }) + const first = await compact(1, "owner"), second = await compact(2, "owner") + const { archive_proof: _v1Proof, ciphertext, ...previous } = second.archive! + const metadata = { ...previous, version: 2 as const } + const archiveProof = await proof("server-poll-retired-archive-record-v2", { ...metadata, identity_kind: "mcp", + storage_state: "compacted", admission_bytes: 32768, charged_rows: 2 }) + second.archive = { ...metadata, ciphertext, archive_proof: archiveProof } + Object.assign(second.row, { version: 2, archive_proof: archiveProof }) + const f = fixture([first, second], v2.verifyCharge) + await f.run() + assert.deepEqual(f.counts(), { factories: 1, charges: 1 }) + second.archive.ciphertext = "A" + ciphertext.slice(1) + await refused(f.run()) + assert.deepEqual(f.counts(), { factories: 2, charges: 2 }) + second.archive.ciphertext = ciphertext + second.row.version = 3 + await refused(f.run()) + assert.deepEqual(f.counts(), { factories: 3, charges: 3 }) +}) + test("empty/global0 and full128 ledgers use complete exact global/owner counters without archive crypto", async () => { const empty = fixture([]); await empty.run(); assert.deepEqual(empty.counts(), { factories: 0, charges: 0 }) const f = fixture(Array.from({ length: 128 }, (_, n) => full(n + 1, "owner-" + Math.floor(n / 8)))) diff --git a/backend/src/services/server-poll-archive-accounting.ts b/backend/src/services/server-poll-archive-accounting.ts index 83c180a6..0a11bc1c 100644 --- a/backend/src/services/server-poll-archive-accounting.ts +++ b/backend/src/services/server-poll-archive-accounting.ts @@ -3,6 +3,7 @@ import type { ProbeStorageSql } from "./server-probe-journal-maintenance.js" import { SERVER_POLL_STORAGE_POLICY as policy, PollStorageRefused } from "./server-poll-storage-model.js" import { createServerPollArchiveChargeVerifier, type verifyServerPollArchiveCharge } from "./server-poll-terminal-archive.js" +import type { verifyServerPollRetiredArchiveCharge } from "./server-poll-retired-archive.js" const BATCH = 128 const MAX_ENTRIES = Math.floor(policy.globalBytes / policy.admissionBytes) @@ -15,6 +16,8 @@ function integer(value: unknown): number { } interface Total { heavy: number; compact: number; archiveBytes: number; rows: number; bytes: number } const zero = (): Total => ({ heavy: 0, compact: 0, archiveBytes: 0, rows: 0, bytes: 0 }) +const verifyRetiredCharge: typeof verifyServerPollRetiredArchiveCharge = async row => + (await import("./server-poll-retired-archive.js")).verifyServerPollRetiredArchiveCharge(row) const QUERY = `SELECT a.receipt_id AS admission_receipt_id,a.identity_kind,a.execution_id AS admission_execution_id, a.connection_id AS admission_connection_id,a.owner_id AS admission_owner_id,a.binding_proof AS admission_binding_proof, a.context_proof AS admission_context_proof,a.storage_state,a.compact_at,a.archive_bytes AS admission_archive_bytes, @@ -34,7 +37,8 @@ const QUERY = `SELECT a.receipt_id AS admission_receipt_id,a.identity_kind,a.exe FROM server_poll_storage_admissions a LEFT JOIN server_poll_archives r ON r.receipt_id=a.receipt_id WHERE ($1::uuid IS NULL OR a.receipt_id>$1::uuid) ORDER BY a.receipt_id LIMIT 128` -async function verify(sql: ProbeStorageSql, factory: () => Promise) { +async function verify(sql: ProbeStorageSql, factory: () => Promise, + retiredCharge: typeof verifyServerPollRetiredArchiveCharge) { const global = zero(), owners = new Map() const seenReceipts = new Set() let cursor: string | null = null, count = 0, charge: typeof verifyServerPollArchiveCharge | undefined @@ -61,8 +65,13 @@ async function verify(sql: ProbeStorageSql, factory: () => Promise Promise { - try { await verify(sql, createServerPollArchiveChargeVerifier) } catch { return fail() } + try { await verify(sql, createServerPollArchiveChargeVerifier, verifyRetiredCharge) } catch { return fail() } } -export async function assertServerPollArchiveAccountingForTests(sql: ProbeStorageSql, factory: () => Promise): Promise { - try { await verify(sql, factory) } catch { return fail() } +export async function assertServerPollArchiveAccountingForTests(sql: ProbeStorageSql, factory: () => Promise, + retiredCharge = verifyRetiredCharge): Promise { + try { await verify(sql, factory, retiredCharge) } catch { return fail() } } diff --git a/backend/src/services/server-poll-history-reader.test.ts b/backend/src/services/server-poll-history-reader.test.ts index d3691fc9..30373e6f 100644 --- a/backend/src/services/server-poll-history-reader.test.ts +++ b/backend/src/services/server-poll-history-reader.test.ts @@ -5,7 +5,8 @@ import assert from "node:assert/strict" import { createCipheriv, createDecipheriv, createHmac, randomBytes, randomUUID } from "node:crypto" import { test } from "node:test" import { fingerprint } from "../modules/mcp/security.js" -import { readServerPollArchivedHistoryForTests } from "./server-poll-history-reader.js" +import { readRetiredServerPollHistoryForTests, readServerPollArchivedHistoryForTests } from "./server-poll-history-reader.js" +import type { openServerPollRetiredArchive } from "./server-poll-retired-archive.js" import type { openServerPollTerminalArchive, ServerPollArchiveSource } from "./server-poll-terminal-archive.js" function fixture(){ @@ -110,3 +111,61 @@ test("caller mutation while decrypt awaits cannot replace authenticated lookup i }}),true) assert.equal(f.calls[0].values[0],f.raw().receiptId) }) + +function retiredFixture() { + const now = Date.now(), operationId = randomUUID(), connectionId = randomUUID(), planId = randomUUID(), ownerId = "retired-owner" + const identity = { operationId, connectionId, ownerId } + const row = { plan_id: planId, target_server_ids: ["2", "1"], gc_not_before: "2026-10-31T01:00:00.123456Z" } + const failure = Error("authority revoked") + let queries = 0, opens = 0, checks = 0 + let mode = "" + const run = () => readRetiredServerPollHistoryForTests(identity, async () => { + checks++ + if (mode === "revoked-after-open" && opens) throw failure + }, { deadlineAt: now + 1000 }, { + now: () => now, + query: async (sql, parameters) => { + queries++ + assert.deepEqual(parameters, [operationId, connectionId, ownerId]) + assert.match(sql, /mcp_operation_replay_tombstones/) + assert.match(sql, /r\.version=2/) + assert.match(sql, /NOT EXISTS\(SELECT 1 FROM public\.mcp_operations/) + if (mode === "gone" && queries === 2) return [] + if (mode === "changed" && queries === 2) return [{ ...row, target_server_ids: ["1"] }] + return [row] + }, + open: async () => { + opens++ + if (mode === "bad-proof") throw Error("private proof failure") + return { metadata: { execution_id: operationId, connection_id: connectionId, owner_id: ownerId }, + history: { planId, affectedServerIds: [1, 2], gcNotBefore: "2026-10-31T01:00:00.123456Z" } } as Awaited> + }, + }) + return { row, failure, run, mode: (value: string) => { mode = value }, counts: () => ({ queries, opens, checks }) } +} + +test("retired lookup verifies a current tombstone, archive proof and a second SQL snapshot before returning only scope", async () => { + const f = retiredFixture() + assert.deepEqual(await f.run(), { affectedServerIds: [1, 2] }) + assert.deepEqual(f.counts(), { queries: 2, opens: 1, checks: 4 }) +}) + +test("retired lookup refuses proof, plan, scope or retention drift and GC between reads", async () => { + for (const mode of ["bad-proof", "changed", "gone"]) { + const f = retiredFixture() + f.mode(mode) + assert.equal(await f.run(), undefined) + } + for (const row of [{ plan_id: randomUUID() }, { target_server_ids: ["1", "1"] }, { gc_not_before: "2026-10-31T01:00:00.123457Z" }]) { + const f = retiredFixture() + Object.assign(f.row, row) + assert.equal(await f.run(), undefined) + } +}) + +test("retired lookup propagates exact current authorization failure after archive open", async () => { + const f = retiredFixture() + f.mode("revoked-after-open") + await assert.rejects(f.run(), error => error === f.failure) + assert.equal(f.counts().queries, 1) +}) diff --git a/backend/src/services/server-poll-history-reader.ts b/backend/src/services/server-poll-history-reader.ts index f37a5041..21559538 100644 --- a/backend/src/services/server-poll-history-reader.ts +++ b/backend/src/services/server-poll-history-reader.ts @@ -3,7 +3,8 @@ * Original credentials authenticate history only; they are never renewed. */ import { z } from "zod" import { decryptSecret, fingerprint, privateSecretFingerprint } from "../modules/mcp/security.js" -import { openServerPollTerminalArchive } from "./server-poll-terminal-archive.js" +import { canonicalPollArchiveTimestamp, openServerPollTerminalArchive } from "./server-poll-terminal-archive.js" +import { openServerPollRetiredArchive } from "./server-poll-retired-archive.js" export interface ServerPollHistoryReadWindow { deadlineAt: number; signal?: AbortSignal } type Query = (text: string, values: unknown[], options: ServerPollHistoryReadWindow) => Promise[]> @@ -99,3 +100,72 @@ const production:Dependencies={ export const readServerPollArchivedHistory=(raw:Record,authorize:()=>Promise,options:ServerPollHistoryReadWindow)=>read(raw,authorize,options,production) /** Explicit component-only composition. No public input selects these hooks. */ export const readServerPollArchivedHistoryForTests=(raw:Record,authorize:()=>Promise,options:ServerPollHistoryReadWindow,deps:Dependencies)=>read(raw,authorize,options,deps) + +const RETIRED_QUERY = `SELECT r.version,r.receipt_id,r.execution_id,r.connection_id,r.owner_id,r.binding_proof,r.context_proof, + r.source_digest,r.source_sha256,r.terminal_digest,r.decoded_bytes,r.compressed_bytes,r.archive_proof, + to_char(r.original_activity_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS original_activity_at, + to_char(r.original_expires_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS original_expires_at, + to_char(a.gc_not_before AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS gc_not_before, + a.retention_days,a.safety_ms,octet_length(r.ciphertext) AS ciphertext_bytes, + pg_catalog.encode(pg_catalog.sha256(pg_catalog.convert_to(r.ciphertext,'UTF8')),'hex') AS ciphertext_sha256, + CASE WHEN octet_length(r.ciphertext)<=33000 THEN r.ciphertext END AS ciphertext, + t.plan_id,t.target_server_ids + FROM public.server_poll_archives r JOIN public.server_poll_storage_admissions a USING(receipt_id) + JOIN public.mcp_connections c ON c.id=a.connection_id AND c.owner_id=a.owner_id + JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=a.execution_id AND t.connection_id=a.connection_id + WHERE r.execution_id=$1 AND a.execution_id=$1 AND a.connection_id=$2 AND a.owner_id=$3 + AND a.identity_kind='mcp' AND a.storage_state='compacted' AND r.version=2 + AND r.connection_id=a.connection_id AND r.owner_id=a.owner_id AND r.binding_proof=a.binding_proof AND r.context_proof=a.context_proof + AND r.ciphertext_bytes=a.archive_bytes AND r.ciphertext_bytes=octet_length(r.ciphertext) + AND r.original_activity_at=a.original_activity_at AND r.original_expires_at=a.authority_expires_at + AND t.action_id='servers.poll' AND t.target_global IS TRUE AND cardinality(t.target_server_ids) BETWEEN 1 AND 1000 + AND NOT EXISTS(SELECT 1 FROM public.mcp_operations o WHERE o.id=t.operation_id) + AND NOT EXISTS(SELECT 1 FROM public.mcp_plans p WHERE p.id=t.plan_id) + AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_steps s WHERE s.operation_id=t.operation_id) + LIMIT 2` + +interface RetiredDependencies { + query: Query; now(): number; open: typeof openServerPollRetiredArchive +} +export interface RetiredSourceIdentity { operationId: string; connectionId: string; ownerId: string } + +async function readRetired(identity: RetiredSourceIdentity, currentAuthorize: () => Promise, + rawWindow: ServerPollHistoryReadWindow, deps: RetiredDependencies): Promise<{ affectedServerIds: number[] } | undefined> { + let authFailure: { error: unknown } | undefined + try { + const { operationId, connectionId, ownerId } = identity + if (!uuid.safeParse(operationId).success || !uuid.safeParse(connectionId).success || !ownerId || ownerId.length > 200) fail() + const options = Object.freeze({ ...rawWindow, deadlineAt: Math.min(rawWindow.deadlineAt, deps.now() + 180000) }) + const live = () => { if (!Number.isSafeInteger(options.deadlineAt) || deps.now() >= options.deadlineAt || options.signal?.aborted) fail() } + const authorize = async () => { + live() + try { await currentAuthorize() } catch (error) { authFailure = { error }; throw error } + live() + } + const values = [operationId, connectionId, ownerId] + await authorize() + const rows = await deps.query(RETIRED_QUERY, values, options) + await authorize() + if (rows.length !== 1) return undefined + const row = structuredClone(rows[0]) + const { metadata, history } = await deps.open(row) + await authorize() + if (metadata.execution_id !== operationId || metadata.connection_id !== connectionId || metadata.owner_id !== ownerId + || history.planId !== row.plan_id || history.gcNotBefore !== canonicalPollArchiveTimestamp(row.gc_not_before)) fail() + const scope = z.array(z.union([sid, z.string().regex(/^[1-9]\d{0,15}$/).transform(Number).pipe(sid)])).min(1).max(1000).parse(row.target_server_ids) + if (scope.length !== history.affectedServerIds.length || new Set(scope).size !== scope.length + || scope.some(id => !history.affectedServerIds.includes(id))) fail() + const fresh = await deps.query(RETIRED_QUERY, values, options) + await authorize() + if (fresh.length !== 1 || fingerprint(fresh[0]) !== fingerprint(row)) fail() + return { affectedServerIds: [...history.affectedServerIds] } + } catch { + if (authFailure) throw authFailure.error + return undefined + } +} + +/** The caller owns current authority and independently checks returned scope. */ +export const readRetiredServerPollHistory = (identity: RetiredSourceIdentity, authorize: () => Promise, options: ServerPollHistoryReadWindow) => + readRetired(identity, authorize, options, { query: production.query, now: Date.now, open: openServerPollRetiredArchive }) +export const readRetiredServerPollHistoryForTests = readRetired diff --git a/backend/src/services/server-poll-operation-retirement.owned.test.ts b/backend/src/services/server-poll-operation-retirement.owned.test.ts new file mode 100644 index 00000000..f82aeb36 --- /dev/null +++ b/backend/src/services/server-poll-operation-retirement.owned.test.ts @@ -0,0 +1,183 @@ +/** Owned-only historical fixture. Synthetically seeds a historical timeline + * before signing. The real V1 compactor, V2 retirement, read and GC run with + * production crypto and all database guards enabled. */ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { assertOwnedBandwidthSql } from "./bandwidth-admission-process.sql-helper.js" + +assertOwnedBandwidthSql() +assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN,"true") +const { pool,closePool } = await import("../db/index.js") +const { applySqlMigrations } = await import("../db/migrate.js") +const { createMcpConnection,authenticateMcpToken } = await import("../modules/mcp/store.js") +const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp") +const { invokeMcpTool } = await import("../modules/mcp/runtime.js") +const { encryptSecret,privateSecretFingerprint,fingerprint,redact } = await import("../modules/mcp/security.js") +const { encodeServerPollJournalAnchor } = await import("./server-poll-journal-anchor.js") +const { compactCompletedServerPollStorage,SERVER_POLL_ARCHIVE_READ_SQL } = await import("./server-poll-terminal-compaction.js") +const { openServerPollTerminalArchive } = await import("./server-poll-terminal-archive.js") +const { retireCompletedServerPollOperation } = await import("./server-poll-operation-retirement.js") +const { collectExpiredServerPollStorage } = await import("./server-poll-storage-retention.js") +const { PollStorageUnknown } = await import("./server-poll-storage.js") +const { tickMcpWorkers } = await import("../modules/mcp/workers.js") +const { Client } = await import("pg") +const proof=(domain:string,value:unknown)=>privateSecretFingerprint(fingerprint(value),domain) +const encrypt=(value:unknown)=>encryptSecret(JSON.stringify(value)) + +async function historical(days:number,serverId:number) { + const base=Date.now()-days*86400000, stamp=(offset:number)=>new Date(base+offset).toISOString() + const ownerId=randomUUID(),executionId=randomUUID(),receiptId=randomUUID(),planId=randomUUID() + const actor={id:ownerId,email:"owned-historical-poll@example.test",name:"Owned historical poll",apps:["mm"], + permissions:["mm:servers:write","mm:servers:read"],issuedAt:Date.now()-1000,expiresAt:Date.now()+300000} + const issued=await createMcpConnection(mcpConnectionCreateSchema.parse({name:actor.name,mode:"approval", + permissions:actor.permissions,expiresAt:new Date(actor.expiresAt).toISOString(), + scope:{serverIds:[serverId],operationIds:["servers.poll","operations.global.reconcile"],allowGlobal:true}, + limits:{callsPerMinute:100,mutationsPerHour:10}}),actor) + const current=await authenticateMcpToken(issued.token),connectionId=current.connectionId,credentialId=randomUUID() + const core={version:1 as const,requiresHuman:true as const,scope:"global" as const,request:{serverId,receiptId},around:stamp(0), + beforeProof:"a".repeat(64),authorityProof:"b".repeat(64),maintenance:[],affectedServerIds:[serverId],mutationCost:8} + const plan={...core,bindingProof:await proof("server-poll-production-v1",core)} + const context={identity:{kind:"mcp" as const,ownerId,executionId,connectionId,credentialId,operationName:"servers.poll", + issuedAt:base,expiresAt:base+180000},plan} + const contextProof=await proof("server-poll-journal-v1:context",context) + const anchor=await encodeServerPollJournalAnchor(context,undefined,{proof:(domain,value)=>proof("server-poll-journal-v1:"+domain,value)}) + const binding={version:1,operation:"servers.poll",principal:{connectionId,credentialId,credentialExpiresAt:stamp(240000)}, + ownerId,issuedAt:base,expiresAt:base+180000,plan} + const input={serverId,receiptId,inputDigest:await proof("server-poll-mcp-binding-v1",binding),bindingCiphertext:await encrypt(binding)} + const step={kind:"service",service:"servers.poll",input} + const evidence={target:{scope:"global"},objects:[{kind:"server",id:String(serverId),serverId}],snapshots:[],steps:[step], + before:{proof:plan.beforeProof},mutationCost:8,requiresHuman:true,verificationScope:"configuration",validUntil:stamp(180000), + warnings:["Полный опрос читает устройство и изменяет общие снимки и разделы истории; требуется подтверждение человеком. Автоматический повтор запрещён."]} + const snapshot={id:42,serverId,polledAt:stamp(1000),status:"online" as const,latencyMs:0,rosVersion:null,boardName:null, + uptime:null,cpuLoad:null,freeMemory:null,totalMemory:null,identityName:null} + const result={operationId:executionId,receiptId,nativeReads:4,confirmedPublications:2,completedPublications:2,complete:true, + maintenanceComplete:true,configurationVerified:true,automaticReplay:false,snapshot} + const receipt={receiptId,nativeReads:4,confirmedPublications:2,completedPublications:2,complete:true, + maintenanceComplete:true,configurationVerified:true,afterProof:"f".repeat(64),observation:"online",snapshot} + const db=await pool.connect() + try{ + await db.query("BEGIN") + await db.query(`INSERT INTO mcp_plans(id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at,approved_by,approved_at,created_at) + VALUES($1,$2,'servers.poll',1,1,'owned-historical',$3,$4,'completed',$5,$6,$7,$8)`, + [planId,connectionId,{sealed:await encrypt({serverId})},{sealed:await encrypt(evidence)},stamp(190000),ownerId,stamp(50),stamp(25)]) + await db.query(`INSERT INTO mcp_operations(id,plan_id,connection_id,idempotency_key,state,verification,result,created_at,finished_at) + VALUES($1,$2,$3,$4,'applied','passed',$5,$6,$7)`, + [executionId,planId,connectionId,"owned-historical-"+executionId,{confirmedWrites:1,verificationScope:"configuration"},stamp(100),stamp(1600)]) + await db.query(`INSERT INTO mcp_operation_steps(operation_id,step_index,state,intended_state,result,created_at,finished_at) + VALUES($1,0,'verified',$2,$3,$4,$5)`,[executionId,redact(step),result,stamp(110),stamp(1500)]) + // Historical journal admission is impossible via live writes by design. + // Only this private fixture transaction seeds the signed, pre-existing rows. + await db.query("SET LOCAL session_replication_role='replica'") + await db.query(`INSERT INTO server_poll_storage_admissions(receipt_id,identity_kind,execution_id,connection_id,owner_id,binding_proof, + context_proof,authority_issued_at,authority_expires_at,max_stages,claim_attempted_at,created_at,retention_days,gc_not_before,used_rows,used_bytes) + VALUES($1,'mcp',$2,$3,$4,$5,$6,$7,$8,8,$9,$10,365,$11,14,4000)`, + [receiptId,executionId,connectionId,ownerId,plan.bindingProof,contextProof,stamp(0),stamp(180000),stamp(300),stamp(200),stamp(180000+365*86400000+60000)]) + await db.query(`INSERT INTO server_poll_executions(receipt_id,identity_kind,execution_id,binding_proof,context_ciphertext,context_proof,created_at) + VALUES($1,'mcp',$2,$3,$4,$5,$6)`,[receiptId,executionId,plan.bindingProof,await encryptSecret(anchor),contextProof,stamp(400)]) + const stages=["native.identity","native.resource","native.interfaces","native.addresses","publish.0","publish.1"] + for(const [index,stage] of stages.entries()){ + const beforeProof=index>4?"e".repeat(64):plan.beforeProof,effectProof="c".repeat(64) + const resultProof=index===stages.length-1?receipt.afterProof:"e".repeat(64),outcome="confirmed" + await db.query("INSERT INTO server_poll_intents VALUES($1,$2,$3,$4,$5,$6)", + [receiptId,stage,beforeProof,effectProof,await proof("server-poll-journal-v1:intent",{receiptId,stage,beforeProof,effectProof}),stamp(500+index*100)]) + await db.query("INSERT INTO server_poll_acks VALUES($1,$2,$3,$4,$5,$6)", + [receiptId,stage,resultProof,outcome,await proof("server-poll-journal-v1:ack",{receiptId,stage,resultProof,outcome}),stamp(550+index*100)]) + } + const receiptText=JSON.stringify(receipt) + await db.query("INSERT INTO server_poll_receipts VALUES($1,$2,$3,$4)", + [receiptId,await encryptSecret(receiptText),await proof("server-poll-journal-v1:receipt",receiptText),stamp(1400)]) + await db.query("INSERT INTO server_poll_storage_usage(scope,admissions,reserved_rows,reserved_bytes) VALUES($1,1,4019,8421376)",["owner:"+ownerId]) + await db.query("UPDATE server_poll_storage_usage SET admissions=admissions+1,reserved_rows=reserved_rows+4019,reserved_bytes=reserved_bytes+8421376 WHERE scope='global'") + await db.query("SET LOCAL session_replication_role='origin'") + assert.equal((await db.query("SELECT server_poll_storage_accounting_valid() AS valid")).rows[0].valid,true) + await db.query("COMMIT") + }catch(error){await db.query("ROLLBACK");throw error}finally{db.release()} + return {receiptId,executionId,planId,ownerId,connectionId,current} +} + +try{ + await applySqlMigrations(pool) + await pool.query("UPDATE mcp_settings SET enabled=true,requests_retention_days=365 WHERE id=1") + const {rows:[server]}=await pool.query("INSERT INTO servers(name,host,enabled) VALUES('owned historical poll','127.0.0.1',false) RETURNING id") + const retained=await historical(181,Number(server.id)), expired=await historical(366,Number(server.id)) + for(const fixture of [expired,retained]){ + assert.equal((await compactCompletedServerPollStorage()).compacted,1) + const {rows:[v1]}=await pool.query(SERVER_POLL_ARCHIVE_READ_SQL,[fixture.receiptId]) + await openServerPollTerminalArchive(v1,Date.now()) + const before=(await pool.query("SELECT gc_not_before,authority_expires_at,retention_days,original_activity_at FROM server_poll_storage_admissions WHERE receipt_id=$1",[fixture.receiptId])).rows[0] + await assert.rejects(retireCompletedServerPollOperation({receiptId:fixture.receiptId,oldArchiveProof:"0".repeat(64)})) + const hold=randomUUID() + await pool.query("INSERT INTO mcp_drift(id,operation_id,expected,observed) VALUES($1,$2,'{}','{}')",[hold,fixture.executionId]) + await assert.rejects(retireCompletedServerPollOperation({receiptId:fixture.receiptId,oldArchiveProof:v1.archive_proof})) + assert.equal((await pool.query(SERVER_POLL_ARCHIVE_READ_SQL,[fixture.receiptId])).rows[0].archive_proof,v1.archive_proof) + assert.equal((await pool.query("SELECT server_poll_storage_accounting_valid() AS valid")).rows[0].valid,true) + assert.equal((await pool.query("SELECT 1 FROM server_poll_retirement_claims WHERE receipt_id=$1",[fixture.receiptId])).rows.length,0) + await pool.query("DELETE FROM mcp_drift WHERE id=$1",[hold]) + assert.equal((await retireCompletedServerPollOperation({receiptId:fixture.receiptId,oldArchiveProof:v1.archive_proof})).retired,true) + const {rows:[v2]}=await pool.query(SERVER_POLL_ARCHIVE_READ_SQL,[fixture.receiptId]) + assert.equal(v2.version,2);assert(v2.ciphertext_bytes(error as {code?:string}).code==="NOT_FOUND") + } + assert.equal((await collectExpiredServerPollStorage()).collected,1) + assert.equal((await collectExpiredServerPollStorage()).collected,0) + const {rows}=await pool.query("SELECT r.receipt_id,r.ciphertext_bytes,u.reserved_bytes,u.reserved_rows FROM server_poll_archives r CROSS JOIN server_poll_storage_usage u WHERE u.scope='global'") + assert.equal(rows.length,1);assert.equal(rows[0].receipt_id,retained.receiptId) + assert.equal(Number(rows[0].reserved_bytes),32768+rows[0].ciphertext_bytes);assert.equal(Number(rows[0].reserved_rows),2) + assert.equal((await pool.query("SELECT 1 FROM server_poll_retirement_claims WHERE receipt_id=$1",[expired.receiptId])).rows.length,0) + + const held=await historical(190,Number(server.id)), spent=await historical(189,Number(server.id)) + const ready=await historical(188,Number(server.id)) + // The first two are already compacted. The actual worker has to compact the + // next eligible archive, skip older held/spent V1 rows, then retire it. + assert.equal((await compactCompletedServerPollStorage()).compacted,1) + assert.equal((await compactCompletedServerPollStorage()).compacted,1) + await pool.query("INSERT INTO mcp_drift(id,operation_id,expected,observed) VALUES($1,$2,'{}','{}')",[randomUUID(),held.executionId]) + const {rows:[spentV1]}=await pool.query(SERVER_POLL_ARCHIVE_READ_SQL,[spent.receiptId]) + const originalQuery=Client.prototype.query + let commits=0 + try{ + Client.prototype.query=function(this:InstanceType,...args:unknown[]){ + if((args[0] as {text?:string})?.text==="COMMIT" && ++commits===2) { + // No COMMIT ACK can reach the owner. Closing the real checked-out + // backend rolls back uncommitted work, leaving the earlier claim spent. + return Promise.reject(Error("owned poll retirement COMMIT transport loss")) + } + return Reflect.apply(originalQuery,this,args) + } as typeof Client.prototype.query + await assert.rejects(retireCompletedServerPollOperation({receiptId:spent.receiptId,oldArchiveProof:spentV1.archive_proof}), + (error:unknown)=>error instanceof PollStorageUnknown && error.automaticReplay===false) + }finally{Client.prototype.query=originalQuery} + assert.equal(commits,2) + assert.equal((await pool.query(SERVER_POLL_ARCHIVE_READ_SQL,[spent.receiptId])).rows[0].version,1) + assert.equal((await pool.query("SELECT 1 FROM server_poll_retirement_claims WHERE receipt_id=$1",[spent.receiptId])).rows.length,1) + const workerProjection=async()=>(await pool.query(`SELECT a.receipt_id,r.version,r.archive_proof,a.archive_bytes, + o.id AS live_operation,t.operation_id AS tombstone,c.claim_token + FROM server_poll_storage_admissions a LEFT JOIN server_poll_archives r USING(receipt_id) + LEFT JOIN mcp_operations o ON o.id=a.execution_id + LEFT JOIN mcp_operation_replay_tombstones t ON t.operation_id=a.execution_id + LEFT JOIN server_poll_retirement_claims c USING(receipt_id) + WHERE a.receipt_id=ANY($1::uuid[]) ORDER BY a.receipt_id`,[[held.receiptId,spent.receiptId,ready.receiptId]])).rows + const beforeWorker=await workerProjection() + await tickMcpWorkers() + const afterWorker=await workerProjection() + assert.deepEqual(afterWorker.filter(row=>row.receipt_id!==ready.receiptId),beforeWorker.filter(row=>row.receipt_id!==ready.receiptId)) + const retiredByWorker=afterWorker.find(row=>row.receipt_id===ready.receiptId) + assert.equal(retiredByWorker?.version,2);assert.equal(retiredByWorker?.live_operation,null) + assert.equal(retiredByWorker?.tombstone,ready.executionId);assert(retiredByWorker?.claim_token) + await tickMcpWorkers() + assert.deepEqual(await workerProjection(),afterWorker) + assert.equal((await pool.query("SELECT server_poll_storage_accounting_valid() AS valid")).rows[0].valid,true) + console.log("Owned PostgreSQL: authenticated V1/V2 transition, retained history, once-only refund, and actual worker skips held/spent rows") +}finally{await closePool()} diff --git a/backend/src/services/server-poll-operation-retirement.ts b/backend/src/services/server-poll-operation-retirement.ts new file mode 100644 index 00000000..9f2f37de --- /dev/null +++ b/backend/src/services/server-poll-operation-retirement.ts @@ -0,0 +1,162 @@ +/** Scheduled/internal maintenance only. No public HTTP or MCP registration. */ +import { randomUUID } from "node:crypto" +import { z } from "zod" +import { operationIdempotencyDigest } from "../modules/mcp/operation-replay-tombstones.js" +import { fingerprint } from "../modules/mcp/security.js" +import { assertServerPollArchiveAccounting } from "./server-poll-archive-accounting.js" +import { SERVER_POLL_ARCHIVE_READ_SQL } from "./server-poll-terminal-compaction.js" +import { openServerPollRetiredArchive, projectServerPollRetiredArchive, type LockedServerPollRetirementSource } from "./server-poll-retired-archive.js" +import { PollStorageRefused, runPollStorageTransaction } from "./server-poll-storage.js" +import type { ProbeStorageSql } from "./server-probe-journal-maintenance.js" +import { connectProbeStorage } from "./server-probe-storage-connection.js" +import { sharedDatabaseGate } from "./shared-database-gate.js" + +const request = z.object({ receiptId: z.string().uuid(), oldArchiveProof: z.string().regex(/^[a-f0-9]{64}$/) }).strict() +const requireThat: (value: unknown) => asserts value = value => { + if (!value) throw new PollStorageRefused() +} + +/** The storage owner already holds its global xact advisory lock. Do not wait + * for a connection writer or runner that might need the storage lock. */ +const ELIGIBLE = `FROM public.server_poll_storage_admissions a JOIN public.server_poll_archives r USING(receipt_id) + JOIN public.mcp_operations o ON o.id=a.execution_id AND o.connection_id=a.connection_id + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0 + WHERE a.identity_kind='mcp' AND a.storage_state='compacted' + AND r.version=1 AND r.execution_id=a.execution_id AND r.connection_id=a.connection_id AND r.owner_id=a.owner_id + AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL AND isfinite(o.finished_at) + AND o.finished_at($1::timestamptz,$2::uuid)) + ORDER BY o.finished_at,a.receipt_id LIMIT 8` + +const deps = { connect:connectProbeStorage,now:Date.now } +const transaction = (deadlineAt:number, work:(sql:ProbeStorageSql)=>Promise) => + runPollStorageTransaction(deps,{deadlineAt},async()=>{},work) +type Target = z.infer + +async function prepare(sql: ProbeStorageSql,{receiptId,oldArchiveProof}:Target) { + const identity = (await sql.query(`SELECT a.connection_id,a.execution_id,a.owner_id FROM public.server_poll_storage_admissions a + JOIN public.server_poll_archives r USING(receipt_id) WHERE a.receipt_id=$1 AND a.storage_state='compacted' + AND a.identity_kind='mcp' AND r.version=1 AND r.archive_proof=$2`, [receiptId,oldArchiveProof])).rows + requireThat(identity.length===1) + const { connection_id: connectionId,execution_id: operationId,owner_id: ownerId } = identity[0] + requireThat((await sql.query("SELECT id FROM public.mcp_connections WHERE id=$1 AND owner_id=$2 FOR UPDATE NOWAIT", + [connectionId,ownerId])).rows.length===1) + for (const key of ["mcp-operation:"+operationId,"server-poll-execution-v1:mcp:"+operationId,"server-poll-receipt-v1:"+receiptId]) { + requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked",[key])).rows[0]?.locked===true) + } + const rows = (await sql.query(SOURCE,[receiptId,oldArchiveProof])).rows + requireThat(rows.length===1 && rows[0].operation_id===operationId && rows[0].admission && rows[0].terminal) + const { admission,terminal,plan_id: planId,idempotency_key: key,observed_at: observedAt } = rows[0] + requireThat(typeof key==="string" && typeof connectionId==="string" && typeof planId==="string") + await assertServerPollArchiveAccounting(sql) + const oldArchive = (await sql.query(SERVER_POLL_ARCHIVE_READ_SQL,[receiptId])).rows + requireThat(oldArchive.length===1 && oldArchive[0].archive_proof===oldArchiveProof) + const archived = await projectServerPollRetiredArchive({ archive:oldArchive[0],admission,terminal } as LockedServerPollRetirementSource, + new Date(observedAt as string | Date).getTime()) + requireThat(archived) + const projection = await openServerPollRetiredArchive(archived) + requireThat(projection.metadata.receipt_id===receiptId && projection.metadata.execution_id===operationId + && projection.metadata.connection_id===connectionId && projection.metadata.owner_id===ownerId && projection.history.planId===planId) + return {connectionId,operationId,planId,key,archived,projection,previousBytes:oldArchive[0].ciphertext_bytes} +} + +async function retire(target:Target,deadlineAt:number) { + const {receiptId,oldArchiveProof}=target + const token=randomUUID() + await transaction(deadlineAt,async sql=>{ + // Check the live hold predicates, all locks and both archive proofs before + // spending the irreversible one-shot claim. + await prepare(sql,target) + const claim=await sql.query(`INSERT INTO public.server_poll_retirement_claims(receipt_id,old_archive_proof,claim_token) + VALUES($1,$2,$3) ON CONFLICT DO NOTHING RETURNING receipt_id`,[receiptId,oldArchiveProof,token]) + requireThat(claim.rows.length===1&&claim.rows[0].receipt_id===receiptId) + }) + // No destructive transaction after an unacknowledged claim. A claim is + // never cleared, even when a later checkout/hold/COMMIT is inconclusive. + return transaction(deadlineAt,async sql=>{ + const claimed=await sql.query(`SELECT receipt_id FROM public.server_poll_retirement_claims + WHERE receipt_id=$1 AND old_archive_proof=$2 AND claim_token=$3 AND claim_xid<>pg_current_xact_id()`, + [receiptId,oldArchiveProof,token]) + requireThat(claimed.rows.length===1&&claimed.rows[0].receipt_id===receiptId) + const {connectionId,operationId,planId,key,archived,projection,previousBytes}=await prepare(sql,target) + const tombstone = await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones + (operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global) + VALUES($1,$2,$3,$4,'servers.poll',$5::bigint[],true) RETURNING operation_id`, + [operationId,planId,connectionId,operationIdempotencyDigest(connectionId,key),projection.history.affectedServerIds]) + requireThat(tombstone.rows.length===1 && tombstone.rows[0].operation_id===operationId) + await sql.query("SELECT set_config('mm.poll_retire_receipt',$1,true),set_config('mm.poll_retire_claim',$2,true)",[receiptId,token]) + const updated = await sql.query(`UPDATE public.server_poll_archives SET version=2,source_digest=$3,source_sha256=$4, + decoded_bytes=$5,compressed_bytes=$6,ciphertext=$7,ciphertext_bytes=$8,archive_proof=$9 + WHERE receipt_id=$1 AND version=1 AND archive_proof=$2 RETURNING receipt_id`, + [receiptId,oldArchiveProof,archived.source_digest,archived.source_sha256,archived.decoded_bytes,archived.compressed_bytes, + archived.ciphertext,archived.ciphertext_bytes,archived.archive_proof]) + requireThat(updated.rows.length===1 && updated.rows[0].receipt_id===receiptId) + const charged = await sql.query(`UPDATE public.server_poll_storage_admissions SET archive_bytes=$2 + WHERE receipt_id=$1 AND storage_state='compacted' AND archive_bytes=$3 RETURNING receipt_id`, + [receiptId,archived.ciphertext_bytes,previousBytes]) + requireThat(charged.rows.length===1 && charged.rows[0].receipt_id===receiptId) + const readback = (await sql.query(SERVER_POLL_ARCHIVE_READ_SQL,[receiptId])).rows + requireThat(readback.length===1 && readback[0].archive_proof===archived.archive_proof + && fingerprint(await openServerPollRetiredArchive(readback[0]))===fingerprint(projection)) + const step = await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId]) + requireThat(step.rows.length===1 && step.rows[0].operation_id===operationId) + const operation = await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id", + [operationId,connectionId,planId]) + requireThat(operation.rows.length===1 && operation.rows[0].id===operationId) + const plan = await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id",[planId,connectionId]) + requireThat(plan.rows.length===1 && plan.rows[0].id===planId) + await assertServerPollArchiveAccounting(sql) + return { retired:true as const, automaticReplay:false as const } + }) +} + +/** Exact internal entry used by owned fixtures. It cannot bypass the claim. */ +export function retireCompletedServerPollOperation(raw:Target) { + const target=request.parse(raw) + return sharedDatabaseGate.request(()=>retire(target,Date.now()+20000)) +} + +let cursor:{finishedAt:string;receiptId:string}|undefined +export function retireCompletedServerPollOperations() { + return sharedDatabaseGate.request(async()=>{ + const deadlineAt=Date.now()+20000 + const rows=await transaction(deadlineAt,async sql=>(await sql.query(CANDIDATES, + [cursor?.finishedAt??null,cursor?.receiptId??null])).rows) + requireThat(rows.length<=8) + let retired=0,scanned=0 + for(const row of rows){ + if(Date.now()>=deadlineAt-2000||retired>=2)break + const target=request.parse({receiptId:row.receipt_id,oldArchiveProof:row.archive_proof}) + requireThat(typeof row.finished_at==="string"&&Number.isFinite(Date.parse(row.finished_at))) + cursor={finishedAt:row.finished_at,receiptId:target.receiptId} + scanned++ + try{await retire(target,deadlineAt);retired++} + catch(error){if(!(error instanceof PollStorageRefused))throw error} + } + // Keep the cursor for any unvisited rows. On exhaustion the next tick + // starts at the beginning, revisiting formerly held or busy candidates. + if(scanned===rows.length&&rows.length<8)cursor=undefined + return {scanned,retired,automaticReplay:false as const} + }) +} diff --git a/backend/src/services/server-poll-retired-archive.test.ts b/backend/src/services/server-poll-retired-archive.test.ts new file mode 100644 index 00000000..ffb98f4b --- /dev/null +++ b/backend/src/services/server-poll-retired-archive.test.ts @@ -0,0 +1,125 @@ +import assert from "node:assert/strict" +import { createCipheriv, createDecipheriv, createHmac, randomBytes } from "node:crypto" +import { test } from "node:test" +import { fingerprint } from "../modules/mcp/security.js" +import { createServerPollRetiredArchiveForTests } from "./server-poll-retired-archive.js" +import { createServerPollTerminalArchiveForTests, ServerPollArchiveRefused } from "./server-poll-terminal-archive.js" +import type { VerifiedServerPollHistory } from "./server-poll-history-verification.js" + +const receiptId = "11111111-1111-4111-8111-111111111111" +const executionId = "22222222-2222-4222-8222-222222222222" +const connectionId = "33333333-3333-4333-8333-333333333333" +const planId = "44444444-4444-4444-8444-444444444444" +const ownerId = "owned-retired-archive" +const secret = "ORIGINAL-CREDENTIAL-OR-PAYLOAD-MUST-DISAPPEAR" +const expiredAt = "2026-10-29T03:20:00.123456Z" +const terminalAt = "2026-10-29T03:21:00.123457Z" +const completedAt = "2026-10-29T03:18:00.123458Z" +const createdAt = "2026-10-29T03:17:00.123459Z" +const gcNotBefore = "2026-11-29T03:20:00.123460Z" +const now = Date.parse("2026-11-30T00:00:00Z") +const key = randomBytes(32), mac = randomBytes(32) +const proof = async (purpose: string, value: unknown) => createHmac("sha256", mac).update(purpose).update(fingerprint(value)).digest("hex") +const encrypt = async (plain: Buffer) => { + const iv = randomBytes(12), cipher = createCipheriv("aes-256-gcm", key, iv) + const ciphertext = Buffer.concat([cipher.update(plain), cipher.final()]) + return Buffer.concat([iv, cipher.getAuthTag(), ciphertext]) +} +const decrypt = async (bytes: Buffer) => { + const cipher = createDecipheriv("aes-256-gcm", key, bytes.subarray(0, 12)) + cipher.setAuthTag(bytes.subarray(12, 28)) + return Buffer.concat([cipher.update(bytes.subarray(28)), cipher.final()]) +} +async function fixture(v2Encrypt = encrypt) { + const admission = { storage_state: "full", gc_not_before: gcNotBefore, opaque: secret } + const terminal = { operation: { finished_at: terminalAt, result: secret }, plan: { id: planId, expires_at: expiredAt, evidence: secret }, + step: { intended_state: secret, result: secret } } + const journal = { execution: { created_at: createdAt, context_ciphertext: secret }, receipt: { created_at: completedAt, receipt_ciphertext: secret }, + intents: [{ opaque: randomBytes(2048).toString("hex") }], acks: [{ opaque: secret }], quarantines: [] } + // Tests use genuine V1 codec authentication with a controlled source callback. + // Full production V1 source parity is tested by its existing verifier suite. + const v1 = createServerPollTerminalArchiveForTests({ encrypt, decrypt, proof, verifySource: async () => ({ + receiptId, executionId, connectionId, ownerId, bindingProof: "a".repeat(64), contextProof: "b".repeat(64), terminalDigest: "c".repeat(64), + originalActivityAt: terminalAt, originalExpiresAt: expiredAt, retentionDays: 30, safetyMs: 60000, + }) }) + const archive = await v1.create(JSON.stringify({ admission, terminal, journal }), now) + assert(archive) + const history = { + context: { identity: { kind: "mcp", executionId, connectionId, ownerId, credentialId: secret }, + plan: { request: { receiptId, serverId: 23 }, affectedServerIds: [12, 23], maintenance: [secret] } }, + receipt: { receiptId, complete: true, maintenanceComplete: true, configurationVerified: true, observation: "online", + nativeReads: 4, confirmedPublications: 8, completedPublications: 8, snapshot: { identityName: secret } }, + source: { createdAt, completedAt, stageCount: 12, publications: 8, + nativeOutcomes: Array.from({ length: 4 }, (_, index) => ({ stage: "native." + index, outcome: "confirmed", resultProof: "d".repeat(64) })) }, + } as unknown as VerifiedServerPollHistory + const calls = { decrypt: 0, history: 0 } + const codec = createServerPollRetiredArchiveForTests({ openV1: v1.open, verifyHistory: async () => { calls.history++; return history }, + encrypt: v2Encrypt, proof, decrypt: async bytes => { calls.decrypt++; return decrypt(bytes) } }) + return { codec, history, calls, input: { archive, terminal, admission: { ...admission, storage_state: "compacted", + archive_bytes: archive.ciphertext_bytes, original_activity_at: terminalAt, compact_at: "2026-10-29T03:23:00.123456Z" } } } +} +const refusal = (work: Promise) => assert.rejects(work, (error: unknown) => { + assert(error instanceof ServerPollArchiveRefused && error.automaticReplay === false) + assert(!String(error).includes(secret)) + return true +}) + +test("retirement authenticates V1 and projects only completed history, scope, and original lifetime", async () => { + const { codec, calls, input } = await fixture() + const retired = await codec.project(input, now) + assert(retired) + assert.equal(calls.history, 1) + assert(retired.ciphertext_bytes < input.archive.ciphertext_bytes) + const { history, metadata } = await codec.open(retired) + assert.deepEqual(history, { version: 2, operation: "servers.poll", operationVersion: 1, planId, serverId: 23, affectedServerIds: [12, 23], + gcNotBefore, terminalAt, previous: { archiveProof: input.archive.archive_proof, sourceDigest: input.archive.source_digest, sourceSha256: input.archive.source_sha256 }, + history: { createdAt, completedAt, stageCount: 12, publications: 8, nativeReads: 4, observation: "online", verification: "passed" } }) + assert.equal(metadata.terminal_digest, input.archive.terminal_digest) + assert.equal(metadata.original_activity_at, terminalAt) + assert.equal(metadata.original_expires_at, expiredAt) + assert.equal(metadata.retention_days, input.archive.retention_days) + assert(!JSON.stringify(history).includes(secret)) + assert.equal((await codec.verifyCharge(retired)).bytes, retired.ciphertext_bytes + 32768) +}) + +test("retirement refuses unauthenticated V1 and any current admission or original terminal drift", async () => { + const { codec, input, calls } = await fixture() + const changed = [ + { ...input, archive: { ...input.archive, archive_proof: "e".repeat(64) } }, + { ...input, admission: { ...input.admission, gc_not_before: terminalAt } }, + { ...input, admission: { ...input.admission, original_activity_at: "2026-10-29T03:21:00.123458Z" } }, + { ...input, admission: { ...input.admission, archive_bytes: 1 } }, + { ...input, terminal: { ...input.terminal, step: { result: "changed" } } }, + ] + for (const row of changed) await refusal(codec.project(row, now)) + assert.equal(calls.history, 0) +}) + +test("V2 metadata or ciphertext tamper never reaches V2 decrypt", async () => { + const { codec, input, calls } = await fixture() + const retired = await codec.project(input, now) + assert(retired) + for (const row of [ + { ...retired, version: 1 }, { ...retired, owner_id: "another-owner" }, + { ...retired, original_expires_at: "2026-10-29T03:20:00.123457Z" }, + { ...retired, terminal_digest: "e".repeat(64) }, + { ...retired, ciphertext: (retired.ciphertext[0] === "A" ? "B" : "A") + retired.ciphertext.slice(1) }, + ]) await refusal(codec.open(row)) + assert.equal(calls.decrypt, 0) + await refusal(codec.verifyCharge(input.archive)) + await refusal(codec.verifyCharge({ ...retired, ciphertext_bytes: retired.ciphertext_bytes - 1 })) +}) + +test("retirement refuses unknown history or a scope that omits its original target", async () => { + const { codec, history, input } = await fixture() + history.source.nativeOutcomes[0].outcome = "read-failed" + await refusal(codec.project(input, now)) + history.source.nativeOutcomes[0].outcome = "confirmed" + history.context.plan.affectedServerIds = [12] + await refusal(codec.project(input, now)) +}) + +test("a larger V2 ciphertext keeps the authenticated V1 instead of increasing the charge", async () => { + const { codec, input } = await fixture(async value => encrypt(Buffer.concat([Buffer.alloc(4096), value]))) + assert.equal(await codec.project(input, now), undefined) +}) diff --git a/backend/src/services/server-poll-retired-archive.ts b/backend/src/services/server-poll-retired-archive.ts new file mode 100644 index 00000000..05140231 --- /dev/null +++ b/backend/src/services/server-poll-retired-archive.ts @@ -0,0 +1,168 @@ +/** Private retirement projection. Publication still requires the SQL owner's + * locks, exact compare-and-swap, dependency checks and atomic quota adjustment. */ +import { createHash, timingSafeEqual } from "node:crypto" +import { gzipSync, gunzipSync } from "node:zlib" +import { z } from "zod" +import { canonicalPollArchiveTimestamp, openServerPollTerminalArchive, ServerPollArchiveRefused } from "./server-poll-terminal-archive.js" +import { verifyServerPollHistory } from "./server-poll-history-verification.js" +import { decryptBinary, encryptBinary, fingerprint, privateSecretFingerprint } from "../modules/mcp/security.js" + +const MAX_PAYLOAD_BYTES = 24576 +const MAX_COMPRESSED_BYTES = MAX_PAYLOAD_BYTES +const MAX_CIPHERTEXT_BYTES = 33000 +const RECORD_PURPOSE = "server-poll-retired-archive-record-v2" +const SOURCE_PURPOSE = "server-poll-retired-archive-source-v2" +const hash = z.string().regex(/^[a-f0-9]{64}$/), uuid = z.string().uuid() +const integer = z.number().int().positive().safe() +const owner = z.string().min(1).max(200).refine(value => value.trim() === value && !/[\x00-\x1f\x7f]/.test(value)) +const utc = z.string().transform(canonicalPollArchiveTimestamp) +const metadataSchema = z.object({ + version: z.literal(2), receipt_id: uuid, execution_id: uuid, connection_id: uuid, owner_id: owner, + binding_proof: hash, context_proof: hash, terminal_digest: hash, + original_activity_at: utc, original_expires_at: utc, retention_days: integer.max(365), safety_ms: z.literal(60000), + source_digest: hash, source_sha256: hash, decoded_bytes: integer.max(MAX_PAYLOAD_BYTES), + compressed_bytes: integer.max(MAX_COMPRESSED_BYTES), ciphertext_bytes: integer.max(MAX_CIPHERTEXT_BYTES), ciphertext_sha256: hash, +}) +const projectionSchema = z.object({ + version: z.literal(2), operation: z.literal("servers.poll"), operationVersion: z.literal(1), + planId: uuid, serverId: integer, affectedServerIds: z.array(integer).min(1).max(1000), + gcNotBefore: utc, terminalAt: utc, + previous: z.object({ archiveProof: hash, sourceDigest: hash, sourceSha256: hash }).strict(), + history: z.object({ + createdAt: utc, completedAt: utc, stageCount: integer.max(2008), + publications: integer.max(2008), nativeReads: z.literal(4), + observation: z.literal("online"), verification: z.literal("passed"), + }).strict(), +}).strict().refine(value => value.affectedServerIds.includes(value.serverId) + && value.affectedServerIds.length === new Set(value.affectedServerIds).size + && value.history.createdAt <= value.history.completedAt + && value.history.completedAt <= value.terminalAt) + +export type ServerPollRetiredArchiveMetadata = z.infer +export type ServerPollRetiredHistory = z.infer +export type ServerPollRetiredArchive = ServerPollRetiredArchiveMetadata & { ciphertext: string; archive_proof: string } + +export interface LockedServerPollRetirementSource { + /** Existing V1 reader projection, including retention and actual ciphertext digest. */ + archive: Record + /** Current compacted admission JSON and exact operation, plan and step JSON. */ + admission: Record + terminal: { operation: Record; plan: Record; step: Record } +} + +interface Dependencies { + openV1: typeof openServerPollTerminalArchive + verifyHistory: typeof verifyServerPollHistory + encrypt(value: Buffer): Promise + decrypt(value: Buffer): Promise + proof(purpose: string, value: unknown): Promise +} +const sha = (value: Buffer | string) => createHash("sha256").update(value).digest("hex") +const equal = (left: string, right: string) => timingSafeEqual(Buffer.from(hash.parse(left), "hex"), Buffer.from(hash.parse(right), "hex")) +const refuse = (): never => { throw new ServerPollArchiveRefused() } +const requireThat: (value: unknown) => asserts value = value => { if (!value) refuse() } + +function immutableAdmission(raw: Record) { + const result = { ...raw } + for (const name of ["storage_state", "compact_at", "archive_bytes", "original_activity_at"]) delete result[name] + return result +} + +function codec(deps: Dependencies) { + const proof = async (purpose: string, value: unknown) => hash.parse(await deps.proof(purpose, value)) + const recordProof = (metadata: ServerPollRetiredArchiveMetadata) => proof(RECORD_PURPOSE, { + ...metadata, identity_kind: "mcp", storage_state: "compacted", admission_bytes: 32768, charged_rows: 2, + }) + async function readMetadata(raw: Record) { + const metadata = metadataSchema.parse(raw) + requireThat(equal(hash.parse(raw.archive_proof), await recordProof(metadata))) + return metadata + } + + async function project(raw: LockedServerPollRetirementSource, now: number): Promise { + try { + requireThat(Number.isSafeInteger(now) && now > 0) + const input = structuredClone(raw) + const opened = await deps.openV1(input.archive, now) + const { source, metadata: previous } = opened + const current = input.admission + requireThat(source.admission.storage_state === "full" && current.storage_state === "compacted" + && current.archive_bytes === previous.ciphertext_bytes + && canonicalPollArchiveTimestamp(current.original_activity_at) === previous.original_activity_at + && fingerprint(immutableAdmission(source.admission)) === fingerprint(immutableAdmission(current)) + && fingerprint(source.terminal) === fingerprint(input.terminal)) + canonicalPollArchiveTimestamp(current.compact_at) + + // The existing codec authenticates the entire original. The shared + // verifier supplies its typed history rather than reading opaque fields + // from unverified context or preserving original authority in V2. + const { context, receipt, source: journal } = await deps.verifyHistory(source.journal) + requireThat(context.identity.kind === "mcp" && context.identity.executionId === previous.execution_id + && context.identity.connectionId === previous.connection_id && context.identity.ownerId === previous.owner_id + && context.plan.request.receiptId === previous.receipt_id && receipt.receiptId === previous.receipt_id + && receipt.complete && receipt.maintenanceComplete && receipt.configurationVerified && receipt.observation === "online" + && receipt.confirmedPublications === receipt.completedPublications && journal.publications === receipt.completedPublications + && journal.nativeOutcomes.length === 4 && journal.nativeOutcomes.every(item => item.outcome === "confirmed")) + const terminalAt = [source.terminal.operation.finished_at, source.terminal.plan.expires_at].map(canonicalPollArchiveTimestamp).sort()[1] + const projection = projectionSchema.parse({ + version: 2, operation: "servers.poll", operationVersion: 1, planId: source.terminal.plan.id, + serverId: context.plan.request.serverId, affectedServerIds: context.plan.affectedServerIds, + gcNotBefore: source.admission.gc_not_before, terminalAt, + previous: { archiveProof: input.archive.archive_proof, sourceDigest: previous.source_digest, sourceSha256: previous.source_sha256 }, + history: { createdAt: source.journal.execution.created_at, completedAt: source.journal.receipt.created_at, + stageCount: journal.stageCount, publications: journal.publications, nativeReads: receipt.nativeReads, + observation: receipt.observation, verification: "passed" }, + }) + requireThat(projection.terminalAt <= previous.original_activity_at && previous.original_expires_at <= projection.gcNotBefore) + const plaintext = Buffer.from(JSON.stringify(projection)) + requireThat(plaintext.length <= MAX_PAYLOAD_BYTES) + const compressed = gzipSync(plaintext, { level: 6, maxOutputLength: MAX_COMPRESSED_BYTES }) + const ciphertext = (await deps.encrypt(compressed)).toString("base64") + const bytes = Buffer.byteLength(ciphertext) + if (bytes > previous.ciphertext_bytes || bytes > MAX_CIPHERTEXT_BYTES) return undefined + const sourceSha = sha(plaintext) + const metadata = metadataSchema.parse({ ...previous, version: 2, source_sha256: sourceSha, source_digest: await proof(SOURCE_PURPOSE, sourceSha), + decoded_bytes: plaintext.length, compressed_bytes: compressed.length, ciphertext_bytes: bytes, ciphertext_sha256: sha(ciphertext) }) + return { ...metadata, ciphertext, archive_proof: await recordProof(metadata) } + } catch { return refuse() } + } + + async function verifyCharge(raw: Record) { + try { + const metadata = await readMetadata(structuredClone(raw)) + return { bytes: metadata.ciphertext_bytes + 32768, rows: 2 as const, metadata } + } catch { return refuse() } + } + + async function open(raw: Record) { + try { + const input = structuredClone(raw) + const metadata = await readMetadata(input), ciphertext = input.ciphertext + requireThat(typeof ciphertext === "string" && Buffer.byteLength(ciphertext) === metadata.ciphertext_bytes + && sha(ciphertext) === metadata.ciphertext_sha256) + const binary = Buffer.from(ciphertext, "base64") + requireThat(binary.toString("base64") === ciphertext) + const compressed = await deps.decrypt(binary) + requireThat(compressed.length === metadata.compressed_bytes) + const plaintext = gunzipSync(compressed, { maxOutputLength: MAX_PAYLOAD_BYTES }) + const text = plaintext.toString("utf8") + requireThat(plaintext.length === metadata.decoded_bytes && Buffer.from(text).equals(plaintext) + && sha(plaintext) === metadata.source_sha256 && equal(metadata.source_digest, await proof(SOURCE_PURPOSE, metadata.source_sha256))) + const history = projectionSchema.parse(JSON.parse(text)) + requireThat(history.terminalAt <= metadata.original_activity_at && metadata.original_expires_at <= history.gcNotBefore) + return { metadata, history } + } catch { return refuse() } + } + return Object.freeze({ project, verifyCharge, open }) +} + +const production = codec({ + openV1: openServerPollTerminalArchive, verifyHistory: verifyServerPollHistory, + encrypt: encryptBinary, decrypt: decryptBinary, + proof: (purpose, value) => privateSecretFingerprint(fingerprint(value), purpose), +}) +export const projectServerPollRetiredArchive = production.project +export const verifyServerPollRetiredArchiveCharge = production.verifyCharge +export const openServerPollRetiredArchive = production.open +/** Isolated component crypto composition. No public/runtime caller selects it. */ +export const createServerPollRetiredArchiveForTests = (deps: Dependencies) => codec(Object.freeze({ ...deps })) diff --git a/backend/src/services/server-poll-storage-fresh.sql.test.ts b/backend/src/services/server-poll-storage-fresh.sql.test.ts index 89aed905..ac33a2e8 100644 --- a/backend/src/services/server-poll-storage-fresh.sql.test.ts +++ b/backend/src/services/server-poll-storage-fresh.sql.test.ts @@ -34,8 +34,40 @@ try{ }finally{await other.end();await pool.query(`DROP DATABASE "${name}"`)} const {tickMcpWorkers}=await import("../modules/mcp/workers.js") assert.equal((await pool.query("SELECT enabled FROM mcp_settings WHERE id=1")).rows[0].enabled,false) + const expiredRequestIds = Array.from({ length: 1001 }, () => randomUUID()).sort() + const [runningId, freshId] = [randomUUID(), randomUUID()].sort() + await pool.query(`INSERT INTO mcp_requests(id,tool,status,created_at) + SELECT id,'owned-retention-test','completed',now()-interval '31 days' FROM unnest($1::uuid[]) AS ids(id) + UNION ALL SELECT $2::uuid,'owned-retention-test','running',now()-interval '32 days' + UNION ALL SELECT $3::uuid,'owned-retention-test','completed',now()-interval '29 days'`, [expiredRequestIds, runningId, freshId]) + const expiredAudit = (await pool.query(`INSERT INTO mcp_audit(actor_id,action,created_at) + SELECT 'owned-retention-test','expired',now()-interval '181 days' FROM generate_series(1,1001) + RETURNING id`)).rows.map(row => String(row.id)) + const lastExpiredAuditId = expiredAudit.reduce((last, id) => BigInt(id) > BigInt(last) ? id : last) + const freshAuditId = (await pool.query(`INSERT INTO mcp_audit(actor_id,action,created_at) + VALUES ('owned-retention-test','fresh',now()-interval '179 days') RETURNING id`)).rows[0].id + const artifactConnectionId = randomUUID() + const expiredArtifactIds = Array.from({ length: 1001 }, () => randomUUID()).sort() + const freshArtifactId = randomUUID() + await pool.query(`INSERT INTO mcp_connections(id,name,owner_id,owner_permissions,policy,generation) + VALUES ($1,'owned-retention-test','owned-retention-test','[]','{}','owned-retention-test')`, [artifactConnectionId]) + await pool.query(`INSERT INTO mcp_artifacts(id,connection_id,filename,media_type,ciphertext,expires_at) + SELECT id,$1::uuid,'test','application/octet-stream','test',now()-interval '1 day' FROM unnest($2::uuid[]) AS ids(id) + UNION ALL SELECT $3::uuid,$1::uuid,'test','application/octet-stream','test',now()+interval '1 day'`, [artifactConnectionId, expiredArtifactIds, freshArtifactId]) + const requestIds = async () => (await pool.query("SELECT id FROM mcp_requests WHERE tool='owned-retention-test' ORDER BY id")).rows.map(row => row.id) + const auditIds = async () => (await pool.query("SELECT id FROM mcp_audit WHERE actor_id='owned-retention-test' ORDER BY id")).rows.map(row => String(row.id)) + const artifactIds = async () => (await pool.query("SELECT id FROM mcp_artifacts WHERE connection_id=$1 ORDER BY id", [artifactConnectionId])).rows.map(row => row.id) await tickMcpWorkers() - console.log("Owned fresh poll storage:",JSON.stringify({normalStartupTwice:true,migrations:MIGRATION_MANIFEST.length,globalZeroSeed:true,legacyPrefix:38,nonemptyJournalRefusedBeforeDdl:true,oldRowsPreserved:true,noForgedMarker:true,actualWorkerTickWithMcpDisabled:true})) + assert.deepEqual(await requestIds(), [...expiredRequestIds.slice(-1), runningId, freshId].sort()) + assert.deepEqual(await auditIds(), [lastExpiredAuditId, String(freshAuditId)]) + assert.deepEqual(await artifactIds(), [...expiredArtifactIds.slice(-1), freshArtifactId].sort()) + await tickMcpWorkers() + assert.deepEqual(await requestIds(), [runningId, freshId].sort()) + assert.deepEqual(await auditIds(), [String(freshAuditId)]) + assert.deepEqual(await artifactIds(), [freshArtifactId]) + await pool.query("DELETE FROM mcp_artifacts WHERE id=$1", [freshArtifactId]) + await pool.query("DELETE FROM mcp_connections WHERE id=$1", [artifactConnectionId]) + console.log("Owned fresh poll storage:",JSON.stringify({normalStartupTwice:true,migrations:MIGRATION_MANIFEST.length,globalZeroSeed:true,legacyPrefix:38,nonemptyJournalRefusedBeforeDdl:true,oldRowsPreserved:true,noForgedMarker:true,actualWorkerTickWithMcpDisabled:true,boundedRequestAuditAndArtifactRetention:true})) if (process.env.MMAPP_OWN_SERVER_POLL_SETTINGS_EPOCH === "true") { await (await import("../db/server-poll-settings-epoch.owned-helper.js")).acceptServerPollSettingsEpochMigrationSql(pool) } else if (process.env.MMAPP_OWN_SERVER_POLL_ATOMIC_MAINTENANCE === "true") { diff --git a/backend/src/services/server-poll-storage-retention.test.ts b/backend/src/services/server-poll-storage-retention.test.ts index 9d342bca..3ebb5b12 100644 --- a/backend/src/services/server-poll-storage-retention.test.ts +++ b/backend/src/services/server-poll-storage-retention.test.ts @@ -1,6 +1,8 @@ import assert from "node:assert/strict" -import { randomUUID } from "node:crypto" +import { createHash, createHmac, randomUUID } from "node:crypto" import { test } from "node:test" +import { gzipSync } from "node:zlib" +import { fingerprint } from "../modules/mcp/security.js" import { collectExpiredServerPollStorageForTests } from "./server-poll-storage-retention.js" import { PollStorageRefused, PollStorageUnknown } from "./server-poll-storage.js" import type { ProbeStorageOptions } from "./server-probe-journal-maintenance.js" @@ -8,8 +10,10 @@ import { sharedDatabaseGate } from "./shared-database-gate.js" const tables = ["server_poll_acks", "server_poll_intents", "server_poll_receipts", "server_poll_quarantines", "server_poll_executions", "server_poll_archives", "server_poll_storage_admissions"] const deferred = () => { let resolve!: () => void; const promise = new Promise(done => { resolve = done }); return { promise, resolve } } -function fixture() { - const now = Date.now(), candidates: Record[] = [{ receipt_id: randomUUID(), owner_id: "owned-gc" }, { receipt_id: randomUUID(), owner_id: "owned-gc" }] +function fixture(openRetired?: Parameters[1]) { + const now = Date.now(), candidates: Record[] = [{ receipt_id: randomUUID(), owner_id: "owned-gc", archive_version: null }, + { receipt_id: randomUUID(), owner_id: "owned-gc", archive_version: 1 }] + const retiredRows: Record[] = [] const commands: { text: string; values: unknown[]; timeout: number }[] = [], releases: boolean[] = [], options: ProbeStorageOptions[] = [] let settings: Record[] = [{ requests_retention_days: 30 }], fail = "", residual = "", removed: Record[] | undefined, hold: ReturnType | undefined const entered = deferred() @@ -25,12 +29,13 @@ function fixture() { return { rows: [{ scope: "global", admissions: 0, compacted_admissions: 0, archive_bytes: 0, reserved_rows: 0, reserved_bytes: 0 }] } if (text.includes("requests_retention_days")) { entered.resolve(); if (hold) await hold.promise; return { rows: settings } } if (text.startsWith("SELECT receipt_id,owner_id")) return { rows: structuredClone(candidates) } + if (text.startsWith("SELECT r.version,r.receipt_id")) return { rows: structuredClone(retiredRows) } if (text.startsWith("DELETE FROM server_poll_storage_admissions")) return { rows: removed ?? candidates.map(row => ({ receipt_id: row.receipt_id })) } if (text.startsWith("SELECT receipt_id FROM ") && text.includes("FROM " + residual + " ")) return { rows: [{ receipt_id: candidates[0]?.receipt_id }] } return { rows: [] } }, release(unsafe = false) { releases.push(unsafe) } } - } }) - return { run, now, candidates, commands, releases, options, entered: entered.promise, block: () => { hold = deferred(); return hold }, + } }, openRetired) + return { run, now, candidates, retiredRows, commands, releases, options, entered: entered.promise, block: () => { hold = deferred(); return hold }, setting: (rows: Record[]) => { settings = rows }, fail: (text: string) => { fail = text }, residual: (table: string) => { residual = table }, removed: (rows: Record[]) => { removed = rows } } } @@ -61,6 +66,67 @@ test("selected GC owns one bounded backend, locks current retention, deletes all assert.deepEqual(f.releases, [false]) }) +async function retiredFixture() { + const { createServerPollRetiredArchiveForTests } = await import("./server-poll-retired-archive.js") + const proof = async (purpose: string, value: unknown) => + createHmac("sha256", "gc-fixture-only").update(purpose).update(fingerprint(value)).digest("hex") + const sha = (value: string | Buffer) => createHash("sha256").update(value).digest("hex") + let opened = 0 + const codec = createServerPollRetiredArchiveForTests({ + proof, + decrypt: async value => { opened++; return value }, + encrypt: async () => { throw new Error("Unexpected archive write") }, + openV1: async () => { throw new Error("Unexpected archive fallback") }, + verifyHistory: async () => { throw new Error("Unexpected historical fallback") }, + }) + const f = fixture(codec.open) + const selected = f.candidates[1] + selected.archive_version = 2 + const timestamp = "2026-10-01T00:00:00.123456Z", gcNotBefore = "2026-11-01T00:00:00.123456Z" + const history = { version: 2, operation: "servers.poll", operationVersion: 1, planId: randomUUID(), serverId: 3, + affectedServerIds: [3], gcNotBefore, terminalAt: timestamp, + previous: { archiveProof: "1".repeat(64), sourceDigest: "2".repeat(64), sourceSha256: "3".repeat(64) }, + history: { createdAt: timestamp, completedAt: timestamp, stageCount: 12, publications: 8, + nativeReads: 4, observation: "online", verification: "passed" } } + const plaintext = Buffer.from(JSON.stringify(history)), compressed = gzipSync(plaintext), ciphertext = compressed.toString("base64") + const metadata = { version: 2, receipt_id: selected.receipt_id, execution_id: randomUUID(), connection_id: randomUUID(), owner_id: selected.owner_id, + binding_proof: "a".repeat(64), context_proof: "b".repeat(64), terminal_digest: "c".repeat(64), + original_activity_at: timestamp, original_expires_at: timestamp, retention_days: 30, safety_ms: 60000, + source_digest: await proof("server-poll-retired-archive-source-v2", sha(plaintext)), source_sha256: sha(plaintext), + decoded_bytes: plaintext.length, compressed_bytes: compressed.length, ciphertext_bytes: ciphertext.length, ciphertext_sha256: sha(ciphertext) } + const archiveProof = await proof("server-poll-retired-archive-record-v2", { ...metadata, identity_kind: "mcp", + storage_state: "compacted", admission_bytes: 32768, charged_rows: 2 }) + f.retiredRows.push({ ...metadata, ciphertext, archive_proof: archiveProof, admission_gc_not_before: gcNotBefore }) + return { ...f, opened: () => opened } +} + +test("GC opens only selected V2 under a row lock and authenticates the original lifetime before any DELETE", async () => { + const f = await retiredFixture() + assert.deepEqual(await f.run(), { collected: 2, automaticReplay: false }) + assert.equal(f.opened(), 1) + const index = f.commands.findIndex(command => command.text.startsWith("SELECT r.version,r.receipt_id")) + assert.deepEqual(f.commands[index].values, [f.candidates[1].receipt_id]) + assert(f.commands[index].text.endsWith("FOR UPDATE OF r")) + assert(index < f.commands.findIndex(command => command.text.startsWith("DELETE"))) +}) + +test("missing V2, forged ciphertext or proof, and microsecond lifetime drift refuse before any DELETE", async () => { + for (const fault of ["missing", "ciphertext", "actual-hash", "proof", "lifetime", "expiry", "version"] as const) { + const f = await retiredFixture(), row = f.retiredRows[0] + if (fault === "missing") f.retiredRows.length = 0 + if (fault === "ciphertext") row.ciphertext = "A" + String(row.ciphertext).slice(1) + if (fault === "actual-hash") row.ciphertext_sha256 = "a".repeat(64) + if (fault === "proof") row.archive_proof = "a".repeat(64) + if (fault === "lifetime") row.admission_gc_not_before = "2026-11-01T00:00:00.123455Z" + if (fault === "expiry") row.original_expires_at = "2026-10-01T00:00:00.123455Z" + if (fault === "version") f.candidates[1].archive_version = 3 + await assert.rejects(f.run(), PollStorageRefused) + assert.equal(f.opened(), fault === "lifetime" ? 1 : 0) + assert(!f.commands.some(command => command.text.startsWith("DELETE") || command.text === "COMMIT")) + assert.equal(f.commands.filter(command => command.text === "ROLLBACK").length, 1) + } +}) + test("empty candidate set returns zero without deleting or refunding", async () => { const f = fixture(); f.candidates.length = 0 assert.deepEqual(await f.run(), { collected: 0, automaticReplay: false }) diff --git a/backend/src/services/server-poll-storage-retention.ts b/backend/src/services/server-poll-storage-retention.ts index 93ec7519..aec1924b 100644 --- a/backend/src/services/server-poll-storage-retention.ts +++ b/backend/src/services/server-poll-storage-retention.ts @@ -4,21 +4,50 @@ import { connectProbeStorage } from "./server-probe-storage-connection.js" import { runPollStorageTransaction, type PollStorageDependencies } from "./server-poll-storage.js" import { PollStorageRefused } from "./server-poll-storage-model.js" import { assertServerPollArchiveAccounting } from "./server-poll-archive-accounting.js" +import type { openServerPollRetiredArchive } from "./server-poll-retired-archive.js" const uuid=/^[a-f\d]{8}(?:-[a-f\d]{4}){3}-[a-f\d]{12}$/i const fail=():never=>{throw new PollStorageRefused()} const production:PollStorageDependencies={connect:connectProbeStorage,now:Date.now} -async function collect(deps:PollStorageDependencies){ +const openRetiredArchive:typeof openServerPollRetiredArchive=async row=> + (await import("./server-poll-retired-archive.js")).openServerPollRetiredArchive(row) +const retiredCandidate = `SELECT r.version,r.receipt_id,r.execution_id,r.connection_id,r.owner_id, + r.binding_proof,r.context_proof,r.terminal_digest,r.source_digest,r.source_sha256, + r.decoded_bytes,r.compressed_bytes,r.ciphertext_bytes,r.ciphertext,r.archive_proof,a.retention_days,a.safety_ms, + to_char(r.original_activity_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS original_activity_at, + to_char(r.original_expires_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS original_expires_at, + to_char(a.gc_not_before AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS admission_gc_not_before, + pg_catalog.encode(pg_catalog.sha256(pg_catalog.convert_to(r.ciphertext,'UTF8')),'hex') AS ciphertext_sha256 + FROM server_poll_archives r JOIN server_poll_storage_admissions a USING(receipt_id) + WHERE r.receipt_id=$1 AND r.version=2 AND a.storage_state='compacted' AND a.identity_kind='mcp' + AND r.execution_id=a.execution_id AND r.connection_id=a.connection_id AND r.owner_id=a.owner_id + AND r.binding_proof=a.binding_proof AND r.context_proof=a.context_proof + AND r.original_activity_at=a.original_activity_at AND r.original_expires_at=a.authority_expires_at + AND r.ciphertext_bytes=a.archive_bytes AND r.ciphertext_bytes=octet_length(r.ciphertext) + FOR UPDATE OF r` + +async function collect(deps:PollStorageDependencies, openRetired:typeof openServerPollRetiredArchive){ return runPollStorageTransaction(deps,{deadlineAt:deps.now()+20000},async()=>{},async sql=>{ // Hold the authoritative setting against concurrent updates until COMMIT. // Each DELETE guard additionally checks its current value independently. const setting=(await sql.query("SELECT requests_retention_days FROM mcp_settings WHERE id=1 FOR SHARE")).rows if(setting.length!==1||!Number.isSafeInteger(setting[0].requests_retention_days)||Number(setting[0].requests_retention_days)<1||Number(setting[0].requests_retention_days)>365)fail() await sql.query("SET LOCAL mm.poll_storage_gc='on'") - const rows=(await sql.query("SELECT receipt_id,owner_id FROM server_poll_storage_admissions WHERE gc_not_before8||rows.some(row=>typeof row.receipt_id!=="string"||!uuid.test(row.receipt_id)||typeof row.owner_id!=="string")||new Set(rows.map(row=>row.receipt_id)).size!==rows.length)fail() + const rows=(await sql.query(`SELECT receipt_id,owner_id, + (SELECT version FROM server_poll_archives WHERE receipt_id=a.receipt_id) AS archive_version + FROM server_poll_storage_admissions a WHERE gc_not_before8||rows.some(row=>typeof row.receipt_id!=="string"||!uuid.test(row.receipt_id)||typeof row.owner_id!=="string" + || (row.archive_version!==null&&row.archive_version!==1&&row.archive_version!==2))||new Set(rows.map(row=>row.receipt_id)).size!==rows.length)fail() if(!rows.length)return {collected:0,automaticReplay:false as const} await assertServerPollArchiveAccounting(sql) + for(const candidate of rows){ + if(candidate.archive_version!==2)continue + const locked=(await sql.query(retiredCandidate,[candidate.receipt_id])).rows + if(locked.length!==1||locked[0].version!==2||locked[0].receipt_id!==candidate.receipt_id||locked[0].owner_id!==candidate.owner_id)fail() + const {history}=await openRetired(locked[0]) + if(history.gcNotBefore!==locked[0].admission_gc_not_before)fail() + } const ids=rows.map(row=>row.receipt_id),owners=[...new Set(rows.map(row=>row.owner_id))] // No payload-only shortcut: all dependent rows, admission and its exact // trigger-owned refund commit together. Any missing guard/refund aborts. @@ -35,6 +64,7 @@ async function collect(deps:PollStorageDependencies){ return {collected:ids.length,automaticReplay:false as const} }) } -export const collectExpiredServerPollStorage=()=>sharedDatabaseGate.request(()=>collect(production)) +export const collectExpiredServerPollStorage=()=>sharedDatabaseGate.request(()=>collect(production,openRetiredArchive)) /** Explicit owned SQL composition; does not start a worker. */ -export const collectExpiredServerPollStorageForTests=(deps:PollStorageDependencies)=>sharedDatabaseGate.request(()=>collect(deps)) +export const collectExpiredServerPollStorageForTests=(deps:PollStorageDependencies,openRetired=openRetiredArchive)=> + sharedDatabaseGate.request(()=>collect(deps,openRetired)) diff --git a/backend/src/services/server-poll-terminal-archive.owned.test.ts b/backend/src/services/server-poll-terminal-archive.owned.test.ts index 4c2e6ff6..5d5e57ec 100644 --- a/backend/src/services/server-poll-terminal-archive.owned.test.ts +++ b/backend/src/services/server-poll-terminal-archive.owned.test.ts @@ -77,7 +77,7 @@ async function main() { } try { await applySqlMigrations(pool) - assert.equal(MIGRATION_MANIFEST.at(-1)?.file, "0044_mcp_routing_manual_inverse_charges.sql") + assert.equal(MIGRATION_MANIFEST.at(-1)?.file, "0053_server_probe_terminal_proofs.sql") assert.equal(MIGRATION_MANIFEST.find(pin=>pin.file==="0041_server_poll_terminal_archives.sql")?.sha256,"fbb8935ce977cfc507801f01beb8f303d4817ed63c6bdd4c05799768dfaac9d4") assert.equal((await pool.query("SELECT count(*)::int AS n FROM schema_migrations")).rows[0].n, MIGRATION_MANIFEST.length) assert.equal((await pool.query("SELECT count(*)::int AS n FROM server_poll_storage_admissions")).rows[0].n, 0) @@ -242,6 +242,12 @@ async function main() { const archivedRows = (await pool.query(SERVER_POLL_ARCHIVE_READ_SQL, [receiptId])).rows assert.equal(archivedRows.length, 1) const archive = archivedRows[0] + const { retireCompletedServerPollOperation } = await import("./server-poll-operation-retirement.js") + const beforeRetirement = await ledger() + await assert.rejects(retireCompletedServerPollOperation({ receiptId, oldArchiveProof: archive.archive_proof })) + assert.equal(await ledger(), beforeRetirement, "Authenticated V1 younger than 180 days retains exact charge") + assert.equal((await pool.query("SELECT version FROM server_poll_archives WHERE receipt_id=$1", [receiptId])).rows[0].version, 1) + assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_operation_replay_tombstones WHERE operation_id=$1", [a.execution_id])).rows[0].n, 0) assert.equal(Number(archive.actual_ciphertext_bytes), Number(archive.ciphertext_bytes)) assert(Number(archive.ciphertext_bytes) > 0 && Number(archive.ciphertext_bytes) <= SERVER_POLL_ARCHIVE_MAX_BYTES) assert.equal(Number(archive.ciphertext_bytes), Buffer.byteLength(archive.ciphertext)) diff --git a/backend/src/services/server-probe-operation-retirement.owned.test.ts b/backend/src/services/server-probe-operation-retirement.owned.test.ts new file mode 100644 index 00000000..b716d0eb --- /dev/null +++ b/backend/src/services/server-probe-operation-retirement.owned.test.ts @@ -0,0 +1,213 @@ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { assertOwnedBandwidthSql, prepareOwnedBandwidthState } from "./bandwidth-admission-process.sql-helper.js" + +assertOwnedBandwidthSql() +assert.equal(process.env.MMAPP_OWN_PROBE_OPERATION_RETIREMENT, "true") +await prepareOwnedBandwidthState() +const { pool, closePool } = await import("../db/index.js") +const { applySqlMigrations } = await import("../db/migrate.js") +const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp") +const { authenticateMcpToken, createMcpConnection } = await import("../modules/mcp/store.js") +const { applyPlan, invokeMcpTool } = await import("../modules/mcp/runtime.js") +const { encryptSecret, McpError } = await import("../modules/mcp/security.js") +const { serverProbeRetiredFixture } = await import("./server-probe-retired-source.fixture.js") +const { captureServerProbeTerminalProof, collectExpiredRetiredServerProbe, retireCompletedServerProbeOperation, + retireExpiredServerProbeOperation } = await import("./server-probe-operation-retirement.js") +const { createProbeJournalStorage } = await import("./server-probe-journal-maintenance.js") +const { connectProbeStorage } = await import("./server-probe-storage-connection.js") +const { PROBE_STORAGE_LIMITS } = await import("../modules/mcp/diagnostic-cancellation-storage.js") +const { retainServerProbeStorage } = await import("./server-probe-retention.js") +const { tickMcpWorkers } = await import("../modules/mcp/workers.js") + +try { + await applySqlMigrations(pool) + await pool.query("UPDATE mcp_settings SET enabled=true WHERE id=1") + await pool.query("INSERT INTO servers(id,name,host,enabled) VALUES(9,'owned retirement source','127.0.0.1',true)") + async function access() { + const actor = { id: randomUUID(), name: "Owned diagnostic retention", email: "owned-retention@example.test", apps: ["mm"], + permissions: ["mm:servers:read", "mm:servers:write", "mm:mcp:approve"], issuedAt: Date.now()-1000, expiresAt: Date.now()+300000 } + const issued = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: actor.name, mode: "approval", permissions: actor.permissions, + expiresAt: new Date(actor.expiresAt).toISOString(), scope: { serverIds: [9], + operationIds: ["servers.probes.run", "servers.probes.receipt.read", "operations.read", "changes.apply"], allowGlobal: false }, + limits: { callsPerMinute: 100, mutationsPerHour: 10 } }), actor) + return { actor, principal: await authenticateMcpToken(issued.token) } + } + async function verifyLifecycle(mode: "retained" | "purged" | "default-old" | "default-young" | "default-unproven" | "default-spent" | "scheduled-retained") { + const { actor, principal } = await access(), foreign = (await access()).principal + const historical = mode === "retained" ? undefined : { + base: Date.now()-(mode === "default-young" ? 60 : 200)*86400000, retentionDays:mode === "scheduled-retained" ? 365 : 30, + } + const { raw } = await serverProbeRetiredFixture(false, { connectionId: principal.connectionId, credentialId: principal.credentialId, ownerId: actor.id }, historical) + const { original: o, admission: a } = raw, key = randomUUID(), empty = await encryptSecret("[]") + const target = { receiptId: a.receipt_id, operationId: o.operation_id, connectionId: o.connection_id } + const request = { operationId: o.operation_id, receiptId: a.receipt_id, page: 0 } + const own = await pool.connect() + try { + await own.query("BEGIN") + // Only this disposable database gets synthetic historical SQL rows. The + // actual codec signs the historical originals with its owned state key. + await own.query(`INSERT INTO mcp_plans(id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at,approved_by,approved_at,created_at) + VALUES($1,$2,$3,1,1,'owned historical test','{}',$4,'completed',$5,$6,$7,$8)`, + [o.plan_id,o.connection_id,o.operation_name,o.evidence,o.plan_expires_at,o.owner_id,o.plan_approved_at,o.plan_created_at]) + await own.query(`INSERT INTO mcp_operations(id,plan_id,connection_id,idempotency_key,state,verification,result,created_at,finished_at) + VALUES($1,$2,$3,$4,'applied','inconclusive',$5,$6,$7)`, [o.operation_id,o.plan_id,o.connection_id,key,o.operation_result,o.operation_created_at,o.operation_finished_at]) + await own.query(`INSERT INTO mcp_operation_steps(operation_id,step_index,state,intended_state,result,created_at,finished_at) + VALUES($1,0,'acknowledged',$2,$3,$4,$5)`, [o.operation_id,o.intended_state,o.step_result,o.step_created_at,o.step_finished_at]) + await own.query("INSERT INTO mcp_operation_checkpoints(operation_id,step_index,before_sealed,after_sealed) VALUES($1,0,$2,$2)", [o.operation_id,empty]) + await own.query("SET LOCAL session_replication_role='replica'") + await own.query("INSERT INTO server_probe_storage_admissions SELECT * FROM jsonb_populate_record(NULL::server_probe_storage_admissions,$1)", [a]) + await own.query("INSERT INTO server_probe_executions SELECT * FROM jsonb_populate_record(NULL::server_probe_executions,$1)", [raw.anchor]) + await own.query("INSERT INTO server_probe_execution_payloads SELECT * FROM jsonb_populate_record(NULL::server_probe_execution_payloads,$1)", [raw.payload]) + await own.query("INSERT INTO server_probe_execution_stages SELECT * FROM jsonb_populate_record(NULL::server_probe_execution_stages,$1)", [raw.stages[0]]) + await own.query("UPDATE server_probe_storage_usage SET admissions=1,reserved_files=1,reserved_bytes=294912 WHERE scope='global'") + await own.query("INSERT INTO server_probe_storage_usage VALUES($1,1,1,294912)", ["owner:"+actor.id]) + await own.query("SET LOCAL session_replication_role='origin'") + await own.query("COMMIT") + } finally { own.release(true) } + const live = await invokeMcpTool(principal, "mm_server_probe_receipt_get", request) + const storage = createProbeJournalStorage({ connect:connectProbeStorage, policy:{ ...PROBE_STORAGE_LIMITS,retentionDays:30 }, safetyWindowMs:60000, + async authorize(ownerId,operation,scope) { + assert.equal(ownerId,actor.id); assert.equal(operation,"purge"); assert.equal(scope,undefined) + return "a".repeat(64) + } }) + if (mode === "default-young" || mode === "default-old" || mode === "default-unproven" || mode === "default-spent") { + await assert.rejects(retireExpiredServerProbeOperation(target)) + if (mode === "default-unproven") { + // This original still qualifies for normal payload expiry, but cannot + // be authenticated as a dependency-free retirement source. + await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$2 WHERE operation_id=$1", + [o.operation_id,await encryptSecret('["held"]')]) + } + assert.deepEqual(await retainServerProbeStorage(), { skipped:false,owners:1,collected:1 }) + assert.equal((await pool.query("SELECT reserved_bytes::int AS bytes FROM server_probe_storage_usage WHERE scope='global'")).rows[0].bytes,0) + assert.equal((await pool.query("SELECT 1 FROM server_probe_terminal_proofs WHERE operation_id=$1",[o.operation_id])).rows.length, + mode === "default-unproven" ? 0 : 1) + await assert.rejects(invokeMcpTool(principal,"mm_server_probe_receipt_get",request)) + await assert.rejects(captureServerProbeTerminalProof(target)) + await assert.rejects(retireCompletedServerProbeOperation(target)) + if (mode === "default-spent") { + const spent = (await pool.query(`INSERT INTO server_probe_retirement_claims + (receipt_id,operation_id,plan_id,connection_id,source_proof,claim_token) + SELECT receipt_id,operation_id,$2,$3,source_proof,$4 FROM server_probe_terminal_proofs WHERE operation_id=$1 + RETURNING claim_token`,[o.operation_id,o.plan_id,o.connection_id,randomUUID()])).rows[0] + assert.ok(spent) + await tickMcpWorkers() + await tickMcpWorkers() + assert.deepEqual((await pool.query("SELECT claim_token FROM server_probe_retirement_claims WHERE receipt_id=$1",[a.receipt_id])).rows,[spent]) + assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[o.operation_id])).rows.length,1) + assert.equal((await pool.query("SELECT 1 FROM mcp_operation_replay_tombstones WHERE operation_id=$1",[o.operation_id])).rows.length,0) + return + } + if (mode !== "default-old") { + await tickMcpWorkers() + await assert.rejects(retireExpiredServerProbeOperation(target)) + assert.equal((await pool.query("SELECT 1 FROM server_probe_retirement_claims WHERE receipt_id=$1",[a.receipt_id])).rows.length,0) + assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[o.operation_id])).rows.length,1) + return + } + await pool.query("UPDATE mcp_operations SET result=jsonb_set(result,'{confirmedWrites}','1') WHERE id=$1",[o.operation_id]) + await tickMcpWorkers() + assert.equal((await pool.query("SELECT 1 FROM server_probe_retirement_claims WHERE receipt_id=$1",[a.receipt_id])).rows.length,0) + await pool.query("UPDATE mcp_operations SET result=$2 WHERE id=$1",[o.operation_id,o.operation_result]) + await tickMcpWorkers() + const absence = (await pool.query(`SELECT + NOT EXISTS(SELECT 1 FROM mcp_operations WHERE id=$1) AS op, + NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan, + NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step, + NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint, + NOT EXISTS(SELECT 1 FROM server_probe_terminal_proofs WHERE operation_id=$1) AS proof, + NOT EXISTS(SELECT 1 FROM server_probe_retired_sources WHERE operation_id=$1) AS archive`,[o.operation_id,o.plan_id])).rows[0] + assert.deepEqual(absence,{op:true,plan:true,step:true,checkpoint:true,proof:true,archive:true}) + await assert.rejects(applyPlan(principal,o.plan_id,key),error=>error instanceof McpError&&error.code==="OPERATION_HISTORY_EXPIRED") + await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:o.operation_id}),error=>error instanceof McpError&&error.code==="NOT_FOUND") + return + } + await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$2 WHERE operation_id=$1", [o.operation_id,await encryptSecret('["held"]')]) + await assert.rejects(retireCompletedServerProbeOperation(target)) + assert.equal((await pool.query("SELECT 1 FROM server_probe_retirement_claims WHERE receipt_id=$1", [a.receipt_id])).rows.length, 0) + await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$2 WHERE operation_id=$1", [o.operation_id,empty]) + if (mode === "scheduled-retained") { + await tickMcpWorkers() + await tickMcpWorkers() + assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[o.operation_id])).rows.length,0) + assert.equal((await pool.query("SELECT 1 FROM server_probe_retired_sources WHERE receipt_id=$1",[a.receipt_id])).rows.length,1) + assert.deepEqual(await invokeMcpTool(principal,"mm_server_probe_receipt_get",request),live) + await assert.rejects(applyPlan(principal,o.plan_id,key),error=>error instanceof McpError&&error.code==="OPERATION_HISTORY_EXPIRED") + return + } + assert.deepEqual(await retireCompletedServerProbeOperation(target), { retired:true,automaticReplay:false }) + const proof = (await pool.query(`SELECT + NOT EXISTS(SELECT 1 FROM mcp_operations WHERE id=$1) AS op, + NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan, + NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step, + NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint, + (SELECT receipt_ciphertext=$4 FROM server_probe_execution_payloads WHERE receipt_id=$3) AS payload, + (SELECT reserved_bytes=294912 FROM server_probe_storage_usage WHERE scope='global') AS quota`, + [o.operation_id,o.plan_id,a.receipt_id,raw.payload.receipt_ciphertext])).rows[0] + assert.deepEqual(proof, { op:true,plan:true,step:true,checkpoint:true,payload:true,quota:true }) + assert.deepEqual(await invokeMcpTool(principal, "mm_server_probe_receipt_get", request), live) + await assert.rejects(invokeMcpTool(foreign, "mm_server_probe_receipt_get", request)) + await assert.rejects(applyPlan(principal,o.plan_id,key), error => error instanceof McpError && error.code==="OPERATION_HISTORY_EXPIRED" && error.statusCode===410) + await assert.rejects(applyPlan(principal,randomUUID(),key), error => error instanceof McpError && error.code==="STATE_CONFLICT" && error.statusCode===409) + await assert.rejects(retireCompletedServerProbeOperation(target)) + const gc = await pool.connect() + try { + await gc.query("BEGIN") + await gc.query("SET LOCAL mm.probe_storage_gc='on'") + assert.equal((await gc.query("SELECT server_probe_storage_gc_allowed($1) AS allowed", [a.receipt_id])).rows[0].allowed, false) + await gc.query("ROLLBACK") + } finally { gc.release(true) } + const protectedWrite = await pool.connect() + try { + await protectedWrite.query("BEGIN") + const {rows:[archive]} = await protectedWrite.query("SELECT proof FROM server_probe_retired_sources WHERE receipt_id=$1",[a.receipt_id]) + await protectedWrite.query(`SELECT set_config('mm.probe_storage_gc','on',true),set_config('mm.probe_retired_gc_receipt',$1,true), + set_config('mm.probe_retired_gc_proof',$2,true)`, [a.receipt_id,archive.proof]) + await assert.rejects(protectedWrite.query("DELETE FROM server_probe_retired_sources WHERE receipt_id=$1",[a.receipt_id]),/immutable/) + await protectedWrite.query("ROLLBACK") + } finally { protectedWrite.release(true) } + const raced = await pool.connect() + try { + await raced.query("BEGIN") + await raced.query("SELECT receipt_id FROM server_probe_retired_sources WHERE receipt_id=$1 FOR UPDATE",[a.receipt_id]) + await assert.rejects(collectExpiredRetiredServerProbe(target)) + await raced.query("ROLLBACK") + } finally { raced.release(true) } + assert.deepEqual(await invokeMcpTool(principal, "mm_server_probe_receipt_get", request),live) + if (mode === "purged") { + await pool.query("UPDATE mcp_settings SET requests_retention_days=365 WHERE id=1") + await assert.rejects(collectExpiredRetiredServerProbe(target)) + assert.deepEqual(await invokeMcpTool(principal,"mm_server_probe_receipt_get",request),live) + await pool.query("UPDATE mcp_settings SET requests_retention_days=30 WHERE id=1") + assert.deepEqual(await storage.purge(actor.id,1,{ deadlineAt:Date.now()+10000 }),{ purged:1,automaticReplay:false }) + await assert.rejects(invokeMcpTool(principal,"mm_server_probe_receipt_get",request)) + assert.equal((await pool.query("SELECT reserved_bytes::int AS bytes FROM server_probe_storage_usage WHERE scope='global'")).rows[0].bytes,32768) + } + if (mode === "retained") await tickMcpWorkers() + else assert.deepEqual(await collectExpiredRetiredServerProbe(target), {collected:true,automaticReplay:false}) + const collected = (await pool.query(`SELECT + (SELECT count(*)::int FROM server_probe_retired_sources WHERE receipt_id=$1) AS archive, + (SELECT count(*)::int FROM server_probe_storage_admissions WHERE receipt_id=$1) AS admission, + (SELECT count(*)::int FROM server_probe_executions WHERE receipt_id=$1) AS anchor, + (SELECT count(*)::int FROM server_probe_execution_stages WHERE receipt_id=$1) AS stages, + (SELECT count(*)::int FROM server_probe_execution_payloads WHERE receipt_id=$1) AS payload, + (SELECT reserved_bytes::int FROM server_probe_storage_usage WHERE scope='global') AS quota, + (SELECT count(*)::int FROM server_probe_storage_usage WHERE scope=$2) AS owner_counter`,[a.receipt_id,"owner:"+actor.id])).rows[0] + assert.deepEqual(collected,{archive:0,admission:0,anchor:0,stages:0,payload:0,quota:0,owner_counter:0}) + await assert.rejects(collectExpiredRetiredServerProbe(target)) + await assert.rejects(invokeMcpTool(principal,"mm_server_probe_receipt_get",request)) + await assert.rejects(applyPlan(principal,o.plan_id,key),error=>error instanceof McpError&&error.code==="OPERATION_HISTORY_EXPIRED") + await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{scope,serverIds}','[]') WHERE id=$1", [principal.connectionId]) + await assert.rejects(invokeMcpTool(principal, "mm_server_probe_receipt_get", request)) + await assert.rejects(applyPlan(principal,o.plan_id,key), error => error instanceof McpError && error.code==="NOT_FOUND" && error.statusCode===404) + } + await verifyLifecycle("retained") + await verifyLifecycle("purged") + await verifyLifecycle("default-young") + await verifyLifecycle("default-old") + await verifyLifecycle("default-unproven") + await verifyLifecycle("default-spent") + await verifyLifecycle("scheduled-retained") + console.log("Owned diagnostic retirement: real worker, retained/expired paths, spent claims and holds, public replay/read and exact GC refunds") +} finally { await closePool() } diff --git a/backend/src/services/server-probe-operation-retirement.ts b/backend/src/services/server-probe-operation-retirement.ts new file mode 100644 index 00000000..17eb6d6f --- /dev/null +++ b/backend/src/services/server-probe-operation-retirement.ts @@ -0,0 +1,370 @@ +/** Exact internal maintenance entry. No scheduler, public registration, or native work. */ +import { randomUUID } from "node:crypto" +import { z } from "zod" +import { operationIdempotencyDigest } from "../modules/mcp/operation-replay-tombstones.js" +import { constantEqual, decryptSecretExisting, fingerprint, privateFingerprinterExisting } from "../modules/mcp/security.js" +import { PROBE_PAYLOAD_MAX_BYTES, PROBE_PERMANENT_RESERVATION_BYTES, ProbeStorageRefused, ProbeStorageUnknown, type ProbeStorageSql } from "./server-probe-journal-maintenance.js" +import { findRetiredServerProbeHistory } from "./server-probe-retired-history.js" +import { matchesServerProbeRetiredTerminal, openServerProbeRetiredSource, projectServerProbeRetiredSource, ServerProbeRetirementSourceRefused } from "./server-probe-retired-source.js" +import { connectProbeStorage } from "./server-probe-storage-connection.js" +import { sharedDatabaseGate } from "./shared-database-gate.js" + +const targetSchema = z.object({ receiptId: z.string().uuid(), operationId: z.string().uuid(), connectionId: z.string().uuid() }).strict() +type Target = z.infer +function requireThat(value: unknown): asserts value { if (!value) throw new ProbeStorageRefused() } + +const ORIGINAL = `jsonb_build_object( + 'operation_id',o.id,'connection_id',o.connection_id,'owner_id',c.owner_id,'plan_id',p.id, + 'operation_name',p.operation_id,'operation_version',p.operation_version,'operation_state',o.state, + 'operation_verification',o.verification,'operation_error_code',o.error_code,'operation_result',o.result, + 'plan_state',p.state,'approved_by',p.approved_by,'evidence',p.evidence,'step_index',s.step_index, + 'step_state',s.state,'intended_state',s.intended_state,'step_result',s.result, + 'operation_created_at',o.created_at,'operation_finished_at',o.finished_at,'plan_created_at',p.created_at, + 'plan_approved_at',p.approved_at,'plan_expires_at',p.expires_at,'step_created_at',s.created_at,'step_finished_at',s.finished_at + ) AS original,o.idempotency_key,k.before_sealed,k.after_sealed` +const ORIGINAL_JOINS = `JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id + JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0 + JOIN public.mcp_operation_checkpoints k ON k.operation_id=o.id AND k.step_index=s.step_index` +const ORIGINAL_ELIGIBLE = ` AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL + AND isfinite(o.finished_at) AND o.finished_at0) + AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_checkpoints WHERE operation_id=o.id AND step_index<>0) + AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id) + AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id) + AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id) + AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id) + AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id) + AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id) + AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id) + AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_quarantines WHERE receipt_id=$1 OR(identity_kind='mcp' AND execution_id=o.id)) + AND NOT EXISTS(SELECT 1 FROM public.server_poll_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id) + AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id) +` +const SOURCE = `SELECT ${ORIGINAL},to_jsonb(a) AS admission,to_jsonb(j) AS anchor,to_jsonb(b) AS payload + FROM public.server_probe_storage_admissions a + JOIN public.server_probe_executions j ON j.receipt_id=a.receipt_id + JOIN public.server_probe_execution_payloads b ON b.receipt_id=a.receipt_id + JOIN public.mcp_operations o ON o.id=a.execution_id + ${ORIGINAL_JOINS} + WHERE a.receipt_id=$1 AND o.id=$2 AND c.id=$3 AND c.owner_id=a.owner_id AND a.identity_kind='mcp' + AND a.claim_token IS NULL AND a.purged_at IS NULL AND a.cancelled_at IS NULL + AND (($5::timestamptz IS NULL AND a.authority_expires_at(deadlineAt: number, work: (sql: ProbeStorageSql) => Promise): Promise { + const sql = await connectProbeStorage({ deadlineAt }) + let committing = false, committed = false + let timer: ReturnType | undefined + try { + await sql.query("BEGIN") + await sql.query("SELECT set_config('TimeZone','UTC',true),set_config('statement_timeout','5000ms',true),set_config('lock_timeout','1000ms',true)") + // The same order as admission, publication, cancellation and payload GC. + requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)) AS locked")).rows[0]?.locked === true) + // Bound non-SQL proof work as well. Any late continuation retains only + // this permanently released checkout and cannot issue further SQL. + const expired = new Promise((_resolve,reject) => { + timer = setTimeout(() => { + sql.release(true) + reject(new ProbeStorageRefused()) + }, Math.max(1,deadlineAt-Date.now())) + }) + const result = await Promise.race([work(sql),expired]) + clearTimeout(timer) + requireThat(Date.now()) { + for (const sealed of [row.before_sealed, row.after_sealed]) { + requireThat(typeof sealed === "string") + let decoded: unknown + try { decoded = JSON.parse(await decryptSecretExisting(sealed)) } + catch { throw new ProbeStorageRefused() } + requireThat(Array.isArray(decoded) && decoded.length === 0) + } +} +async function prepare(sql: ProbeStorageSql, target: Target, requireOld = true, expectedFinishedAt?: string) { + const { receiptId, operationId, connectionId } = target + await lockOriginal(sql, target) + const rows = (await sql.query(SOURCE, [receiptId, operationId, connectionId, requireOld, expectedFinishedAt ?? null])).rows + requireThat(rows.length === 1) + const { original, admission, anchor, payload, idempotency_key: key } = rows[0] + requireThat(typeof key === "string") + await assertEmptyCheckpoints(rows[0]) + const stages = (await sql.query(`SELECT to_jsonb(s) AS stage FROM public.server_probe_execution_stages s + WHERE s.receipt_id=$1 ORDER BY s.stage_index LIMIT 12 FOR UPDATE NOWAIT`, [receiptId])).rows.map(row => row.stage) + const signed = await projectServerProbeRetiredSource({ original, admission, anchor, payload, stages, quarantineCount: 0, cancellationCount: 0 }) + requireThat(signed.source.receiptId === receiptId && signed.source.executionId === operationId && signed.source.connectionId === connectionId) + return { signed, key } +} + +const captureHash = z.string().regex(/^[a-f0-9]{64}$/) +async function terminalCaptureProof(sourceProof: string, idempotencyDigest: string) { + const sign = await privateFingerprinterExisting("server-probe-terminal-capture-v1") + return sign(fingerprint({ sourceProof, idempotencyDigest })) +} + +function captureTerminalProof(target: Target, deadlineAt: number, expectedFinishedAt?: string) { + return sharedDatabaseGate.request(() => transaction(deadlineAt, async sql => { + const { signed, key } = await prepare(sql, target, false, expectedFinishedAt) + const digest = operationIdempotencyDigest(target.connectionId, key) + const proof = await terminalCaptureProof(signed.proof, digest) + const saved = (await sql.query(`INSERT INTO public.server_probe_terminal_proofs + (operation_id,receipt_id,source,source_proof,idempotency_digest,capture_proof) + VALUES($1,$2,$3::jsonb,$4,$5,$6) ON CONFLICT DO NOTHING RETURNING receipt_id`, + [target.operationId,target.receiptId,JSON.stringify(signed.source),signed.proof,digest,proof])).rows + requireThat(saved.length === 1 && saved[0].receipt_id === target.receiptId) + return { captured:true as const, automaticReplay:false as const } + })) +} +/** Captures evidence only while the normal retention owner can still read the original. */ +export function captureServerProbeTerminalProof(raw: Target, deadlineAt = Date.now()+20000) { + return captureTerminalProof(targetSchema.parse(raw), deadlineAt) +} + +/** Only the operation runner supplies the positively acknowledged terminal SQL timestamp. */ +export function capturePublishedServerProbeTerminalProof(raw: Target & { finishedAt: string }) { + const { finishedAt, ...identity } = raw + const stamp = z.string().regex(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{6}Z$/).parse(finishedAt) + return captureTerminalProof(targetSchema.parse(identity), Date.now()+1200, stamp) +} + +async function prepareClosed(sql: ProbeStorageSql, target: Target) { + const { receiptId, operationId, connectionId } = target + await lockOriginal(sql, target) + const rows = (await sql.query(CLOSED_SOURCE, [receiptId, operationId, connectionId])).rows + requireThat(rows.length === 1) + const row = rows[0], key = z.string().parse(row.idempotency_key) + await assertEmptyCheckpoints(row) + const proof = captureHash.parse(row.source_proof), digest = operationIdempotencyDigest(connectionId, key) + requireThat(digest === row.idempotency_digest + && constantEqual(captureHash.parse(row.capture_proof), await terminalCaptureProof(proof, digest))) + const source = await openServerProbeRetiredSource({ source:row.source, proof }) + requireThat(source.receiptId === receiptId && source.executionId === operationId && source.connectionId === connectionId + && matchesServerProbeRetiredTerminal(source, row.original)) + return { signed:{source,proof},key } +} + +function retireServerProbeOperation(raw: Target, kind: "retained" | "expired", deadlineAt = Date.now()+20000) { + const target = targetSchema.parse(raw) + const { receiptId, operationId, connectionId } = target + return sharedDatabaseGate.request(async () => { + const token = randomUUID() + const read = kind === "retained" ? prepare : prepareClosed + const claim = await transaction(deadlineAt, async sql => { + const { signed, key } = await read(sql, target) + const digest = operationIdempotencyDigest(connectionId, key) + const result = await sql.query(`INSERT INTO public.server_probe_retirement_claims + (receipt_id,operation_id,plan_id,connection_id,source_proof,claim_token) + VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT DO NOTHING RETURNING receipt_id`, + [receiptId, operationId, signed.source.planId, connectionId, signed.proof, token]) + requireThat(result.rows.length === 1 && result.rows[0].receipt_id === receiptId) + return { proof: signed.proof, planId: signed.source.planId, digest } + }) + return transaction(deadlineAt, async sql => { + const { signed, key } = await read(sql, target), { planId, proof, digest } = claim + requireThat(signed.proof === proof && signed.source.planId === planId && operationIdempotencyDigest(connectionId, key) === digest) + const consumed = await sql.query(`SELECT receipt_id FROM public.server_probe_retirement_claims + WHERE receipt_id=$1 AND operation_id=$2 AND plan_id=$3 AND connection_id=$4 AND source_proof=$5 + AND claim_token=$6 AND claim_xid<>pg_current_xact_id()`, [receiptId, operationId, planId, connectionId, proof, token]) + requireThat(consumed.rows.length === 1 && consumed.rows[0].receipt_id === receiptId) + const tombstone = await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones + (operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global) + VALUES($1,$2,$3,$4,'servers.probes.run',$5::bigint[],false) RETURNING operation_id`, + [operationId, planId, connectionId, digest, signed.source.serverIds]) + requireThat(tombstone.rows.length === 1 && tombstone.rows[0].operation_id === operationId) + await sql.query("SELECT set_config('mm.probe_retirement_claim',$1,true)", [token]) + if (kind === "retained") { + const archive = await sql.query(`INSERT INTO public.server_probe_retired_sources(receipt_id,operation_id,source,proof) + VALUES($1,$2,$3::jsonb,$4) RETURNING receipt_id`, [receiptId, operationId, JSON.stringify(signed.source), proof]) + requireThat(archive.rows.length === 1 && archive.rows[0].receipt_id === receiptId) + } + const released = await sql.query("DELETE FROM public.server_probe_terminal_proofs WHERE operation_id=$1 RETURNING receipt_id", [operationId]) + requireThat(released.rows.length <= 1 && released.rows.every(row => row.receipt_id === receiptId) + && (kind !== "expired" || released.rows.length === 1)) + const checkpoint = await sql.query("DELETE FROM public.mcp_operation_checkpoints WHERE operation_id=$1 AND step_index=0 RETURNING operation_id", [operationId]) + requireThat(checkpoint.rows.length === 1 && checkpoint.rows[0].operation_id === operationId) + const step = await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id", [operationId]) + requireThat(step.rows.length === 1 && step.rows[0].operation_id === operationId) + const operation = await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id", [operationId, connectionId, planId]) + requireThat(operation.rows.length === 1 && operation.rows[0].id === operationId) + const plan = await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id", [planId, connectionId]) + requireThat(plan.rows.length === 1 && plan.rows[0].id === planId) + if (kind === "retained") { + const observed = await findRetiredServerProbeHistory(async (text, values) => (await sql.query(text, values)).rows, + { ...target, ownerId: signed.source.ownerId, scopeServerIds: signed.source.serverIds }) + requireThat(observed && !observed.payloadPurged && fingerprint(observed.source) === fingerprint(signed.source)) + } + return { retired: true as const, automaticReplay: false as const } + }) + }) +} +export function retireCompletedServerProbeOperation(raw: Target) { + return retireServerProbeOperation(raw, "retained") +} +export function retireExpiredServerProbeOperation(raw: Target) { + return retireServerProbeOperation(raw, "expired") +} + +/** Exact internal GC. This entry has no worker or public registration. */ +export function collectExpiredRetiredServerProbe(raw: Target, deadlineAt = Date.now()+20000) { + const target = targetSchema.parse(raw) + const { receiptId, operationId, connectionId } = target + return sharedDatabaseGate.request(() => transaction(deadlineAt, async sql => { + const settings = (await sql.query("SELECT requests_retention_days FROM public.mcp_settings WHERE id=1 FOR SHARE NOWAIT")).rows + const retentionDays = settings[0]?.requests_retention_days + requireThat(settings.length === 1 && typeof retentionDays === "number" && Number.isInteger(retentionDays) + && retentionDays >= 1 && retentionDays <= 365) + const pinned = (await sql.query(`SELECT c.owner_id,r.proof,t.target_server_ids::text[] AS server_ids FROM public.server_probe_retired_sources r + JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id + JOIN public.mcp_connections c ON c.id=t.connection_id + JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.owner_id=c.owner_id + JOIN public.server_probe_executions j ON j.receipt_id=r.receipt_id + WHERE r.receipt_id=$1 AND r.operation_id=$2 AND c.id=$3 FOR UPDATE OF c,r,t,a,j NOWAIT`, + [receiptId, operationId, connectionId])).rows + const identity = z.object({ owner_id: z.string().min(1).max(200), proof: z.string().regex(/^[a-f0-9]{64}$/), + server_ids: z.array(z.string().regex(/^[1-9]\d{0,15}$/).transform(Number).pipe(z.number().int().positive().safe())).min(1).max(2) }).strict().parse(pinned[0]) + requireThat(pinned.length === 1) + const observed = await findRetiredServerProbeHistory(async (text, values) => (await sql.query(text, values)).rows, + { ...target, ownerId: identity.owner_id, scopeServerIds: identity.server_ids }) + requireThat(observed) + const stageRows = (await sql.query(`SELECT to_jsonb(s) AS stage FROM public.server_probe_execution_stages s + WHERE s.receipt_id=$1 ORDER BY s.stage_index LIMIT 12 FOR UPDATE NOWAIT`, [receiptId])).rows + requireThat(stageRows.length >= 1 && stageRows.length <= 11 + && fingerprint(stageRows.map(row => row.stage)) === observed.source.stageSnapshotDigest) + await sql.query(`SELECT set_config('mm.probe_storage_gc','on',true),set_config('mm.probe_retired_gc_receipt',$1,true), + set_config('mm.probe_retired_gc_proof',$2,true)`, [receiptId, identity.proof]) + requireThat((await sql.query("SELECT public.server_probe_storage_gc_allowed($1) AS allowed", [receiptId])).rows[0]?.allowed === true) + const deletedPayload = (await sql.query("DELETE FROM public.server_probe_execution_payloads WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows + const files = observed.payloadPurged ? 0 : 1 + requireThat(deletedPayload.length === files && deletedPayload.every(row => row.receipt_id === receiptId)) + const deletedStages = (await sql.query("DELETE FROM public.server_probe_execution_stages WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows + requireThat(deletedStages.length === stageRows.length && deletedStages.every(row => row.receipt_id === receiptId)) + const deletedAnchor = (await sql.query("DELETE FROM public.server_probe_executions WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows + requireThat(deletedAnchor.length === 1 && deletedAnchor[0].receipt_id === receiptId) + // Admission deletion cascades to the exact authenticated archive only. + const deletedAdmission = (await sql.query("DELETE FROM public.server_probe_storage_admissions WHERE receipt_id=$1 AND owner_id=$2 RETURNING receipt_id", + [receiptId, identity.owner_id])).rows + requireThat(deletedAdmission.length === 1 && deletedAdmission[0].receipt_id === receiptId) + const scopes = ["global", "owner:" + identity.owner_id], bytes = PROBE_PERMANENT_RESERVATION_BYTES + files * PROBE_PAYLOAD_MAX_BYTES + const released = (await sql.query(`UPDATE public.server_probe_storage_usage SET admissions=admissions-1, + reserved_files=reserved_files-$2,reserved_bytes=reserved_bytes-$3 WHERE scope=ANY($1::text[]) + AND admissions>=1 AND reserved_files>=$2 AND reserved_bytes>=$3 RETURNING scope`, [scopes, files, bytes])).rows + requireThat(released.length === 2 && scopes.every(scope => released.some(row => row.scope === scope))) + await sql.query("DELETE FROM public.server_probe_storage_usage WHERE scope=$1 AND admissions=0 AND reserved_files=0 AND reserved_bytes=0", [scopes[1]]) + const state = (await sql.query(`SELECT + NOT EXISTS(SELECT 1 FROM public.server_probe_retired_sources WHERE receipt_id=$1) AS archive_gone, + EXISTS(SELECT 1 FROM public.mcp_operation_replay_tombstones WHERE operation_id=$2 AND connection_id=$3) AS replay_retained, + EXISTS(SELECT 1 FROM public.server_probe_retirement_claims WHERE receipt_id=$1 AND operation_id=$2 AND connection_id=$3) AS claim_retained`, + [receiptId, operationId, connectionId])).rows[0] + requireThat(state?.archive_gone === true && state.replay_retained === true && state.claim_retained === true) + return { collected: true as const, automaticReplay: false as const } + })) +} + +const RETIRE_CANDIDATES = `SELECT COALESCE(a.receipt_id,f.receipt_id) AS receipt_id,o.id AS operation_id,o.connection_id, + o.finished_at::text AS ordered_at,CASE WHEN a.receipt_id IS NULL THEN 'expired' ELSE 'retained' END AS kind + FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id + LEFT JOIN public.server_probe_storage_admissions a ON a.identity_kind='mcp' AND a.execution_id=o.id AND a.owner_id=c.owner_id + LEFT JOIN public.server_probe_terminal_proofs f ON f.operation_id=o.id + WHERE o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL + AND isfinite(o.finished_at) AND o.finished_at($1::timestamptz,$2::uuid)) + ORDER BY o.finished_at,o.id LIMIT 8` +const GC_CANDIDATES = `SELECT r.receipt_id,r.operation_id,t.connection_id,a.gc_not_before::text AS ordered_at,'gc' AS kind + FROM public.server_probe_retired_sources r + JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.execution_id=r.operation_id + JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id + JOIN public.mcp_connections c ON c.id=t.connection_id AND c.owner_id=a.owner_id + WHERE isfinite(a.gc_not_before) AND a.gc_not_before($1::timestamptz,$2::uuid)) + ORDER BY a.gc_not_before,r.operation_id LIMIT 8` + +type Cursor = { orderedAt: string; operationId: string } +let retireCursor: Cursor | undefined, gcCursor: Cursor | undefined +const workerRow = z.object({ receipt_id:z.string().uuid(),operation_id:z.string().uuid(),connection_id:z.string().uuid(), + ordered_at:z.string().refine(value=>Number.isFinite(Date.parse(value))),kind:z.enum(["retained","expired","gc"]) }).strict() + +/** Invoked only under the existing scheduler owner. Keep expiry and one-shot + * retirement independent; a claimed operation is never selected again. */ +export function maintainServerProbeOperationRetirement() { + return sharedDatabaseGate.request(async () => { + const deadlineAt = Date.now()+20000 + const scan = async (lane: "retire" | "gc") => { + const cursor = lane === "retire" ? retireCursor : gcCursor + const setCursor = (value: Cursor | undefined) => { if (lane === "retire") retireCursor=value; else gcCursor=value } + const rows = await transaction(deadlineAt,async sql => (await sql.query(lane === "retire" ? RETIRE_CANDIDATES : GC_CANDIDATES, + [cursor?.orderedAt ?? null,cursor?.operationId ?? null])).rows) + requireThat(rows.length<=8) + let scanned=0, completed=0 + for (const raw of rows) { + if (Date.now()>=deadlineAt-2000 || completed>=2) break + const row=workerRow.parse(raw) + requireThat((row.kind === "gc") === (lane === "gc")) + const target={receiptId:row.receipt_id,operationId:row.operation_id,connectionId:row.connection_id} + setCursor({orderedAt:row.ordered_at,operationId:row.operation_id}); scanned++ + try { + if (row.kind === "gc") await collectExpiredRetiredServerProbe(target,deadlineAt) + else await retireServerProbeOperation(target,row.kind,deadlineAt) + completed++ + } catch (error) { + if (!(error instanceof ProbeStorageRefused || error instanceof ServerProbeRetirementSourceRefused || error instanceof z.ZodError)) throw error + } + } + if (scanned === rows.length && rows.length < 8) setCursor(undefined) + return completed + } + const collected=await scan("gc") + const retired=Date.now()[] + try { + const sql = await connectProbeStorage({ deadlineAt: Math.min(deadlineAt, Date.now()+500) }) + let read = false + try { + candidates = (await sql.query(`/* probe-retention:capture-candidates */ + SELECT a.receipt_id,o.id AS operation_id,o.connection_id + FROM public.server_probe_storage_admissions a + JOIN public.server_probe_execution_payloads b USING(receipt_id) + JOIN public.mcp_operations o ON o.id=a.execution_id + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id AND c.owner_id=a.owner_id + WHERE a.identity_kind='mcp' AND a.purged_at IS NULL AND a.cancelled_at IS NULL AND a.claim_token IS NULL + AND a.claim_attempted_at IS NOT NULL AND a.authority_expires_at= deadlineAt || candidates.length > 8) break + if (typeof row.receipt_id !== "string" || typeof row.operation_id !== "string" || typeof row.connection_id !== "string") continue + try { + await captureServerProbeTerminalProof({ receiptId:row.receipt_id, operationId:row.operation_id, connectionId:row.connection_id }, + Math.min(deadlineAt, Date.now()+550)) + } catch { /* Never hold the payload or retry a capture in this window. */ } + } +} + +function createRetention(connect: typeof connectProbeStorage, preCapture?: () => Promise) { let running = false, cursor = "" return async () => { if (running) return { skipped: true, owners: 0, collected: 0 } running = true try { return await sharedDatabaseGate.request(async () => { + await preCapture?.() const options: ProbeStorageOptions = { deadlineAt: Date.now() + 20000 } const physical = await connect(options) let active = true, broken = false, released = false, currentOwner = "" @@ -69,6 +110,6 @@ function createRetention(connect: typeof connectProbeStorage) { } } -export const retainServerProbeStorage = createRetention(connectProbeStorage) +export const retainServerProbeStorage = createRetention(connectProbeStorage, captureBeforeExpiry) /** Trusted owned-test composition only; never an HTTP/MCP parameter. */ export const createServerProbeRetentionForTests = createRetention diff --git a/backend/src/services/server-probe-retired-history.ts b/backend/src/services/server-probe-retired-history.ts new file mode 100644 index 00000000..374d99a3 --- /dev/null +++ b/backend/src/services/server-probe-retired-history.ts @@ -0,0 +1,67 @@ +/** Fixed private source lookup. The action owns the current authority and + * protection fence, and the existing history reader authenticates payload. */ +import { z } from "zod" +import { fingerprint } from "../modules/mcp/security.js" +import { openServerProbeRetiredSource, ServerProbeRetirementSourceRefused } from "./server-probe-retired-source.js" + +type Query=(sql:string,values:unknown[])=>Promise[]> +export interface ServerProbeRetiredLookup { + receiptId:string + operationId:string + connectionId:string + ownerId:string + scopeServerIds:readonly number[] +} + +const SQL=`/* probe-history:retired-source */ SELECT r.source,r.proof,t.plan_id,t.target_server_ids::text[] AS tombstone_server_ids, + j.source_server_id,j.destination_server_id,j.tool,j.mcp_credential_id,j.authority_issued_at, + j.receipt_proof,a.binding_proof,a.retention_days,a.safety_ms, + to_char(a.authority_expires_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS authority_expires_at, + to_char(a.gc_not_before AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS gc_not_before, + a.purged_at IS NOT NULL AS payload_purged,p.receipt_proof AS payload_proof, + CASE WHEN p.receipt_ciphertext IS NOT NULL THEN pg_catalog.encode(pg_catalog.sha256(pg_catalog.convert_to(p.receipt_ciphertext,'UTF8')),'hex') END AS payload_sha256 + FROM public.server_probe_retired_sources r + JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id AND t.connection_id=$3 + JOIN public.mcp_connections c ON c.id=t.connection_id AND c.owner_id=$4 + JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id + JOIN public.server_probe_executions j ON j.receipt_id=a.receipt_id AND j.execution_id=a.execution_id AND j.identity_kind=a.identity_kind + LEFT JOIN public.server_probe_execution_payloads p ON p.receipt_id=a.receipt_id + WHERE r.receipt_id=$1 AND r.operation_id=$2 AND a.execution_id=r.operation_id AND a.identity_kind='mcp' + AND a.owner_id=c.owner_id AND a.claim_token IS NULL AND isfinite(a.claim_attempted_at) AND a.cancelled_at IS NULL + AND (a.purged_at IS NULL OR isfinite(a.purged_at)) + AND j.actor_id=c.owner_id AND j.mcp_connection_id=c.id AND j.authentication='mcp' AND j.request_id IS NULL + AND j.binding_proof=a.binding_proof AND j.authority_expires_at=a.authority_expires_at AND j.state='completed' + AND t.action_id='servers.probes.run' AND NOT t.target_global AND cardinality(t.target_server_ids) BETWEEN 1 AND 2 + AND j.source_server_id=ANY($5::int[]) AND (j.destination_server_id IS NULL OR j.destination_server_id=ANY($5::int[])) + AND NOT EXISTS(SELECT 1 FROM public.mcp_operations WHERE id=r.operation_id) + AND NOT EXISTS(SELECT 1 FROM public.mcp_plans WHERE id=t.plan_id) + AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_steps WHERE operation_id=r.operation_id) + AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_quarantines q + WHERE q.receipt_id=r.receipt_id OR(q.identity_kind='mcp' AND q.execution_id=r.operation_id)) + LIMIT 2` +const sid=z.number().int().positive().safe() +const ids=z.array(z.string().regex(/^[1-9]\d{0,15}$/).transform(Number).pipe(sid)).min(1).max(2) + +export async function findRetiredServerProbeHistory(query:Query,input:ServerProbeRetiredLookup) { + const rows=await query(SQL,[input.receiptId,input.operationId,input.connectionId,input.ownerId,input.scopeServerIds]) + if(rows.length!==1)return + const row=rows[0] + const source=await openServerProbeRetiredSource({source:row.source,proof:row.proof}).catch(error=>{ + if(error instanceof ServerProbeRetirementSourceRefused)return + throw error + }) + const parsedScope=ids.safeParse(row.tombstone_server_ids) + if(!source||!parsedScope.success)return + const scope=parsedScope.data.sort((a,b)=>a-b) + if(source.receiptId!==input.receiptId||source.executionId!==input.operationId||source.connectionId!==input.connectionId|| + source.ownerId!==input.ownerId||source.planId!==row.plan_id||fingerprint(source.serverIds)!==fingerprint(scope)|| + source.serverIds.some(id=>!input.scopeServerIds.includes(id))||source.sourceServerId!==row.source_server_id|| + (source.destinationServerId??null)!==row.destination_server_id||source.tool!==row.tool|| + source.originalCredentialId!==row.mcp_credential_id||String(source.issuedAt)!==String(row.authority_issued_at)|| + source.bindingProof!==row.binding_proof||source.receiptProof!==row.receipt_proof||source.retentionDays!==row.retention_days|| + source.safetyMs!==row.safety_ms||source.authorityExpiresAt!==row.authority_expires_at||source.gcNotBefore!==row.gc_not_before)return + if(row.payload_purged===true){ + if(row.payload_proof!==null||row.payload_sha256!==null)return + }else if(row.payload_purged!==false||row.payload_proof!==source.receiptProof||row.payload_sha256!==source.ciphertextSha256)return + return {source,payloadPurged:row.payload_purged} +} diff --git a/backend/src/services/server-probe-retired-source.fixture.ts b/backend/src/services/server-probe-retired-source.fixture.ts new file mode 100644 index 00000000..bf63028f --- /dev/null +++ b/backend/src/services/server-probe-retired-source.fixture.ts @@ -0,0 +1,50 @@ +import { randomUUID } from "node:crypto" +import { encryptSecret,fingerprint,privateSecretFingerprint,redact } from "../modules/mcp/security.js" +import { serverProbeReceiptMetadata } from "../modules/mcp/server-probe-actions.js" +import type { ServerProbeReceipt } from "./server-probe-execution.js" +import type { ServerProbeHistoryBinding } from "./server-probe-journal-history.js" +export async function serverProbeRetiredFixture(twoSides=false, identity?:{connectionId:string;credentialId:string;ownerId:string}, + historical={base:Date.parse("2025-01-02T00:00:00Z"),retentionDays:365}){ + const operationId=randomUUID(),receiptId=randomUUID(),connectionId=identity?.connectionId??randomUUID(),credentialId=identity?.credentialId??randomUUID(),ownerId=identity?.ownerId??randomUUID() + const {base,retentionDays}=historical,stamp=(ms:number)=>new Date(base+ms).toISOString() + const originalTimes={issuedAt:base,authorityExpiresAt:base+15000} + const tool=twoSides?"bandwidth":"ping",sourceServerId=9,destination=twoSides?{destinationServerId:11}:{} + const prepared={receiptId,tool,serverId:sourceServerId,...destination,expiresAt:stamp(15000),bindingCiphertext:"inert-private-inner-binding"} + const binding={version:1,operation:"servers.probes.run",principal:{connectionId,credentialId,credentialExpiresAt:stamp(30000)}, + ownerId,...originalTimes,prepared,bindingProof:"a".repeat(64),stageBudget:twoSides?1:2} + const proof=(domain:string,value:unknown)=>privateSecretFingerprint(fingerprint(value),domain) + const input={serverId:sourceServerId,receiptId,inputDigest:await proof("server-probe-mcp-binding-v1",binding), + bindingCiphertext:await encryptSecret(JSON.stringify(binding))} + const step={kind:"service",service:binding.operation,input} + const serverIds=twoSides?[9,11]:[9] + const evidence={steps:[step],requiresHuman:true,verificationScope:"acknowledgement",mutationCost:binding.stageBudget, + target:{scope:"servers",serverIds},validUntil:prepared.expiresAt} + const context:ServerProbeHistoryBinding["context"]={identity:{kind:"mcp",operationName:"servers.probes.run",ownerId, + connectionId,credentialId,executionId:operationId,issuedAt:base,expiresAt:base+15000},sourceServerId,...destination,tool} + const receipt:ServerProbeReceipt={receiptId,serverId:sourceServerId,...destination,tool,state:"completed",httpStatus:200, + attemptedStages:1,completedStages:1,output:"INERT_RESULT_ONLY_IN_ORIGINAL_PAYLOAD",data:{kind:tool},automaticReplay:false,forwardingVerified:false} + const metadata=serverProbeReceiptMetadata(receipt,operationId,binding.stageBudget) + const plaintext=JSON.stringify({version:1,context,bindingProof:binding.bindingProof,receipt}) + const payload={receipt_id:receiptId,receipt_proof:await proof("server-probe-journal-v1:receipt",plaintext),receipt_ciphertext:await encryptSecret(plaintext)} + const anchor={receipt_id:receiptId,binding_proof:binding.bindingProof,identity_kind:"mcp",execution_id:operationId, + actor_id:ownerId,authentication:"mcp",mcp_connection_id:connectionId,mcp_credential_id:credentialId,request_id:null, + authority_issued_at:String(base),authority_expires_at:stamp(15000),source_server_id:sourceServerId, + destination_server_id:destination.destinationServerId??null,tool,state:"completed",receipt_proof:payload.receipt_proof, + attempted_stages:1,completed_stages:1,created_at:stamp(500),finished_at:stamp(1000)} + const raw={ + original:{operation_id:operationId,connection_id:connectionId,owner_id:ownerId,plan_id:randomUUID(), + operation_name:binding.operation,operation_version:1,operation_state:"applied",operation_verification:"inconclusive",operation_error_code:null, + operation_result:{confirmedWrites:0,verificationScope:"acknowledgement",completedServiceSteps:1,diagnosticReceipt:metadata}, + plan_state:"completed",approved_by:ownerId,evidence:{sealed:await encryptSecret(JSON.stringify(evidence))}, + step_index:0,step_state:"acknowledged",intended_state:redact(step),step_result:metadata, + operation_created_at:stamp(100),operation_finished_at:stamp(1500),plan_created_at:stamp(10),plan_approved_at:stamp(50), + plan_expires_at:stamp(16000),step_created_at:stamp(200),step_finished_at:stamp(1400)}, + admission:{receipt_id:receiptId,owner_id:ownerId,identity_kind:"mcp",execution_id:operationId,binding_proof:binding.bindingProof, + authority_expires_at:stamp(15000),claim_token:null,claim_attempted_at:stamp(400),purged_at:null,cancelled_at:null,created_at:stamp(300), + retention_days:retentionDays,safety_ms:60000,gc_not_before:stamp(15000+retentionDays*86400000+60000)}, + anchor,payload,stages:[{receipt_id:receiptId,stage_index:1,kind:twoSides?"native-bandwidth":"native-ping",timeout_ms:1000, + state:"received",outcome:"received",created_at:stamp(600),received_at:stamp(900)}],quarantineCount:0,cancellationCount:0, + } + return {raw,context,receipt,serverIds,stamp} +} + diff --git a/backend/src/services/server-probe-retired-source.test.ts b/backend/src/services/server-probe-retired-source.test.ts new file mode 100644 index 00000000..89e00c3c --- /dev/null +++ b/backend/src/services/server-probe-retired-source.test.ts @@ -0,0 +1,77 @@ +import assert from "node:assert/strict" +import { findRetiredServerProbeHistory } from "./server-probe-retired-history.js" +import { randomUUID } from "node:crypto" +import { mkdtemp,rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { test } from "node:test" +import { serverProbeRetiredFixture as fixture } from "./server-probe-retired-source.fixture.js" +import { createServerProbeJournalHistoryReader } from "./server-probe-journal-history.js" +import { openServerProbeRetiredSource,projectServerProbeRetiredSource,ServerProbeRetirementSourceRefused } from "./server-probe-retired-source.js" + +test("production proofs authenticate the minimal projection and keep the original payload readable only with the current fence",async()=>{ + const directory=await mkdtemp(path.join(tmpdir(),"owned-probe-retirement-")) + const previous=process.env.MCP_STATE_DIR + process.env.MCP_STATE_DIR=directory + try{ + for(const twoSides of [false,true]){ + const {raw,context,receipt,serverIds}=await fixture(twoSides) + const exact=structuredClone(raw),signed=await projectServerProbeRetiredSource(raw) + assert.deepEqual(raw,exact) + const opened=await openServerProbeRetiredSource(JSON.parse(JSON.stringify(signed))) + assert.deepEqual(opened,signed.source) + assert.deepEqual(opened.serverIds,serverIds) + assert.equal(opened.authorityExpiresAt,"2025-01-02T00:00:15.000000Z") + assert.equal(opened.terminalAt,"2025-01-02T00:00:16.000000Z") + assert.equal(opened.activityAt,opened.terminalAt) + assert.equal(opened.retentionDays,365) + assert.equal(opened.gcNotBefore,"2026-01-02T00:01:15.000000Z") + const lookup={operationId:opened.executionId,receiptId:opened.receiptId,connectionId:opened.connectionId,ownerId:opened.ownerId,scopeServerIds:serverIds} + const stored={...signed,...raw.anchor,plan_id:raw.original.plan_id,tombstone_server_ids:serverIds.map(String), + retention_days:raw.admission.retention_days,safety_ms:raw.admission.safety_ms, + gc_not_before:raw.admission.gc_not_before.replace("Z","000Z"),authority_expires_at:raw.anchor.authority_expires_at.replace("Z","000Z"), + payload_purged:false,payload_proof:raw.payload.receipt_proof,payload_sha256:opened.ciphertextSha256} + const read=(row:Record)=>findRetiredServerProbeHistory(async (_sql,values)=>{ + assert.deepEqual(values,[lookup.receiptId,lookup.operationId,lookup.connectionId,lookup.ownerId,lookup.scopeServerIds]) + return [row] + },lookup) + assert.deepEqual(await read(stored),{source:opened,payloadPurged:false}) + assert.deepEqual(await read({...stored,payload_purged:true,payload_proof:null,payload_sha256:null}),{source:opened,payloadPurged:true}) + for(const alteration of [{tombstone_server_ids:["800"]},{plan_id:randomUUID()},{gc_not_before:raw.admission.authority_expires_at}, + {mcp_credential_id:randomUUID()},{payload_sha256:"d".repeat(64)},{payload_purged:true}]) + assert.equal(await read({...stored,...alteration}),undefined) + assert.equal(await read({...stored,proof:"d".repeat(64)}),undefined) + const envelope=JSON.stringify(signed) + for(const secret of [raw.payload.receipt_ciphertext,raw.original.evidence.sealed,receipt.output!,"inert-private-inner-binding"]) + assert(!envelope.includes(secret)) + let permitted=true + const reader=createServerProbeJournalHistoryReader({ + assertReadAuthority:async()=>{if(!permitted)throw Error("current scope or protection denied")}, + loadArchive:async()=>[{...raw.anchor,receipt_ciphertext:raw.payload.receipt_ciphertext,quarantine_count:0}], + }) + const binding={context,receiptId:opened.receiptId,bindingProof:opened.bindingProof} + assert.deepEqual(await reader.read(binding),receipt) + permitted=false + await assert.rejects(reader.read(binding),/current scope or protection denied/) + for(const alteration of [{retentionDays:364},{activityAt:"2025-01-02T00:00:17.000000Z"},{ownerId:randomUUID()}]) + await assert.rejects(openServerProbeRetiredSource({...signed,source:{...signed.source,...alteration}}),ServerProbeRetirementSourceRefused) + } + const {raw}=await fixture() + for(const alter of [ + (copy:typeof raw)=>{copy.original.evidence.sealed="broken"}, + (copy:typeof raw)=>{copy.payload.receipt_proof="b".repeat(64)}, + (copy:typeof raw)=>{copy.original.step_result.sha256="b".repeat(64)}, + (copy:typeof raw)=>{copy.stages[0].received_at="2025-01-02T00:00:00.000001Z"}, + (copy:typeof raw)=>{copy.admission.authority_expires_at="2025-01-02T00:00:15.000001Z"}, + (copy:typeof raw)=>{copy.quarantineCount=1}, + (copy:typeof raw)=>{copy.cancellationCount=1}, + ]){ + const copy=structuredClone(raw);alter(copy) + await assert.rejects(projectServerProbeRetiredSource(copy),ServerProbeRetirementSourceRefused) + } + }finally{ + if(previous===undefined)delete process.env.MCP_STATE_DIR + else process.env.MCP_STATE_DIR=previous + await rm(directory,{recursive:true,force:true}) + } +}) diff --git a/backend/src/services/server-probe-retired-source.ts b/backend/src/services/server-probe-retired-source.ts new file mode 100644 index 00000000..e2213e8f --- /dev/null +++ b/backend/src/services/server-probe-retired-source.ts @@ -0,0 +1,147 @@ +/** Private codec only. A future caller must own a bounded, locked SQL + * snapshot, and readers must still prove current scope and protections. */ +import { createHash } from "node:crypto" +import { z } from "zod" +import { authenticateServerProbeExecutionSourceRow } from "../modules/mcp/diagnostic-cancellation-source.js" +import { serverProbeReceiptMetadata } from "../modules/mcp/server-probe-actions.js" +import { serviceDiagnosticReceipt } from "../modules/mcp/service-diagnostic-progress.js" +import { constantEqual, decryptSecretExisting, fingerprint, privateFingerprinterExisting } from "../modules/mcp/security.js" +import { createLegacyServerProbeVerifier } from "./server-probe-legacy-verifier.js" +import { validateServerProbeJournalReceipt } from "./server-probe-journal.js" +import type { ServerProbeReceipt } from "./server-probe-execution.js" +import { canonicalPollArchiveTimestamp } from "./server-poll-terminal-archive.js" + +const purpose="server-probe-retired-source-v1" +const uuid=z.string().uuid(),hash=z.string().regex(/^[0-9a-f]{64}$/),sid=z.number().int().positive().safe() +const time=z.string().transform(canonicalPollArchiveTimestamp) +const tool=z.enum(["ping","traceroute","bandwidth","dns","route","mtu"]) +const record=z.record(z.string(),z.unknown()) +const originalSchema=z.object({ + operation_id:uuid,connection_id:uuid,owner_id:z.string().min(1).max(200),plan_id:uuid, + operation_name:z.literal("servers.probes.run"),operation_version:z.literal(1),operation_state:z.literal("applied"), + operation_verification:z.literal("inconclusive"),operation_error_code:z.null(),operation_result:z.unknown(), + plan_state:z.literal("completed"),approved_by:z.string().min(1).max(200),evidence:record, + step_index:z.literal(0),step_state:z.literal("acknowledged"),intended_state:record,step_result:record, + operation_created_at:time,operation_finished_at:time,plan_created_at:time,plan_approved_at:time, + plan_expires_at:time,step_created_at:time,step_finished_at:time, +}).strict() +const admissionSchema=z.object({ + receipt_id:uuid,owner_id:z.string().min(1).max(200),identity_kind:z.literal("mcp"),execution_id:uuid,binding_proof:hash, + authority_expires_at:time,claim_token:z.null(),claim_attempted_at:time,purged_at:z.null(),cancelled_at:z.null(),created_at:time, + retention_days:z.number().int().min(1).max(365),safety_ms:z.number().int().min(60000).max(86400000),gc_not_before:time, +}).strict() +const sourceSchema=z.object({ + original:originalSchema,admission:admissionSchema,anchor:record, + payload:z.object({receipt_id:uuid,receipt_proof:hash,receipt_ciphertext:z.string().min(1).max(262144)}).strict(), + stages:z.array(record).min(1).max(11),quarantineCount:z.literal(0),cancellationCount:z.literal(0), +}).strict() +const signedSchema=z.object({ + version:z.literal(1),receiptId:uuid,executionId:uuid,connectionId:uuid,ownerId:z.string().min(1).max(200),planId:uuid, + operation:z.literal("servers.probes.run"),operationVersion:z.literal(1),originalCredentialId:uuid, + sourceServerId:sid,destinationServerId:sid.optional(),tool,serverIds:z.array(sid).min(1).max(2), + issuedAt:z.number().int().nonnegative().safe(),authorityExpiresAt:time,terminalAt:time,activityAt:time,gcNotBefore:time, + retentionDays:z.number().int().min(1).max(365),safetyMs:z.number().int().min(60000).max(86400000), + bindingProof:hash,sourceDigest:hash,terminalDigest:hash,stageSnapshotDigest:hash,receiptProof:hash,ciphertextSha256:hash, +}).strict().refine(value=>{ + const expected=[...new Set([value.sourceServerId,...(value.destinationServerId===undefined?[]:[value.destinationServerId])])].sort((a,b)=>a-b) + return fingerprint(expected)===fingerprint(value.serverIds) + && (value.tool==="bandwidth" ? value.destinationServerId!==undefined&&value.destinationServerId!==value.sourceServerId : value.destinationServerId===undefined) + && value.issuedAt=micros(value.authorityExpiresAt)+BigInt(value.retentionDays*86400000+value.safetyMs)*1000n +}) + +export type ServerProbeRetiredSource = z.output +/** Match the exact canonical live terminal projection after payload expiry. */ +export function matchesServerProbeRetiredTerminal(source: ServerProbeRetiredSource, raw: unknown): boolean { + const parsed = originalSchema.safeParse(raw) + return parsed.success && parsed.data.operation_id === source.executionId && parsed.data.plan_id === source.planId + && parsed.data.connection_id === source.connectionId && parsed.data.owner_id === source.ownerId + && fingerprint(parsed.data) === source.terminalDigest +} +export interface SignedServerProbeRetiredSource { source: ServerProbeRetiredSource; proof: string } +export class ServerProbeRetirementSourceRefused extends Error { + readonly automaticReplay=false + constructor(){super("Diagnostic retirement source cannot be authenticated");this.name="ServerProbeRetirementSourceRefused"} +} +function requireThat(value:unknown):asserts value {if(!value)throw new ServerProbeRetirementSourceRefused()} +function micros(value:string){return BigInt(Date.parse(value))*1000n+BigInt(value.slice(23,26))} +const maximum=(...values:string[])=>values.sort().at(-1)! + +/** Produces no operation/payload/cancellation mutation or public authorization. + * Only settled successful sources with the original retained payload qualify. */ +export async function projectServerProbeRetiredSource(raw:unknown):Promise{ + try{ + requireThat(Buffer.byteLength(JSON.stringify(raw))<=3*1048576) + const source=sourceSchema.parse(structuredClone(raw)),{original,admission,anchor,payload,stages}=source + const authenticated=await authenticateServerProbeExecutionSourceRow(original,original.operation_id, + {connectionId:original.connection_id,ownerId:original.owner_id}) + requireThat(authenticated.terminal&&authenticated.sourceState==="applied") + const context=authenticated.originalContext,id=context.identity + requireThat(admission.receipt_id===authenticated.scope.receiptId&&admission.execution_id===id.executionId&& + admission.owner_id===id.ownerId&&admission.binding_proof===authenticated.scope.bindingProof&& + micros(admission.authority_expires_at)===BigInt(id.expiresAt)*1000n&&time.parse(anchor.authority_expires_at)===admission.authority_expires_at&& + payload.receipt_id===admission.receipt_id&& + payload.receipt_proof===anchor.receipt_proof&&anchor.state==="completed") + const verify=createLegacyServerProbeVerifier({loadMcpSource:async operationId=>{ + requireThat(operationId===id.executionId) + return authenticated + }}) + const verified=await verify({identityKind:"mcp",executionId:id.executionId, + anchor:{...anchor,receipt_ciphertext:payload.receipt_ciphertext},stages}) + requireThat(verified.status==="verified"&&verified.authentication==="receipt"&&verified.bindingAuthenticated&& + verified.progressAuthenticated&&verified.terminal&&!verified.quarantined&&!verified.cancelled) + // The same ciphertext has already passed the existing receipt MAC and + // original-context checks above. Match its actual public effect as well. + const envelope=JSON.parse(await decryptSecretExisting(payload.receipt_ciphertext)) as {receipt:ServerProbeReceipt} + const receipt=validateServerProbeJournalReceipt(context,admission.receipt_id,envelope.receipt) + requireThat(receipt.state==="completed") + const metadata=serverProbeReceiptMetadata(receipt,id.executionId,authenticated.stageBudget) + const confirmed=serviceDiagnosticReceipt(metadata,true,id.executionId,authenticated.stageBudget) + requireThat(confirmed&&fingerprint(original.step_result)===fingerprint(metadata)&&fingerprint(original.operation_result)===fingerprint({ + confirmedWrites:0,verificationScope:"acknowledgement",completedServiceSteps:1,diagnosticReceipt:confirmed, + })) + // The original receipt authenticates context and progress, not SQL row + // timestamps. Sign the exact observed locked snapshot and its chronology, + // without upgrading the historical journal's stage or timestamp proofs. + const anchorCreated=time.parse(anchor.created_at),anchorFinished=time.parse(anchor.finished_at) + requireThat(original.plan_created_at<=original.plan_approved_at&&original.plan_approved_at<=original.operation_created_at&& + original.operation_created_at{ + try{ + requireThat(Buffer.byteLength(JSON.stringify(raw))<=8192) + const {source,proof}=z.object({source:signedSchema,proof:hash}).strict().parse(structuredClone(raw)) + const sign=await privateFingerprinterExisting(purpose) + requireThat(constantEqual(proof,sign(fingerprint(source)))) + return source + }catch{throw new ServerProbeRetirementSourceRefused()} +} diff --git a/backend/src/services/wireguard-interface-delete.ts b/backend/src/services/wireguard-interface-delete.ts index 142a0d41..1bf671d8 100644 --- a/backend/src/services/wireguard-interface-delete.ts +++ b/backend/src/services/wireguard-interface-delete.ts @@ -4,7 +4,7 @@ import { pool } from "../db/index.js" import { getServerRowById, type ServerRow } from "../modules/servers/repository/servers-repository.js" import { fingerprint, McpError, privateFingerprinter } from "../modules/mcp/security.js" import { readWireguardPeerDeletionState, wireguardPeerDeletionTarget, wireguardConfigurationRows, type WireguardPeerDeletionState } from "./wireguard-peer-delete.js" -import { greDependencyPaths, greStableRows, assertGreDeleteDependencies } from "./gre-mutation-state.js" +import { greStableRows, assertGreDeleteDependencies } from "./gre-mutation-state.js" import { observedBoolean } from "./ospf-observed-values.js" import { MikrotikClient } from "./mikrotik.js" import { withApplicationMutationLock, withConfigMutationLock } from "./config-mutation-lock.js" @@ -12,6 +12,8 @@ import { executeGuardedApplicationWrite } from "./config-mutation-policy.js" import { captureAndAppendRevision } from "./config-revisions.js" import { captureWireguardSnapshot } from "./wireguard-live.js" import { sameWireguardEndpoint } from "./wireguard-source.js" +import { wireguardInterfaceDependencyPaths, wireguardInterfaceRuntimeDependencyPaths } from "./routeros-native-paths.js" +export { wireguardInterfaceDependencyPaths } from "./routeros-native-paths.js" type Row = Record export interface WireguardInterfaceDeleteState { @@ -19,8 +21,7 @@ export interface WireguardInterfaceDeleteState { filters: { id: number; gatewayTunnelId: string; gateway: string }[]; proof: string; } export interface WireguardInterfaceDeleteCommand { method: "PATCH" | "DELETE"; path: string; resource: "interfaces" | "peers" | "ipv4" | "ipv6"; id: string; body?: Record } -const runtimePaths = new Set(["/routing/bgp/session", "/routing/ospf/neighbor", "/routing/ospf/interface", "/routing/bfd/session"]) -export const wireguardInterfaceDependencyPaths = [...new Set([...greDependencyPaths, "/interface/vxlan", "/interface/vrrp", "/interface/bonding", "/interface/eoip", "/interface/gre", "/interface/ipip", "/ip/dhcp-server", "/ip/dhcp-client", "/ipv6/dhcp-server", "/ipv6/dhcp-client", "/ip/ipsec/peer", "/ip/ipsec/policy", ...runtimePaths])] +const runtimePaths = new Set(wireguardInterfaceRuntimeDependencyPaths) const fail = (code: string, message: string) => new McpError(code, message, ["STATE_UNAVAILABLE", "MUTATION_OUTCOME_UNKNOWN", "VERIFICATION_FAILED"].includes(code) ? 502 : 409) const automaticLinkLocal = (row: Row) => observedBoolean(row.dynamic) === true && /^fe80:/i.test(row.address ?? "") const ordered = (path: string) => ["/ip/vrf", "/routing/rule", "/routing/filter/rule", "/routing/bfd/configuration"].includes(path) || /^\/(?:ip|ipv6)\/firewall\//.test(path) diff --git a/backend/src/test/traffic-flow-chr-capture-check.test.ts b/backend/src/test/traffic-flow-chr-capture-check.test.ts index ebff717c..9e8dcbd3 100644 --- a/backend/src/test/traffic-flow-chr-capture-check.test.ts +++ b/backend/src/test/traffic-flow-chr-capture-check.test.ts @@ -66,3 +66,54 @@ test("same-tuple pre-refresh packet still decodes with previous template and ref assert.throws(() => checkChrCapture(input), /reused the previous template/) assert.equal(checkChrCapture(fixture()).captureReplayMatched, true) }) + +function concatenateSets(version: number, first: string, second: string, reversed = false) { + const header = version === 9 ? 20 : 16 + const left = Buffer.from(first, "hex"), right = Buffer.from(second, "hex") + const values = [left.subarray(header), right.subarray(header)] + if (reversed) values.reverse() + const packet = Buffer.concat([left.subarray(0, header), ...values]) + packet.writeUInt16BE(version === 9 ? 2 : packet.length, 2) + return packet +} + +function bundledFixture() { + const input = fixture() + input.frames = [0, 1, 4, 5].map((index, ordinal) => { + const first = input.frames[index], second = input.frames[index + 2] + return { ...first, ordinal: ordinal + 1, expected: second.expected, + payloadHex: concatenateSets(input.exporters[first.exporter].version, first.payloadHex, second.payloadHex).toString("hex") } + }) + return input +} + +test("v9 and IPFIX template plus data datagrams prove both phases with the current product parser", () => { + const result = checkChrCapture(bundledFixture()) + assert.equal(result.frames, 4) + assert.deepEqual(result.exporters.map(row => [row.baselineRecords, row.refreshedRecords]), [[1, 1], [1, 1]]) + const multipleData = fixture() + const row = multipleData.frames[3] + row.payloadHex = concatenateSets(10, row.payloadHex, row.payloadHex).toString("hex") + row.expected = [...row.expected, ...row.expected] + assert.equal(checkChrCapture(multipleData).exporters[1].baselineRecords, 2) +}) + +test("rejects stale data before a refresh, foreign sets, trailing bytes and v9 counts that hide a set", () => { + const original = fixture() + const reversed = bundledFixture() + reversed.frames[2].payloadHex = concatenateSets(9, original.frames[4].payloadHex, original.frames[6].payloadHex, true).toString("hex") + assert.throws(() => checkChrCapture(reversed), /template order/) + const foreign = bundledFixture() + const foreignPacket = Buffer.from(foreign.frames[1].payloadHex, "hex") + foreignPacket.writeUInt16BE(257, 16 + foreignPacket.readUInt16BE(18)) + foreign.frames[1].payloadHex = foreignPacket.toString("hex") + assert.throws(() => checkChrCapture(foreign), /data set ID/) + const trailing = bundledFixture() + trailing.frames[0].payloadHex += "00" + assert.throws(() => checkChrCapture(trailing), /set count or header/) + const hidden = bundledFixture() + const hiddenPacket = Buffer.from(hidden.frames[0].payloadHex, "hex") + hiddenPacket.writeUInt16BE(1, 2) + hidden.frames[0].payloadHex = hiddenPacket.toString("hex") + assert.throws(() => checkChrCapture(hidden), /v9 set coverage/) +}) diff --git a/backend/src/test/traffic-flow-chr-capture-check.ts b/backend/src/test/traffic-flow-chr-capture-check.ts index ce5eb53c..540b8a10 100644 --- a/backend/src/test/traffic-flow-chr-capture-check.ts +++ b/backend/src/test/traffic-flow-chr-capture-check.ts @@ -6,7 +6,7 @@ import { resolve } from "node:path" import { fileURLToPath } from "node:url" import { isDeepStrictEqual } from "node:util" import { z } from "zod" -import { clearFlowTemplateCache, FlowTemplateRefused, parseFlowPacket } from "../services/traffic-flow-parse.js" +import { clearFlowTemplateCache, FLOW_TEMPLATE_LIMITS, FlowTemplateRefused, parseFlowPacket } from "../services/traffic-flow-parse.js" const maxInput = 512 * 1024 const ip = z.string().max(45).refine(value => isIP(value) !== 0) @@ -28,20 +28,65 @@ const capture = z.object({ payloadHex: z.string().min(40).max(8192).regex(/^(?:[0-9a-f]{2})+$/), kind: z.enum(["template", "data", "refresh", "pre-refresh-probe"]), expected: z.array(flow).max(64), - }).strict()).min(8).max(64), + }).strict()).min(4).max(64), }).strict() function requireEvidence(condition: unknown, reason: string): asserts condition { if (!condition) throw Error(reason) } +type Exporter = z.infer +const zeroPadding = (bytes: Buffer) => bytes.length <= 3 && bytes.every(value => value === 0) + +/** Check the entire datagram envelope without decoding record values. */ +function inspectSets(packet: Buffer, source: Exporter, changesTemplate: boolean, previousWidth: number) { + const header = source.version === 9 ? 20 : 16 + requireEvidence(packet.length >= header + 4 && packet.readUInt16BE(0) === source.version && + packet.readUInt32BE(header - 4) === source.domain && + (source.version === 9 || packet.readUInt16BE(2) === packet.length), "Packet does not match the declared exporter") + let width = previousWidth, templateSeen = false, records = 0, sets = 0 + for (let offset = header; offset < packet.length;) { + requireEvidence(packet.length - offset >= 4 && ++sets <= 32, "Invalid set count or header") + const id = packet.readUInt16BE(offset), length = packet.readUInt16BE(offset + 2) + requireEvidence(length >= 4 && offset + length <= packet.length, "Invalid set length") + const body = packet.subarray(offset + 4, offset + length) + offset += length + if (id === (source.version === 9 ? 0 : 2)) { + requireEvidence(changesTemplate && !templateSeen && records === 0 && body.length >= 4 && + body.readUInt16BE(0) === source.templateId, "Unexpected template identity or order") + const fieldCount = body.readUInt16BE(2) + requireEvidence(fieldCount > 0 && fieldCount <= FLOW_TEMPLATE_LIMITS.perTemplateFields, "Invalid template field count") + let at = 4 + width = 0 + for (let field = 0; field < fieldCount; field++) { + requireEvidence(body.length - at >= 4, "Truncated template field") + const type = body.readUInt16BE(at), size = body.readUInt16BE(at + 2) + requireEvidence(size > 0 && width + size <= 4096, "Variable or oversized replay template") + width += size + at += source.version === 10 && (type & 0x8000) !== 0 ? 8 : 4 + requireEvidence(at <= body.length, "Truncated enterprise template field") + } + requireEvidence(zeroPadding(body.subarray(at)), "Unexpected additional template or padding") + templateSeen = true + } else { + requireEvidence(id === source.templateId && (!changesTemplate || templateSeen), "Unexpected data set ID or template order") + requireEvidence(width > 0 && body.length >= width && zeroPadding(body.subarray(Math.floor(body.length / width) * width)), "Invalid data set records or padding") + records += Math.floor(body.length / width) + requireEvidence(records <= 64, "Replay record budget exceeded") + } + } + requireEvidence(templateSeen === changesTemplate && (source.version !== 9 || packet.readUInt16BE(2) >= sets), + "Template transition or v9 set coverage differs") + return { width, records } +} + /** Synchronous, isolated replay. Call in an offline process with no live parser. */ export function checkChrCapture(input: unknown) { const parsed = capture.safeParse(input) requireEvidence(parsed.success, "Invalid capture projection") const { destination, exporters, frames } = parsed.data requireEvidence(exporters[0].address !== exporters[1].address && exporters.every(row => row.address !== destination), "Expected two distinct exporter addresses and one destination") - const state = exporters.map(() => ({ phase: 0, baselineRecords: 0, refreshedRecords: 0, missingTemplateObserved: false })) + const state = exporters.map(() => ({ phase: 0, width: 0, baselineRecords: 0, refreshedRecords: 0, missingTemplateObserved: false })) let lastOrdinal = -1, lastReceivedMs = -1 clearFlowTemplateCache() try { @@ -51,19 +96,13 @@ export function checkChrCapture(input: unknown) { lastReceivedMs = frame.receivedMs const source = exporters[frame.exporter], current = state[frame.exporter] const packet = Buffer.from(frame.payloadHex, "hex") - const header = source.version === 9 ? 20 : 16 - requireEvidence(packet.length >= header + 8 && packet.readUInt16BE(0) === source.version && - packet.readUInt32BE(header - 4) === source.domain && - (source.version === 9 || packet.readUInt16BE(2) === packet.length) && - packet.readUInt16BE(header + 2) === packet.length - header, "Packet does not match the declared exporter or single-set envelope") const isTemplate = frame.kind === "template" || frame.kind === "refresh" - requireEvidence(packet.readUInt16BE(header) === (isTemplate ? source.version === 9 ? 0 : 2 : source.templateId), "Unexpected set ID") if (isTemplate) { - requireEvidence(packet.readUInt16BE(header + 4) === source.templateId && frame.expected.length === 0, "Unexpected template ID or template records") requireEvidence(frame.kind === "template" ? current.phase === 0 : current.phase === 1 && current.baselineRecords > 0, "Unexpected template transition") } else { requireEvidence(current.phase > 0, "Data preceding baseline template") } + const envelope = inspectSets(packet, source, isTemplate, current.width) const transport = { sourcePort: source.port, collectorSocketId: "offline_capture_socket" } if (frame.kind === "pre-refresh-probe") { requireEvidence(current.phase === 1 && current.baselineRecords > 0 && !current.missingTemplateObserved && frame.expected.length === 0, "Unexpected pre-refresh probe") @@ -77,10 +116,13 @@ export function checkChrCapture(input: unknown) { continue } const actual = parseFlowPacket(packet, source.address, transport).map(({ src, dst, bytes, packets, inIface, outIface }) => ({ src, dst, bytes, packets, inIface, outIface })) - requireEvidence(isDeepStrictEqual(actual, frame.expected), "Decoded flow records differ from the expected capture projection") - if (isTemplate) current.phase++ - else { - requireEvidence(actual.length > 0 && actual.every(row => row.bytes > 0 && row.packets > 0), "Expected positive traffic after the template") + requireEvidence(actual.length === envelope.records && isDeepStrictEqual(actual, frame.expected), "Decoded flow records differ from the expected capture projection") + if (isTemplate) { + current.phase++ + current.width = envelope.width + } + if (actual.length) { + requireEvidence(actual.every(row => row.bytes > 0 && row.packets > 0), "Expected positive traffic after the template") if (current.phase === 1) current.baselineRecords += actual.length else current.refreshedRecords += actual.length } diff --git a/backend/src/test/traffic-flow-chr-capture-process.test.ts b/backend/src/test/traffic-flow-chr-capture-process.test.ts new file mode 100644 index 00000000..ae45c28c --- /dev/null +++ b/backend/src/test/traffic-flow-chr-capture-process.test.ts @@ -0,0 +1,111 @@ +import assert from "node:assert/strict" +import { EventEmitter } from "node:events" +import { PassThrough } from "node:stream" +import { test } from "node:test" +import { captureOwnedChrPacketsForTests } from "./traffic-flow-chr-capture-process.js" + +const scope = { interfaceName: "owned0", sources: ["192.0.2.1", "192.0.2.2"] as const, destination: "192.0.2.100", port: 4739 } +function fixturePcap(linkType: "RAW" | "EN10MB" = "RAW") { + const link = linkType === "EN10MB" ? 14 : 0, frame = 60 + link + const bytes = Buffer.alloc(24 + 2 * frame) + bytes.writeUInt32LE(0xa1b2c3d4) + bytes.writeUInt16LE(2, 4); bytes.writeUInt16LE(4, 6) + bytes.writeUInt32LE(4124 + link, 16); bytes.writeUInt32LE(link ? 1 : 101, 20) + for (const [index, at] of [24, 24 + frame].entries()) { + bytes.writeUInt32LE(1700000000, at); bytes.writeUInt32LE(44 + link, at + 8); bytes.writeUInt32LE(44 + link, at + 12) + if (link) bytes.writeUInt16BE(0x0800, at + 16 + 12) + const ip = bytes.subarray(at + 16 + link, at + frame) + ip[0] = 0x45; ip.writeUInt16BE(44, 2); ip[9] = 17 + Buffer.from([192, 0, 2, index + 1, 192, 0, 2, 100]).copy(ip, 12) + ip.writeUInt16BE(2055, 20); ip.writeUInt16BE(scope.port, 22); ip.writeUInt16BE(24, 24) + ip.writeUInt16BE(10, 28); ip.writeUInt16BE(16, 30) + } + return bytes +} +const ready = "tcpdump: listening on owned0, link-type RAW (Raw IP), snapshot length 4124 bytes\n" +type Link = "RAW" | "EN10MB" +function fixture({ mode = "normal", drop = 0, linkType = "RAW", reportedLink = linkType, fileLink = linkType }: + { mode?: "normal" | "early" | "wrong-interface"; drop?: number; linkType?: Link; reportedLink?: Link; fileLink?: Link } = {}) { + const child = Object.assign(new EventEmitter(), { stdout: new PassThrough(), stderr: new PassThrough(), + kill(signal: "SIGINT" | "SIGKILL") { + events.push(signal) + if (signal === "SIGINT") { + child.stdout.end(fixturePcap(fileLink)) + child.stderr.end("2 packets captured\n2 packets received by filter\n" + drop + " packets dropped by kernel\n") + } + queueMicrotask(() => child.emit("close", signal === "SIGINT" ? 0 : null, signal === "SIGINT" ? null : signal)) + return true + }, + }) + const events: string[] = [] + const launch = (args: string[]) => { + events.push("launch") + assert.deepEqual(args, ["-n", "-p", "-U", "-i", "owned0", "-y", linkType, "-s", linkType === "RAW" ? "4124" : "4138", "-c", "64", "-w", "-", + "ip and udp and dst host 192.0.2.100 and dst port 4739 and (src host 192.0.2.1 or src host 192.0.2.2)"]) + queueMicrotask(() => { + if (mode === "early") child.emit("close", 0, null) + else { + const line = reportedLink === "RAW" ? ready : ready.replace("RAW (Raw IP)", "EN10MB (Ethernet)").replace("4124", "4138") + child.stderr.write(mode === "normal" ? line : line.replace("owned0", "foreign0")) + } + }) + return child + } + return { child, launch, events } +} + +test("confined capture waits for exact readiness, then checks complete acknowledged two-source pcap", async () => { + const f = fixture() + const result = await captureOwnedChrPacketsForTests(scope, { + deadlineAt: Date.now() + 1000, assertCurrent: async () => { f.events.push("current") }, + exercise: async signal => { assert.equal(signal.aborted, false); f.events.push("exercise") }, + }, f.launch) + assert.deepEqual(f.events, ["current", "launch", "current", "exercise", "current", "SIGINT"]) + assert.deepEqual(result.counts, { captured: 2, received: 2, dropped: 0 }) + assert.deepEqual(result.bytes, fixturePcap()) +}) + +test("wrong-interface readiness and early exit never dispatch exercise", async () => { + for (const mode of ["wrong-interface", "early"] as const) { + const f = fixture({ mode }) + await assert.rejects(captureOwnedChrPacketsForTests(scope, { + deadlineAt: Date.now() + 100, assertCurrent: async () => {}, + exercise: async () => { f.events.push("exercise") }, + }, f.launch)) + assert.deepEqual(f.events, mode === "early" ? ["launch"] : ["launch", "SIGKILL"]) + } +}) + +test("capture drops refuse evidence and scope revocation prevents exercise", async () => { + const dropped = fixture({ drop: 1 }) + await assert.rejects(captureOwnedChrPacketsForTests(scope, { + deadlineAt: Date.now() + 1000, assertCurrent: async () => {}, exercise: async () => {}, + }, dropped.launch), /completeness/) + const revoked = fixture() + let checks = 0 + await assert.rejects(captureOwnedChrPacketsForTests(scope, { + deadlineAt: Date.now() + 1000, assertCurrent: async () => { if (++checks === 2) throw Error("revoked") }, + exercise: async () => { revoked.events.push("exercise") }, + }, revoked.launch), /revoked/) + assert.deepEqual(revoked.events, ["launch", "SIGKILL"]) +}) + +test("EN10MB launch, readiness, and file must agree independently", async () => { + const ethScope = { ...scope, linkType: "EN10MB" } as const + const f = fixture({ linkType: "EN10MB" }) + const result = await captureOwnedChrPacketsForTests(ethScope, { + deadlineAt: Date.now() + 1000, assertCurrent: async () => {}, exercise: async () => {}, + }, f.launch) + assert.deepEqual(result.bytes, fixturePcap("EN10MB")) + + const wrongReady = fixture({ linkType: "EN10MB", reportedLink: "RAW" }) + await assert.rejects(captureOwnedChrPacketsForTests(ethScope, { + deadlineAt: Date.now() + 100, assertCurrent: async () => {}, exercise: async () => { wrongReady.events.push("exercise") }, + }, wrongReady.launch)) + assert.deepEqual(wrongReady.events, ["launch", "SIGKILL"]) + + const wrongFile = fixture({ linkType: "EN10MB", fileLink: "RAW" }) + await assert.rejects(captureOwnedChrPacketsForTests(ethScope, { + deadlineAt: Date.now() + 1000, assertCurrent: async () => {}, exercise: async () => {}, + }, wrongFile.launch), /pinned link type EN10MB/) +}) diff --git a/backend/src/test/traffic-flow-chr-capture-process.ts b/backend/src/test/traffic-flow-chr-capture-process.ts new file mode 100644 index 00000000..09bc009a --- /dev/null +++ b/backend/src/test/traffic-flow-chr-capture-process.ts @@ -0,0 +1,135 @@ +/** Owns a single confined tcpdump child. Used only by an admitted native runner. */ +import { spawn } from "node:child_process" +import type { Readable } from "node:stream" +import { isIPv4 } from "node:net" +import { chrCaptureLayout, projectChrWire } from "./traffic-flow-chr-wire.js" +import { prepareChrReplay } from "./traffic-flow-chr-offline-replay.js" + +type Scope = Parameters[1] & { interfaceName: string } +type Child = { + stdout: Pick + stderr: Pick + once(event: "error", listener: (error: Error) => void): unknown + once(event: "close", listener: (code: number | null, signal: NodeJS.Signals | null) => void): unknown + kill(signal: "SIGINT" | "SIGKILL"): boolean +} +type Launch = (args: string[]) => Child +type Options = { + deadlineAt: number + assertCurrent: () => Promise + exercise: (signal: AbortSignal) => Promise +} +const maxCapture = 512 * 1024 +const maxDiagnostics = 8192 +const packetCap = 64 +function must(value: unknown, reason: string): asserts value { + if (!value) throw Error(reason) +} +function argsFor(scope: Scope, layout: ReturnType) { + must(/^[A-Za-z0-9][A-Za-z0-9_.-]{0,14}$/.test(scope.interfaceName) && scope.interfaceName !== "any" && scope.sources.length === 2 && + [...scope.sources, scope.destination].every(isIPv4) && new Set([...scope.sources, scope.destination]).size === 3 && + Number.isInteger(scope.port) && scope.port >= 1 && scope.port <= 65535, "Invalid confined capture scope") + const [first, second] = scope.sources + return ["-n", "-p", "-U", "-i", scope.interfaceName, "-y", layout.linkType, "-s", String(layout.snapLength), "-c", String(packetCap), "-w", "-", + "ip and udp and dst host " + scope.destination + " and dst port " + scope.port + " and (src host " + first + " or src host " + second + ")"] +} +function counters(stderr: string) { + const exact = (label: string) => { + const matches = stderr.split(/\r?\n/).map(line => new RegExp("^(\\d+) packets? " + label + "$").exec(line)).filter(match => match !== null) + must(matches.length === 1, "Capture counters are missing or ambiguous") + const value = Number(matches[0][1]) + must(Number.isSafeInteger(value) && value >= 0, "Capture counter exceeds integer range") + return value + } + return { captured: exact("captured"), received: exact("received by filter"), dropped: exact("dropped by kernel") } +} + +async function run(scope: Scope, options: Options, launch: Launch) { + const pinned = structuredClone(scope), layout = chrCaptureLayout(pinned.linkType) + const args = argsFor(pinned, layout), deadline = options.deadlineAt + must(Number.isSafeInteger(deadline) && deadline > Date.now() && deadline - Date.now() <= 90000, "Invalid capture deadline") + const controller = new AbortController() + const live = () => must(!controller.signal.aborted && Date.now() < deadline, "Capture deadline or ownership ended") + await options.assertCurrent() + live() + const child = launch(args), chunks: Buffer[] = [] + let bytes = 0, diagnosticBytes = 0, stderr = "", requestedStop = false, closed = false, failed = false + let ready!: () => void, terminate!: () => void, rejectWork!: (error: Error) => void + const readiness = new Promise(resolve => { ready = resolve }) + const workFailed = new Promise((_resolve, reject) => { rejectWork = reject }) + // The promise can reject during cleanup after an earlier error was observed. + void workFailed.catch(() => {}) + const stopped = new Promise(resolve => { terminate = resolve }) + const fail = (reason: string) => { + if (failed) return + failed = true + controller.abort() + rejectWork(Error(reason)) + } + child.stdout.on("data", (part: Buffer) => { + bytes += part.length + if (bytes > maxCapture) { fail("Capture exceeded byte limit"); return } + chunks.push(Buffer.from(part)) + }) + child.stderr.on("data", (part: Buffer) => { + diagnosticBytes += part.length + if (diagnosticBytes > maxDiagnostics) { fail("Capture diagnostics exceeded byte limit"); return } + stderr += part.toString("utf8") + const expected = "tcpdump: listening on " + pinned.interfaceName + ", link-type " + layout.linkType + if (stderr.split(/\r?\n/).some(line => line.startsWith(expected + " (") && line.endsWith("snapshot length " + layout.snapLength + " bytes"))) ready() + }) + child.once("error", () => fail("Capture child failed to start or signal")) + child.once("close", (code, signal) => { + closed = true + if (!requestedStop || code !== 0 || signal !== null) fail("Capture closed outside its acknowledged stop") + terminate() + }) + const timer = setTimeout(() => fail("Capture deadline expired"), Math.max(1, deadline - Date.now())) + let cleanupTimer: ReturnType | undefined + try { + await Promise.race([readiness, workFailed]) + live() + await Promise.race([options.assertCurrent(), workFailed]) + live() + await Promise.race([options.exercise(controller.signal), workFailed]) + live() + await Promise.race([options.assertCurrent(), workFailed]) + live() + requestedStop = true + must(child.kill("SIGINT"), "Capture stop signal was not delivered") + await Promise.race([stopped, workFailed]) + live() + const counts = counters(stderr), capture = Buffer.concat(chunks, bytes) + const wire = projectChrWire(capture, { sources: pinned.sources, destination: pinned.destination, port: pinned.port, ...(pinned.linkType === undefined ? {} : { linkType: pinned.linkType }) }) + must(counts.dropped === 0 && counts.captured === counts.received && counts.captured === wire.frames.length && counts.captured < packetCap, + "Capture completeness was not established") + return { bytes: capture, sha256: wire.captureSha256, dropped: counts.dropped, complete: true as const, counts, wire } + } finally { + clearTimeout(timer) + controller.abort() + if (!closed) { + child.kill("SIGKILL") + const exited = await Promise.race([stopped.then(() => true), new Promise(resolve => { cleanupTimer = setTimeout(() => resolve(false), 2000) })]) + if (cleanupTimer) clearTimeout(cleanupTimer) + must(exited, "Capture child exit could not be confirmed") + } + } +} + +export const captureOwnedChrPackets = (scope: Scope, options: Options) => { + must(process.platform === "linux", "Confined tcpdump capture requires Linux") + return run(scope, options, args => + spawn("tcpdump", args, { shell: false, stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, LC_ALL: "C" } })) +} + +/** Adapter for the capture callback of executeOwnedChrCapture. */ +export function createConfinedChrCapture(scope: Scope, options: Omit) { + const pinned = structuredClone(scope) + return async (exercise: (signal?: AbortSignal) => Promise) => { + const capture = await captureOwnedChrPackets(pinned, { ...options, exercise }) + return { ...capture, ...prepareChrReplay(capture.wire) } + } +} + +/** Explicit inert transport seam. */ +export const captureOwnedChrPacketsForTests = run diff --git a/backend/src/test/traffic-flow-chr-capture-timing.test.ts b/backend/src/test/traffic-flow-chr-capture-timing.test.ts new file mode 100644 index 00000000..7fc18ecc --- /dev/null +++ b/backend/src/test/traffic-flow-chr-capture-timing.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { assessChrCaptureTiming, planChrMeasuredCaptureWindow } from "./traffic-flow-chr-capture-timing.js" +import { CHR_TOPOLOGY_CATALOG_PAIR_GETS, CHR_TOPOLOGY_CATALOG_READ_LIMIT } from "./traffic-flow-chr-topology-inventory.js" +import { CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS } from "./traffic-flow-chr-target-harness.js" + +const current = { pairGets: CHR_TOPOLOGY_CATALOG_PAIR_GETS, catalogueReadLimit: CHR_TOPOLOGY_CATALOG_READ_LIMIT, + captureMs: 90000, targetMinimumRemainingMs: CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS } + +test("reduced exact catalogue count fits the existing cap but does not authorize writes without measurement", () => { + assert.equal(current.pairGets, 86) + const result = assessChrCaptureTiming(current) + assert.equal(result.status, "measurement-required") + assert.equal(result.minimumNodeCatalogueGets, 9460) + const max = assessChrCaptureTiming({ ...current, forwardWrites: 6 }) + assert.equal(max.minimumNodeCatalogueGets, 10492) + assert.equal(max.pairsBeforeGenerator, 22) + assert.equal(result.pairsBeforeFirstTargetWindow, 2) + assert.equal(result.setupBudgetMs, 26000) + assert.equal(assessChrCaptureTiming({ ...current, catalogueReadLimit: 8000 }).status, "catalogue-read-limit") +}) + +const measured = { + childPairMs: 12023, parentPairMs: 12023, directSetupMs: 4000, + collectorBeforeCaptureMs: 45000, collectorAfterCaptureMs: 55000, + beforeCaptureMs: 1400000, afterCaptureMs: 1500000, counterSpanMs: 120000, remainingAuthorityMs: 3600000, +} + +test("proposes a finite measured capture window and reserves authority through inverse", () => { + assert.deepEqual(planChrMeasuredCaptureWindow(measured), { + status: "candidate-window", captureMs: 205000, requiredAtCaptureStartMs: 1810000, requiredBeforeFirstWriteMs: 3315000, + requiredCollectorMs: 515000, + }) + assert.equal(planChrMeasuredCaptureWindow({ ...measured, remainingAuthorityMs: 3315000 }).status, "authority-window-exceeded") + assert.deepEqual(planChrMeasuredCaptureWindow({ ...measured, childPairMs: 100, parentPairMs: 100 }), { + status: "candidate-window", captureMs: 90000, requiredAtCaptureStartMs: 1695000, requiredBeforeFirstWriteMs: 3200000, + requiredCollectorMs: 400000, + }) +}) + +test("refuses missing or excessive measurements and preserves the counter evidence horizon", () => { + assert.equal(planChrMeasuredCaptureWindow({ ...measured, parentPairMs: undefined }).status, "measurement-required") + assert.equal(planChrMeasuredCaptureWindow({ ...measured, directSetupMs: undefined }).status, "measurement-required") + assert.equal(planChrMeasuredCaptureWindow({ ...measured, childPairMs: 30000 }).status, "capture-window-exceeded") + assert.equal(planChrMeasuredCaptureWindow({ ...measured, counterSpanMs: 180001 }).status, "capture-window-exceeded") + assert.throws(() => planChrMeasuredCaptureWindow({ ...measured, childPairMs: Infinity }), /Invalid CHR measured capture inputs/) + assert.throws(() => planChrMeasuredCaptureWindow({ ...measured, remainingAuthorityMs: 3600001 }), /Invalid CHR measured capture inputs/) +}) + +test("a long fixture lease still requires measured work within the short MCP credential", () => { + assert.equal(planChrMeasuredCaptureWindow({ ...measured, collectorAfterCaptureMs: undefined }).status, "collector-measurement-required") + assert.deepEqual(planChrMeasuredCaptureWindow({ ...measured, collectorAfterCaptureMs: 140000 }), { + status: "collector-window-exceeded", requiredCollectorMs: 600000, + }) +}) + +test("a larger catalogue budget still requires real representative measurement and refuses observed latency", () => { + const options = { ...current, catalogueReadLimit: 20000 } + assert.equal(assessChrCaptureTiming(options).status, "measurement-required") + // Minimum recorded GET on CHR A/B times 43 each. This is a projection, + // not an assertion about the next live relay transaction. + assert.equal(assessChrCaptureTiming({ ...options, pairedReadMs: 43 * (108 + 263) }).status, "capture-setup-window") + assert.equal(assessChrCaptureTiming({ ...options, pairedReadMs: 100 }).status, "parent-measurement-required") + assert.equal(assessChrCaptureTiming({ ...options, pairedReadMs: 100, parentPairedReadMs: 100 }).status, "budget-check-passed") +}) diff --git a/backend/src/test/traffic-flow-chr-capture-timing.ts b/backend/src/test/traffic-flow-chr-capture-timing.ts new file mode 100644 index 00000000..899e793f --- /dev/null +++ b/backend/src/test/traffic-flow-chr-capture-timing.ts @@ -0,0 +1,88 @@ +import { CHR_COLLECTOR_RESULT_POLL_MS, CHR_STAGED_COLLECTOR_APPROVAL_MS, CHR_STAGED_TARGET_CAPTURE_MAX_MS, + CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS } from "./traffic-flow-chr-target-timing-profile.js" + +/** This model is a lower bound for the current serial read schedule. It can + * refuse admission. Its success never substitutes for live scope or authority. */ +export function assessChrCaptureTiming(options: { + pairGets: number + catalogueReadLimit: number + captureMs: number + targetMinimumRemainingMs: number + forwardWrites?: number + pairedReadMs?: number + parentPairedReadMs?: number +}) { + const { pairGets, catalogueReadLimit, captureMs, targetMinimumRemainingMs, forwardWrites = 4, pairedReadMs, parentPairedReadMs } = options + if (![pairGets, catalogueReadLimit, captureMs, targetMinimumRemainingMs].every(value => Number.isSafeInteger(value) && value > 0) || + ![4, 5, 6].includes(forwardWrites) || [pairedReadMs, parentPairedReadMs].some(value => + value !== undefined && (!Number.isFinite(value) || value <= 0))) { + throw Error("Invalid CHR timing inputs") + } + // Each flow setting/target write takes three fresh reads and a fresh + // topology proof after each. Eight outer pairs include the timing sample. + const pairsBeforeFirstTargetWindow = 2 + const pairsBeforeGenerator = pairsBeforeFirstTargetWindow + 3 * forwardWrites + 2 + const topologyAcquireAndRestorePairs = 2 * (1 + 2 * 18) + const minimumNodePairs = topologyAcquireAndRestorePairs + pairsBeforeGenerator + 3 * forwardWrites + 8 + const minimumNodeCatalogueGets = minimumNodePairs * pairGets + const minimumParentPairsBeforeGenerator = 2 * forwardWrites + const minimumParentCatalogueGetsBeforeGenerator = minimumParentPairsBeforeGenerator * pairGets + const setupBudgetMs = captureMs - targetMinimumRemainingMs + // A measured pair omits parent identity/catalogue extras. Use it only to + // veto an already underbudget capture, with margin for local scheduling. + const projectedSetupMs = pairedReadMs === undefined ? undefined : + Math.ceil((pairsBeforeGenerator * pairedReadMs + minimumParentPairsBeforeGenerator * (parentPairedReadMs ?? 0)) * 1.25) + const status = minimumNodeCatalogueGets > catalogueReadLimit ? "catalogue-read-limit" : + setupBudgetMs <= 0 || (projectedSetupMs !== undefined && projectedSetupMs >= setupBudgetMs) ? "capture-setup-window" : + pairedReadMs === undefined ? "measurement-required" : + parentPairedReadMs === undefined ? "parent-measurement-required" : "budget-check-passed" + return { status, minimumNodeCatalogueGets, minimumParentCatalogueGetsBeforeGenerator, + pairsBeforeFirstTargetWindow, pairsBeforeGenerator, setupBudgetMs, projectedSetupMs } as const +} + +/** Offline proposal for the staged disabled-baseline path. Before/after phase + * budgets cover the complete reviewed schedule outside capture, with one + * separate bounded human approval in each phase. */ +export function planChrMeasuredCaptureWindow(options: { + childPairMs?: number + parentPairMs?: number + directSetupMs?: number + collectorBeforeCaptureMs?: number + collectorAfterCaptureMs?: number + beforeCaptureMs: number + afterCaptureMs: number + counterSpanMs: number + remainingAuthorityMs: number +}) { + const { childPairMs, parentPairMs, directSetupMs, collectorBeforeCaptureMs, collectorAfterCaptureMs, beforeCaptureMs, afterCaptureMs, + counterSpanMs, remainingAuthorityMs } = options + const valid = (value: number) => Number.isSafeInteger(value) && value > 0 && value <= 3600000 + if (![beforeCaptureMs, afterCaptureMs, counterSpanMs, remainingAuthorityMs].every(valid) || + [childPairMs, parentPairMs, directSetupMs, collectorBeforeCaptureMs, collectorAfterCaptureMs] + .some(value => value !== undefined && !valid(value))) { + throw Error("Invalid CHR measured capture inputs") + } + if (childPairMs === undefined || parentPairMs === undefined || directSetupMs === undefined) { + return { status: "measurement-required" } as const + } + // Two enables, each with full pre-dispatch and post-ACK snapshots on both + // independent paths, plus the child's complete pre-generation fence. + const setupMs = (5 * childPairMs + 4 * parentPairMs + directSetupMs) * 1.25 + const captureMs = Math.max(90000, + (Math.floor((CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS + setupMs) / 1000) + 1) * 1000) + if (captureMs > CHR_STAGED_TARGET_CAPTURE_MAX_MS || counterSpanMs > 180000) { + return { status: "capture-window-exceeded", captureMs } as const + } + const operationMs = CHR_STAGED_COLLECTOR_APPROVAL_MS + CHR_COLLECTOR_RESULT_POLL_MS + if (collectorBeforeCaptureMs === undefined || collectorAfterCaptureMs === undefined) { + return { status: "collector-measurement-required" } as const + } + const requiredCollectorMs = captureMs + 2 * operationMs + collectorBeforeCaptureMs + collectorAfterCaptureMs + if (requiredCollectorMs >= 600000) return { status: "collector-window-exceeded", requiredCollectorMs } as const + const requiredAtCaptureStartMs = captureMs + afterCaptureMs + operationMs + const requiredBeforeFirstWriteMs = beforeCaptureMs + operationMs + requiredAtCaptureStartMs + if (requiredBeforeFirstWriteMs >= remainingAuthorityMs) { + return { status: "authority-window-exceeded", requiredBeforeFirstWriteMs } as const + } + return { status: "candidate-window", captureMs, requiredAtCaptureStartMs, requiredBeforeFirstWriteMs, requiredCollectorMs } as const +} diff --git a/backend/src/test/traffic-flow-chr-disabled-baseline.test.ts b/backend/src/test/traffic-flow-chr-disabled-baseline.test.ts new file mode 100644 index 00000000..06f49af3 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-disabled-baseline.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { createDisabledChrFlowHarness } from "./traffic-flow-chr-disabled-baseline.js" + +function fixture(lose = false, idless = false) { + const baseline = { + deviceId: "fixture", flow: { ...(idless ? {} : { ".id": "*F" }), enabled: "false", interfaces: "all", untouched: "1m" }, + ipfix: { ...(idless ? {} : { ".id": "*A" }), bytes: "false", packets: "true", "src-address": "true", "dst-address": "true", + protocol: "true", "src-port": "true", "dst-port": "true", "in-interface": "true", "out-interface": "true" }, + targets: [] as Record[], + } + const state = structuredClone(baseline) + const writes: Record[] = [] + const journal: string[] = [] + const dispatches: string[] = [] + async function change(method: string, path: string, body: Record, + fence?: () => Promise): Promise { + await fence?.() + dispatches.push(method + " " + path) + assert.ok(Object.values(body).every(value => typeof value === "string")) + writes.push(body as Record) + const row = path.includes("/ipfix/") ? state.ipfix : state.flow + Object.assign(row, Object.fromEntries(Object.entries(body).map(([key, value]) => + [key, key === "interfaces" ? value : value === "yes" ? "true" : "false"]))) + if (lose) throw Error("lost ACK") + return undefined as T + } + const harness = createDisabledChrFlowHarness({ + baseline, transitInterface: "transit-test", read: async () => state, authorize: async () => {}, + checkpoint: async entry => { journal.push(entry.step + "." + entry.phase) }, + native: { + patch: (path, body, _ms, fence) => change("PATCH", path, body, fence), + post: (path, body, _ms, _signal, fence) => change("POST", path, body, fence), + }, + }) + return { baseline, state, writes, journal, harness, dispatches } +} + +test("changes only required fields and one transit interface, then restores the exact disabled inventory", async () => { + const { baseline, state, writes, journal, harness } = fixture() + assert.equal((await harness.enable()).flow.enabled, "true") + assert.deepEqual(await harness.restore(), { restored: true, automaticReplay: false }) + assert.deepEqual(state, baseline) + assert.deepEqual(writes, [{ bytes: "yes" }, { enabled: "yes", interfaces: "transit-test" }, + { enabled: "no", interfaces: "all" }, { bytes: "no" }]) + assert.deepEqual(journal.filter(value => value.endsWith(".verified")), + ["ipfix.enable.verified", "flow.enable.verified", "flow.restore.verified", "ipfix.restore.verified"]) +}) + +test("missing singleton IDs use only fixed POST /set and require ACK before inverse", async () => { + const { baseline, state, dispatches, harness } = fixture(false, true) + await harness.enable() + await harness.restore() + assert.deepEqual(state, baseline) + assert.deepEqual(dispatches, ["POST /ip/traffic-flow/ipfix/set", "POST /ip/traffic-flow/set", + "POST /ip/traffic-flow/set", "POST /ip/traffic-flow/ipfix/set"]) + const lost = fixture(true, true) + await assert.rejects(lost.harness.enable(), /requires reconciliation/) + await assert.rejects(lost.harness.restore(), /No verified/) + assert.deepEqual(lost.dispatches, ["POST /ip/traffic-flow/ipfix/set"]) +}) + +test("refuses restore while a target remains and refuses replay after unknown acknowledgement", async () => { + const known = fixture() + await known.harness.enable() + known.state.targets.push({ ".id": "*1" }) + await assert.rejects(known.harness.restore(), /baseline changed/) + assert.equal(known.writes.length, 2) + const unknown = fixture(true) + await assert.rejects(unknown.harness.enable(), /requires reconciliation/) + await assert.rejects(unknown.harness.enable(), /already attempted/) + await assert.rejects(unknown.harness.restore(), /No verified/) + assert.equal(unknown.writes.length, 1) +}) diff --git a/backend/src/test/traffic-flow-chr-disabled-baseline.ts b/backend/src/test/traffic-flow-chr-disabled-baseline.ts new file mode 100644 index 00000000..c6aa6615 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-disabled-baseline.ts @@ -0,0 +1,116 @@ +import { runChrWrite, type ChrWriteAdmission } from "./traffic-flow-chr-write-admission.js" +import { isDeepStrictEqual } from "node:util" +import type { MikrotikClient } from "../services/mikrotik.js" +import { encodeRosId } from "../services/routeros-native-paths.js" + +type Row = Record +type Inventory = { deviceId: string; flow: Row; ipfix: Row; targets: Row[] } +type Step = "ipfix.enable" | "flow.enable" | "flow.restore" | "ipfix.restore" +type Options = { + baseline: Inventory + relayWrite?: { side: "A" | "B"; admission: ChrWriteAdmission } + transitInterface: string + read: () => Promise + authorize: () => Promise + verifyContext?: () => Promise + native: Pick + checkpoint: (entry: { step: Step; phase: "intent" | "acknowledged" | "verified"; + method: "PATCH" | "POST"; path: string; id?: string }) => Promise +} +const fields = ["bytes", "packets", "src-address", "dst-address", "protocol", "src-port", "dst-port", "in-interface", "out-interface"] +const yes = (value: unknown) => value === true || value === "true" || value === "yes" +const no = (value: unknown) => value === false || value === "false" || value === "no" +const base = "/ip/traffic-flow" +function must(value: unknown, message: string): asserts value { + if (!value) throw Error(message) +} +function idOf(row: Row) { + if (!Object.hasOwn(row, ".id")) return undefined + const id = row[".id"] + must(typeof id === "string" && /^\*[a-f0-9]{1,16}$/i.test(id), "Exact Traffic Flow singleton ID required") + return id +} + +export function chrDisabledFlowPatchCount(ipfix: Row) { + must(fields.every(key => yes(ipfix[key]) || no(ipfix[key])), "Exact restorable IPFIX fields required") + return fields.some(key => no(ipfix[key])) ? 2 : 1 +} + +/** Scope is an already proven isolated transit interface. This component owns + * singleton changes only. The separate target owner must remove its ACKed row + * and conserve both singletons before restore is admitted. */ +export function createDisabledChrFlowHarness(options: Options) { + const baseline = structuredClone(options.baseline) + const { transitInterface, authorize, verifyContext, read, native, checkpoint, relayWrite } = options + must(baseline.deviceId && baseline.targets.length === 0 && no(baseline.flow.enabled), "Disabled target-free baseline required") + must(/^[a-zA-Z0-9_.-]{1,64}$/.test(transitInterface) && transitInterface !== "all", "Exact transit interface required") + const flowId = idOf(baseline.flow), ipfixId = idOf(baseline.ipfix) + must(typeof baseline.flow.interfaces === "string" && baseline.flow.interfaces.length > 0, "Exact restorable settings required") + chrDisabledFlowPatchCount(baseline.ipfix) + const missing = fields.filter(key => no(baseline.ipfix[key])) + const enableFields = Object.fromEntries(missing.map(key => [key, "yes"])) + const restoreFields = Object.fromEntries(missing.map(key => [key, "no"])) + let phase: "new" | "blocked" | "enabled" | "restored" = "new" + let active: Inventory | undefined + async function current() { + await authorize() + const result = structuredClone(await read()) + await verifyContext?.() + await authorize() + return result + } + async function change(step: Step, kind: "flow" | "ipfix", before: Inventory, body: Record) { + const id = kind === "flow" ? flowId : ipfixId + const path = (kind === "flow" ? base : base + "/ipfix") + "/" + (id ? encodeRosId(id) : "set") + const method = id ? "PATCH" : "POST" + const record = { step, path, method, ...(id && { id }) } as const + must(isDeepStrictEqual(await current(), before), "Traffic Flow baseline changed") + await checkpoint({ ...record, phase: "intent" }) + let dispatched = false + try { + await runChrWrite(relayWrite?.admission, relayWrite && { step: "flow:" + relayWrite.side + ":" + step, + side: relayWrite.side, deviceId: baseline.deviceId, method, path, body }, async arm => { + const beforeDispatch = async () => { + must(!dispatched && isDeepStrictEqual(await current(), before) && !dispatched, "Traffic Flow dispatch refused") + dispatched = true + await arm() + } + if (id) await native.patch(path, body, 15000, beforeDispatch) + else await native.post(path, body, 15000, undefined, beforeDispatch) + }) + must(dispatched, "Traffic Flow dispatch fence omitted") + await checkpoint({ ...record, phase: "acknowledged" }) + const after = await current() + const changed = after[kind] + must(Object.entries(body).every(([key, value]) => key === "interfaces" ? changed[key] === value : + value === "yes" ? yes(changed[key]) : no(changed[key])), "Traffic Flow settings differ") + const conserved = structuredClone(after) + for (const key of Object.keys(body)) conserved[kind][key] = before[kind][key] + must(isDeepStrictEqual(conserved, before), "Unrelated Traffic Flow settings changed") + await checkpoint({ ...record, phase: "verified" }) + return after + } catch (error) { + if (dispatched) throw Error("Traffic Flow write requires reconciliation. Automatic replay forbidden.") + throw error + } + } + return { + async enable() { + must(phase === "new", "Traffic Flow enable already attempted") + phase = "blocked" + const before = missing.length ? await change("ipfix.enable", "ipfix", baseline, enableFields) : baseline + active = await change("flow.enable", "flow", before, { enabled: "yes", interfaces: transitInterface }) + phase = "enabled" + return structuredClone(active) + }, + async restore() { + must(phase === "enabled" && active, "No verified enabled state to restore") + phase = "blocked" + const after = await change("flow.restore", "flow", active, { enabled: "no", interfaces: String(baseline.flow.interfaces) }) + const restored = missing.length ? await change("ipfix.restore", "ipfix", after, restoreFields) : after + must(isDeepStrictEqual(restored, baseline), "Original Traffic Flow baseline not restored") + phase = "restored" + return { restored: true as const, automaticReplay: false as const } + }, + } +} diff --git a/backend/src/test/traffic-flow-chr-disabled-runner.test.ts b/backend/src/test/traffic-flow-chr-disabled-runner.test.ts new file mode 100644 index 00000000..fa525a45 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-disabled-runner.test.ts @@ -0,0 +1,116 @@ +import assert from "node:assert/strict" +import { mkdtemp, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { test } from "node:test" +import { chrExportDrainMs, executeDisabledChrCapture, type DisabledChrCaptureOptions } from "./traffic-flow-chr-disabled-runner.js" + +test("export drain requires a bounded inactive-flow timeout plus two seconds", () => { + assert.equal(chrExportDrainMs("15s"), 17000) + assert.equal(chrExportDrainMs("00:00:30"), 32000) + for (const value of ["", "0s", "1m1s", "300", undefined]) { + assert.throws(() => chrExportDrainMs(value), /Inactive flow timeout/) + } +}) + +const cases = { + "create ACK loss": /Target creation outcome unknown/, + "unsealed direct capture": /capture producer stopped/, + "staged capture ACK missing": /capture acknowledgement unavailable/, + "staged enable ACK loss": /Target enable outcome unknown/, + "unsealed staged capture": /capture producer stopped/, +} as const + +for (const mode of Object.keys(cases) as (keyof typeof cases)[]) test(mode, async () => { + const staged = mode.includes("staged") + const dir = await mkdtemp(join(tmpdir(), "mm-chr-disabled-")) + const actions: string[] = [] + const cleanup: boolean[] = [] + let topologyChecks = 0, topologyChecksAtCapture = 0 + function router(index: number) { + const name = "nfa012bcde" + (index === 0 ? "ea" : "eb") + const baseline = { deviceId: String(index), flow: { ".id": "*F", enabled: "false", interfaces: "all", "inactive-flow-timeout": "15s" }, + ipfix: { ".id": "*A", ...Object.fromEntries(["bytes", "packets", "src-address", "dst-address", "protocol", "src-port", + "dst-port", "in-interface", "out-interface"].map(key => [key, "true"])) }, targets: [] as Record[] } + const state = structuredClone(baseline) + return { baseline, transitInterface: name, interfaces: [{ id: "*1", name, ifIndex: "1" }, + { id: "*2", name: "nfa012bcde" + (index === 0 ? "ma" : "mb"), ifIndex: "2" }] as const, + read: async () => state, counters: async () => { + if (staged) actions.push("counters:" + index) + return { startedAt: 1, observedAt: 1, rows: [] } + }, + native: { async post() { throw Error("unexpected POST") }, + async patch(_path: string, body: Record, _timeout?: number, fence?: () => Promise) { + await fence?.() + actions.push("flow:" + index) + Object.assign(state.flow, body, { enabled: "true" }) + return undefined as T + } }, + targetNative: { async put(_path: string, body: Record, _timeout?: number, fence?: () => Promise): Promise { + await fence?.() + actions.push("target:" + index) + if (mode === "create ACK loss") throw Error("lost ACK") + assert.equal(body.disabled, staged ? "yes" : undefined) + state.targets.push({ ...body, ".id": "*C", disabled: body.disabled ?? "false" }) + return { ".id": "*C" } as T + }, async patch(path: string, body: Record, _timeout?: number, fence?: () => Promise) { + await fence?.() + assert.equal(path, "/ip/traffic-flow/target/*C") + assert.deepEqual(body, { disabled: "no" }) + assert.equal(state.targets[0]?.disabled, "yes") + state.targets[0].disabled = "false" + actions.push("enable:" + index) + if (mode === "staged enable ACK loss") throw Error("lost ACK") + return { ".id": "*C" } as T + }, async delete() { actions.push("unexpected inverse") } } } + } + const journalPath = join(dir, "owned.jsonl") + try { + const options: DisabledChrCaptureOptions = { + journalPath, deadlineAt: Date.now() + 180000, ...(staged ? { stagedTarget: true as const } : {}), + scope: { sources: ["192.0.2.1", "192.0.2.2"], destination: "192.0.2.3", port: 50000 }, + expected: [], serverIds: [1, 2], routers: [router(0), router(1)], + assertTopology: async () => { topologyChecks++ }, authorize: async () => {}, + capture: async exercise => { + if (staged) actions.push("capture") + topologyChecksAtCapture = topologyChecks + await exercise(new AbortController().signal, staged && mode !== "staged capture ACK missing" + ? { kind: "staged-ipfix-timeout-20s", captureDeadlineAt: Date.now() + 120000 } : undefined) + throw Error("unexpected capture") + }, + generate: async () => { + if (staged) { + assert.equal(topologyChecks - topologyChecksAtCapture, 5) // Two per enable, one before generation. + actions.push("generate") + } + throw Error("capture producer stopped before seal") + }, + drainCollector: async () => { actions.push("unexpected drain") }, + reconcile: async () => {}, reportRouterCleanup: clean => cleanup.push(clean), + } + await assert.rejects(executeDisabledChrCapture(options), cases[mode]) + await assert.rejects(executeDisabledChrCapture(options), { code: "EEXIST" }) + const prefix = ["flow:0", "target:0", "flow:1", "target:1"] + assert.deepEqual(actions, { + "create ACK loss": prefix.slice(0, 2), + "unsealed direct capture": prefix, + "staged capture ACK missing": [...prefix, "capture"], + "staged enable ACK loss": [...prefix, "capture", "enable:0"], + "unsealed staged capture": [...prefix, "capture", "enable:0", "enable:1", "counters:0", "counters:1", "generate"], + }[mode]) + assert.deepEqual(cleanup, [false]) + const journal = (await readFile(journalPath, "utf8")).trim().split("\n").map(line => JSON.parse(line)) + if (staged && mode !== "staged capture ACK missing") { + assert(journal.find(entry => entry.phase === "capture-ready")) + } + assert.deepEqual(journal.at(-1), { + "create ACK loss": { device: 0, action: "create", phase: "intent" }, + "unsealed direct capture": { device: 1, action: "create", phase: "verified", id: "*C" }, + "staged capture ACK missing": { phase: "targets-staged-disabled" }, + "staged enable ACK loss": { device: 0, action: "enable", phase: "intent", id: "*C" }, + "unsealed staged capture": { device: 1, action: "enable", phase: "verified", id: "*C" }, + }[mode]) + } finally { + await rm(dir, { recursive: true, force: true }) + } +}) diff --git a/backend/src/test/traffic-flow-chr-disabled-runner.ts b/backend/src/test/traffic-flow-chr-disabled-runner.ts new file mode 100644 index 00000000..73cecc4e --- /dev/null +++ b/backend/src/test/traffic-flow-chr-disabled-runner.ts @@ -0,0 +1,224 @@ +import { open } from "node:fs/promises" +import { dirname } from "node:path" +import { setTimeout as delay } from "node:timers/promises" +import { createDisabledChrFlowHarness } from "./traffic-flow-chr-disabled-baseline.js" +import { assertChrTargetTimerWindow, createChrTargetHarness, type ChrStagedTargetTimerProfile, type ChrTargetTimerProfile } from "./traffic-flow-chr-target-harness.js" +import { verifyChrOwnedWgEvidence } from "./traffic-flow-chr-owned-wg-evidence.js" +import { projectChrWire } from "./traffic-flow-chr-wire.js" + +import type { ChrWriteAdmission } from "./traffic-flow-chr-write-admission.js" + +type Flow = Parameters[0] +type Target = Parameters[0] +type Evidence = Parameters[0] +type Counters = Evidence["exporters"][0]["before"] +type InterfacePins = Evidence["exporters"][0]["interfaces"] +type Router = Pick & { + targetNative: Target["native"] + interfaces: InterfacePins + counters: () => Promise +} +type Capture = Pick +export type DisabledChrCaptureOptions = { + journalPath: string + deadlineAt: number + writeAdmission?: ChrWriteAdmission + targetTimerProfile?: Omit + stagedTarget?: true + scope: Evidence["scope"] + expected: Evidence["expected"] + serverIds: readonly [number, number] + routers: readonly [Router, Router] + assertTopology: () => Promise + authorize: () => Promise + capture: (exercise: (signal: AbortSignal, timer?: Omit) => Promise) => Promise + generate: (signal: AbortSignal) => Promise<{ lastDatagramAt: number }> + drainCollector: () => Promise + reconcile: (wire: ReturnType, serverIds: readonly [number, number]) => Promise + reportRouterCleanup: (clean: boolean) => void +} +function must(value: unknown, reason: string): asserts value { if (!value) throw Error(reason) } + +export function chrExportDrainMs(value: unknown) { + must(typeof value === "string", "Inactive flow timeout unavailable") + const compact = /^(?:(\d{1,2})m)?(?:(\d{1,2})s)?$/.exec(value) + const clock = /^(\d{2}):(\d{2}):(\d{2})$/.exec(value) + const seconds = compact ? Number(compact[1] ?? 0) * 60 + Number(compact[2] ?? 0) : + clock ? Number(clock[1]) * 3600 + Number(clock[2]) * 60 + Number(clock[3]) : 0 + must(seconds >= 1 && seconds <= 60, "Inactive flow timeout outside owned capture budget") + return seconds * 1000 + 2000 +} + +/** Trusted local runner composition. The caller owns topology and collector. + * A post-dispatch ambiguity is never converted into a synthetic cleanup ACK. */ +export async function executeDisabledChrCapture(options: DisabledChrCaptureOptions) { + const scope = structuredClone(options.scope), expected = structuredClone(options.expected) + const serverIds = structuredClone(options.serverIds), deadline = options.deadlineAt + const writeAdmission = options.writeAdmission + const targetTimerProfile = options.targetTimerProfile && { ...options.targetTimerProfile } + const staged = options.stagedTarget === true + const routers = options.routers.map(router => ({ ...router, baseline: structuredClone(router.baseline), + interfaces: structuredClone(router.interfaces) })) + const live = () => must(Number.isSafeInteger(deadline) && Date.now() < deadline, "Disabled capture authority expired") + const authorize = async () => { + live() + await options.authorize() + live() + } + const verifyTopology = async () => { + await authorize() + await options.assertTopology() + await authorize() + } + live() + must(deadline - Date.now() <= 600000 && routers.length === 2 && routers[0].baseline.deviceId !== routers[1].baseline.deviceId && + serverIds.length === 2 && new Set(serverIds).size === 2 && serverIds.every(id => Number.isSafeInteger(id) && id > 0) && + routers.every(router => router.interfaces.some(pin => pin.name === router.transitInterface && /e[ab]$/.test(pin.name))), + "Invalid disabled capture fixture") + must(!staged || (!targetTimerProfile && routers.every(router => "patch" in router.targetNative && typeof router.targetNative.patch === "function" && + router.baseline.targets.length === 0 && ["15s", "00:00:15"].some(value => value === router.baseline.flow["inactive-flow-timeout"]))), + "Invalid staged target fixture") + const drainMs = Math.max(...routers.map(router => chrExportDrainMs(router.baseline.flow["inactive-flow-timeout"]))) + const assertTargetTimersFit = () => { + if (!targetTimerProfile) return + must(targetTimerProfile.captureDeadlineAt <= deadline, "IPFIX capture timer exceeds the owner deadline") + for (const router of routers) assertChrTargetTimerWindow(targetTimerProfile, router.baseline.flow["inactive-flow-timeout"]) + } + assertTargetTimersFit() + const journal = await open(options.journalPath, "wx", 0o600) + const append = async (record: object) => { + await journal.writeFile(JSON.stringify(record) + "\n") + await journal.sync() + } + const states = routers.map(() => ({ clean: true, inverse: undefined as (() => Promise) | undefined })) + let work: Promise | undefined, closed = false + let before: Counters[] | undefined, finalExport: { lastDatagramAt: number; completedAt: number } | undefined + let captureSealed = false + let captureSignal: AbortSignal | undefined + const routerAuthority = async () => { + must(captureSealed || (!closed && !captureSignal?.aborted), "Disabled capture cancelled") + await authorize() + must(captureSealed || (!closed && !captureSignal?.aborted), "Disabled capture cancelled") + } + const routerContext = async () => { + await routerAuthority() + await options.assertTopology() + await routerAuthority() + } + const stageRouters = async () => { + const targets: ReturnType[] = [] + for (const [index, router] of routers.entries()) { + const state = states[index] + const checkpoint = (entry: object) => append({ device: index, ...entry }) + const relayWrite = writeAdmission && { side: index === 0 ? "A" as const : "B" as const, admission: writeAdmission } + const flow = createDisabledChrFlowHarness({ ...router, authorize: routerAuthority, verifyContext: routerContext, checkpoint, relayWrite }) + await routerAuthority() + state.clean = false + const active = await flow.enable() + state.inverse = async () => { await flow.restore(); state.clean = true } + const targetOptions = { baseline: active, read: router.read, + source: scope.sources[index], destination: scope.destination, port: scope.port, authorize: routerAuthority, + verifyContext: routerContext, checkpoint, relayWrite } + const native = router.targetNative + const target = staged ? (() => { + must("patch" in native && typeof native.patch === "function", "Staged target enable transport unavailable") + return createChrTargetHarness({ ...targetOptions, stagedDisabled: true, native }) + })() : createChrTargetHarness({ ...targetOptions, native, timerProfile: targetTimerProfile }) + await routerAuthority() + state.inverse = undefined + await target.create() + state.inverse = async () => { await target.remove(); await flow.restore(); state.clean = true } + targets.push(target) + } + return targets + } + try { + if (process.platform !== "win32") { + const dir = await open(dirname(options.journalPath), "r") + try { await dir.sync() } finally { await dir.close() } + } + await verifyTopology() + await append({ phase: "admitted", scope, serverIds }) + const stagedTargets = staged ? await stageRouters() : undefined + if (stagedTargets) await append({ phase: "targets-staged-disabled" }) + const capture = await options.capture((signal, captureTimer) => { + must(!work && !closed && !signal.aborted, "Disabled capture exercise already dispatched or cancelled") + work = (async () => { + captureSignal = signal + const timer = captureTimer && { ...captureTimer } + must(staged ? timer?.kind === "staged-ipfix-timeout-20s" && timer.captureDeadlineAt <= deadline : !timer, + "Staged capture acknowledgement unavailable") + if (timer) for (const router of routers) assertChrTargetTimerWindow(timer, router.baseline.flow["inactive-flow-timeout"]) + if (stagedTargets) await routerAuthority() + else await routerContext() + if (!stagedTargets) before = await Promise.all(routers.map(router => router.counters())) + assertTargetTimersFit() + const targets = stagedTargets ?? await stageRouters() + if (timer) { + await append({ phase: "capture-ready", captureDeadlineAt: timer.captureDeadlineAt }) + for (const target of targets) await target.enable(timer) + } + await routerAuthority() + if (stagedTargets) { + await routerContext() + for (const target of stagedTargets) await target.assertStagedTimerReady() + before = await Promise.all(routers.map(router => router.counters())) + } else if (targetTimerProfile) { + for (const target of targets) await target.assertTimerReady() + } + await routerAuthority() + const generatedAfter = Date.now() + const { lastDatagramAt } = await options.generate(signal) + must(Number.isSafeInteger(lastDatagramAt) && lastDatagramAt >= generatedAfter && lastDatagramAt <= Date.now() && + lastDatagramAt + drainMs < Math.min(deadline, timer?.captureDeadlineAt ?? targetTimerProfile?.captureDeadlineAt ?? deadline), + "Final export drain cannot fit the owned window") + await append({ phase: "final-export-drain", lastDatagramAt, drainMs }) + await delay(Math.max(0, lastDatagramAt + drainMs - Date.now()), undefined, { signal }) + // The staged path takes its complete post-traffic snapshot after the + // host seals capture, with no native write between the two points. + if (stagedTargets) await routerAuthority() + else await routerContext() + const completedAt = Date.now() + must(completedAt >= lastDatagramAt + drainMs && (!timer || completedAt < timer.captureDeadlineAt), "Final export inactivity unconfirmed") + await append({ phase: "final-export-drain-complete", lastDatagramAt, drainMs, completedAt }) + finalExport = { lastDatagramAt, completedAt } + })() + return work + }) + must(work, "Capture omitted the disabled exercise") + await work + if (!staged) await verifyTopology() + await authorize() + const after = await Promise.all(routers.map(router => router.counters())) + if (staged) await verifyTopology() + must(before?.length === 2 && after.length === 2 && finalExport && states.every(state => state.inverse), "Disabled capture evidence incomplete") + const evidence = verifyChrOwnedWgEvidence({ ...capture, scope, expected, requireRefresh: true, + ...(staged ? { counterWindow: "traffic" as const } : {}), exporters: [ + { interfaces: routers[0].interfaces, before: before[0], after: after[0] }, + { interfaces: routers[1].interfaces, before: before[1], after: after[1] }, + ] }) + await append({ phase: "wire-bound", captureSha256: evidence.captureSha256, ...finalExport }) + captureSealed = true + // The parent independently proves export inactivity and sealed capture + // before authorizing these exact one-shot inverses. + for (const state of [...states].reverse()) { + const inverse = state.inverse + state.inverse = undefined + must(inverse, "Verified router inverse unavailable") + await inverse() + } + must(states.every(state => state.clean), "Disabled capture router cleanup requires reconciliation") + await verifyTopology() + await options.drainCollector() + await verifyTopology() + await options.reconcile(projectChrWire(capture.bytes, scope), serverIds) + await append({ phase: "reconciled" }) + return { ...evidence, componentSequenceComplete: true as const } + } finally { + closed = true + // Capture may report child failure before its exercise settles. + if (work) await work.catch(() => undefined) + try { options.reportRouterCleanup(states.every(state => state.clean)) } + finally { await journal.close() } + } +} diff --git a/backend/src/test/traffic-flow-chr-disposable-collector.ts b/backend/src/test/traffic-flow-chr-disposable-collector.ts new file mode 100644 index 00000000..93b207c4 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-disposable-collector.ts @@ -0,0 +1,153 @@ +import { randomUUID } from "node:crypto" +import { networkInterfaces } from "node:os" +import { setTimeout as delay } from "node:timers/promises" +import type { McpPrincipal } from "../modules/mcp/store.js" +import { readChrLinuxNamespaceIdentity, readChrLinuxUdpOwner } from "./traffic-flow-chr-linux-udp-owner.js" +import { CHR_COLLECTOR_RESULT_POLL_MS } from "./traffic-flow-chr-target-timing-profile.js" + +type Namespace = Awaited> +type Options = { + runId: string + deadlineAt: number + namespace: Namespace + hostNamespace: Namespace + port: number + serverIds: readonly [number, number] + principal: McpPrincipal + database: { name: string; oid: string; role: string } + assertCurrent: () => Promise + /** Waits for a separate operator's ordinary approval of this exact new plan. */ + awaitHumanApproval: (planId: string, enabled: boolean, deadlineAt: number) => Promise +} +const address = "198.18.242.3" +const uuid = (value: unknown): value is string => + typeof value === "string" && /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(value) +function must(value: unknown, reason: string): asserts value { if (!value) throw Error(reason) } + +/** Runs inside the disposable backend process. The reviewed parent owns its + * launch, admission, process exit and namespace removal. */ +export async function createChrDisposableCollector(options: Options) { + const { runId, deadlineAt, port, assertCurrent, awaitHumanApproval } = options + const { namespace, hostNamespace, serverIds, database, principal } = structuredClone({ + namespace: options.namespace, hostNamespace: options.hostNamespace, serverIds: options.serverIds, + database: options.database, principal: options.principal, + }) + const instance = "netflow-" + runId + const interfaceName = "nf" + runId + "cv" + must(process.platform === "linux" && process.env.MMAPP_OWN_CHR_NETFLOW_COLLECTOR === "true" && + /^[a-f0-9]{8}$/.test(runId) && Number.isSafeInteger(deadlineAt) && + Date.now() < deadlineAt && deadlineAt - Date.now() <= 600000 && + Number.isSafeInteger(port) && port >= 49152 && port <= 65535 && + [namespace.device, namespace.inode, hostNamespace.device, hostNamespace.inode, database.oid].every(value => /^[1-9][0-9]{0,19}$/.test(value)) && + (namespace.device !== hostNamespace.device || namespace.inode !== hostNamespace.inode) && + database.name === "mcp_netflow_" + runId && database.role === "mcp_test" && + serverIds.length === 2 && new Set(serverIds).size === 2 && serverIds.every(value => Number.isSafeInteger(value) && value > 0), + "Invalid disposable collector boundary") + const fixtureRoot = "/tmp/mcp-netflow-" + runId + const dbUrl = new URL(process.env.DATABASE_URL ?? "about:blank") + must(["postgres:", "postgresql:"].includes(dbUrl.protocol) && dbUrl.hostname === "localhost" && + dbUrl.username === database.role && /^[a-f0-9]{64}$/.test(dbUrl.password) && !dbUrl.port && !dbUrl.hash && + dbUrl.pathname === "/" + database.name && [...dbUrl.searchParams.keys()].join() === "host" && + dbUrl.searchParams.get("host") === fixtureRoot + "/pg-socket" && + process.env.MCP_STATE_DIR === fixtureRoot + "/state", "Disposable database or MCP state endpoint is unpinned") + const local = async () => { + must(Date.now() < deadlineAt, "Disposable collector deadline ended") + await assertCurrent() + const actual = await readChrLinuxNamespaceIdentity() + must(Date.now() < deadlineAt && actual.device === namespace.device && actual.inode === namespace.inode && + (networkInterfaces()[interfaceName] ?? []).filter(row => row.family === "IPv4" && row.address === address).length === 1 && + (!process.env.FLOW_LISTEN_HOST?.trim() || process.env.FLOW_LISTEN_HOST.trim() === address) && + !process.env.FLOW_AGENT_SOCKET?.trim(), "Disposable namespace or listener source changed") + } + await local() + const [{ pool }, { invokeMcpTool, applyPlan, loadPlan }, { validatePrincipal }, ingest, { createFlowSettingsStore }, + { flowSettingsPrivateProof }] = await Promise.all([ + import("../db/index.js"), import("../modules/mcp/runtime.js"), import("../modules/mcp/store.js"), + import("../services/traffic-flow-ingest.js"), import("../services/traffic-flow-settings-store.js"), + import("../services/traffic-flow-host-keys-service.js"), + ]) + const authorize = async () => { + await local() + await validatePrincipal(principal) + await local() + } + await authorize() + const db = (await pool.query("SELECT current_database() AS name,current_user AS role,oid::text AS oid," + + "inet_server_addr() IS NULL AS unix_socket FROM pg_database WHERE datname=current_database()")).rows + const servers = (await pool.query("SELECT id FROM servers ORDER BY id LIMIT 3")).rows.map(row => Number(row.id)) + must(db.length === 1 && db[0].name === database.name && db[0].role === database.role && db[0].oid === database.oid && + db[0].unix_socket === true && servers.length === 2 && serverIds.every(id => servers.includes(id)), "Disposable database identity differs") + const configStore = createFlowSettingsStore(pool, authorize, () => {}) + const config = await configStore.read() + must(config.enabled === false && config.collectorIp === address && config.flowListenPort === port, + "Expected disabled exact collector configuration") + const proof = (value: typeof config) => flowSettingsPrivateProof({ ...value, enabled: false }) + const invariantProof = await proof(config) + const observedUdp = () => readChrLinuxUdpOwner({ namespace, address, port }) + const read = async () => { + await authorize() + const configuration = await configStore.read() + const udpOwners = await observedUdp() + const listener = ingest.getFlowListenerState(), worker = ingest.getFlowWorkerHealth() + const currentProof = await proof(configuration) + must(currentProof === invariantProof && configuration.collectorIp === address && + configuration.flowListenPort === port && typeof configuration.enabled === "boolean", "Disposable collector settings changed") + await authorize() + return { instance, pid: process.pid, invariantProof: currentProof, listenAddress: configuration.collectorIp, + listenPort: configuration.flowListenPort, enabled: configuration.enabled, workerAlive: worker.alive, + boundAddress: listener.bound ? listener.address : null, udpOwners } + } + const initial = await read() + must(initial.udpOwners.length === 0 && !initial.workerAlive && initial.boundAddress === null, "Disposable collector not idle") + let enableAttempted = false, disableAttempted = false, enabledVerified = false, disableVerified = false + const applyEnabled = async (enabled: boolean, beforeDispatch: () => Promise) => { + must(enabled ? !enableAttempted : enabledVerified && !disableAttempted, "Collector mutation already attempted or unknown") + if (enabled) enableAttempted = true + else disableAttempted = true + await authorize() + const current = await read() + must(current.enabled !== enabled, "Unexpected collector enable state") + const prepared = await invokeMcpTool(principal, "mm_netflow_settings_update_plan", { settings: { enabled } }) as { + state?: string; planId?: string + } + must(prepared.state === "awaiting-approval" && uuid(prepared.planId), "Collector plan did not await human approval") + await awaitHumanApproval(prepared.planId, enabled, deadlineAt) + await authorize() + const approved = await loadPlan(prepared.planId) + must(approved.connection_id === principal.connectionId && approved.operation_id === "netflow.settings.update" && + approved.state === "approved" && approved.approved_by && approved.expires_at.getTime() > Date.now(), "Collector approval unconfirmed") + await beforeDispatch() + await authorize() + const operation = await applyPlan(principal, prepared.planId, randomUUID()) + must(uuid(operation.id), "Collector operation acknowledgement missing") + const pollUntil = Math.min(deadlineAt, Date.now() + CHR_COLLECTOR_RESULT_POLL_MS) + while (Date.now() < pollUntil) { + await authorize() + const rows = (await pool.query("SELECT o.state,o.verification,s.state AS step_state,s.result " + + "FROM mcp_operations o LEFT JOIN mcp_operation_steps s ON s.operation_id=o.id " + + "WHERE o.id=$1 AND o.connection_id=$2 AND o.plan_id=$3 LIMIT 2", + [operation.id, principal.connectionId, prepared.planId])).rows + const [row] = rows + if (rows.length === 1 && ["pending", "running"].includes(row.state)) { await delay(100); continue } + must(rows.length === 1 && row.state === "applied" && row.verification === "passed" && row.step_state === "verified" && + row.result?.configurationConfirmed === true && row.result.receiptConfirmed === true && + row.result.listener === "request-acknowledged" && row.result.automaticReplay === false && + uuid(row.result.receiptId), "Collector operation result requires reconciliation") + const receipt = (await pool.query("SELECT phase,state FROM flow_settings_intents " + + "WHERE receipt_id=$1 ORDER BY phase", [row.result.receiptId])).rows + must(receipt.length === 2 && receipt[0].phase === "configuration" && receipt[0].state === "completed" && + receipt[1].phase === "listener" && receipt[1].state === "completed", "Collector durable listener acknowledgement missing") + await authorize() + if (enabled) enabledVerified = true + else disableVerified = true + return { operationId: operation.id, acknowledged: true } + } + throw Error("Collector operation outcome unknown. Automatic replay forbidden.") + } + const assertDrained = async () => { + const observed = await read() + must(disableVerified && !observed.enabled && !observed.workerAlive && + observed.boundAddress === null && observed.udpOwners.length === 0, "Collector retirement unconfirmed") + } + return { interfaceName, pins: { instance, pid: process.pid, invariantProof }, collector: { authorize, read, applyEnabled, assertDrained } } +} diff --git a/backend/src/test/traffic-flow-chr-firewall-preflight.test.ts b/backend/src/test/traffic-flow-chr-firewall-preflight.test.ts new file mode 100644 index 00000000..9785d1bf --- /dev/null +++ b/backend/src/test/traffic-flow-chr-firewall-preflight.test.ts @@ -0,0 +1,88 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { verifyChrFirewallPreflight } from "./traffic-flow-chr-firewall-preflight.js" + +type Row = Record +const rule = { ".id": "*A", chain: "forward", action: "accept", disabled: "false", protocol: "udp", + "src-address": "198.18.240.0/24", "dst-address": "198.18.240.2/32", + "in-interface": "edge-a", "out-interface": "middle-a", "dst-port": "50003,50101-50103", + "connection-state": "new,established", log: "false", "log-prefix": "" } + +test("owned WG policy admits complete empty snapshots only and requires matching path profile", async () => { + const { config, rules } = fixture() + rules.filter.length = 0 + const profile = "owned-wg-empty-firewall" as const + const owned = { ...config, profile, readPathEvidence: async () => ({ + ...await config.readPathEvidence(), profile, ownedInterfaces: ["edge-a", "middle-a"], + ownedRoutes: [{ destination: "198.18.240.1", interfaceName: "edge-a" }, { destination: "198.18.240.2", interfaceName: "middle-a" }], + udpPairs: ["198.18.240.1:50100/198.18.240.2:50101"], + }) } + assert.equal((await verifyChrFirewallPreflight(owned)).policyVerified, true) + await assert.rejects(verifyChrFirewallPreflight({ ...owned, packets: [{ ...config.packets[0], destinationPort: 50102 }] }), /only owned WireGuard/) + await assert.rejects(verifyChrFirewallPreflight({ ...owned, readPathEvidence: config.readPathEvidence }), /reachability/) + rules.raw.push({ ".id": "*A", chain: "output", action: "accept", disabled: "true" }) + await assert.rejects(verifyChrFirewallPreflight(owned), /requires empty firewall/) +}) +function fixture() { + const calls: string[] = [] + const rules = { raw: [] as Row[], mangle: [] as Row[], nat: [] as Row[], filter: [rule] as Row[] } + const config = { + identity: "router-a", version: "7.24.5 (stable)", + packets: [{ direction: "forward" as const, source: "198.18.240.1", destination: "198.18.240.2", + sourcePort: 50100, destinationPort: 50101, ingress: "edge-a", egress: "middle-a" }], + assertCurrent: async () => {}, + readPathEvidence: async () => ({ routesVerified: true, freshConntrackTuples: true, fingerprint: "a".repeat(64) }), + client: { async get(path: string, timeout?: number, _signal?: AbortSignal, bytes?: number, fence?: () => Promise) { + assert.ok(timeout && timeout <= 2000) + assert.equal(bytes, 128 * 1024) + await fence?.() + calls.push(path) + if (path === "/system/identity") return [{ name: "router-a" }] as T + if (path === "/system/resource") return [{ version: "7.24.5 (stable)" }] as T + const table = path.replace("/ip/firewall/", "") as keyof typeof rules + return rules[table].map(row => ({ ...row, packets: String(calls.length), bytes: "100" })) as T + } }, + } + return { config, rules, calls } +} + +test("reads only fixed bounded native inventories twice and accepts exact simple UDP policy", async () => { + const { config, calls, rules } = fixture() + rules.filter.unshift({ ".id": "*B", chain: "forward", action: "drop", protocol: "tcp", "in-interface-list": "any" }) + const paths = ["/system/identity", "/system/resource", "/ip/firewall/raw", "/ip/firewall/mangle", + "/ip/firewall/nat", "/ip/firewall/filter"] + assert.deepEqual(await verifyChrFirewallPreflight(config), { policyVerified: true, packetScopes: 1, actualIngressVerified: false }) + assert.deepEqual(calls, [...paths, ...paths]) +}) + +test("rejects matching FastTrack, NAT, jumps, unsupported matchers and incomplete filter admission", async () => { + for (const [table, inserted] of [ + ["filter", { chain: "forward", action: "fasttrack-connection", "connection-state": "established" }], + ["nat", { chain: "srcnat", action: "masquerade" }], + ["raw", { chain: "prerouting", action: "jump", "jump-target": "policy" }], + ["filter", { chain: "forward", action: "accept", "src-address-list": "unknown" }], + ] as const) { + const { config, rules } = fixture() + rules[table].unshift({ ".id": "*B", ...inserted }) + await assert.rejects(verifyChrFirewallPreflight(config), /matching|unsupported/) + } + const { config, rules } = fixture() + rules.filter.length = 0 + await assert.rejects(verifyChrFirewallPreflight(config), /no explicit filter allow/) +}) + +test("rejects inventory drift, unavailable route or conntrack proof, and lost authorization", async () => { + const stale = fixture() + stale.config.readPathEvidence = async () => { + if (stale.calls.length === 6) stale.rules.filter[0] = { ...rule, comment: "changed" } + return { routesVerified: true, freshConntrackTuples: true, fingerprint: "a".repeat(64) } + } + await assert.rejects(verifyChrFirewallPreflight(stale.config), /stale inventory/) + const unknown = fixture() + unknown.config.readPathEvidence = async () => ({ routesVerified: true, freshConntrackTuples: false, fingerprint: "a".repeat(64) }) + await assert.rejects(verifyChrFirewallPreflight(unknown.config), /unknown active reachability/) + const expired = fixture() + expired.config.assertCurrent = async () => { throw Error("ownership expired") } + await assert.rejects(verifyChrFirewallPreflight(expired.config), /ownership expired/) + assert.equal(expired.calls.length, 0) +}) diff --git a/backend/src/test/traffic-flow-chr-firewall-preflight.ts b/backend/src/test/traffic-flow-chr-firewall-preflight.ts new file mode 100644 index 00000000..af01245d --- /dev/null +++ b/backend/src/test/traffic-flow-chr-firewall-preflight.ts @@ -0,0 +1,179 @@ +import { isIPv4 } from "node:net" +import { performance } from "node:perf_hooks" +import { isDeepStrictEqual } from "node:util" +import type { MikrotikClient } from "../services/mikrotik.js" + +type Row = Record +const tables = ["raw", "mangle", "nat", "filter"] as const +type Table = (typeof tables)[number] +type Packet = { + direction: "input" | "forward" | "output" + source: string + destination: string + sourcePort: number + destinationPort: number + ingress?: string + egress?: string +} +type Options = { + client: Pick + identity: string + version: string + packets: readonly Packet[] + assertCurrent: () => Promise + /** Trusted reader of exact active routes and address/interface bindings. */ + profile?: "owned-wg-empty-firewall" + readPathEvidence: () => Promise<{ routesVerified: boolean; freshConntrackTuples: boolean; fingerprint: string; + profile?: "owned-wg-empty-firewall"; ownedInterfaces?: readonly string[]; + ownedRoutes?: readonly { destination: string; interfaceName: string }[]; udpPairs?: readonly string[] }> +} +const paths: Record> = { + raw: { input: ["prerouting"], forward: ["prerouting"], output: ["output"] }, + mangle: { input: ["prerouting", "input"], forward: ["prerouting", "forward", "postrouting"], output: ["output", "postrouting"] }, + nat: { input: ["dstnat", "input"], forward: ["dstnat", "srcnat"], output: ["output", "srcnat"] }, + filter: { input: ["input"], forward: ["forward"], output: ["output"] }, +} +const meta = new Set([".id", "action", "chain", "comment", "disabled", "dynamic", "invalid"]) +const matchKeys = new Set(["protocol", "src-address", "dst-address", "src-port", "dst-port", "in-interface", "out-interface", "connection-state"]) +const states = ["new", "established", "related", "invalid", "untracked"] as const +const deny = (reason: string): never => { throw Error("RouterOS fixture preflight refused: " + reason) } +const validPort = (value: number) => Number.isInteger(value) && value > 0 && value <= 65535 +const flag = (value: unknown) => { + if (value === undefined || value === false || value === "false" || value === "no") return false + if (value === true || value === "true" || value === "yes") return true + return deny("unsupported rule flag") +} +const ipv4Number = (address: string) => address.split(".").reduce((sum, octet) => sum * 256 + Number(octet), 0) +function addressMatches(value: string, ip: string) { + const [base, bits = "32", ...extra] = value.split("/") + if (!isIPv4(base) || !/^(0|[1-9]|[12][0-9]|3[0-2])$/.test(bits) || extra.length) return undefined + const size = 2 ** (32 - Number(bits)) + return Math.floor(ipv4Number(base) / size) === Math.floor(ipv4Number(ip) / size) +} +function portMatches(value: string, port: number) { + const ranges = value.split(",").map(range => /^([1-9][0-9]{0,4})(?:-([1-9][0-9]{0,4}))?$/.exec(range)) + if (ranges.some(range => !range || !validPort(Number(range[1])) || !validPort(Number(range[2] ?? range[1])) || + Number(range[1]) > Number(range[2] ?? range[1]))) return undefined + return ranges.some(range => range && Number(range[1]) <= port && port <= Number(range[2] ?? range[1])) +} +function matcher(key: string, value: unknown, packet: Packet, state: "new" | "established", chain: string) { + if (typeof value !== "string" || !value) return undefined + switch (key) { + case "protocol": + return /^(udp|tcp|icmp|gre|ipsec-esp|17|6|1|47|50)$/.test(value) ? value === "udp" || value === "17" : undefined + case "src-address": return addressMatches(value, packet.source) + case "dst-address": return addressMatches(value, packet.destination) + case "src-port": return portMatches(value, packet.sourcePort) + case "dst-port": return portMatches(value, packet.destinationPort) + case "in-interface": + return !["postrouting", "srcnat"].includes(chain) && packet.ingress && /^[\w.-]{1,64}$/.test(value) + ? value === packet.ingress : undefined + case "out-interface": + return !["prerouting", "dstnat"].includes(chain) && packet.egress && /^[\w.-]{1,64}$/.test(value) + ? value === packet.egress : undefined + case "connection-state": { + const values = value.split(",") + return values.every(item => (states as readonly string[]).includes(item)) ? values.includes(state) : undefined + } + default: return undefined + } +} +function matches(rule: Row, packet: Packet, state: "new" | "established", chain: string, table: Table) { + const results = Object.entries(rule).filter(([key]) => !meta.has(key)) + .filter(([key, value]) => !(key === "log" && (value === false || value === "false" || value === "no") || + key === "log-prefix" && value === "")) + .map(([key, value]) => matchKeys.has(key) && !(table === "raw" && key === "connection-state") + ? matcher(key, value, packet, state, chain) : undefined) + if (results.includes(false)) return false + if (results.includes(undefined)) deny("unsupported potentially matching rule") + if (flag(rule.dynamic) || flag(rule.invalid)) deny("dynamic or invalid matching rule") + return true +} +function verifyRules(inventory: Record, packet: Packet) { + for (const state of ["new", "established"] as const) { + for (const table of tables) { + for (const chain of paths[table][packet.direction]) { + let accepted = false + for (const rule of inventory[table]) { + if (rule.chain !== chain || flag(rule.disabled) || !matches(rule, packet, state, chain, table)) continue + if (rule.action !== "accept") deny("matching non-accept action") + accepted = true + break + } + // Require an explicit filter allow even though RouterOS may implicitly accept. + if (table === "filter" && !accepted) deny("no explicit filter allow") + } + } + } +} +function rows(value: unknown, limit: number): Row[] { + const result = Array.isArray(value) ? value : [value] + if (result.length > limit || result.some(row => !row || typeof row !== "object" || Array.isArray(row))) return deny("invalid inventory") + return result +} + +/** Bounded double read via the existing RouterOS client. This proves a narrow + * policy subset at observation time. It does not prove packet delivery. */ +export async function verifyChrFirewallPreflight(options: Options) { + const { client, assertCurrent, readPathEvidence, profile } = options + const { identity, version, packets } = structuredClone({ identity: options.identity, version: options.version, packets: options.packets }) + if (!identity || !version || !packets.length || packets.length > 32 || + packets.some(packet => !Object.hasOwn(paths.filter, packet.direction) || + !isIPv4(packet.source) || !isIPv4(packet.destination) || !validPort(packet.sourcePort) || !validPort(packet.destinationPort) || + (packet.direction !== "output" && !packet.ingress) || (packet.direction !== "input" && !packet.egress) || + [packet.ingress, packet.egress].some(name => name !== undefined && !/^[\w.-]{1,64}$/.test(name)))) deny("invalid packet scope") + const deadline = performance.now() + 15000 + async function fence() { + if (performance.now() >= deadline) deny("observation expired") + await assertCurrent() + if (performance.now() >= deadline) deny("observation expired") + } + async function get(path: string, limit: number) { + await fence() + const result = rows(await client.get(path, Math.min(2000, Math.ceil(deadline - performance.now())), + undefined, 128 * 1024, fence), limit) + await fence() + return result + } + async function snapshot() { + const [device] = await get("/system/identity", 1) + const [resource] = await get("/system/resource", 1) + if (device?.name !== identity || resource?.version !== version) deny("device identity changed") + const inventory = {} as Record + for (const table of tables) { + inventory[table] = (await get("/ip/firewall/" + table, 256)).map(({ bytes: _bytes, packets: _packets, ...rule }) => { + if (typeof rule.chain !== "string" || !rule.chain || typeof rule.action !== "string" || + typeof rule[".id"] !== "string" || !/^\*[a-f0-9]{1,16}$/i.test(rule[".id"])) deny("invalid rule identity") + return structuredClone(rule) + }) + if (new Set(inventory[table].map(rule => rule[".id"])).size !== inventory[table].length) deny("duplicate rule ID") + } + await fence() + const evidence = await readPathEvidence() + await fence() + if (evidence.routesVerified !== true || evidence.freshConntrackTuples !== true || evidence.profile !== profile || + !/^[a-f0-9]{64}$/.test(evidence.fingerprint)) deny("unknown active reachability or existing connection") + if (profile && (evidence.ownedInterfaces?.length !== 2 || + !packets.some(packet => packet.direction === "forward") || + packets.some(packet => packet.direction === "forward" && + (packet.ingress === packet.egress || !evidence.ownedInterfaces?.includes(packet.ingress!) || + !evidence.ownedInterfaces?.includes(packet.egress!) || + !evidence.ownedRoutes?.some(route => route.destination === packet.destination && route.interfaceName === packet.egress) || + !evidence.ownedRoutes?.some(route => route.destination === packet.source && route.interfaceName === packet.ingress) || + !evidence.udpPairs?.includes([packet.source + ":" + packet.sourcePort, packet.destination + ":" + packet.destinationPort].sort().join("/")))))) { + deny("forwarding does not use only owned WireGuard") + } + return { inventory, fingerprint: evidence.fingerprint, ownedInterfaces: evidence.ownedInterfaces, + ownedRoutes: evidence.ownedRoutes, udpPairs: evidence.udpPairs } + } + const before = await snapshot() + const after = await snapshot() + if (!isDeepStrictEqual(before, after)) deny("stale inventory or path") + if (profile === "owned-wg-empty-firewall") { + if (tables.some(table => after.inventory[table].length !== 0)) deny("owned WireGuard profile requires empty firewall") + } else { + for (const packet of packets) verifyRules(after.inventory, packet) + } + await fence() + return { policyVerified: true as const, packetScopes: packets.length, actualIngressVerified: false as const } +} diff --git a/backend/src/test/traffic-flow-chr-host-ipc.ts b/backend/src/test/traffic-flow-chr-host-ipc.ts new file mode 100644 index 00000000..84b3ef98 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-host-ipc.ts @@ -0,0 +1,158 @@ +import { createHash } from "node:crypto" +import { constants } from "node:fs" +import { lstat, open } from "node:fs/promises" +import { createConnection } from "node:net" +import { CHR_STAGED_TARGET_CAPTURE_MAX_MS } from "./traffic-flow-chr-target-timing-profile.js" +import type { ChrStagedTargetTimerProfile } from "./traffic-flow-chr-target-harness.js" + +type Operation = "topology-ready" | "generate" | "capture-start" | "capture-stop" | "cancel" +type Phase = "calibration" | "initial" | "refresh" +type Identity = { device: string; inode: string } +const limit = 4096 +const captureLimit = 512 * 1024 +function must(value: unknown, reason: string): asserts value { if (!value) throw Error(reason) } + +export function decodeChrHostReply(raw: Buffer, sequence: number, op: Operation, stagedStart = false) { + must(raw.length > 0 && raw.length <= limit && raw.at(-1) === 10 && raw.indexOf(10) === raw.length - 1, + "Invalid host IPC frame") + const value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(raw)) + must(value && value.sequence === sequence && value.op === op && value.ok === true, "Host IPC acknowledgement unavailable") + const extra = op === "generate" ? ["lastDatagramAt"] : op === "capture-stop" ? ["sha256", "complete", "dropped"] : + op === "capture-start" && stagedStart ? ["captureDeadlineAt"] : [] + must(Object.keys(value).sort().join() === ["sequence", "op", "ok", ...extra].sort().join(), "Invalid host IPC reply shape") + if (op === "generate") must(Number.isSafeInteger(value.lastDatagramAt) && value.lastDatagramAt > 0, "Host UDP time unconfirmed") + if (op === "capture-stop") must(value.complete === true && value.dropped === 0 && typeof value.sha256 === "string" && + /^[a-f0-9]{64}$/.test(value.sha256), "Host capture unconfirmed") + if (op === "capture-start" && stagedStart) must(Number.isSafeInteger(value.captureDeadlineAt) && value.captureDeadlineAt > 0, + "Host capture deadline unconfirmed") + return value as { lastDatagramAt?: number; sha256?: string; captureDeadlineAt?: number } +} + +export function bindChrStagedHostCaptureTimer(captureDeadlineAt: number | undefined, captureMs: number, + ownerDeadline: number, requestedAt: number, acknowledgedAt: number): Omit { + must(Number.isSafeInteger(captureMs) && captureMs >= 90000 && captureMs <= CHR_STAGED_TARGET_CAPTURE_MAX_MS && + [ownerDeadline, requestedAt, acknowledgedAt].every(value => Number.isSafeInteger(value) && value > 0) && + captureDeadlineAt !== undefined && Number.isSafeInteger(captureDeadlineAt) && requestedAt <= acknowledgedAt && + captureDeadlineAt > acknowledgedAt && captureDeadlineAt <= ownerDeadline && + captureDeadlineAt >= requestedAt + captureMs && captureDeadlineAt <= acknowledgedAt + captureMs, + "Host capture deadline outside reviewed window") + return { kind: "staged-ipfix-timeout-20s", captureDeadlineAt } +} + +export async function createChrHostControl(options: { + runId: string; nonce: string; deadlineAt: number; socket: Identity; assertCurrent: () => Promise +}) { + const { runId, nonce, deadlineAt, assertCurrent } = options + const pin = structuredClone(options.socket) + const gid = process.getegid?.() + must(process.platform === "linux" && gid && /^[a-f0-9]{8}$/.test(runId) && + /^[a-f0-9]{32}$/.test(nonce) && nonce.startsWith(runId), "Host IPC ownership unavailable") + const directory = "/tmp/mcp-netflow-" + runId + "/ipc" + const socketPath = directory + "/control.sock" + const fence = async () => { + await assertCurrent() + const [dir, socket] = await Promise.all([lstat(directory, { bigint: true }), lstat(socketPath, { bigint: true })]) + must(Date.now() < deadlineAt && dir.isDirectory() && dir.uid === 0n && dir.gid === BigInt(gid) && + (dir.mode & 0o777n) === 0o710n && socket.isSocket() && socket.uid === 0n && socket.gid === BigInt(gid) && + (socket.mode & 0o777n) === 0o660n && String(socket.dev) === pin.device && String(socket.ino) === pin.inode, + "Host IPC ownership changed") + } + await fence() + let sequence = 0, busy = false, failed = false, phase = "new" + const reached = (expected: string) => phase === expected + async function exchange(op: Operation, signal?: AbortSignal, generation?: Phase, stagedStart = false) { + must(!failed && !busy && !signal?.aborted && sequence < 8, "Host IPC request refused") + busy = true + try { + await fence() + const current = ++sequence + const started = Date.now() + const raw = await new Promise((resolve, reject) => { + const sock = createConnection(socketPath) + let reply = Buffer.alloc(0) + const fail = () => { sock.destroy(); reject(Error("Host IPC outcome unknown")) } + const timer = setTimeout(fail, Math.max(1, Math.min(15000, deadlineAt - Date.now()))) + const cleanup = () => { clearTimeout(timer); signal?.removeEventListener("abort", fail) } + signal?.addEventListener("abort", fail, { once: true }) + if (signal?.aborted) fail() + sock.once("connect", () => sock.write(JSON.stringify({ runId, nonce, sequence: current, op, + ...(generation && { phase: generation }) }) + "\n")) + sock.on("data", chunk => { + if (reply.length + chunk.length > limit) return fail() + reply = Buffer.concat([reply, chunk]) + if (reply.includes(10)) { cleanup(); resolve(reply); sock.destroy() } + }) + sock.once("error", fail) + sock.once("close", () => { cleanup(); reject(Error("Host IPC closed before acknowledgement")) }) + }) + const value = decodeChrHostReply(raw, current, op, stagedStart) + await fence() + if (op === "generate") must(value.lastDatagramAt! >= started && value.lastDatagramAt! <= Date.now(), + "Host UDP receipt is outside request window") + return value + } catch (error) { failed = true; throw error } + finally { busy = false } + } + async function capture(deadline: number, exercise: (signal: AbortSignal, timer?: Omit) => Promise, + stagedMs?: number) { + const requestedAt = Date.now() + must(phase === "calibration" && requestedAt < deadline && deadline <= deadlineAt, "Host capture phase differs") + const startSignal = AbortSignal.timeout(deadline - requestedAt) + const started = await exchange("capture-start", startSignal, undefined, stagedMs !== undefined) + phase = "capturing" + try { + const acknowledgedAt = Date.now() + const timer = stagedMs === undefined ? undefined : + bindChrStagedHostCaptureTimer(started.captureDeadlineAt, stagedMs, deadline, requestedAt, acknowledgedAt) + const signal = timer ? AbortSignal.timeout(timer.captureDeadlineAt - acknowledgedAt) : startSignal + if (timer) await exercise(signal, timer) + else await exercise(signal) + must(reached("refresh") && !signal.aborted, "Host capture exercise incomplete") + const { sha256 } = await exchange("capture-stop", signal) + phase = "finished" + const file = await open(directory + "/capture.pcap", constants.O_RDONLY | constants.O_NOFOLLOW) + try { + const row = await file.stat() + must(row.isFile() && row.uid === 0 && row.gid === gid && (row.mode & 0o777) === 0o640 && + row.nlink === 1 && row.size > 0 && row.size <= captureLimit, "Host capture ownership unavailable") + const bytes = Buffer.alloc(row.size + 1) + let offset = 0 + while (offset < bytes.length) { + const result = await file.read(bytes, offset, bytes.length - offset, null) + if (!result.bytesRead) break + offset += result.bytesRead + } + const capture = bytes.subarray(0, offset) + must(offset === row.size && createHash("sha256").update(capture).digest("hex") === sha256, "Host capture bytes differ") + return { bytes: capture, sha256: sha256!, complete: true, dropped: 0 } + } finally { await file.close() } + } finally { + if (!failed && phase !== "finished") { + phase = "cancelled" + // Parent cancellation intentionally refuses the session. The + // original capture failure remains the terminal cause. + await exchange("cancel").catch(() => {}) + } + } + } + return { + async topologyReady() { + must(phase === "new", "Host topology acknowledgement already attempted") + await exchange("topology-ready") + phase = "topology" + }, + async generate(generation: Phase, signal?: AbortSignal) { + const expected = generation === "calibration" ? "topology" : generation === "initial" ? "capturing" : "initial" + must(phase === expected, "Host UDP phase order differs") + const value = await exchange("generate", signal, generation) + phase = generation + return { lastDatagramAt: value.lastDatagramAt! } + }, + capture: (deadline: number, exercise: (signal: AbortSignal) => Promise) => capture(deadline, exercise), + captureStaged: (captureMs: number, exercise: Parameters[1]) => { + must(Number.isSafeInteger(captureMs) && captureMs >= 90000 && captureMs <= CHR_STAGED_TARGET_CAPTURE_MAX_MS && + Date.now() + captureMs < deadlineAt, "Staged host capture window unavailable") + return capture(deadlineAt, exercise, captureMs) + }, + } +} diff --git a/backend/src/test/traffic-flow-chr-independent-ipfix.test.ts b/backend/src/test/traffic-flow-chr-independent-ipfix.test.ts new file mode 100644 index 00000000..a4e890d0 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-independent-ipfix.test.ts @@ -0,0 +1,66 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { decodeOwnedIpfix } from "./traffic-flow-chr-independent-ipfix.js" + +const fields = [[1, 8], [2, 8], [4, 1], [7, 2], [8, 4], [10, 4], [11, 2], [12, 4], [14, 4]] +function set(id: number, body: Buffer) { + const header = Buffer.alloc(4) + header.writeUInt16BE(id); header.writeUInt16BE(body.length + 4, 2) + return Buffer.concat([header, body]) +} +function packet(sets: Buffer[]) { + const header = Buffer.alloc(16) + header.writeUInt16BE(10) + header.writeUInt16BE(16 + sets.reduce((total, value) => total + value.length, 0), 2) + header.writeUInt32BE(7, 12) + return Buffer.concat([header, ...sets]).toString("hex") +} +function fixture() { + const template = Buffer.alloc(4 + fields.length * 4) + template.writeUInt16BE(256); template.writeUInt16BE(fields.length, 2) + fields.forEach(([field, size], index) => { template.writeUInt16BE(field, 4 + index * 4); template.writeUInt16BE(size, 6 + index * 4) }) + const data = Buffer.alloc(37) + data.writeBigUInt64BE(9007199254740993n); data.writeBigUInt64BE(3n, 8); data[16] = 17 + data.writeUInt16BE(9000, 17); Buffer.from([192, 0, 2, 1]).copy(data, 19); data.writeUInt32BE(11, 23) + data.writeUInt16BE(9001, 27); Buffer.from([198, 51, 100, 1]).copy(data, 29); data.writeUInt32BE(12, 33) + const frames = [0, 1].map((exporter, index) => ({ + ordinal: index + 1, receivedMs: 1700000000000, receivedNs: "1700000000000000000", exporter, sourcePort: 2055, + version: 10, domain: 7, payloadHex: packet([set(2, template), set(256, data)]), + })) + return { template, data, input: { captureSha256: "a".repeat(64), scope: { sources: ["192.0.2.10", "192.0.2.20"], destination: "192.0.2.100", port: 4739 }, frames } } +} + +test("independent IPv4 IPFIX decoder preserves exact counters beyond JavaScript safe integer range", () => { + const { input } = fixture() + const result = decodeOwnedIpfix(input) + assert.equal(result.captureSha256, input.captureSha256) + assert.equal(result.nativeAcceptance, false) + for (const row of result.frames) { + assert.deepEqual(row.setIds, [2, 256]) + assert.deepEqual(row.flows, [{ src: "192.0.2.1", dst: "198.51.100.1", proto: 17, srcPort: 9000, dstPort: 9001, + inIface: "11", outIface: "12", bytes: "9007199254740993", packets: "3" }]) + } +}) + +test("refuses template borrowing across exporters and UDP source ports", () => { + const { input, data } = fixture() + input.frames[1].payloadHex = packet([set(256, data)]) + assert.throws(() => decodeOwnedIpfix(input), /Missing IPFIX template/) + input.frames[1].exporter = 0 + input.frames[1].sourcePort = 2056 + assert.throws(() => decodeOwnedIpfix(input), /Missing IPFIX template/) +}) + +test("refuses duplicate required fields, ambiguous padding and unsupported variable fields", () => { + const duplicate = fixture() + duplicate.template.writeUInt16BE(1, 8) + duplicate.input.frames[0].payloadHex = packet([set(2, duplicate.template), set(256, duplicate.data)]) + assert.throws(() => decodeOwnedIpfix(duplicate.input), /Ambiguous/) + const variable = fixture() + variable.template.writeUInt16BE(65535, 6) + variable.input.frames[0].payloadHex = packet([set(2, variable.template), set(256, variable.data)]) + assert.throws(() => decodeOwnedIpfix(variable.input), /Variable/) + const padded = fixture() + padded.input.frames[0].payloadHex = packet([set(2, padded.template), set(256, Buffer.concat([padded.data, Buffer.from([1])]))]) + assert.throws(() => decodeOwnedIpfix(padded.input), /padding/) +}) diff --git a/backend/src/test/traffic-flow-chr-independent-ipfix.ts b/backend/src/test/traffic-flow-chr-independent-ipfix.ts new file mode 100644 index 00000000..707b2457 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-independent-ipfix.ts @@ -0,0 +1,135 @@ +/** Narrow independent decoder for the reviewed IPv4 RouterOS IPFIX fixture. + * Deliberately imports no product flow parser or SQL aggregation. */ +import { createHash } from "node:crypto" +import { open } from "node:fs/promises" +import { isIPv4 } from "node:net" +import { resolve } from "node:path" +import { fileURLToPath } from "node:url" +import { z } from "zod" + +const sha = z.string().regex(/^[a-f0-9]{64}$/) +const ip = z.string().max(15).refine(isIPv4) +const port = z.number().int().min(1).max(65535) +const projection = z.object({ + captureSha256: sha, + scope: z.object({ sources: z.tuple([ip, ip]), destination: ip, port, linkType: z.enum(["RAW", "EN10MB"]).optional() }).strict(), + frames: z.array(z.object({ + ordinal: z.number().int().positive().safe(), receivedMs: z.number().int().nonnegative().safe(), + receivedNs: z.string().regex(/^\d{1,20}$/), exporter: z.union([z.literal(0), z.literal(1)]), + sourcePort: port, version: z.literal(10), domain: z.number().int().min(0).max(0xffffffff), + payloadHex: z.string().min(40).max(8192).regex(/^(?:[0-9a-f]{2})+$/), + }).strict()).min(2).max(64), +}).strict() +const required = new Map([[1, [1, 2, 4, 8]], [2, [1, 2, 4, 8]], [4, [1]], [7, [2]], [8, [4]], [10, [1, 2, 4]], [11, [2]], [12, [4]], [14, [1, 2, 4]]]) +type Field = { offset: number; size: number } +type Template = { fields: Map; width: number } +function must(value: unknown, error: string): asserts value { + if (!value) throw Error(error) +} +function padding(bytes: Buffer) { + return bytes.length <= 3 && bytes.every(value => value === 0) +} + +export function decodeOwnedIpfix(raw: unknown) { + const input = projection.parse(raw) + must(new Set([...input.scope.sources, input.scope.destination]).size === 3, "Invalid exporter scope") + const templates = new Map() + let last = 0, records = 0, lastTimestamp = -1n + const positive = new Set() + const frames = input.frames.map(frame => { + const timestamp = BigInt(frame.receivedNs) + must(frame.ordinal === last + 1 && timestamp >= lastTimestamp && timestamp / 1000000n === BigInt(frame.receivedMs), "Invalid wire ordinal or timestamp") + last = frame.ordinal + lastTimestamp = timestamp + const data = Buffer.from(frame.payloadHex, "hex") + must(data.readUInt16BE(0) === 10 && data.readUInt16BE(2) === data.length && data.readUInt32BE(12) === frame.domain, "IPFIX header disagrees with wire metadata") + const session = [frame.exporter, frame.sourcePort, frame.domain].join(":") + const flows = [] + const setIds: number[] = [] + const templateIds: number[] = [] + for (let offset = 16; offset < data.length;) { + must(data.length - offset >= 4 && setIds.length < 32, "Invalid IPFIX set count or header") + const id = data.readUInt16BE(offset), length = data.readUInt16BE(offset + 2) + must(length >= 4 && offset + length <= data.length, "Invalid IPFIX set length") + const body = data.subarray(offset + 4, offset + length) + offset += length + setIds.push(id) + if (id === 2) { + for (let at = 0; at < body.length && !padding(body.subarray(at));) { + must(body.length - at >= 4, "Truncated IPFIX template") + const templateId = body.readUInt16BE(at), fieldCount = body.readUInt16BE(at + 2) + must(templateId >= 256 && fieldCount >= required.size && fieldCount <= 64, "Unsupported template or withdrawal") + at += 4 + const fields = new Map() + let width = 0 + for (let index = 0; index < fieldCount; index++) { + must(body.length - at >= 4, "Truncated IPFIX field") + const field = body.readUInt16BE(at), size = body.readUInt16BE(at + 2) + at += 4 + must(size > 0 && width + size <= 4096, "Variable or oversized IPFIX field") + if (field & 0x8000) { + must(body.length - at >= 4, "Truncated enterprise field") + at += 4 + } else if (required.has(field)) { + must(!fields.has(field) && required.get(field)!.includes(size), "Ambiguous or unsupported required field") + fields.set(field, { offset: width, size }) + } + width += size + } + must(fields.size === required.size, "Required IPv4 flow fields are missing") + templates.set(session + ":" + templateId, { fields, width }) + must(templates.size <= 32, "Template budget exceeded") + templateIds.push(templateId) + } + continue + } + must(id >= 256, "Unsupported IPFIX control set") + const template = templates.get(session + ":" + id) + must(template && body.length >= template.width, "Missing IPFIX template or data") + let at = 0 + for (; at + template.width <= body.length; at += template.width) { + must(++records <= 200 && flows.length < 64, "Independent record budget exceeded") + const part = (field: number) => { + const item = template.fields.get(field)! + return body.subarray(at + item.offset, at + item.offset + item.size) + } + const unsigned = (field: number) => part(field).reduce((value, byte) => value * 256n + BigInt(byte), 0n) + const bytes = unsigned(1), packets = unsigned(2) + must(bytes > 0n && packets > 0n && bytes <= 9223372036854775807n && packets <= 9223372036854775807n, "Counter outside positive SQL bigint range") + flows.push({ src: [...part(8)].join("."), dst: [...part(12)].join("."), proto: Number(unsigned(4)), + srcPort: Number(unsigned(7)), dstPort: Number(unsigned(11)), inIface: String(unsigned(10)), outIface: String(unsigned(14)), + bytes: String(bytes), packets: String(packets) }) + } + must(padding(body.subarray(at)), "Ambiguous IPFIX data padding") + } + if (flows.length) positive.add(frame.exporter) + return { ordinal: frame.ordinal, flows, setIds, templateIds } + }) + must(positive.size === 2, "Both exporters need independently decoded positive traffic") + return { captureSha256: input.captureSha256, frames, source: "independent-fixed-ipv4-ipfix-v1", nativeAcceptance: false } +} + +async function main() { + const [path, expectedSha, ...extra] = process.argv.slice(2) + must(path && !extra.length && sha.safeParse(expectedSha).success, "Expected pinned wire projection") + const handle = await open(path, "r") + try { + const info = await handle.stat() + must(info.isFile() && info.size > 0 && info.size <= 512 * 1024, "Wire projection outside file bound") + const buffer = Buffer.alloc(info.size + 1) + let size = 0 + while (size < buffer.length) { + const { bytesRead } = await handle.read(buffer, size, buffer.length - size, size) + if (!bytesRead) break + size += bytesRead + } + const body = buffer.subarray(0, size) + must(size === info.size && createHash("sha256").update(body).digest("hex") === expectedSha, "Wire projection changed or digest differs") + console.log(JSON.stringify({ wireProjectionSha256: expectedSha, ...decodeOwnedIpfix(JSON.parse(body.toString("utf8"))) })) + } finally { + await handle.close() + } +} +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch(() => { console.error("Independent IPFIX projection refused"); process.exitCode = 1 }) +} diff --git a/backend/src/test/traffic-flow-chr-ingress-discovery.test.ts b/backend/src/test/traffic-flow-chr-ingress-discovery.test.ts new file mode 100644 index 00000000..db3781a8 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-ingress-discovery.test.ts @@ -0,0 +1,79 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { discoverChrIngressEndpoint } from "./traffic-flow-chr-ingress-discovery.js" + +type Options = Parameters[0] +function fixture() { + const reads: string[] = [], toolReads: string[] = [] + const collectors = { enabled: true, collectorIp: "192.0.2.100", flowListenPort: 4740, + listenerBound: true, listenerAddress: "192.0.2.100:4740" } + let revoked = false + const mappings = new Map([["192.0.2.10", 1], ["192.0.2.20", 2]]) + const native = (serverId: number) => { + const identity = "owned-" + serverId, version = "7.24.5 (stable)", source = "192.0.2." + serverId * 10 + const table: Record = { + "/system/identity": [{ name: identity }], + "/system/resource": { version }, + "/ip/address": [{ address: source + "/32", disabled: "false" }], + "/ip/traffic-flow": [{ enabled: "true" }], + "/ip/traffic-flow/target": [{ ".id": "*A", disabled: "false", "dst-address": "192.0.2.100", + "src-address": serverId === 1 ? source : "0.0.0.0", port: "4740", version: "ipfix" }], + } + const device: Options["devices"][0] = { + identity, version, source, serverId, + assertPinnedTransport: async () => { if (revoked) throw Error("transport pin lost") }, + client: { async get(path: string, timeout?: number, _signal?: AbortSignal, max?: number, before?: () => Promise) { + assert(timeout! > 0 && timeout! <= 5000) + assert.equal(max, 65536) + assert(before) + await before() + reads.push(serverId + ":" + path) + assert(Object.hasOwn(table, path)) + return structuredClone(table[path]) as T + } }, + } + return { table, device } + } + const a = native(1), b = native(2) + const options: Options = { destination: collectors.collectorIp, baselinePort: 4740, testPort: 4739, + deadlineAt: Date.now() + 5000, devices: [a.device, b.device], assertCurrent: async () => {}, + callTool: async (name, input) => { assert.deepEqual(input, {}); toolReads.push(name); return collectors }, + resolveExporter: async address => mappings.get(address) ?? null, + } + return { options, reads, toolReads, a, b, mappings, collectors, revoke: () => { revoked = true } } +} + +test("discovers exact configured two-CHR baseline endpoint without claiming observed ingress", async () => { + const f = fixture() + const result = await discoverChrIngressEndpoint(f.options) + assert.deepEqual(result.endpoint, { sources: ["192.0.2.10", "192.0.2.20"], destination: "192.0.2.100", port: 4740 }) + assert.deepEqual(result.observations.map(row => row.sourceConfiguredExplicitly), [true, false]) + assert.equal(result.ingressVerified, false) + assert.equal(f.reads.length, 10) + assert.deepEqual(f.toolReads, ["mm_netflow_settings_get", "mm_netflow_settings_get"]) +}) + +test("refuses absent exact target, competing test port, source drift and attribution mismatch", async () => { + for (const [mutate, reason] of [ + [(f: ReturnType) => { f.b.table["/ip/traffic-flow/target"] = [] }, /baseline IPFIX target/], + [(f: ReturnType) => { (f.a.table["/ip/traffic-flow/target"] as object[]).push({ ".id": "*B", "dst-address": "192.0.2.100", port: "4739" }) }, /test endpoint/], + [(f: ReturnType) => { f.b.table["/ip/address"] = [{ address: "192.0.2.99/32", disabled: false }] }, /source is not assigned/], + [(f: ReturnType) => { f.mappings.set("192.0.2.20", 1) }, /attribution differs/], + ] as const) { + const f = fixture() + mutate(f) + await assert.rejects(discoverChrIngressEndpoint(f.options), reason) + } +}) + +test("stops on wrong collector bind, lost transport pin and changed CHR identity", async () => { + const wrong = fixture(); wrong.collectors.listenerAddress = "0.0.0.0:4740" + await assert.rejects(discoverChrIngressEndpoint(wrong.options), /collector endpoint/) + assert.deepEqual(wrong.reads, []) + const revoked = fixture(); revoked.revoke() + await assert.rejects(discoverChrIngressEndpoint(revoked.options), /transport pin lost/) + assert.deepEqual(revoked.reads, []) + const changed = fixture(); changed.a.table["/system/identity"] = [{ name: "foreign" }] + await assert.rejects(discoverChrIngressEndpoint(changed.options), /identity or version/) + assert.deepEqual(changed.reads, ["1:/system/identity", "1:/system/resource"]) +}) diff --git a/backend/src/test/traffic-flow-chr-ingress-discovery.ts b/backend/src/test/traffic-flow-chr-ingress-discovery.ts new file mode 100644 index 00000000..77764485 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-ingress-discovery.ts @@ -0,0 +1,123 @@ +/** Exact read-only prerequisite for a passive two-CHR ingress capture. */ +import { isIPv4 } from "node:net" +import { setTimeout as delay } from "node:timers/promises" +import type { MikrotikClient } from "../services/mikrotik.js" +import { captureOwnedChrPackets } from "./traffic-flow-chr-capture-process.js" + +type Row = Record +type Device = { + client: Pick + identity: string; version: string; source: string; serverId: number + assertPinnedTransport: () => Promise +} +type Options = { + destination: string; baselinePort: number; testPort: number + devices: readonly [Device, Device] + callTool: (name: "mm_netflow_settings_get", input: Record) => Promise + resolveExporter: (address: string) => Promise + assertCurrent: () => Promise + deadlineAt: number + signal?: AbortSignal +} +const targetPath = "/ip/traffic-flow/target" +const trueValue = (value: unknown) => value === true || value === "true" || value === "yes" +const falseValue = (value: unknown) => value === false || value === "false" || value === "no" +function must(value: unknown, reason: string): asserts value { + if (!value) throw Error(reason) +} +function rows(value: unknown, limit: number): Row[] { + const result = Array.isArray(value) ? value : [value] + must(result.length <= limit && result.every(row => row && typeof row === "object" && !Array.isArray(row)), "Invalid bounded native observation") + return result as Row[] +} +function singleton(value: unknown) { + const result = rows(value, 1) + must(result.length === 1, "Missing native singleton observation") + return result[0] +} + +export async function discoverChrIngressEndpoint(options: Options) { + const { destination, baselinePort, testPort, deadlineAt, signal } = options + const devices = options.devices.map(device => ({ ...device })) + const port = (value: number) => Number.isInteger(value) && value >= 1 && value <= 65535 + must(devices.length === 2 && new Set([...devices.map(device => device.source), destination]).size === 3 && + [...devices.map(device => device.source), destination].every(isIPv4) && new Set(devices.map(device => device.serverId)).size === 2 && + new Set(devices.map(device => device.identity)).size === 2 && + devices.every(device => Number.isSafeInteger(device.serverId) && device.serverId > 0 && device.identity.length > 0 && + device.identity.length <= 128 && device.version.length > 0 && device.version.length <= 128) && + port(baselinePort) && port(testPort) && baselinePort !== testPort && + Number.isSafeInteger(deadlineAt) && deadlineAt > Date.now() && deadlineAt - Date.now() <= 90000, "Invalid pinned ingress scope") + const live = () => { + signal?.throwIfAborted() + must(Date.now() < deadlineAt, "Ingress discovery deadline ended") + } + const check = async () => { live(); await options.assertCurrent(); live() } + const collector = async () => { + await check() + const value = singleton(await options.callTool("mm_netflow_settings_get", {})) + await check() + must(value.enabled === true && value.collectorIp === destination && value.flowListenPort === baselinePort && + value.listenerBound === true && value.listenerAddress === destination + ":" + baselinePort, "Current baseline collector endpoint is not confirmed") + } + await collector() + const observations = [] + for (const device of devices) { + const fence = async () => { await check(); await device.assertPinnedTransport(); await check() } + const get = async (path: string) => { + await fence() + const value = await device.client.get(path, Math.min(5000, deadlineAt - Date.now()), signal, 64 * 1024, fence) + await fence() + return value + } + const identity = singleton(await get("/system/identity")), resource = singleton(await get("/system/resource")) + must(identity.name === device.identity && resource.version === device.version, "Pinned CHR identity or version differs") + const addresses = rows(await get("/ip/address"), 256) + must(addresses.filter(row => typeof row.address === "string" && row.address.split("/")[0] === device.source && falseValue(row.disabled)).length === 1, + "Pinned exporter source is not assigned exactly once") + await fence() + must(await options.resolveExporter(device.source) === device.serverId, "Collector exporter attribution differs") + await fence() + const flow = singleton(await get("/ip/traffic-flow")) + must(trueValue(flow.enabled), "Traffic Flow is disabled or unconfirmed") + const targets = rows(await get(targetPath), 64) + must(targets.every(row => typeof row[".id"] === "string" && /^\*[a-f0-9]{1,16}$/i.test(row[".id"])) && + new Set(targets.map(row => row[".id"])).size === targets.length, "Target identity is malformed or ambiguous") + const matches = targets.filter(row => row["dst-address"] === destination && String(row.port) === String(baselinePort)) + must(matches.length === 1 && falseValue(matches[0].disabled) && matches[0].version === "ipfix", + "Exact enabled baseline IPFIX target is missing or ambiguous") + must(!targets.some(row => row["dst-address"] === destination && String(row.port) === String(testPort)), + "Proposed test endpoint is already configured") + const source = matches[0]["src-address"] + must(source === device.source || source === "0.0.0.0", "Baseline target source conflicts with the pinned address") + observations.push({ serverId: device.serverId, identity: device.identity, targetId: matches[0][".id"], + sourceAddress: device.source, sourceConfiguredExplicitly: source === device.source }) + } + await collector() + return { + endpoint: { sources: [devices[0].source, devices[1].source] as const, destination, port: baselinePort }, + testPort, observations, passiveCaptureCandidate: true, + ingressVerified: false, actualNativeAcceptance: false, + } +} + +/** Passive capture entry for the pinned Linux operator. The exercise only waits. */ +export async function observeChrBaselineIngress(options: Options & { interfaceName: string; sampleMs: number }) { + must(process.platform === "linux" && Number.isInteger(options.sampleMs) && options.sampleMs >= 1000 && options.sampleMs <= 30000, + "Invalid passive capture platform or window") + const discovered = await discoverChrIngressEndpoint(options) + must(Date.now() + options.sampleMs < options.deadlineAt, "Passive capture window exceeds the reviewed deadline") + const capture = await captureOwnedChrPackets({ ...discovered.endpoint, interfaceName: options.interfaceName }, { + deadlineAt: options.deadlineAt, + assertCurrent: async () => { + options.signal?.throwIfAborted() + await options.assertCurrent() + for (const device of options.devices) await device.assertPinnedTransport() + options.signal?.throwIfAborted() + }, + exercise: async signal => { + await delay(options.sampleMs, undefined, { signal: options.signal ? AbortSignal.any([signal, options.signal]) : signal }) + }, + }) + return { discovery: discovered, ingress: { bytes: capture.bytes, sha256: capture.sha256, existingPort: discovered.endpoint.port }, + counts: capture.counts, nativeAcceptance: false } +} diff --git a/backend/src/test/traffic-flow-chr-interface-counters.test.ts b/backend/src/test/traffic-flow-chr-interface-counters.test.ts new file mode 100644 index 00000000..109ea896 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-interface-counters.test.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { createChrInterfaceCounterReader } from "./traffic-flow-chr-interface-counters.js" + +function fixture() { + const interfaces = [{ id: "*A", name: "nfa012bcdeea", ifIndex: "12" }, + { id: "*B", name: "nfa012bcdema", ifIndex: "13" }] as const + const calls: string[] = [] + const catalog: Record> = { + "/system/identity": { name: "chr-a" }, + "/system/resource": { version: "7.24.5 (stable)" }, + } + for (const pin of interfaces) catalog["/interface/" + pin.id] = { + ".id": pin.id, name: pin.name, type: "wg", ifindex: pin.ifIndex, running: "true", disabled: "false", + "rx-packet": "10", "tx-packet": "11", "fp-rx-packet": "0", "fp-tx-packet": "0", + "fp-rx-byte": "0", "fp-tx-byte": "0", opaque: "must stay private", + } + const config = { identity: "chr-a", version: "7.24.5 (stable)", interfaces, assertCurrent: async () => {}, + client: { async get(path: string, timeout?: number, _signal?: AbortSignal, bytes?: number, fence?: () => Promise) { + assert.ok(timeout && timeout <= 1500) + assert.equal(bytes, 16384) + await fence?.() + calls.push(path) + assert.ok(Object.hasOwn(catalog, path)) + return structuredClone(catalog[path]) as T + } }, + } + return { config, catalog, calls } +} + +test("reads only the exact native IDs with bounded identity checks and emits selected counters", async () => { + const { config, calls } = fixture() + const read = createChrInterfaceCounterReader(config) + const result = await read() + assert.deepEqual(calls, ["/system/identity", "/system/resource", "/interface/*A", "/interface/*B", "/system/identity", "/system/resource"]) + assert.ok(result.startedAt <= result.observedAt) + assert.deepEqual(result.rows.map(row => row.ifindex), ["12", "13"]) + assert.ok(result.rows.every(row => row["fp-rx-packet"] === "0" && !Object.hasOwn(row, "opaque"))) +}) + +test("refuses missing and overflowing counters, changed identity and index guesses", async () => { + for (const change of [{ "fp-rx-packet": undefined }, { "fp-rx-byte": "18446744073709551616" }, + { ifindex: undefined }, { ifindex: "10" }, { ".id": "*F" }]) { + const { config, catalog } = fixture() + Object.assign(catalog["/interface/*A"], change) + await assert.rejects(createChrInterfaceCounterReader(config)(), /counter unavailable|identity or state differs/) + } + const stale = fixture() + stale.config.assertCurrent = async () => { if (stale.calls.length >= 4) throw Error("expired") } + await assert.rejects(createChrInterfaceCounterReader(stale.config)(), /expired/) +}) diff --git a/backend/src/test/traffic-flow-chr-interface-counters.ts b/backend/src/test/traffic-flow-chr-interface-counters.ts new file mode 100644 index 00000000..8f5ab5b6 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-interface-counters.ts @@ -0,0 +1,70 @@ +import { performance } from "node:perf_hooks" +import { encodeRosId, type MikrotikClient } from "../services/mikrotik.js" +import { isChrWireGuardInterfaceType } from "./traffic-flow-chr-transit-fixture.js" + +type Row = Record +type InterfacePin = { id: string; name: string; ifIndex: string } +type Options = { + client: Pick + identity: string + version: string + interfaces: readonly [InterfacePin, InterfacePin] + assertCurrent: () => Promise +} + +export const ownedWgFastPathFields = ["fp-rx-byte", "fp-tx-byte", "fp-rx-packet", "fp-tx-packet"] as const +export const ownedWgPacketFields = ["rx-packet", "tx-packet"] as const +const fields = [...ownedWgFastPathFields, ...ownedWgPacketFields] +const validId = (id: string) => /^\*[a-f0-9]{1,16}$/i.test(id) +const index = (value: unknown) => typeof value === "string" && /^[1-9][0-9]{0,9}$/.test(value) && Number(value) <= 0xffffffff +const observedUint = (value: unknown) => typeof value === "string" && /^(0|[1-9][0-9]{0,19})$/.test(value) && + BigInt(value) <= 0xffffffffffffffffn +const deny = (reason: string): never => { throw Error("RouterOS owned counter read refused: " + reason) } + +/** Uses only fixed read operations. Missing counter/index fields refuse the + * profile rather than substituting zero or guessing the wire ifIndex. */ +export function createChrInterfaceCounterReader(options: Options) { + const { client, assertCurrent, identity, version } = options + const interfaces = structuredClone(options.interfaces) + if (!identity || !version || interfaces.length !== 2 || interfaces.some(pin => + !validId(pin.id) || !/^nf[a-f0-9]{8}(ea|ma|mb|eb)$/.test(pin.name) || !index(pin.ifIndex)) || + interfaces[0].id === interfaces[1].id || interfaces[0].ifIndex === interfaces[1].ifIndex || + interfaces[0].name.slice(0, -2) !== interfaces[1].name.slice(0, -2) || + !["ea,ma", "eb,mb"].includes(interfaces.map(pin => pin.name.slice(-2)).sort().join(","))) deny("invalid interface pins") + return async () => { + const startedAt = Date.now(), deadline = performance.now() + 10000 + const fence = async () => { + if (performance.now() >= deadline) deny("observation expired") + await assertCurrent() + if (performance.now() >= deadline) deny("observation expired") + } + const read = async (path: string) => { + await fence() + const result = await client.get(path, Math.min(1500, Math.ceil(deadline - performance.now())), undefined, 16384, fence) + await fence() + const rows = Array.isArray(result) ? result : [result] + if (rows.length !== 1 || !rows[0] || typeof rows[0] !== "object" || Array.isArray(rows[0])) deny("incomplete response") + return rows[0] as Row + } + const verifyIdentity = async () => { + const device = await read("/system/identity") + const resource = await read("/system/resource") + if (device.name !== identity || resource.version !== version) deny("device identity changed") + } + await verifyIdentity() + const rows: Row[] = [] + for (const pin of interfaces) { + const row = await read("/interface/" + encodeRosId(pin.id)) + if (row[".id"] !== pin.id || row.name !== pin.name || !isChrWireGuardInterfaceType(row.type) || row.ifindex !== pin.ifIndex || + ![true, "true", "yes"].includes(row.running as string | boolean) || + ![false, "false", "no"].includes(row.disabled as string | boolean)) deny("interface identity or state differs") + if (!fields.every(field => observedUint(row[field]))) deny("required counter unavailable") + rows.push({ ".id": pin.id, name: pin.name, type: "wg", ifindex: pin.ifIndex, + ...Object.fromEntries(fields.map(field => [field, row[field]])) }) + } + await verifyIdentity() + const observedAt = Date.now() + if (observedAt < startedAt || observedAt - startedAt > 10000) deny("observation wall clock changed") + return { startedAt, observedAt, rows } + } +} diff --git a/backend/src/test/traffic-flow-chr-linux-udp-owner.test.ts b/backend/src/test/traffic-flow-chr-linux-udp-owner.test.ts new file mode 100644 index 00000000..403269cd --- /dev/null +++ b/backend/src/test/traffic-flow-chr-linux-udp-owner.test.ts @@ -0,0 +1,20 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { parseChrLinuxUdpTable } from "./traffic-flow-chr-linux-udp-owner.js" + +const header = "sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode" +const row = (address: string, port: string, inode: string) => + ` 14: ${address}:${port} 00000000:0000 07 00000000:00000000 00:00000000 00000000 1000 0 ${inode} 2 0000000000000000 0` + +test("parses exact kernel IPv4 socket inode and leaves unrelated UDP ports out", () => { + const text = [header, row("03F212C6", "C355", "42"), row("00000000", "C356", "43")].join("\n") + assert.deepEqual(parseChrLinuxUdpTable(text, 4, 50005), [{ address: "198.18.242.3", inode: "42" }]) + assert.deepEqual(parseChrLinuxUdpTable(text, 4, 50007), []) +}) + +test("fails closed for malformed rows, ownerless sockets and IPv6 conflicts", () => { + assert.throws(() => parseChrLinuxUdpTable("header", 4, 50005), /kernel table/) + assert.throws(() => parseChrLinuxUdpTable(header + "\n14: malformed", 4, 50005), /kernel socket row/) + assert.throws(() => parseChrLinuxUdpTable(header + "\n" + row("00000000", "C355", "0"), 4, 50005), /unattributed/) + assert.throws(() => parseChrLinuxUdpTable(header + "\n" + row("0".repeat(32), "C355", "42"), 6, 50005), /ambiguous IPv6/) +}) diff --git a/backend/src/test/traffic-flow-chr-linux-udp-owner.ts b/backend/src/test/traffic-flow-chr-linux-udp-owner.ts new file mode 100644 index 00000000..5ac4c16d --- /dev/null +++ b/backend/src/test/traffic-flow-chr-linux-udp-owner.ts @@ -0,0 +1,86 @@ +import { open, opendir, readlink, stat } from "node:fs/promises" + +const MAX_TABLE_BYTES = 256 * 1024 +const MAX_ROWS = 2048 +const deny = (reason: string): never => { throw Error("Collector UDP ownership refused: " + reason) } +type UdpOwner = { pid: number; address: string; port: number } + +export function parseChrLinuxUdpTable(text: string, family: 4 | 6, port: number) { + const lines = text.trim().split("\n") + if (Buffer.byteLength(text) > MAX_TABLE_BYTES || lines.length > MAX_ROWS || + !/^sl\s+local_address\s+(?:rem_address|remote_address)\s+st\s/.test(lines[0])) deny("unrecognized or oversized kernel table") + const width = family === 4 ? 8 : 32 + const entries: { address: string; inode: string }[] = [] + for (const line of lines.slice(1)) { + const values = line.trim().split(/\s+/) + const socket = values[1]?.split(":") + if (!/^\d+:$/.test(values[0]) || socket?.length !== 2 || socket[0].length !== width || + !/^[0-9A-F]+$/.test(socket[0]) || !/^[0-9A-F]{4}$/.test(socket[1]) || + !/^(0|[1-9][0-9]{0,19})$/.test(values[9] ?? "")) deny("incomplete kernel socket row") + if (Number.parseInt(socket[1], 16) !== port) continue + if (family !== 4 || values[9] === "0") deny("ambiguous IPv6 or unattributed socket") + const address = socket[0].match(/../g)!.reverse().map(value => Number.parseInt(value, 16)).join(".") + entries.push({ address, inode: values[9] }) + } + return entries +} + +async function boundedRead(path: "/proc/self/net/udp" | "/proc/self/net/udp6") { + const file = await open(path, "r") + try { + const bytes = Buffer.alloc(MAX_TABLE_BYTES + 1) + let length = 0 + while (length < bytes.length) { + const { bytesRead } = await file.read(bytes, length, bytes.length - length, null) + if (bytesRead === 0) break + length += bytesRead + } + if (length > MAX_TABLE_BYTES) deny("kernel table exceeds budget") + return new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length)) + } finally { + await file.close() + } +} + +export async function readChrLinuxNamespaceIdentity() { + if (process.platform !== "linux") deny("Linux required") + const identity = await stat("/proc/self/ns/net", { bigint: true }) + return { device: String(identity.dev), inode: String(identity.ino) } +} + +/** Self-process observation in the collector namespace, with no process + * execution, pathname overrides or inference from a missing socket owner. */ +export async function readChrLinuxUdpOwner(options: { + namespace: { device: string; inode: string }; port: number; address: string +}): Promise { + const { namespace, port, address } = options + const current = async () => { + const identity = await readChrLinuxNamespaceIdentity() + if (identity.device !== namespace.device || identity.inode !== namespace.inode) deny("network namespace changed") + } + await current() + const scan = async () => { + const ipv4 = parseChrLinuxUdpTable(await boundedRead("/proc/self/net/udp"), 4, port) + parseChrLinuxUdpTable(await boundedRead("/proc/self/net/udp6"), 6, port) + if (ipv4.length > 1 || ipv4.some(row => row.address !== address)) deny("competing or wildcard UDP bind") + return ipv4[0] + } + const candidate = await scan() + if (candidate) { + let matches = 0, visited = 0 + const directory = await opendir("/proc/self/fd") + for await (const entry of directory) { + if (++visited > 2048 || !/^\d+$/.test(entry.name)) deny("process descriptor inventory exceeds budget") + try { + if (await readlink("/proc/self/fd/" + entry.name) === `socket:[${candidate.inode}]`) matches++ + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + } + } + if (matches !== 1) deny("UDP socket is not exclusively identified in the collector process") + } + const recheck = await scan() + if (candidate?.inode !== recheck?.inode) deny("UDP socket changed during observation") + await current() + return candidate ? [{ pid: process.pid, address, port }] : [] +} diff --git a/backend/src/test/traffic-flow-chr-namespace-relay-policy.ts b/backend/src/test/traffic-flow-chr-namespace-relay-policy.ts new file mode 100644 index 00000000..ca22cfb6 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-namespace-relay-policy.ts @@ -0,0 +1,26 @@ +import { wireguardInterfaceDependencyPaths } from "../services/routeros-native-paths.js" + +const readPaths = new Set(["/system/identity", "/system/resource", "/interface", "/interface/wireguard", + "/interface/wireguard/peers", "/interface/bridge", "/interface/list/member", "/ip/address", "/ip/route", + "/ipv6/address", "/ipv6/route", "/routing/route", "/routing/rule", "/routing/table", "/ip/vrf", "/ip/settings", + "/ip/firewall/filter", "/ip/firewall/raw", "/ip/firewall/mangle", "/ip/firewall/nat", "/ipv6/firewall/filter", + "/ip/firewall/connection/tracking", "/ip/firewall/connection", "/ip/traffic-flow", "/ip/traffic-flow/ipfix", + "/ip/traffic-flow/target", ...wireguardInterfaceDependencyPaths]) +const rowMenus = ["/interface/wireguard", "/interface/wireguard/peers", "/ip/address", "/ip/route", "/ip/traffic-flow/target"] +export const targetMenu = "/ip/traffic-flow/target" +const singletonMenus = ["/ip/traffic-flow", "/ip/traffic-flow/ipfix"] +export const nativeId = /^\*[0-9a-f]{1,16}$/i +const deny = () => { throw Error("Owned CHR namespace relay refused") } + +export function chrNamespaceRelayOperation(method: string, rest: string) { + if (!rest.startsWith("/rest/")) return deny() + const path = rest.slice(5) + const last = path.lastIndexOf("/") + const menu = path.slice(0, last), token = path.slice(last + 1) + if (method === "GET" && (readPaths.has(path) || menu === "/interface" && nativeId.test(token))) return { path, write: false } + if (method === "PUT" && rowMenus.includes(path) || + method === "DELETE" && rowMenus.includes(menu) && nativeId.test(token) || + method === "PATCH" && (singletonMenus.includes(menu) || menu === targetMenu) && nativeId.test(token) || + method === "POST" && singletonMenus.includes(menu) && token === "set") return { path, write: true } + return deny() +} diff --git a/backend/src/test/traffic-flow-chr-offline-replay.test.ts b/backend/src/test/traffic-flow-chr-offline-replay.test.ts new file mode 100644 index 00000000..2e8acd53 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-offline-replay.test.ts @@ -0,0 +1,142 @@ +import assert from "node:assert/strict" +import { execFile } from "node:child_process" +import { createHash } from "node:crypto" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { promisify } from "node:util" +import { test } from "node:test" +import { replayRecordedChrEvidence } from "./traffic-flow-chr-offline-replay.js" +import { verifyChrOwnedWgEvidence } from "./traffic-flow-chr-owned-wg-evidence.js" + +const digest = (value: Buffer | string) => createHash("sha256").update(value).digest("hex") +function fixture(ethernet = false, earlyTemplates = false) { + const link = ethernet ? 14 : 0 + const seconds = 1700000010 + const bucket = Math.floor(seconds / 60) * 60000 + const from = new Date(bucket).toISOString(), until = new Date(bucket + 60000).toISOString() + const scope = { sources: ["192.0.2.10", "192.0.2.20"], destination: "192.0.2.100", port: 4739, ...(ethernet ? { linkType: "EN10MB" as const } : {}) } + const fields = [[1, 4], [2, 4], [4, 1], [7, 2], [8, 4], [10, 4], [11, 2], [12, 4], [14, 4]] + const template = Buffer.alloc(8 + fields.length * 4) + template.writeUInt16BE(2); template.writeUInt16BE(template.length, 2) + template.writeUInt16BE(256, 4); template.writeUInt16BE(fields.length, 6) + fields.forEach(([id, width], index) => { template.writeUInt16BE(id, 8 + index * 4); template.writeUInt16BE(width, 10 + index * 4) }) + const data = Buffer.alloc(33) + data.writeUInt16BE(256); data.writeUInt16BE(data.length, 2) + data.writeUInt32BE(1200, 4); data.writeUInt32BE(3, 8); data[12] = 17 + data.writeUInt16BE(9000, 13); Buffer.from([192, 0, 2, 1]).copy(data, 15); data.writeUInt32BE(11, 19) + data.writeUInt16BE(9001, 23); Buffer.from([198, 51, 100, 1]).copy(data, 25); data.writeUInt32BE(12, 29) + const ipfix = Buffer.alloc(16) + ipfix.writeUInt16BE(10); ipfix.writeUInt16BE(ipfix.length + template.length + data.length, 2); ipfix.writeUInt32BE(7, 12) + const header = Buffer.alloc(24) + header.writeUInt32LE(0xa1b2c3d4); header.writeUInt16LE(2, 4); header.writeUInt16LE(4, 6) + header.writeUInt32LE(4124 + link, 16); header.writeUInt32LE(ethernet ? 1 : 101, 20) + const frames = [10, 20, 10, 20].map((octet, index) => ({ + octet, at: seconds, micros: index * 1000, sequence: Math.floor(index / 2), + sets: earlyTemplates && index < 2 ? [data] : [template, data], + })) + if (earlyTemplates) frames.unshift(...[10, 20].map(octet => + ({ octet, at: seconds - 1, micros: 0, sequence: 0, sets: [template] }))) + const packets = frames.map(({ octet, at, micros, sequence, sets }) => { + const payload = Buffer.concat([ipfix, ...sets]) + payload.writeUInt16BE(payload.length, 2) + const packet = Buffer.alloc(44 + link + payload.length) + packet.writeUInt32LE(at, 0); packet.writeUInt32LE(micros, 4) + packet.writeUInt32LE(packet.length - 16, 8); packet.writeUInt32LE(packet.length - 16, 12) + if (ethernet) packet.writeUInt16BE(0x0800, 28) + const ip = packet.subarray(16 + link) + ip[0] = 0x45; ip.writeUInt16BE(ip.length, 2); ip[9] = 17 + Buffer.from([192, 0, 2, octet, 192, 0, 2, 100]).copy(ip, 12) + ip.writeUInt16BE(2055, 20); ip.writeUInt16BE(scope.port, 22); ip.writeUInt16BE(ip.length - 20, 24) + payload.copy(ip, 28) + ip.writeUInt32BE(seconds, 32) + ip.writeUInt32BE(sequence, 36) + return packet + }) + const pcap = Buffer.concat([header, ...packets]) + const raw = [1, 2].map(serverId => ({ serverId, bucketAt: from, src: "192.0.2.1", dst: "198.51.100.1", + proto: 17, srcPort: 9000, dstPort: 9001, inIface: "11", outIface: "12", bytes: "2400", packets: "6" })) + const minute = raw.map(({ serverId, bucketAt, bytes, packets }) => ({ serverId, bucketAt, bytes, packets })) + const evidence = { + captureSha256: digest(pcap), complete: true, dropped: 0, scope, serverIds: [1, 2], from, until, minutes: 2, + interfaceLabels: [1, 2].map(serverId => ({ serverId, index: "11", label: "ether-" + serverId })), + before: { raw, minute }, after: structuredClone({ raw, minute }), + talkers: raw.map(row => ({ serverId: row.serverId, result: { + serverIds: [row.serverId], from: new Date(bucket - 60000).toISOString(), to: until, + topConversationsLimit: 20, total: 1, totalConversations: 1, observedExporters: 1, retainedRowsBytes: 2400, nextOffset: null, + coverageVerified: false, rowSumIsNetworkTotal: false, deduplicatedAcrossInterfaces: false, forwardingVerified: false, + items: [{ ...row, serverId: String(row.serverId), inIface: "ether-" + row.serverId, inIfaceIndex: "11", bytes: 2400, packets: 6, bps: 160 }], + } })), + } + return { pcap, evidence } +} + +test("owned WG evidence requires exact IPFIX tuples and unchanged FastPath counters", () => { + const { pcap, evidence } = fixture() + const makeExporter = (roles: [string, string]) => { + const interfaces = roles.map((role, index) => ({ id: "*" + (index + 1), name: "nfa012bcde" + role, + ifIndex: String(index + 11) })) as [{ id: string; name: string; ifIndex: string }, { id: string; name: string; ifIndex: string }] + const snapshot = (observedAt: number, packetCount: string) => ({ startedAt: observedAt, observedAt, rows: interfaces.map(item => ({ + ".id": item.id, name: item.name, type: "wg", ifindex: item.ifIndex, "rx-packet": packetCount, "tx-packet": packetCount, + "fp-rx-byte": "0", "fp-tx-byte": "0", "fp-rx-packet": "0", "fp-tx-packet": "0", + })) }) + return { interfaces, before: snapshot(1700000010000, "0"), after: snapshot(1700000011000, "6") } + } + const input = { bytes: pcap, sha256: evidence.captureSha256, complete: true, dropped: 0, + scope: { ...evidence.scope, sources: [evidence.scope.sources[0], evidence.scope.sources[1]] as [string, string] }, + exporters: [makeExporter(["ea", "ma"]), makeExporter(["mb", "eb"])] as const, + expected: ([0, 1] as const).map(exporter => ({ exporter, src: "192.0.2.1", dst: "198.51.100.1", + srcPort: 9000, dstPort: 9001, inIface: "11", outIface: "12", packets: 6 })), + } + assert.equal(verifyChrOwnedWgEvidence(input).nativeAcceptance, false) + const initialAndRefreshed = input.expected.flatMap(row => (["initial", "refreshed"] as const) + .map(phase => ({ ...row, phase, packets: 3 }))) + assert.equal(verifyChrOwnedWgEvidence({ ...input, expected: initialAndRefreshed, requireRefresh: true }).exactIpfixMatched, true) + const staged = fixture(false, true) + const stagedInput = { ...input, bytes: staged.pcap, sha256: staged.evidence.captureSha256, + expected: initialAndRefreshed, requireRefresh: true } + assert.throws(() => verifyChrOwnedWgEvidence(stagedInput), /counter window/) + assert.equal(verifyChrOwnedWgEvidence({ ...stagedInput, counterWindow: "traffic" }).exactIpfixMatched, true) + input.exporters[0].before.observedAt++ + assert.throws(() => verifyChrOwnedWgEvidence({ ...stagedInput, counterWindow: "traffic" }), /counter window/) + input.exporters[0].before.observedAt-- + assert.throws(() => verifyChrOwnedWgEvidence({ ...input, requireRefresh: true }), /initial and refreshed/) + assert.throws(() => verifyChrOwnedWgEvidence({ ...input, expected: input.expected.map(row => ({ ...row, packets: 5 })) }), /totals/) + input.exporters[0].after.rows[0]["fp-rx-packet"] = "1" + assert.throws(() => verifyChrOwnedWgEvidence(input), /FastPath/) +}) + +test("offline CLI joins original multi-set IPFIX pcap, product parser and recorded SQL/MCP evidence", async () => { + const { pcap, evidence } = fixture() + const dir = await mkdtemp(join(fileURLToPath(new URL("../../../tmp/", import.meta.url)), "mm-chr-offline-")) + try { + const packetPath = join(dir, "owned.pcap"), evidencePath = join(dir, "owned.json"), body = JSON.stringify(evidence) + await Promise.all([writeFile(packetPath, pcap), writeFile(evidencePath, body)]) + const { stdout } = await promisify(execFile)(process.execPath, ["--import", "tsx", + fileURLToPath(new URL("./traffic-flow-chr-offline-replay.ts", import.meta.url)), packetPath, digest(pcap), evidencePath, digest(body)], + { timeout: 15000, windowsHide: true }) + const result = JSON.parse(stdout) + assert.equal(result.recordedEvidenceMatched, true) + assert.equal(result.replay.frames, 4) + assert.equal(result.reconciliation.rawRows, 2) + assert.equal(result.actualNativeAcceptance, false) + } finally { + await rm(dir, { recursive: true, force: true }) + } +}) + +test("refuses a mixed capture pin or drift in post-MCP SQL evidence", async () => { + const { pcap, evidence } = fixture() + await assert.rejects(replayRecordedChrEvidence(pcap, "0".repeat(64), evidence), /Capture SHA/) + evidence.after.minute[0].packets = "5" + await assert.rejects(replayRecordedChrEvidence(pcap, digest(pcap), evidence), /persisted SQL/) +}) + +test("offline replay preserves an explicit Ethernet capture scope across both decoders", async () => { + const { pcap, evidence } = fixture(true) + const result = await replayRecordedChrEvidence(pcap, digest(pcap), evidence) + assert.equal(result.recordedEvidenceMatched, true) + assert.equal(result.replay.frames, 4) + await assert.rejects(replayRecordedChrEvidence(pcap, digest(pcap), { ...evidence, + scope: { ...evidence.scope, linkType: "RAW" } }), /pinned link type RAW/) +}) diff --git a/backend/src/test/traffic-flow-chr-offline-replay.ts b/backend/src/test/traffic-flow-chr-offline-replay.ts new file mode 100644 index 00000000..6c7c1736 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-offline-replay.ts @@ -0,0 +1,129 @@ +/** Offline comparison of one pcap with recorded SQL and MCP evidence. */ +import { createHash } from "node:crypto" +import { open } from "node:fs/promises" +import { isIPv4 } from "node:net" +import { resolve } from "node:path" +import { fileURLToPath } from "node:url" +import { z } from "zod" +import { checkChrCapture } from "./traffic-flow-chr-capture-check.js" +import { decodeOwnedIpfix } from "./traffic-flow-chr-independent-ipfix.js" +import { reconcileOwnedChrFlow } from "./traffic-flow-chr-reconcile.js" +import { projectChrWire } from "./traffic-flow-chr-wire.js" + +const sha = z.string().regex(/^[a-f0-9]{64}$/) +const ip = z.string().max(15).refine(isIPv4) +const serverId = z.number().int().positive().safe() +const snapshot = z.object({ raw: z.array(z.unknown()).max(200), minute: z.array(z.unknown()).max(20) }).strict() +const evidenceSchema = z.object({ + captureSha256: sha, complete: z.literal(true), dropped: z.literal(0), + scope: z.object({ sources: z.tuple([ip, ip]), destination: ip, port: z.number().int().min(1).max(65535), linkType: z.enum(["RAW", "EN10MB"]).optional() }).strict(), + serverIds: z.tuple([serverId, serverId]), + from: z.string().max(24), until: z.string().max(24), minutes: z.number().int().min(1).max(15), + interfaceLabels: z.array(z.object({ serverId, index: z.string().regex(/^(?:0|[1-9]\d{0,9})$/), label: z.string().min(1).max(200) }).strict()).max(200), + before: snapshot, after: snapshot, + talkers: z.tuple([z.object({ serverId, result: z.unknown() }).strict(), z.object({ serverId, result: z.unknown() }).strict()]), +}).strict() +function must(value: unknown, reason: string): asserts value { + if (!value) throw Error(reason) +} +function replayCounter(value: string) { + const integer = BigInt(value) + must(integer <= BigInt(Number.MAX_SAFE_INTEGER), "Product replay requires exact JavaScript integer counters") + return Number(integer) +} + +export function prepareChrReplay(wire: ReturnType) { + const decoded = decodeOwnedIpfix(wire) + const exporters = wire.scope.sources.map((address, index) => { + const frames = wire.frames.filter(frame => frame.exporter === index) + const first = frames[0] + must(first && frames.every(frame => frame.sourcePort === first.sourcePort && frame.version === first.version && frame.domain === first.domain), "Exporter session changed") + const ids = new Set(decoded.frames.flatMap((frame, ordinal) => wire.frames[ordinal].exporter === index ? frame.templateIds : [])) + must(ids.size === 1, "Replay requires one pinned template ID per exporter") + return { address, port: first.sourcePort, version: first.version, domain: first.domain, templateId: [...ids][0] } + }) + const phase = [0, 0] + const frames = wire.frames.map((frame, index) => { + const { flows, templateIds } = decoded.frames[index] + let kind = "data" + if (templateIds.length) kind = phase[frame.exporter]++ === 0 ? "template" : "refresh" + const expected = flows.map(({ src, dst, bytes, packets, inIface, outIface }) => + ({ src, dst, bytes: replayCounter(bytes), packets: replayCounter(packets), inIface, outIface })) + return { exporter: frame.exporter, ordinal: frame.ordinal, receivedMs: frame.receivedMs, payloadHex: frame.payloadHex, kind, expected } + }) + return { decoded, projection: { destination: wire.scope.destination, exporters, frames } } +} + +export async function replayRecordedChrEvidence(bytes: Buffer, expectedSha: string, rawEvidence: unknown) { + const evidence = evidenceSchema.parse(rawEvidence) + const wire = projectChrWire(bytes, evidence.scope) + must(wire.captureSha256 === expectedSha && evidence.captureSha256 === expectedSha, "Capture SHA-256 does not match recorded evidence") + const { decoded, projection } = prepareChrReplay(wire) + const replay = checkChrCapture(projection) + const labels = new Map() + for (const row of evidence.interfaceLabels) { + const key = row.serverId + ":" + row.index + must(evidence.serverIds.includes(row.serverId) && !labels.has(key), "Interface evidence identity differs") + labels.set(key, row.label) + } + must(evidence.talkers.every((row, index) => row.serverId === evidence.serverIds[index]), "Public evidence order or scope differs") + let snapshots = 0, publicReads = 0 + const reconciliation = await reconcileOwnedChrFlow({ + wire, decoded, serverIds: evidence.serverIds, from: evidence.from, until: evidence.until, minutes: evidence.minutes, + interfaceLabel: (id, index) => { + const label = labels.get(id + ":" + index) + must(label, "Missing independently observed interface label") + return label + }, + readOnlySnapshot: async work => { + const recorded = [evidence.before, evidence.after][snapshots++] + must(recorded, "Unexpected SQL evidence read") + return work(async sql => { + if (sql.includes("FROM flow_buckets")) return recorded.raw + must(sql.includes("FROM flow_minute_stats"), "Unexpected SQL evidence query") + return recorded.minute + }) + }, + callTool: async (tool, request) => { + const recorded = evidence.talkers[publicReads++] + must(tool === "mm_netflow_talkers_get" && recorded?.serverId === request.serverId, "Unexpected recorded MCP scope") + return recorded.result + }, + assertCurrent: async () => { /* Offline comparison of immutable recorded evidence. */ }, + }) + return { captureSha256: wire.captureSha256, replay, reconciliation, recordedEvidenceMatched: true, actualNativeAcceptance: false } +} + +async function readPinned(path: string, digest: string) { + sha.parse(digest) + const file = await open(path, "r") + try { + const info = await file.stat() + must(info.isFile() && info.size > 0 && info.size <= 512 * 1024, "Offline evidence file outside size bound") + const bytes = Buffer.alloc(info.size + 1) + let read = 0 + while (read < bytes.length) { + const { bytesRead } = await file.read(bytes, read, bytes.length - read, read) + if (!bytesRead) break + read += bytesRead + } + const body = bytes.subarray(0, read) + must(read === info.size && createHash("sha256").update(body).digest("hex") === digest, "Offline evidence hash differs") + return body + } finally { + await file.close() + } +} +async function main() { + const args = process.argv.slice(2) + must(args.length === 4, "Expected pcap and recorded evidence pins") + const [capture, captureSha, evidence, evidenceSha] = args + const [bytes, raw] = await Promise.all([readPinned(capture, captureSha), readPinned(evidence, evidenceSha)]) + console.log(JSON.stringify({ evidenceSha256: evidenceSha, ...await replayRecordedChrEvidence(bytes, captureSha, JSON.parse(raw.toString("utf8"))) })) +} +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch(() => { + console.error("Recorded CHR evidence refused") + process.exitCode = 1 + }) +} diff --git a/backend/src/test/traffic-flow-chr-owned-sequence.test.ts b/backend/src/test/traffic-flow-chr-owned-sequence.test.ts index e3fa383d..aef94da2 100644 --- a/backend/src/test/traffic-flow-chr-owned-sequence.test.ts +++ b/backend/src/test/traffic-flow-chr-owned-sequence.test.ts @@ -89,6 +89,17 @@ test("owned journal is synchronized before target dispatch and captures target c assert.deepEqual(events, ["start", "create:A", "create:B", "exercise", "delete:B", "delete:A", "stop", "restore", "reconcile"]) await assert.rejects(executeOwnedChrCapture(options), { code: "EEXIST" }) assert.equal(events.length, 9) + bound = true + await assert.rejects(executeOwnedChrCapture({ + ...options, journalPath: join(directory, "aborted.jsonl"), + capture: async run => { + const owner = new AbortController() + owner.abort() + await run(owner.signal) + return fixtures(Date.now()) + }, + }), /ownership ended/) + assert.equal(events.length, 9) } finally { await rm(directory, { recursive: true, force: true }) } diff --git a/backend/src/test/traffic-flow-chr-owned-sequence.ts b/backend/src/test/traffic-flow-chr-owned-sequence.ts index 273959c5..c47da919 100644 --- a/backend/src/test/traffic-flow-chr-owned-sequence.ts +++ b/backend/src/test/traffic-flow-chr-owned-sequence.ts @@ -6,7 +6,7 @@ import { createChrTargetHarness } from "./traffic-flow-chr-target-harness.js" import { projectChrWire } from "./traffic-flow-chr-wire.js" import { checkChrCapture } from "./traffic-flow-chr-capture-check.js" -type Target = Parameters[0] +type Target = Extract[0], { stagedDisabled?: false }> type Scope = Parameters[1] type Wire = ReturnType type Projection = { @@ -15,10 +15,19 @@ type Projection = { frames: readonly { exporter: number; ordinal: number; receivedMs: number; payloadHex: string }[] } type Capture = { bytes: Buffer; sha256: string; dropped: number; complete: boolean; projection: Projection } +type Ingress = { bytes: Buffer; sha256: string; existingPort: number } function must(value: unknown, reason: string): asserts value { if (!value) throw Error(reason) } +export function checkChrBaselineIngress(input: Ingress, scope: Scope, now = Date.now()) { + const wire = projectChrWire(input.bytes, { sources: scope.sources, destination: scope.destination, port: input.existingPort }) + must(wire.captureSha256 === input.sha256, "Baseline ingress SHA-256 differs") + must(wire.frames.every(frame => frame.receivedMs >= now - 10 * 60_000 && frame.receivedMs <= now), + "Baseline ingress is outside the current evidence window") + return wire +} + /** Bind every reviewed row to its original wire record and require complete * capture coverage for conservation. Sample-only replay uses the separate CLI. */ export function bindChrCapture(capture: Capture, scope: Scope) { @@ -42,11 +51,11 @@ type Options = { journalPath: string deadlineAt: number scope: Scope - ingress: { bytes: Buffer; sha256: string; existingPort: number } + ingress: Ingress serverIds: readonly [number, number] targets: readonly [Omit, Omit] collector: { authorize: () => Promise; assertBound: () => Promise; restore: () => Promise } - capture: (exercise: () => Promise) => Promise + capture: (exercise: (signal?: AbortSignal) => Promise) => Promise exercise: () => Promise reconcile: (wire: Wire, serverIds: readonly [number, number]) => Promise } @@ -54,17 +63,13 @@ type Options = { export async function executeOwnedChrCapture(options: Options) { const scope = structuredClone(options.scope), serverIds = Object.freeze([...options.serverIds]) as readonly [number, number] const deadlineAt = options.deadlineAt - let active = true - const live = () => must(active && Number.isSafeInteger(deadlineAt) && Date.now() < deadlineAt, "Owned capture deadline expired") + let active = true, captureSignal: AbortSignal | undefined + const live = () => must(active && !captureSignal?.aborted && Number.isSafeInteger(deadlineAt) && Date.now() < deadlineAt, "Owned capture deadline or ownership ended") live() must(deadlineAt - Date.now() <= 10 * 60_000 && serverIds.length === 2 && new Set(serverIds).size === 2 && serverIds.every(id => Number.isSafeInteger(id) && id > 0) && options.targets.length === 2 && options.targets[0].baseline.deviceId !== options.targets[1].baseline.deviceId, "Invalid owned capture identity") - const ingress = projectChrWire(options.ingress.bytes, { ...scope, port: options.ingress.existingPort }) - must(ingress.captureSha256 === options.ingress.sha256, "Baseline ingress SHA-256 differs") - const admittedAt = Date.now() - must(ingress.frames.every(frame => frame.receivedMs >= admittedAt - 10 * 60_000 && frame.receivedMs <= admittedAt), - "Baseline ingress is outside the current evidence window") + const ingress = checkChrBaselineIngress(options.ingress, scope) const file = await open(options.journalPath, "wx", 0o600) const append = async (record: object) => { live() @@ -93,9 +98,10 @@ export async function executeOwnedChrCapture(options: Options) { await authorize() let started = false, exercised = false const captureStartedAt = Date.now() - const capture = await options.capture(async () => { + const capture = await options.capture(async signal => { must(!started, "Capture exercise was already attempted") started = true + captureSignal = signal await authorize() for (const target of targets) await target.create() await authorize() @@ -106,6 +112,7 @@ export async function executeOwnedChrCapture(options: Options) { }) const captureEndedAt = Date.now() must(exercised, "Capture omitted or did not complete its exercise") + captureSignal = undefined await authorize() await append({ phase: "collector-restore-intent" }) await options.collector.authorize() diff --git a/backend/src/test/traffic-flow-chr-owned-wg-evidence.ts b/backend/src/test/traffic-flow-chr-owned-wg-evidence.ts new file mode 100644 index 00000000..7f26d1b9 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-owned-wg-evidence.ts @@ -0,0 +1,92 @@ +import { decodeOwnedIpfix } from "./traffic-flow-chr-independent-ipfix.js" +import { projectChrWire } from "./traffic-flow-chr-wire.js" +import { ownedWgFastPathFields as fp, ownedWgPacketFields as packets } from "./traffic-flow-chr-interface-counters.js" +import { isChrWireGuardInterfaceType } from "./traffic-flow-chr-transit-fixture.js" + +type Scope = Parameters[1] +type Counters = { startedAt: number; observedAt: number; rows: readonly Record[] } +type Interface = { id: string; name: string; ifIndex: string } +type Expected = { exporter: 0 | 1; src: string; dst: string; srcPort: number; dstPort: number; + inIface: string; outIface: string; packets: number; phase?: "initial" | "refreshed" } +function must(value: unknown, reason: string): asserts value { if (!value) throw Error(reason) } +function counter(row: Record, name: string) { + const value = row[name] + must(typeof value === "string" && /^(0|[1-9][0-9]{0,19})$/.test(value), "Owned interface counter unavailable") + return BigInt(value) +} + +/** Requires real before/after counters from the runner's pinned native reader + * and original pcap bytes. Artifact agreement alone confers no live authority. */ +export function verifyChrOwnedWgEvidence(options: { + bytes: Buffer; sha256: string; complete: boolean; dropped: number; scope: Scope + exporters: readonly [{ interfaces: readonly [Interface, Interface]; before: Counters; after: Counters }, + { interfaces: readonly [Interface, Interface]; before: Counters; after: Counters }] + expected: readonly Expected[] + requireRefresh?: boolean + counterWindow?: "traffic" +}) { + const { scope, expected, exporters } = options + const wire = projectChrWire(options.bytes, scope) + must(options.complete === true && options.dropped === 0 && options.sha256 === wire.captureSha256, "Owned capture provenance differs") + const decoded = decodeOwnedIpfix(wire) + must(options.counterWindow !== "traffic" || options.requireRefresh, "Staged traffic counters require refreshed data") + const matches = expected.map(() => 0n) + must(expected.length >= 2 && expected.length <= 16 && [0, 1].every(exporter => expected.some(flow => flow.exporter === exporter)) && + expected.every(flow => Number.isSafeInteger(flow.packets) && flow.packets > 0), "Expected owned flow scope is incomplete") + const sessions = new Map() + if (options.requireRefresh) must([0, 1].every(exporter => + ["initial", "refreshed"].every(phase => expected.some(pin => pin.exporter === exporter && pin.phase === phase))), + "Both exporters need initial and refreshed flow pins") + for (const [index, { flows, templateIds }] of decoded.frames.entries()) { + const { exporter, sourcePort, domain } = wire.frames[index] + if (options.requireRefresh) { + const key = sourcePort + "/" + domain + const session = sessions.get(exporter) + must(!session || session.key === key, "Owned exporter session changed") + if (templateIds.length) { + must(templateIds.length === 1 && (!session || session.templateId === templateIds[0]), "Owned template identity changed") + sessions.set(exporter, { key, templateId: templateIds[0], phase: session ? "refreshed" : "initial" }) + } + } + for (const flow of flows) { + const matching = expected.map((pin, offset) => ({ pin, offset })).filter(({ pin }) => + pin.exporter === exporter && pin.src === flow.src && pin.dst === flow.dst && pin.srcPort === flow.srcPort && + pin.dstPort === flow.dstPort && pin.inIface === flow.inIface && pin.outIface === flow.outIface && flow.proto === 17 && + (!options.requireRefresh || sessions.get(exporter)?.phase === pin.phase)) + must(matching.length === 1, "Unscoped or ambiguous owned IPFIX flow") + matches[matching[0].offset] += BigInt(flow.packets) + } + } + must(expected.every((pin, offset) => matches[offset] === BigInt(pin.packets)), "Owned IPFIX packet totals differ") + for (const [index, { before, after, interfaces }] of exporters.entries()) { + // A staged target can emit a template as soon as it is enabled. Only + // record-bearing frames are relevant to the forwarding counter interval. + const timestamps = wire.frames.filter((frame, offset) => frame.exporter === index && + (options.counterWindow !== "traffic" || decoded.frames[offset].flows.length > 0)).map(frame => frame.receivedMs) + must([before.startedAt, before.observedAt, after.startedAt, after.observedAt].every(Number.isSafeInteger) && + before.startedAt <= before.observedAt && after.startedAt <= after.observedAt && + timestamps.length > 0 && before.observedAt <= Math.min(...timestamps) && after.startedAt >= Math.max(...timestamps) && + after.observedAt - before.startedAt <= 180000 && + interfaces[0].id !== interfaces[1].id && interfaces[0].name !== interfaces[1].name && + interfaces[0].ifIndex !== interfaces[1].ifIndex && + interfaces.every(pin => /^\*[a-f0-9]{1,16}$/i.test(pin.id) && /^nf[a-f0-9]{8}(ea|ma|mb|eb)$/.test(pin.name) && + /^[1-9][0-9]{0,9}$/.test(pin.ifIndex)) && + expected.filter(flow => flow.exporter === index).every(flow => flow.inIface !== flow.outIface && + interfaces.some(pin => pin.ifIndex === flow.inIface) && interfaces.some(pin => pin.ifIndex === flow.outIface)), + "Owned counter window or interface scope differs") + for (const pin of interfaces) { + const read = (sample: Counters) => { + const rows = sample.rows.filter(row => row[".id"] === pin.id || row.name === pin.name) + must(rows.length === 1 && rows[0][".id"] === pin.id && rows[0].name === pin.name && isChrWireGuardInterfaceType(rows[0].type) && + rows[0].ifindex === pin.ifIndex, + "Owned interface counter identity differs") + return rows[0] + } + const old = read(before), current = read(after) + must(fp.every(name => counter(old, name) === counter(current, name)), "Owned interface FastPath counter changed") + must(packets.every(name => counter(current, name) > counter(old, name)), "Owned interface did not observe bidirectional packets") + } + } + return { captureSha256: wire.captureSha256, exactIpfixMatched: true as const, + ownedFastPathDeltasZero: true as const, nativeAcceptance: false as const } +} diff --git a/backend/src/test/traffic-flow-chr-parent-admission-socket.ts b/backend/src/test/traffic-flow-chr-parent-admission-socket.ts new file mode 100644 index 00000000..e78c3bbd --- /dev/null +++ b/backend/src/test/traffic-flow-chr-parent-admission-socket.ts @@ -0,0 +1,127 @@ +import { fstatSync } from "node:fs" +import { Socket } from "node:net" +import type { Duplex } from "node:stream" +import { createChrParentOrchestration } from "./traffic-flow-chr-parent-orchestration.js" + +type Parent = Awaited> + +/** The root launcher inherits one private socketpair end from its credential + * verifying AF_UNIX acceptor. The collector's pathname never reaches Node. */ +function inheritedSocket(descriptor: number) { + if (process.platform !== "linux" || process.geteuid?.() !== 0 || + !Number.isSafeInteger(descriptor) || descriptor < 3 || descriptor > 1024 || + !fstatSync(descriptor).isSocket()) { + throw Error("Inherited parent CHR admission socket refused") + } + return new Socket({ fd: descriptor, readable: true, writable: true, allowHalfOpen: true }) +} + +export async function serveChrParentInheritedAdmission(parent: Parent, descriptor: number) { + let socket: Socket | undefined + try { + socket = inheritedSocket(descriptor) + return await parent.serve({ input: socket, output: socket }) + } finally { + socket?.destroy() + parent.retire() + } +} + +/** Ask the host owner to recheck actual bridge milestones. This channel is + * separate from the collector connection and has no mutation operations. */ +export function openChrParentHostPhaseChecks(descriptor: number, deadlineAt: number, staged = false) { + return openChrParentConnectedPhaseChecks(inheritedSocket(descriptor), deadlineAt, staged) +} + +export function openChrParentConnectedPhaseChecks(socket: Duplex, deadlineAt: number, staged = false) { + socket.on("error", () => {}) + const responses = socket[Symbol.asyncIterator]() + let sequence = 0, pending = false + const check = async (phase: "preflight" | "stage" | "ready" | "drain" | "inverse" | "timer") => { + if (pending || socket.destroyed || ++sequence > (staged ? 99 : 91) || Date.now() >= deadlineAt) { + socket.destroy() + throw Error("Host phase query refused") + } + pending = true + const timer = setTimeout(() => socket.destroy(), Math.min(8000, deadlineAt - Date.now())) + try { + const expected = sequence + ":ok" + await new Promise((resolve, reject) => { + socket.write(sequence + ":" + phase + "\n", error => error ? reject(error) : resolve()) + }) + let received = Buffer.alloc(0) + while (received.at(-1) !== 10) { + const { value, done } = await responses.next() + if (done || !Buffer.isBuffer(value) || received.length + value.length > 64) throw Error("Host phase query refused") + received = Buffer.concat([received, value]) + } + const text = received.toString("ascii") + if (phase !== "timer") { + if (text !== expected + "\n") throw Error("Host phase query refused") + return + } + const deadline = text.startsWith(expected + ":") ? text.slice(expected.length + 1) : "" + const value = Number(deadline.trim()) + if (!/^[1-9][0-9]{12}\n$/.test(deadline) || !Number.isSafeInteger(value) || value <= Date.now() || + value > Math.min(deadlineAt, Date.now() + 240000)) throw Error("Host capture deadline refused") + return value + } catch (error) { socket.destroy(); throw error } + finally { clearTimeout(timer); pending = false } + } + const fence = async (phase: Exclude[0], "timer">) => { await check(phase) } + return Object.freeze({ + assertTimingPreflight: () => fence("preflight"), + ...(staged ? { assertTransitAndCollectorStagingReady: () => fence("stage") } : {}), + assertTransitAndCollectorReady: () => fence("ready"), + assertCaptureAndDrainComplete: () => fence("drain"), + assertLinuxInverseReady: () => fence("inverse"), + async captureDeadline() { + if (!staged) throw Error("Staged host capture deadline unavailable") + const result = await check("timer") + if (result === undefined) throw Error("Staged host capture deadline unavailable") + return result + }, + close: () => socket.destroy(), + }) +} + + + + +type HostCheck = keyof Omit, "close" | "captureDeadline"> + +/** The host launches one root Node with two private descriptors. Call serve + * after opening the CHR relay with the returned hooks and sealing its pins. */ +export async function openChrParentAdmissionTransport( + options: Omit[0], HostCheck>, + descriptors: { admission: number; phase: number }, +) { + if (descriptors.admission === descriptors.phase) throw Error("Parent admission descriptors overlap") + const socket = inheritedSocket(descriptors.admission) + socket.on("error", () => {}) + let phases: ReturnType | undefined + let parent: Parent | undefined + try { + phases = openChrParentHostPhaseChecks(descriptors.phase, options.deadlineAt, !!options.stagedTarget) + parent = await createChrParentOrchestration({ ...options, ...phases }) + const owner = parent + const close = () => { owner.retire(); socket.destroy(); phases?.close() } + return Object.freeze({ + authorizeWrite: owner.authorizeWrite, + recordWriteAcknowledgement: owner.recordWriteAcknowledgement, + async serve() { + try { return await owner.serve({ input: socket, output: socket }) } + finally { close() } + }, + close, + }) + } catch (error) { + parent?.retire() + socket.destroy() + phases?.close() + throw error + } +} + + + diff --git a/backend/src/test/traffic-flow-chr-parent-native-boundary.ts b/backend/src/test/traffic-flow-chr-parent-native-boundary.ts new file mode 100644 index 00000000..7d3aa625 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-parent-native-boundary.ts @@ -0,0 +1,68 @@ +import type { Duplex } from "node:stream" +import type { createChrParentOrchestration } from "./traffic-flow-chr-parent-orchestration.js" +import type { ChrWriteIntent } from "./traffic-flow-chr-write-admission.js" + +type Owner = Awaited> +type Hooks = Pick +const frameLimit = 16384 +const nativeId = /^\*[a-f0-9]{1,16}$/i +const fail = () => { throw Error("Parent native boundary refused. Automatic replay forbidden.") } +const object = (raw: unknown): raw is Record => !!raw && typeof raw === "object" && !Array.isArray(raw) +const shape = (raw: Record, keys: string[]) => + Object.keys(raw).length === keys.length && keys.every(key => Object.hasOwn(raw, key)) + +async function* frames(socket: Duplex) { + let buffered = Buffer.alloc(0) + for await (const data of socket) { + const bytes = Buffer.isBuffer(data) ? data : Buffer.from(data) + if (bytes.length > frameLimit * 2) return fail() + for (let offset = 0; offset < bytes.length;) { + const end = bytes.indexOf(10, offset) + const length = (end < 0 ? bytes.length : end) - offset + if (buffered.length + length > frameLimit) return fail() + buffered = Buffer.concat([buffered, bytes.subarray(offset, offset + length)]) + if (end < 0) break + const value: unknown = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(buffered)) + if (!object(value)) return fail() + yield value + buffered = Buffer.alloc(0) + offset = end + 1 + } + } + if (buffered.length) return fail() +} + +/** Host obtains a consumed grant before sending native bytes. Only its + * authenticated peer can submit the subsequent complete response projection. */ +export async function serveChrParentNativeBoundary(socket: Duplex, hooks: Hooks, assertCurrent: () => Promise) { + let sequence = 0 + let pending: ChrWriteIntent | undefined + socket.on("error", () => {}) + for await (const value of frames(socket)) { + await assertCurrent() + const { side, method, path } = value + if (value.sequence !== ++sequence || sequence > 100 || side !== "A" && side !== "B" || + !["PUT", "PATCH", "POST", "DELETE"].includes(String(method)) || typeof path !== "string" || path.length > 128) return fail() + if (value.op === "consume") { + if (pending || !shape(value, ["sequence", "op", "side", "method", "path", "body"]) || !object(value.body) || + !Object.values(value.body).every(item => typeof item === "string")) return fail() + pending = await hooks.authorizeWrite({ side, method: String(method), path, body: value.body as Record }) + if (pending.side !== side || pending.method !== method || pending.path !== path) return fail() + } else if (value.op === "ack") { + const { status, id } = value + if (!pending || pending.side !== side || pending.method !== method || pending.path !== path || + !shape(value, ["sequence", "op", "side", "method", "path", "status", ...(Object.hasOwn(value, "id") ? ["id"] : [])]) || + typeof status !== "number" || !Number.isInteger(status) || status < 100 || status > 599) return fail() + const needsId = (method === "PUT" && (status === 200 || status === 201)) || + (pending.step === "target:" + side + ":enable" && status === 200) + if (needsId ? typeof id !== "string" || !nativeId.test(id) : Object.hasOwn(value, "id")) return fail() + hooks.recordWriteAcknowledgement({ intent: pending, status, body: needsId ? { ".id": id } : undefined }) + pending = undefined + } else return fail() + await assertCurrent() + await new Promise((resolve, reject) => { + socket.write(JSON.stringify({ sequence, ok: true }) + "\n", error => error ? reject(error) : resolve()) + }) + } + return fail() +} diff --git a/backend/src/test/traffic-flow-chr-parent-orchestration.test.ts b/backend/src/test/traffic-flow-chr-parent-orchestration.test.ts new file mode 100644 index 00000000..c87ec1e2 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-parent-orchestration.test.ts @@ -0,0 +1,8 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { runChrParentSynthetic } from "./traffic-flow-chr-parent-synthetic.js" + +for (const mode of ["direct", "staged", "wrong-ack"] as const) test("paired catalogue and native ACK parent profile: " + mode, async () => { + const result = await runChrParentSynthetic(mode) + assert.equal(result.writes, mode === "direct" ? 48 : mode === "staged" ? 50 : 25) +}) diff --git a/backend/src/test/traffic-flow-chr-parent-orchestration.ts b/backend/src/test/traffic-flow-chr-parent-orchestration.ts new file mode 100644 index 00000000..a15f0a3e --- /dev/null +++ b/backend/src/test/traffic-flow-chr-parent-orchestration.ts @@ -0,0 +1,131 @@ +import type { MikrotikClient } from "../services/mikrotik.js" +import { createChrParentAckMailbox, createChrParentTrustedStages } from "./traffic-flow-chr-parent-stages.js" +import { createChrTransitTopologyInventory } from "./traffic-flow-chr-topology-inventory.js" +import { createChrParentWriteAdmission, type ChrWriteIntent } from "./traffic-flow-chr-write-admission.js" +import { serveChrParentAdmissionPipe } from "./traffic-flow-chr-write-admission-ipc.js" +import { chrFixtureDeadlineMaxMs, type ChrStagedFixtureLease } from "./traffic-flow-chr-target-timing-profile.js" + +type StagesOptions = Parameters[0] +type Side = "A" | "B" +type Row = Record +type Options = Pick & { + leaseProfile?: ChrStagedFixtureLease + routers: Record }> + assertTimingPreflight(): Promise + assertRouterIdentity(side: Side, deviceId: string): Promise + assertTransitAndCollectorReady(): Promise + assertTransitAndCollectorStagingReady?(): Promise + assertCaptureAndDrainComplete(): Promise + assertLinuxInverseReady(): Promise +} + +const fail = () => { throw Error("Parent CHR composition refused. Automatic replay forbidden.") } +const isRow = (value: unknown): value is Row => !!value && typeof value === "object" && !Array.isArray(value) + +export async function createChrParentOrchestration(options: Options) { + const { deadlineAt, leaseProfile, authorize, assertRouterIdentity } = options + const stagedTarget = options.stagedTarget && { ...options.stagedTarget } + const stageReady = options.assertTransitAndCollectorStagingReady + if ((stagedTarget && !stageReady) || (leaseProfile && !stagedTarget)) return fail() + const routers = { A: { ...options.routers.A }, B: { ...options.routers.B } } + const sides = ["A", "B"] as const + const check = () => { + const left = deadlineAt - Date.now() + if (!Number.isSafeInteger(deadlineAt) || left <= 0 || left > chrFixtureDeadlineMaxMs(leaseProfile)) return fail() + return Math.min(8000, left) + } + const assertCurrent = async (side: Side, deviceId: string) => { + check() + if (routers[side].deviceId !== deviceId) return fail() + await authorize() + await assertRouterIdentity(side, deviceId) + check() + } + const assertPair = async () => { + for (const side of sides) await assertCurrent(side, routers[side].deviceId) + } + const assertStageAuthority = async (intent: Readonly) => { + await assertPair() + if (intent.step.startsWith("topology:delete:")) { + await options.assertLinuxInverseReady() + } else if (intent.step.endsWith(".restore") || intent.step.startsWith("target:") && intent.method === "DELETE") { + await options.assertCaptureAndDrainComplete() + } else if (stageReady && stagedTarget && (intent.step.endsWith(".enable") || intent.step.startsWith("target:") && intent.method === "PUT")) { + await stageReady() + } else if (!intent.step.startsWith("topology:create:")) { + await options.assertTransitAndCollectorReady() + } + check() + } + const exactGet = async (side: Side, path: string, singleton = false) => { + const { client } = routers[side] + const fence = async () => { check(); await authorize(); check() } + await fence() + const timeout = check() + const raw = await client.get(path, timeout, AbortSignal.timeout(timeout), 65536, fence) + await fence() + const rows = Array.isArray(raw) ? raw : singleton && isRow(raw) ? [raw] : undefined + if (!rows || Buffer.byteLength(JSON.stringify(rows)) > 65536 || !rows.every(isRow)) return fail() + return rows + } + const readFlow = async (side: Side) => { + const deviceId = routers[side].deviceId + await assertCurrent(side, deviceId) + const [flow, ipfix, targets] = await Promise.all([ + exactGet(side, "/ip/traffic-flow", true), exactGet(side, "/ip/traffic-flow/ipfix", true), + exactGet(side, "/ip/traffic-flow/target"), + ]) + await assertCurrent(side, deviceId) + if (flow.length !== 1 || ipfix.length !== 1 || targets.length > 1) return fail() + return { deviceId, flow: flow[0], ipfix: ipfix[0], targets } + } + check() + await assertPair() + await options.assertTimingPreflight() + check() + const scope = structuredClone(options.scope) + const inventory = createChrTransitTopologyInventory({ + runId: scope[0], deadlineAt, authorize: assertPair, authorizeRequest: authorize, parallelism: 2, routers, + }) + const mailbox = createChrParentAckMailbox() + const trusted = await createChrParentTrustedStages({ + scope, linux: options.linux, deadlineAt, timerProfile: options.timerProfile, stagedTarget, authorize: assertPair, + routers: { + A: { ...routers.A, read: () => inventory.read("A"), readFlow: () => readFlow("A") }, + B: { ...routers.B, read: () => inventory.read("B"), readFlow: () => readFlow("B") }, + }, + assertOutsideScope: inventory.assertOutsideScope, assertStageAuthority, takeAcknowledgement: mailbox.take, + }) + const gate = createChrParentWriteAdmission({ + deadlineAt, profile: trusted.profile, deviceIds: { A: routers.A.deviceId, B: routers.B.deviceId }, assertCurrent, assertIntentCurrent: assertStageAuthority, + }) + let started = false, retired = false + let activeInput: Parameters[0]["input"] | undefined + const retire = () => { + retired = true + gate.retire() + mailbox.retire() + trusted.retire() + activeInput?.destroy() + } + return Object.freeze({ + authorizeWrite: gate.authorizeWrite, + recordWriteAcknowledgement: mailbox.record, + retire, + async serve(pipes: Parameters[0]) { + if (started || retired) return fail() + started = true + activeInput = pipes.input + try { + const result = await serveChrParentAdmissionPipe(pipes, gate, trusted.stages, deadlineAt, trusted.profile, leaseProfile) + if (result.completed !== trusted.stages.length || gate.consumed !== result.completed) return fail() + return result + } finally { retire() } + }, + }) +} + + + + + diff --git a/backend/src/test/traffic-flow-chr-parent-stages.test.ts b/backend/src/test/traffic-flow-chr-parent-stages.test.ts new file mode 100644 index 00000000..501c3336 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-parent-stages.test.ts @@ -0,0 +1,121 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { chrTransitNames } from "./traffic-flow-chr-transit-fixture.js" + + +const runId = "a012bcde" +const ports = { linuxA: 50101, edgeA: 50102, middleA: 50103, middleB: 50104, edgeB: 50105, linuxB: 50106, + collector: 50107, calibrationSource: 50112, calibrationDestination: 50113, phase1Source: 50108, phase1Destination: 50109, phase2Source: 50110, phase2Destination: 50111 } +const directions = ["A-to-B", "B-to-A", "export-A", "export-B", "underlay-linuxA-to-A", "underlay-A-to-linuxA", + "underlay-A-to-B", "underlay-B-to-A", "underlay-B-to-linuxB", "underlay-linuxB-to-B"] as const +const isolation = { checkedPorts: Object.values(ports), checkedNames: Object.values(chrTransitNames(runId)), + checkedCidrs: ["198.18.240.1/32", "198.18.240.2/32", "198.18.241.0/30", "198.18.241.4/30", "198.18.241.8/30", "198.18.242.0/29"], + occupiedPorts: [], overlappingCidrs: [], occupiedNames: [], + policies: directions.map(direction => ({ direction, verdict: "allow", nat: "none", fastTrack: "none", ambiguousRules: 0 } as const)) } +import { createChrParentAckMailbox, createChrParentTrustedStages } from "./traffic-flow-chr-parent-stages.js" +import type { ChrWriteIntent } from "./traffic-flow-chr-write-admission.js" + +const menus = { "/interface/wireguard": "wireguard", "/interface/wireguard/peers": "peer", "/ip/address": "address", "/ip/route": "route" } as const +async function fixture(fault?: "foreign" | "ack" | "singleton" | "target" | "authority") { + const makeRows = () => ({ wireguard: [] as Row[], peer: [] as Row[], address: [] as Row[], route: [] as Row[] }) + const ipfix = Object.fromEntries(["bytes", "packets", "src-address", "dst-address", "protocol", "src-port", "dst-port", "in-interface", "out-interface"].map(key => [key, "no"])) + const rows = { A: makeRows(), B: makeRows() } + const makeFlow = (side: "A" | "B") => ({ + deviceId: side, flow: { ...(side === "B" ? { ".id": "*F" } : {}), enabled: "no", interfaces: "all", + "inactive-flow-timeout": "15s" }, ipfix: { ...ipfix }, targets: [] as Row[], + }) + const flow = { A: makeFlow("A"), B: makeFlow("B") } + function router(side: "A" | "B", endpoint: string) { + return { deviceId: side, endpoint, read: async () => ({ deviceId: side, complete: true as const, rows: structuredClone(rows[side]) }), + readFlow: async () => structuredClone(flow[side]) } + } + let count = 0 + const writes: ChrWriteIntent[] = [] + const result = await createChrParentTrustedStages({ + scope: [runId, ports, isolation], deadlineAt: 100000, now: () => 10000, + timerProfile: { kind: "bounded-ipfix-timeout-20s", captureDeadlineAt: 99000, now: () => 10000 }, + routers: { A: router("A", "192.0.2.1"), B: router("B", "192.0.2.2") }, + linux: { endpoint: "192.0.2.3", publicKeys: { linuxA: "X".repeat(43) + "=", linuxB: "Y".repeat(43) + "=" } }, + authorize: async () => {}, assertOutsideScope: async () => {}, + async assertStageAuthority(intent) { + if (fault === "authority" && intent.step === "flow:A:ipfix.enable") throw Error("Owner phase refused") + }, + async takeAcknowledgement(intent) { + const recorded = writes.at(-1)! + const id = "*" + (++count).toString(16) + const slash = recorded.path.lastIndexOf("/") + const menuPath = recorded.method === "PUT" ? recorded.path : recorded.path.slice(0, slash) + const menu = menus[menuPath as keyof typeof menus] + if (menu) { + if (recorded.method === "PUT") { + rows[recorded.side][menu].push({ ...recorded.body, ".id": id, ...(menu === "wireguard" ? { "public-key": String.fromCharCode(64 + count).repeat(43) + "=" } : {}) }) + if (fault === "foreign") rows.B.address.push({ ".id": "*FF" }) + } else rows[recorded.side][menu] = rows[recorded.side][menu].filter(row => row[".id"] !== decodeURIComponent(recorded.path.slice(slash + 1))) + } else if (menuPath === "/ip/traffic-flow/target") { + if (recorded.method === "PUT") { + flow[recorded.side].targets.push({ ".id": id, disabled: false, ...recorded.body, + ...(fault === "target" ? { "v9-template-timeout": "21s" } : {}) }) + } else flow[recorded.side].targets = [] + } else { + Object.assign(menuPath.endsWith("/ipfix") ? flow[recorded.side].ipfix : flow[recorded.side].flow, recorded.body) + if (fault === "singleton") flow[recorded.side].flow.interfaces = "foreign" + } + return { intent: { ...intent, ...(fault === "ack" ? { side: "B" } : {}) }, + status: recorded.method === "PUT" ? 201 : 204, body: recorded.method === "PUT" ? { ".id": id } : undefined } + }, + }) + return { ...result, rows, flow, writes, async step(index: number) { + const stage = result.stages[index] + const intent = await stage.intent() + writes.push(intent) + await stage.verify(intent) + } } +} + +test("parent resolves exact 48 stages from real proof owners and restores both baselines", async () => { + const f = await fixture() + assert.equal(f.stages.length, 48) + for (let i = 0; i < f.stages.length; i++) await f.step(i) + assert(f.writes.some(write => write.step === "flow:B:flow.enable" && write.method === "PATCH" && write.path === "/ip/traffic-flow/*F")) + assert(f.writes.some(write => write.step === "flow:A:flow.enable" && write.method === "POST" && write.path === "/ip/traffic-flow/set")) + assert(f.writes.some(write => write.step === "target:A:create" && write.body["v9-template-timeout"] === "20s")) + assert.deepEqual(Object.values(f.rows).flatMap(side => Object.values(side).flat()), []) + assert.equal(f.flow.B.flow.enabled, "no") +}) + +test("parent refuses ACK identity, peer drift, singleton drift and wrong target timer without releasing next stage", async () => { + for (const [fault, failedAt] of [["ack", 0], ["foreign", 0], ["singleton", 18], ["target", 20]] as const) { + const f = await fixture(fault) + for (let i = 0; i < failedAt; i++) await f.step(i) + await assert.rejects(f.step(failedAt)) + await assert.rejects(f.stages[failedAt + 1].intent()) + await assert.rejects(f.stages[failedAt].intent()) + assert.equal(f.writes.length, failedAt + 1) + } +}) + + +test("outer relay ACK mailbox allows one exact take and latches missing or duplicate data", async () => { + const f = await fixture() + const intent = await f.stages[0].intent() + const ack = { intent, status: 201, body: { ".id": "*A" } } + f.retire() + const mailbox = createChrParentAckMailbox() + mailbox.record(ack) + assert.deepEqual(await mailbox.take(intent), ack) + await assert.rejects(mailbox.take(intent)) + assert.throws(() => mailbox.record(ack)) + const duplicate = createChrParentAckMailbox() + duplicate.record(ack) + assert.throws(() => duplicate.record(ack)) + await assert.rejects(duplicate.take(intent)) +}) + + +test("parent readiness refusal blocks the export phase before any singleton or target grant", async () => { + const f = await fixture("authority") + for (let i = 0; i < 18; i++) await f.step(i) + await assert.rejects(f.stages[18].intent()) + await assert.rejects(f.stages[19].intent()) + assert.equal(f.writes.length, 18) +}) diff --git a/backend/src/test/traffic-flow-chr-parent-stages.ts b/backend/src/test/traffic-flow-chr-parent-stages.ts new file mode 100644 index 00000000..504a6c63 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-parent-stages.ts @@ -0,0 +1,280 @@ +import { isDeepStrictEqual } from "node:util" +import { createDisabledChrFlowHarness } from "./traffic-flow-chr-disabled-baseline.js" +import { assertChrTargetTimerWindow, createChrTargetHarness, type ChrTargetTimerProfile, type ChrStagedTargetTimerProfile } from "./traffic-flow-chr-target-harness.js" +import { createChrTransitTopologyOwner } from "./traffic-flow-chr-topology-owner.js" +import { planChrTransitFixture } from "./traffic-flow-chr-transit-fixture.js" +import { chrWriteScheduleLimit, type ChrWriteAdmission, type ChrWriteIntent } from "./traffic-flow-chr-write-admission.js" +import type { serveChrParentAdmissionPipe } from "./traffic-flow-chr-write-admission-ipc.js" + +type Topology = Parameters[0] +type Flow = Parameters[0] +type Side = "A" | "B" +type Stage = Parameters[2][number] +export type ChrParentNativeAck = { intent: ChrWriteIntent; status: number; body: unknown } +type Options = Pick & { + routers: Record & { readFlow: Flow["read"] }> + timerProfile?: ChrTargetTimerProfile + stagedTarget?: { captureTimer(): Promise } + /** Assert the parent-owned phase permits this stage (for example, isolated transit ready before export). */ + assertStageAuthority(intent: Readonly): Promise + takeAcknowledgement(intent: Readonly): Promise +} + +const roles = ["edgeA", "middleA", "middleB", "edgeB"] as const +const sides = ["A", "B"] as const +const exportFields = ["bytes", "packets", "src-address", "dst-address", "protocol", "src-port", "dst-port", "in-interface", "out-interface"] +const disabled = (value: unknown) => value === false || value === "false" || value === "no" +const refuse = (): never => { throw Error("Parent native stage refused. Automatic replay forbidden.") } +const require = (value: unknown) => { if (!value) refuse() } + +function deferred() { + let resolve!: (value: T) => void, reject!: (error: unknown) => void + const promise = new Promise((yes, no) => { resolve = yes; reject = no }) + void promise.catch(() => {}) + return { promise, resolve, reject } +} + +/** The parent executes the same bounded proof owners using independent native reads. + * Its synthetic transport cannot send a write and can only consume the outer relay ACK. */ +export async function createChrParentTrustedStages(options: Options) { + const scope = structuredClone(options.scope) + const plan = planChrTransitFixture(...scope) + const routers = { A: { ...options.routers.A }, B: { ...options.routers.B } } + const { deadlineAt, now = Date.now, authorize, takeAcknowledgement, assertOutsideScope, assertStageAuthority } = options + const timerProfile = options.timerProfile && { ...options.timerProfile } + const stagedTarget = options.stagedTarget && { ...options.stagedTarget } + require(!stagedTarget || !timerProfile) + const profile = stagedTarget ? "staged-disabled-target" : "direct-target" + let failed = false, cursor = 0 + const live = () => require(!failed && Number.isSafeInteger(deadlineAt) && now() < deadlineAt) + const readFlow = async (side: Side) => { + live() + await authorize() + const value = structuredClone(await routers[side].readFlow()) + await authorize() + live() + require(value.deviceId === routers[side].deviceId && value.flow && value.ipfix && + Array.isArray(value.targets) && value.targets.length <= 1 && Buffer.byteLength(JSON.stringify(value)) <= 65536) + return value + } + const baseline = { A: await readFlow("A"), B: await readFlow("B") } + if (timerProfile) { + require(timerProfile.captureDeadlineAt <= deadlineAt) + for (const side of sides) assertChrTargetTimerWindow(timerProfile, baseline[side].flow["inactive-flow-timeout"]) + } + const expected = structuredClone(baseline) + const hasIpfixChange = (side: Side) => exportFields.some(field => disabled(baseline[side].ipfix[field])) + const keys = [...roles, ...roles.map(role => "address:" + role), ...roles.map(role => "peer:" + role), + ...plan.routes.filter(route => route.node === "A" || route.node === "B").map((route, i) => [route.node, "route:" + i].join(":"))] + const topologyStep = (action: string, key: string) => { + const side = key.startsWith("A:route:") || key.startsWith("B:route:") ? key[0] : key.at(-1) + return "topology:" + action + ":" + side + ":" + key.replace(/^[AB]:(?=route:)/, "") + } + const labels = keys.map(key => topologyStep("create", key)) + for (const side of sides) { + if (hasIpfixChange(side)) labels.push("flow:" + side + ":ipfix.enable") + labels.push("flow:" + side + ":flow.enable", "target:" + side + ":create") + } + if (stagedTarget) for (const side of sides) labels.push("target:" + side + ":enable") + for (const side of [...sides].reverse()) { + labels.push("target:" + side + ":delete", "flow:" + side + ":flow.restore") + if (hasIpfixChange(side)) labels.push("flow:" + side + ":ipfix.restore") + } + labels.push(...[...keys].reverse().map(key => topologyStep("delete", key))) + require(keys.length === 18 && labels.length <= chrWriteScheduleLimit(profile) && new Set(labels).size === labels.length) + type Slot = { + intent: ChrWriteIntent + acknowledgement: ReturnType> + verified: ReturnType> + requested: boolean + supplied: boolean + } + const ready = labels.map(() => deferred()) + const requested = labels.map(() => deferred()) + let slot: Slot | undefined + let timer: ReturnType | undefined + let launched = false + const abort = (error: unknown) => { + failed = true + clearTimeout(timer) + slot?.acknowledgement.reject(error) + slot?.verified.reject(error) + for (const entry of ready) entry.reject(error) + for (const entry of requested) entry.reject(error) + } + const expectFlow = async (side: Side) => require(isDeepStrictEqual(await readFlow(side), expected[side])) + const otherSide = (side: Side) => side === "A" ? "B" : "A" + const conserveFlow = async (intent: ChrWriteIntent) => { + await expectFlow(otherSide(intent.side)) + if (intent.step.startsWith("topology:")) await expectFlow(intent.side) + } + + const broker: ChrWriteAdmission = { + async run(intent: ChrWriteIntent, send: (arm: () => Promise) => Promise): Promise { + live() + require(!slot && labels[cursor] === intent.step) + await requested[cursor].promise + live() + slot = { intent: structuredClone(intent), acknowledgement: deferred(), verified: deferred(), requested: false, supplied: false } + return send(async () => { + live() + require(slot?.intent.step === intent.step) + await assertStageAuthority(structuredClone(intent)) + await conserveFlow(intent) + if (!intent.step.startsWith("topology:")) await topology.assertCurrent() + live() + ready[cursor].resolve(slot!) + }) + }, + } + const nativeFor = () => { + const simulate = async (method: ChrWriteIntent["method"], path: string, body: unknown, before?: () => Promise) => { + const entry = slot + require(entry && before && isDeepStrictEqual({ method, path, body }, { + method: entry?.intent.method, path: entry?.intent.path, body: entry?.intent.body, + })) + await before!() + return await entry!.acknowledgement.promise as T + } + return { + put: (path: string, body: unknown, _timeout?: number, before?: () => Promise) => simulate("PUT", path, body, before), + patch: (path: string, body: unknown, _timeout?: number, before?: () => Promise) => simulate("PATCH", path, body, before), + post: (path: string, body: unknown, _timeout?: number, _headers?: unknown, before?: () => Promise) => simulate("POST", path, body, before), + delete: async (path: string, _timeout?: number, before?: () => Promise) => { await simulate("DELETE", path, {}, before) }, + } + } + const checkpoint = async (entry: { phase: string }) => { + if (entry.phase !== "verified") return + const verified = slot + require(verified?.supplied) + const intent = verified!.intent + await conserveFlow(intent) + if (!intent.step.startsWith("topology:")) { + await topology.assertCurrent() + // The harness already performed exact local readback. See checkedRead below. + require(nextFlow && nextFlow.side === intent.side) + expected[intent.side] = nextFlow!.value + nextFlow = undefined + } + slot = undefined + cursor++ + verified!.verified.resolve() + } + let nextFlow: { side: Side; value: Flow["baseline"] } | undefined + const checkedRead = async (side: Side) => { + const value = await readFlow(side) + if (slot?.supplied) nextFlow = { side, value } + return value + } + const native = nativeFor() + const topology = createChrTransitTopologyOwner({ + scope, linux: options.linux, deadlineAt, now, authorize, assertOutsideScope, + routers: { A: { ...routers.A, native }, B: { ...routers.B, native } }, + writeAdmission: broker, checkpoint, + assertFlowRestored: async () => { await expectFlow("A"); await expectFlow("B"); require(isDeepStrictEqual(expected, baseline)) }, + }) + const flows = Object.fromEntries(sides.map(side => [side, createDisabledChrFlowHarness({ + baseline: baseline[side], transitInterface: "nf" + plan.runId + (side === "A" ? "ea" : "eb"), + authorize, read: () => checkedRead(side), native, checkpoint, relayWrite: { side, admission: broker }, + })])) as Record> + + const workflow = async () => { + await topology.acquire() + const targets: Partial>> = {} + for (const side of sides) { + const active = await flows[side].enable() + const targetOptions = { + baseline: active, source: side === "A" ? plan.address.exportA : plan.address.exportB, + destination: plan.address.collector, port: plan.ports.collector, + authorize, read: () => checkedRead(side), native, checkpoint, relayWrite: { side, admission: broker }, + } + const target = stagedTarget + ? createChrTargetHarness({ ...targetOptions, stagedDisabled: true }) + : createChrTargetHarness({ ...targetOptions, timerProfile }) + await target.create() + targets[side] = target + } + if (stagedTarget) for (const side of sides) { + await requested[cursor].promise + live() + const timer = await stagedTarget.captureTimer() + require(timer.kind === "staged-ipfix-timeout-20s" && timer.captureDeadlineAt <= deadlineAt) + await targets[side]!.enable(timer) + } + for (const side of [...sides].reverse()) { + await targets[side]!.remove() + await flows[side].restore() + } + await topology.restore() + await expectFlow("A") + await expectFlow("B") + require(cursor === labels.length && isDeepStrictEqual(expected, baseline)) + } + let completed: Promise | undefined + const retire = () => abort(Error("Parent native stage retired")) + const stages: Stage[] = labels.map((step, index) => Object.freeze({ + step, + async intent() { + try { + live() + require(cursor === index) + if (!launched) { + launched = true + timer = setTimeout(retire, Math.max(1, deadlineAt - now())) + completed = workflow() + void completed.then(() => clearTimeout(timer), abort) + } + requested[index].resolve() + const entry = await ready[index].promise + live() + require(slot === entry && !entry.requested) + entry.requested = true + return structuredClone(entry.intent) + } catch (error) { abort(error); throw error } + }, + async verify(intent: Readonly) { + try { + live() + const entry = slot + require(cursor === index && entry?.requested && !entry.supplied && isDeepStrictEqual(entry.intent, intent)) + const ack = await takeAcknowledgement(intent) + live() + require(ack && isDeepStrictEqual(ack.intent, intent) && + (intent.method === "PUT" ? [200, 201].includes(ack.status) : + intent.step === "target:" + intent.side + ":enable" ? ack.status === 200 : [200, 204].includes(ack.status))) + entry!.supplied = true + entry!.acknowledgement.resolve(structuredClone(ack.body)) + await entry!.verified.promise + if (index === labels.length - 1) await completed + live() + } catch (error) { abort(error); throw error } + }, + })) + return Object.freeze({ profile, stages: Object.freeze(stages), retire }) +} + + +/** Connect directly to the owned parent relay. Each response is disposable and + * usable once. Only exact consumed gate intent and projected ACK travel here. */ +export function createChrParentAckMailbox() { + let pending: ChrParentNativeAck | undefined, retired = false + return Object.freeze({ + record(ack: ChrParentNativeAck) { + if (retired || pending) { retired = true; pending = undefined; return refuse() } + pending = structuredClone(ack) + }, + async take(intent: Readonly) { + const ack = pending + pending = undefined + if (retired || !ack || !isDeepStrictEqual(ack.intent, intent)) { retired = true; return refuse() } + return ack + }, + retire() { retired = true; pending = undefined }, + }) +} + + + + + + diff --git a/backend/src/test/traffic-flow-chr-parent-synthetic.ts b/backend/src/test/traffic-flow-chr-parent-synthetic.ts new file mode 100644 index 00000000..07e0a324 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-parent-synthetic.ts @@ -0,0 +1,237 @@ +import assert from "node:assert/strict" +import { Duplex, PassThrough } from "node:stream" +import { serveChrParentNativeBoundary } from "./traffic-flow-chr-parent-native-boundary.js" +import { chrTransitNames } from "./traffic-flow-chr-transit-fixture.js" + + +const runId = "a012bcde" +const ports = { linuxA: 50101, edgeA: 50102, middleA: 50103, middleB: 50104, edgeB: 50105, linuxB: 50106, + collector: 50107, calibrationSource: 50112, calibrationDestination: 50113, phase1Source: 50108, phase1Destination: 50109, phase2Source: 50110, phase2Destination: 50111 } +const directions = ["A-to-B", "B-to-A", "export-A", "export-B", "underlay-linuxA-to-A", "underlay-A-to-linuxA", + "underlay-A-to-B", "underlay-B-to-A", "underlay-B-to-linuxB", "underlay-linuxB-to-B"] as const +const isolation = { checkedPorts: Object.values(ports), checkedNames: Object.values(chrTransitNames(runId)), + checkedCidrs: ["198.18.240.1/32", "198.18.240.2/32", "198.18.241.0/30", "198.18.241.4/30", "198.18.241.8/30", "198.18.242.0/29"], + occupiedPorts: [], overlappingCidrs: [], occupiedNames: [], + policies: directions.map(direction => ({ direction, verdict: "allow", nat: "none", fastTrack: "none", ambiguousRules: 0 } as const)) } +import { createChrParentOrchestration } from "./traffic-flow-chr-parent-orchestration.js" +import { createChrChildAdmissionPipe } from "./traffic-flow-chr-write-admission-ipc.js" +import { parseChrVrfPrefix } from "./chr-vrf-companion-address-space.js" +import type { ChrWriteIntent } from "./traffic-flow-chr-write-admission.js" + +const admittedMenus = ["/interface/wireguard", "/ip/address", "/interface/wireguard/peers", "/ip/route", "/ip/traffic-flow/target"] +const roles = ["edgeA", "middleA", "middleB", "edgeB"] +const roleSuffix: Record = { edgeA: "ea", middleA: "ma", middleB: "mb", edgeB: "eb" } +const names = Object.fromEntries(roles.map(role => [role, "nf" + runId + roleSuffix[role]])) +const marker = "netflow-" + runId +const exportFields = ["bytes", "packets", "src-address", "dst-address", "protocol", "src-port", "dst-port", "in-interface", "out-interface"] +const singleton = "/ip/traffic-flow", ipfix = singleton + "/ipfix", target = singleton + "/target" +type Row = Record +type Side = "A" | "B" + +function state(side: Side) { + return { + [singleton]: [{ ...(side === "B" ? { ".id": "*F" } : {}), enabled: "no", interfaces: "all", "inactive-flow-timeout": "15s" }], + [ipfix]: [Object.fromEntries(exportFields.map(key => [key, "no"]))], + "/interface": [{ ".id": "*F0", name: "ether1", type: "ether" }], + } as Record +} + +export async function runChrParentSynthetic(mode: "direct" | "staged" | "wrong-ack", density: "empty" | "dense" = "empty") { + const staged = mode !== "direct" + const phaseCalls: string[] = [] + const rows = { A: state("A"), B: state("B") }, baseline = structuredClone(rows) + const stressPaths = new Set(["/ip/firewall/filter", "/ip/firewall/nat", "/ip/firewall/raw", "/ip/firewall/mangle", + "/ipv6/firewall/filter", "/ipv6/firewall/nat", "/ipv6/firewall/raw", "/ipv6/firewall/mangle"]) + const denseBody = density === "dense" ? JSON.stringify(Array.from({ length: 700 }, (_, i) => ({ + ".id": "*" + (i + 4096).toString(16), chain: "forward", action: "accept", disabled: "true", + comment: "synthetic-stress-" + i + "-" + "x".repeat(1150), + }))) : "" + const catalogueFixtureBytesPerSide = Buffer.byteLength(denseBody) * stressPaths.size + const now = Date.now() + let writes = 0, staging = false, ready = false, drained = false, inverse = false + const reads: Record = { A: 0, B: 0 } + const makeRouter = (side: Side, endpoint: string) => ({ + endpoint, deviceId: side, + client: { async get(path: string, _timeout?: number, _signal?: AbortSignal, _max?: number, before?: () => Promise) { + await before?.() + reads[side]++ + if (denseBody && stressPaths.has(path)) return JSON.parse(denseBody) as T + return JSON.parse(JSON.stringify(rows[side][path] ?? [])) as T + } }, + }) + const orchestration = await createChrParentOrchestration({ + scope: [runId, ports, isolation], deadlineAt: now + (density === "dense" ? 580000 : 100000), + linux: { endpoint: "192.0.2.3", publicKeys: { linuxA: "X".repeat(43) + "=", linuxB: "Y".repeat(43) + "=" } }, + routers: { A: makeRouter("A", "192.0.2.1"), B: makeRouter("B", "192.0.2.2") }, + ...(staged ? { stagedTarget: { async captureTimer() { + assert(ready) + return { kind: "staged-ipfix-timeout-20s" as const, captureDeadlineAt: Math.min(Date.now() + 180000, now + (density === "dense" ? 580000 : 100000)) } + } } } : { timerProfile: { kind: "bounded-ipfix-timeout-20s" as const, captureDeadlineAt: now + 89000 } }), + assertTransitAndCollectorStagingReady: async () => { phaseCalls.push("staging"); assert(staging && !ready) }, + authorize: async () => {}, assertTimingPreflight: async () => { phaseCalls.push("preflight") }, assertRouterIdentity: async (side, id) => assert.equal(side, id), + assertTransitAndCollectorReady: async () => { phaseCalls.push("ready"); assert(ready) }, + assertCaptureAndDrainComplete: async () => { phaseCalls.push("drained"); assert(drained) }, + assertLinuxInverseReady: async () => { phaseCalls.push("inverse"); assert(inverse) }, + }) + const toParent = new PassThrough(), toChild = new PassThrough() + const done = orchestration.serve({ input: toParent, output: toChild }) + void done.catch(() => {}) + const child = createChrChildAdmissionPipe({ input: toChild, output: toParent }, staged ? "staged-disabled-target" : "direct-target") + const nativeInput = new PassThrough(), nativeOutput = new PassThrough() + const nativeStream = Duplex.from({ readable: nativeInput, writable: nativeOutput }) + const physicalBoundary = serveChrParentNativeBoundary(nativeStream, orchestration, async () => {}) + void physicalBoundary.catch(() => { nativeStream.destroy(); nativeOutput.destroy() }) + let nativeSequence = 0 + const physicalExchange = async (data: object) => { + const sequence = ++nativeSequence + await new Promise((resolve, reject) => { + const receive = (reply: Buffer) => { + nativeOutput.off("close", closed) + try { assert.deepEqual(JSON.parse(reply.toString()), { sequence, ok: true }); resolve() } + catch (error) { reject(error) } + } + const closed = () => { nativeOutput.off("data", receive); reject(Error("Synthetic native boundary closed")) } + nativeOutput.once("data", receive) + nativeOutput.once("close", closed) + nativeInput.write(JSON.stringify({ sequence, ...data }) + "\n") + }) + } + const owned = new Map() + const execute = async (intent: ChrWriteIntent) => { + await child.run(intent, async arm => { + await arm() + const projection = { side: intent.side, method: intent.method, path: intent.path } + await physicalExchange({ op: "consume", ...projection, body: intent.body }) + const catalogue = rows[intent.side] + const id = "*" + (++writes).toString(16) + const slash = intent.path.lastIndexOf("/") + const menu = intent.method === "PUT" ? intent.path : intent.path.slice(0, slash) + const list = catalogue[menu] ??= [] + let body: unknown + if (intent.method === "PUT") { + assert(admittedMenus.includes(menu)) + const row: Row = { ...intent.body, ".id": id } + if (menu === "/interface/wireguard") { + row["public-key"] = String.fromCharCode(64 + writes).repeat(43) + "=" + catalogue["/interface"].push({ ".id": id, name: row.name, type: "wireguard", comment: marker, disabled: false }) + } + if (menu === "/ip/address") { + row.dynamic = false + row.vrf = "main" + const prefix = parseChrVrfPrefix(row.address, 4)! + const ip = [24n, 16n, 8n, 0n].map(shift => Number((prefix.first >> shift) & 255n)).join(".") + const route = { ".id": "*a" + id.slice(1), "dst-address": ip + "/" + prefix.prefixLength, gateway: row.interface, + "routing-table": "main", connect: true, distance: "0", scope: "10", "target-scope": "5" } + ;(catalogue["/ip/route"] ??= []).push({ ...route, dynamic: true }) + ;(catalogue["/routing/route"] ??= []).push({ ...route, afi: "ip4", "belongs-to": "connected", + "local-address": String(row.address).split("/")[0] + "%" + row.interface }) + } + if (menu === target) row.disabled = staged ? "yes" : false + list.push(row) + owned.set(intent.step, { intent, id }) + body = { ".id": id } + } else if (intent.method === "DELETE") { + const previous = decodeURIComponent(intent.path.slice(slash + 1)) + assert.equal(list.filter(row => row[".id"] === previous).length, 1) + catalogue[menu] = list.filter(row => row[".id"] !== previous) + if (menu === "/interface/wireguard") catalogue["/interface"] = catalogue["/interface"].filter(row => row[".id"] !== previous) + if (menu === "/ip/address") for (const path of ["/ip/route", "/routing/route"]) { + catalogue[path] = catalogue[path].filter(row => row[".id"] !== "*a" + previous.slice(1)) + } + } else if (menu === target) { + const previous = decodeURIComponent(intent.path.slice(slash + 1)) + const matching = list.filter(row => row[".id"] === previous) + assert.equal(matching.length, 1) + assert.equal(matching[0].disabled, "yes") + Object.assign(matching[0], intent.body) + body = { ".id": mode === "wrong-ack" ? "*FFFF" : previous } + } else Object.assign(list[0], intent.body) + await physicalExchange({ op: "ack", ...projection, status: intent.method === "PUT" ? 201 : body ? 200 : 204, + ...(body && typeof body === "object" ? { id: (body as Row)[".id"] } : {}) }) + }) + } + const send = (step: string, side: Side, method: ChrWriteIntent["method"], path: string, body: ChrWriteIntent["body"]) => + execute({ step, side, deviceId: side, method, path, body }) + const create = (key: string, side: Side, path: string, body: Record) => + send("topology:create:" + side + ":" + key, side, "PUT", path, { ...body, disabled: "false", comment: marker }) + const addr: Record = { edgeA: "198.18.241.2/30", middleA: "198.18.241.5/30", middleB: "198.18.241.6/30", edgeB: "198.18.241.9/30" } + const sideOf = (role: string): Side => role.endsWith("A") ? "A" : "B" + const portOf = (role: string) => String(ports[role as keyof typeof ports]) + try { + for (const role of roles) await create(role, sideOf(role), "/interface/wireguard", { name: names[role], "listen-port": portOf(role), mtu: "1420" }) + for (const role of roles) await create("address:" + role, sideOf(role), "/ip/address", { interface: names[role], address: addr[role] }) + const peerSpecs = [ + ["edgeA", "linuxA", "192.0.2.3", "X", "198.18.240.1/32,198.18.242.3/32"], + ["middleA", "middleB", "192.0.2.2", "C", "198.18.240.2/32"], + ["middleB", "middleA", "192.0.2.1", "B", "198.18.240.1/32"], + ["edgeB", "linuxB", "192.0.2.3", "Y", "198.18.240.2/32,198.18.242.3/32"], + ] + for (const [role, opposite, endpoint, key, allowed] of peerSpecs) await create("peer:" + role, sideOf(role), "/interface/wireguard/peers", { + interface: names[role], "public-key": key.repeat(43) + "=", "allowed-address": allowed, + "endpoint-address": endpoint, "endpoint-port": portOf(opposite), "persistent-keepalive": "10", + }) + const routes = [ + ["edgeA", "198.18.240.1", "198.18.241.1"], ["middleA", "198.18.240.2", "198.18.241.6"], ["edgeA", "198.18.242.3", "198.18.241.1"], + ["edgeB", "198.18.240.2", "198.18.241.10"], ["middleB", "198.18.240.1", "198.18.241.5"], ["edgeB", "198.18.242.3", "198.18.241.10"], + ] + for (const [i, [role, destination, gateway]] of routes.entries()) await create("route:" + i, sideOf(role), "/ip/route", { + "dst-address": destination + "/32", gateway: gateway + "%" + names[role], "routing-table": "main", + }) + staging = staged + ready = !staged + const changeIpfix = (side: Side, enable: boolean) => send("flow:" + side + ":ipfix." + (enable ? "enable" : "restore"), + side, "POST", ipfix + "/set", Object.fromEntries(exportFields.map(key => [key, enable ? "yes" : "no"]))) + const changeFlow = (side: Side, enable: boolean) => send("flow:" + side + ":flow." + (enable ? "enable" : "restore"), + side, side === "B" ? "PATCH" : "POST", singleton + (side === "B" ? "/*F" : "/set"), + { enabled: enable ? "yes" : "no", interfaces: enable ? names["edge" + side] : "all" }) + for (const side of ["A", "B"] as const) { + await changeIpfix(side, true) + await changeFlow(side, true) + await send("target:" + side + ":create", side, "PUT", target, { "dst-address": "198.18.242.3", + "src-address": side === "A" ? "198.18.241.2" : "198.18.241.9", port: portOf("collector"), version: "ipfix", "v9-template-timeout": "20s", ...(staged ? { disabled: "yes" } : {}) }) + } + if (staged) { + ready = true + for (const side of ["A", "B"] as const) { + const enabling = send("target:" + side + ":enable", side, "PATCH", target + "/" + owned.get("target:" + side + ":create")!.id, { disabled: "no" }) + if (mode === "wrong-ack") { + await assert.rejects(enabling) + await assert.rejects(done) + assert.equal(writes, 25) + assert.equal(rows.B[target][0].disabled, "yes") + return { completed: 0, writes, reads, catalogueFixtureBytesPerSide, nativeFrames: nativeSequence } + } + await enabling + } + } + drained = true + for (const side of ["B", "A"] as const) { + await send("target:" + side + ":delete", side, "DELETE", target + "/" + owned.get("target:" + side + ":create")!.id, {}) + await changeFlow(side, false) + await changeIpfix(side, false) + } + inverse = true + for (const { intent, id } of [...owned.values()].reverse().filter(value => value.intent.step.startsWith("topology:"))) { + await send(intent.step.replace(":create:", ":delete:"), intent.side, "DELETE", intent.path + "/" + id, {}) + } + toParent.end() + assert.deepEqual(await done, { completed: staged ? 50 : 48 }) + assert.equal(writes, staged ? 50 : 48) + assert(reads.A > 100 && reads.B > 100) + assert.deepEqual([...new Set(phaseCalls)], ["preflight", ...(staged ? ["staging"] : []), "ready", "drained", "inverse"]) + for (const side of ["A", "B"] as const) { + for (const [path, records] of Object.entries(rows[side])) assert.deepEqual(records, baseline[side][path] ?? []) + } + assert.equal(nativeSequence, writes * 2) + return { completed: writes, writes, reads, catalogueFixtureBytesPerSide, nativeFrames: nativeSequence } + } finally { + nativeStream.destroy() + nativeInput.destroy() + nativeOutput.destroy() + toParent.destroy() + toChild.destroy() + orchestration.retire() + } +} + + diff --git a/backend/src/test/traffic-flow-chr-readiness.test.ts b/backend/src/test/traffic-flow-chr-readiness.test.ts new file mode 100644 index 00000000..a7dae7d8 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-readiness.test.ts @@ -0,0 +1,72 @@ +import assert from "node:assert/strict" +import { createHash } from "node:crypto" +import { test } from "node:test" +import { inspectChrReadiness } from "./traffic-flow-chr-readiness.js" + +const now = 1700000005000 +function fixture() { + const devices = [1, 2].map(serverId => ({ + managementAddress: "198.51.100." + serverId, deviceId: "fixture-" + serverId, + version: "7.24.5 (stable)", exporterSource: "192.0.2." + serverId, serverId, + })) + const pins = { vpsHostKeySha256: "a".repeat(64), collectorInstance: "owned-backend", collectorAddress: "192.0.2.100", + interfaceName: "test0", testPort: 4739, baselinePort: 4740, devices } + const evidence = { observedAt: now - 1, vpsHostKeySha256: pins.vpsHostKeySha256, collectorInstance: pins.collectorInstance, + interfaceName: pins.interfaceName, interfaceAddresses: [pins.collectorAddress], collectorPid: 42, + udpOwners: [{ pid: 42, address: pins.collectorAddress, port: pins.testPort }], + worker: { alive: true, bound: true, address: "192.0.2.100:4739" }, + settings: { enabled: true, collectorIp: pins.collectorAddress, flowListenPort: pins.testPort, + listenerBound: true, listenerAddress: "192.0.2.100:4739" }, devices: structuredClone(devices) } + return { pins, evidence } +} +function capture() { + const bytes = Buffer.alloc(24 + 2 * 60) + bytes.writeUInt32LE(0xa1b2c3d4) + bytes.writeUInt16LE(2, 4); bytes.writeUInt16LE(4, 6) + bytes.writeUInt32LE(4124, 16); bytes.writeUInt32LE(101, 20) + for (const [index, at] of [24, 84].entries()) { + bytes.writeUInt32LE(1700000000, at); bytes.writeUInt32LE(44, at + 8); bytes.writeUInt32LE(44, at + 12) + const ip = bytes.subarray(at + 16, at + 60) + ip[0] = 0x45; ip.writeUInt16BE(44, 2); ip[9] = 17 + Buffer.from([192, 0, 2, index + 1, 192, 0, 2, 100]).copy(ip, 12) + ip.writeUInt16BE(2055, 20); ip.writeUInt16BE(4740, 22); ip.writeUInt16BE(24, 24) + ip.writeUInt16BE(10, 28); ip.writeUInt16BE(16, 30) + } + return { bytes, sha256: createHash("sha256").update(bytes).digest("hex") } +} + +test("pinned identity and owned bind do not fill a missing ingress observation", () => { + const { pins, evidence } = fixture() + const result = inspectChrReadiness(pins, evidence, undefined, now) + assert.equal(result.identityMatched, true) + assert.equal(result.collectorBindMatched, true) + assert.equal(result.ingressStatus, "missing") + assert.equal(result.readyForReview, false) + assert.deepEqual(result.targetScopes.map(row => [row.deviceId, row.fields["src-address"], row.fields.port]), [ + ["fixture-1", "192.0.2.1", "4739"], ["fixture-2", "192.0.2.2", "4739"], + ]) +}) + +test("requires fresh pinned observations, exact listener address and exclusive UDP owner", () => { + for (const mutate of [ + (f: ReturnType) => { f.evidence.devices[0].version = "different" }, + (f: ReturnType) => { f.evidence.observedAt = now - 120001 }, + (f: ReturnType) => { f.evidence.udpOwners[0].pid = 999 }, + (f: ReturnType) => { f.evidence.udpOwners[0].address = "0.0.0.0" }, + (f: ReturnType) => { f.evidence.udpOwners.push({ ...f.evidence.udpOwners[0] }) }, + (f: ReturnType) => { f.evidence.worker.alive = false }, + ]) { + const f = fixture(); mutate(f) + assert.equal(inspectChrReadiness(f.pins, f.evidence, capture(), now).readyForReview, false) + } +}) + +test("two-source scoped baseline pcap is checked separately from the owned test port", () => { + const { pins, evidence } = fixture(), input = capture() + const result = inspectChrReadiness(pins, evidence, input, now) + assert.equal(result.ingressStatus, "matched") + assert.equal(result.readyForReview, true) + assert.equal(result.actualNativeAcceptance, false) + assert.equal(inspectChrReadiness(pins, evidence, { ...input, sha256: "0".repeat(64) }, now).ingressStatus, "refused") + assert.equal(inspectChrReadiness(pins, { ...evidence, observedAt: now + 600001 }, input, now + 600001).ingressStatus, "refused") +}) diff --git a/backend/src/test/traffic-flow-chr-readiness.ts b/backend/src/test/traffic-flow-chr-readiness.ts new file mode 100644 index 00000000..e4a87923 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-readiness.ts @@ -0,0 +1,104 @@ +/** Local preflight report from reviewed VPS/CHR observations and optional pcap. */ +import { createHash } from "node:crypto" +import { open } from "node:fs/promises" +import { isIPv4 } from "node:net" +import { resolve } from "node:path" +import { fileURLToPath } from "node:url" +import { isDeepStrictEqual } from "node:util" +import { z } from "zod" +import { projectChrWire } from "./traffic-flow-chr-wire.js" + +const ip = z.string().max(15).refine(isIPv4) +const port = z.number().int().min(1).max(65535) +const id = z.string().min(1).max(128) +const sha = z.string().regex(/^[a-f0-9]{64}$/) +const device = z.object({ managementAddress: ip, deviceId: id, version: id, exporterSource: ip, serverId: z.number().int().positive().safe() }).strict() +const pinsSchema = z.object({ + vpsHostKeySha256: sha, collectorInstance: id, collectorAddress: ip, interfaceName: z.string().regex(/^[A-Za-z0-9_.-]{1,15}$/), + testPort: port, baselinePort: port, devices: z.tuple([device, device]), +}).strict() +const evidenceSchema = z.object({ + observedAt: z.number().int().nonnegative().safe(), + vpsHostKeySha256: sha, collectorInstance: id, interfaceName: z.string().max(15), + interfaceAddresses: z.array(ip).max(32), + collectorPid: z.number().int().positive().max(0x7fffffff), + udpOwners: z.array(z.object({ pid: z.number().int().positive().max(0x7fffffff), address: ip, port }).strict()).max(16), + worker: z.object({ alive: z.boolean(), bound: z.boolean(), address: z.string().max(32).nullable() }).strict(), + settings: z.object({ enabled: z.boolean(), collectorIp: ip, flowListenPort: port, listenerBound: z.boolean(), listenerAddress: z.string().max(32).nullable() }).strict(), + devices: z.tuple([device, device]), +}).strict() + +type Ingress = { bytes: Buffer; sha256: string } +export function inspectChrReadiness(rawPins: unknown, rawEvidence: unknown, ingress?: Ingress, now = Date.now()) { + const pins = pinsSchema.parse(rawPins), evidence = evidenceSchema.parse(rawEvidence) + const distinct = new Set([...pins.devices.map(row => row.exporterSource), pins.collectorAddress]).size === 3 && + new Set(pins.devices.map(row => row.deviceId)).size === 2 && new Set(pins.devices.map(row => row.managementAddress)).size === 2 && + new Set(pins.devices.map(row => row.serverId)).size === 2 && pins.testPort !== pins.baselinePort + const fresh = evidence.observedAt <= now && now - evidence.observedAt <= 120000 + const identityMatched = distinct && fresh && pins.vpsHostKeySha256 === evidence.vpsHostKeySha256 && + pins.collectorInstance === evidence.collectorInstance && isDeepStrictEqual(pins.devices, evidence.devices) + const endpoint = pins.collectorAddress + ":" + pins.testPort + const owners = evidence.udpOwners.filter(row => row.port === pins.testPort) + const collectorBindMatched = identityMatched && pins.interfaceName === evidence.interfaceName && + evidence.interfaceAddresses.includes(pins.collectorAddress) && owners.length === 1 && + owners[0].pid === evidence.collectorPid && owners[0].address === pins.collectorAddress && + evidence.worker.alive && evidence.worker.bound && evidence.worker.address === endpoint && + evidence.settings.enabled && evidence.settings.collectorIp === pins.collectorAddress && + evidence.settings.flowListenPort === pins.testPort && evidence.settings.listenerBound && evidence.settings.listenerAddress === endpoint + let ingressStatus: "missing" | "refused" | "matched" = ingress ? "refused" : "missing" + if (identityMatched && ingress) { + try { + const wire = projectChrWire(ingress.bytes, { + sources: [pins.devices[0].exporterSource, pins.devices[1].exporterSource], destination: pins.collectorAddress, port: pins.baselinePort, + }) + if (wire.captureSha256 === ingress.sha256 && + wire.frames.every(frame => frame.receivedMs <= now && frame.receivedMs >= now - 600000)) ingressStatus = "matched" + } catch { + // Keep the failed evidence state explicit without including packet data. + } + } + return { + identityMatched, collectorBindMatched, ingressStatus, + readyForReview: collectorBindMatched && ingressStatus === "matched", + targetScopes: pins.devices.map(row => ({ deviceId: row.deviceId, serverId: row.serverId, method: "PUT", path: "/ip/traffic-flow/target", + fields: { "dst-address": pins.collectorAddress, "src-address": row.exporterSource, port: String(pins.testPort), version: "ipfix" } })), + actualNativeAcceptance: false, + } +} + +async function pinnedFile(path: string, digest: string, max: number) { + sha.parse(digest) + const file = await open(path, "r") + try { + const info = await file.stat() + if (!info.isFile() || info.size < 1 || info.size > max) throw Error("Input outside file bound") + const bytes = Buffer.alloc(info.size + 1) + let size = 0 + while (size < bytes.length) { + const { bytesRead } = await file.read(bytes, size, bytes.length - size, size) + if (!bytesRead) break + size += bytesRead + } + const body = bytes.subarray(0, size) + if (size !== info.size || createHash("sha256").update(body).digest("hex") !== digest) throw Error("Input digest differs") + return body + } finally { + await file.close() + } +} + +async function main() { + const args = process.argv.slice(2) + if (args.length !== 4 && args.length !== 6) throw Error("Expected pinned inputs") + const [pins, pinsSha, evidence, evidenceSha, capture, captureSha] = args + const report = inspectChrReadiness( + JSON.parse((await pinnedFile(pins, pinsSha, 16384)).toString("utf8")), + JSON.parse((await pinnedFile(evidence, evidenceSha, 16384)).toString("utf8")), + capture ? { bytes: await pinnedFile(capture, captureSha, 512 * 1024), sha256: captureSha } : undefined, + ) + console.log(JSON.stringify(report)) + if (!report.readyForReview) process.exitCode = 2 +} +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch(() => { console.error("CHR readiness inputs refused"); process.exitCode = 1 }) +} diff --git a/backend/src/test/traffic-flow-chr-route-conntrack.test.ts b/backend/src/test/traffic-flow-chr-route-conntrack.test.ts new file mode 100644 index 00000000..4fced343 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-route-conntrack.test.ts @@ -0,0 +1,112 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { createChrRouteConntrackReader } from "./traffic-flow-chr-route-conntrack.js" + +function fixture() { + const calls: string[] = [] + const catalog: Record[]> = { + "/system/identity": [{ name: "chr-a" }], + "/system/resource": [{ version: "7.24.5 (stable)" }], + "/ip/settings": [{ "ip-forward": "true", "allow-fast-path": "false" }], + "/routing/rule": [], + "/ip/vrf": [{ ".id": "*1", name: "main", interfaces: "all" }], + "/routing/table": [{ ".id": "*1", name: "main", fib: "true" }], + "/interface": [{ ".id": "*2", name: "nfedge", type: "wg", running: "true", disabled: "false" }], + "/ip/address": [{ ".id": "*3", address: "198.18.241.2/30", interface: "nfedge", disabled: "false" }], + "/routing/route": [{ ".id": "*4", "routing-table": "main", "dst-address": "198.18.240.2/32", + gateway: "198.18.241.1", "immediate-gw": "198.18.241.1%nfedge", active: "true", disabled: "false" }], + "/ip/firewall/connection/tracking": [{ enabled: "yes" }], + "/ip/firewall/connection": [], + } + const config = { + identity: "chr-a", version: "7.24.5 (stable)", + routes: [{ routeId: "*4", destination: "198.18.240.2", nextHop: "198.18.241.1", + interfaceName: "nfedge", interfaceId: "*2", addressId: "*3", interfaceCidr: "198.18.241.2/30" }], + udp: [{ source: { address: "198.18.240.1", port: 50001 }, destination: { address: "198.18.240.2", port: 50002 } }], + assertCurrent: async () => {}, + client: { async get(path: string, timeout?: number, _signal?: AbortSignal, bytes?: number, fence?: () => Promise) { + assert.ok(timeout && timeout <= 1800) + assert.equal(bytes, 512 * 1024) + await fence?.() + calls.push(path) + assert.ok(Object.hasOwn(catalog, path)) + return structuredClone(catalog[path]) as T + } }, + } + return { calls, catalog, config, read: createChrRouteConntrackReader(config) } +} +const foreignUdp = { ".id": "*1", protocol: "udp", "src-address": "192.0.2.1:50001", "dst-address": "192.0.2.2:50002", + "reply-src-address": "192.0.2.2:50002", "reply-dst-address": "192.0.2.1:50001" } + +test("owned two-interface profile permits observed FastPath flag and auto only with empty conntrack", async () => { + const { catalog, config } = fixture() + const [route] = config.routes + const edge = "nfa012bcdeea", middle = "nfa012bcdema" + const owned = createChrRouteConntrackReader({ ...config, profile: "owned-wg-empty-firewall", routes: [ + { ...route, interfaceName: edge }, + { ...route, interfaceName: middle, interfaceId: "*5", addressId: "*6", routeId: "*7", + interfaceCidr: "198.18.241.5/30", destination: "198.18.240.1", nextHop: "198.18.241.6" }, + ] }) + catalog["/ip/settings"][0]["allow-fast-path"] = "true" + catalog["/ip/firewall/connection/tracking"][0].enabled = "auto" + catalog["/interface"][0].name = edge + catalog["/interface"].push({ ...catalog["/interface"][0], ".id": "*5", name: middle }) + catalog["/ip/address"][0].interface = edge + catalog["/ip/address"].push({ ...catalog["/ip/address"][0], ".id": "*6", interface: middle, address: "198.18.241.5/30" }) + catalog["/routing/route"][0]["immediate-gw"] = "198.18.241.1%" + edge + catalog["/routing/route"].push({ ...catalog["/routing/route"][0], ".id": "*7", "dst-address": "198.18.240.1/32", + gateway: "198.18.241.6", "immediate-gw": "198.18.241.6%" + middle }) + assert.deepEqual((await owned()).ownedInterfaces, [edge, middle]) + catalog["/ip/firewall/connection"].push(foreignUdp) + await assert.rejects(owned(), /not observed empty/) +}) + +test("reads complete bounded native catalogs with identity fences and stable exact path fingerprint", async () => { + const { calls, catalog, read } = fixture() + const first = await read() + assert.equal(first.routesVerified, true) + assert.equal(first.freshConntrackTuples, true) + assert.match(first.fingerprint, /^[a-f0-9]{64}$/) + assert.deepEqual(calls, [...Object.keys(catalog).slice(0, 5), "/interface", "/ip/address", "/routing/table", + "/routing/route", "/ip/firewall/connection/tracking", "/ip/firewall/connection", "/system/identity", "/system/resource"]) + catalog["/ip/firewall/connection"].push(foreignUdp) + assert.deepEqual(await read(), first) + catalog["/routing/route"][0].distance = "2" + assert.notEqual((await read()).fingerprint, first.fingerprint) +}) + +test("accepts the owned scoped gateway and skips explicitly typed non-IP link routes", async () => { + const { catalog, read } = fixture() + catalog["/routing/route"][0].gateway = "198.18.241.1%nfedge" + catalog["/routing/route"].push({ ".id": "*5", "routing-table": "main", afi: "link", "dst-address": "nfedge" }) + assert.equal((await read()).routesVerified, true) + catalog["/routing/route"][1].afi = "ip" + await assert.rejects(read(), /invalid prefix/) +}) + +test("refuses original or reply tuple reuse and malformed nonmatching UDP rows", async () => { + for (const fields of [ + { "src-address": "198.18.240.1:50001", "dst-address": "198.18.240.2:50002" }, + { "reply-src-address": "198.18.240.2:50002", "reply-dst-address": "198.18.240.1:50001" }, + { "reply-src-address": undefined }, + ]) { + const { catalog, read } = fixture() + catalog["/ip/firewall/connection"].push({ ...foreignUdp, ...fields }) + await assert.rejects(read(), /tracked|incomplete UDP/) + } +}) + +test("refuses competing routes, routing policy, unproven conntrack and changed device identity", async () => { + const competitor = fixture() + competitor.catalog["/routing/route"].push({ ...competitor.catalog["/routing/route"][0], ".id": "*5" }) + await assert.rejects(competitor.read(), /exact active host route/) + const policy = fixture() + policy.catalog["/routing/rule"].push({ ".id": "*1", disabled: "false" }) + await assert.rejects(policy.read(), /active routing policy/) + const tracking = fixture() + tracking.catalog["/ip/firewall/connection/tracking"][0].enabled = "auto" + await assert.rejects(tracking.read(), /not explicitly enabled/) + const identity = fixture() + identity.catalog["/system/identity"][0].name = "foreign" + await assert.rejects(identity.read(), /device identity changed/) +}) diff --git a/backend/src/test/traffic-flow-chr-route-conntrack.ts b/backend/src/test/traffic-flow-chr-route-conntrack.ts new file mode 100644 index 00000000..8e0be358 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-route-conntrack.ts @@ -0,0 +1,160 @@ +import { createHash } from "node:crypto" +import { isIP, isIPv4 } from "node:net" +import { performance } from "node:perf_hooks" +import type { MikrotikClient } from "../services/mikrotik.js" +import { isChrWireGuardInterfaceType } from "./traffic-flow-chr-transit-fixture.js" + +type Row = Record +type Socket = { address: string; port: number } +type UdpTuple = { source: Socket; destination: Socket } +type RoutePin = { routeId: string; destination: string; nextHop: string; interfaceName: string; + interfaceId: string; addressId: string; interfaceCidr: string } +type Options = { + client: Pick + identity: string + version: string + routes: readonly RoutePin[] + udp: readonly UdpTuple[] + assertCurrent: () => Promise + profile?: "owned-wg-empty-firewall" +} + +const deny = (message: string): never => { throw Error("RouterOS route/conntrack proof refused: " + message) } +const id = (value: unknown) => typeof value === "string" && /^\*[a-f0-9]{1,16}$/i.test(value) +const yes = (value: unknown) => value === true || value === "true" || value === "yes" +const no = (value: unknown) => value === false || value === "false" || value === "no" +const port = (value: number) => Number.isInteger(value) && value >= 1 && value <= 65535 +const socketKey = (value: Socket) => value.address + ":" + value.port +const endpointsKey = (a: string, b: string) => [a, b].sort().join("/") +const canonicalRow = (row: Row) => Object.fromEntries(Object.keys(row).sort().map(key => [key, row[key]])) +const ipNumber = (address: string) => address.split(".").reduce((sum, octet) => sum * 256 + Number(octet), 0) + +function prefix(value: unknown) { + if (typeof value !== "string") return deny("invalid prefix") + const [address, length, ...rest] = value.split("/") + const family = isIP(address) + if (!family || rest.length || !/^(0|[1-9][0-9]{0,2})$/.test(length) || Number(length) > (family === 4 ? 32 : 128)) { + return deny("invalid prefix") + } + return { address, length: Number(length), family } +} +function endpoint(value: unknown) { + if (typeof value !== "string") return deny("incomplete UDP conntrack row") + const match = /^([0-9.]+):([1-9][0-9]{0,4})$/.exec(value) + if (!match || !isIPv4(match[1]) || !port(Number(match[2]))) return deny("unsupported UDP conntrack endpoint") + return socketKey({ address: match[1], port: Number(match[2]) }) +} +function sameSubnet(cidr: string, nextHop: string) { + const net = prefix(cidr) + const size = 2 ** (32 - net.length) + return net.family === 4 && net.length < 32 && net.address !== nextHop && + Math.floor(ipNumber(net.address) / size) === Math.floor(ipNumber(nextHop) / size) +} + +/** Narrow main-table reader for an already provisioned fixture. An authoritative + * caller brackets policy evaluation with independent fresh calls to this reader. */ +export function createChrRouteConntrackReader(options: Options) { + const { client, assertCurrent, profile } = options + const { identity, version, routes, udp } = structuredClone({ + identity: options.identity, version: options.version, routes: options.routes, udp: options.udp, + }) + if (!identity || !version || !routes.length || routes.length > 16 || !udp.length || udp.length > 32 || + new Set(routes.map(pin => pin.destination)).size !== routes.length || routes.some(pin => + !id(pin.routeId) || !id(pin.interfaceId) || !id(pin.addressId) || + !isIPv4(pin.destination) || !isIPv4(pin.nextHop) || !/^[\w.-]{1,64}$/.test(pin.interfaceName) || + !sameSubnet(pin.interfaceCidr, pin.nextHop)) || + udp.some(tuple => [tuple.source, tuple.destination].some(value => !isIPv4(value.address) || !port(value.port)))) deny("invalid fixture scope") + const pinnedTuples = new Set(udp.map(tuple => endpointsKey(socketKey(tuple.source), socketKey(tuple.destination)))) + const ownedWg = profile === "owned-wg-empty-firewall" + const ownedInterfaces = [...new Set(routes.map(pin => pin.interfaceName))].sort() + if (ownedWg && (routes.some(pin => !/^nf[a-f0-9]{8}(ea|ma|mb|eb)$/.test(pin.interfaceName)) || + new Set(routes.map(pin => pin.interfaceName.slice(0, -2))).size !== 1 || + !["ea,ma", "eb,mb"].includes(ownedInterfaces.map(name => name.slice(-2)).join(",")) || + new Set(routes.map(pin => pin.interfaceId)).size !== 2)) deny("owned WireGuard interface scope differs") + + return async function readPathEvidence() { + const deadline = performance.now() + 12000 + async function fence() { + if (performance.now() >= deadline) deny("expired evidence") + await assertCurrent() + if (performance.now() >= deadline) deny("expired evidence") + } + async function read(path: string, limit: number, single = false) { + await fence() + const response = await client.get(path, Math.min(1800, Math.ceil(deadline - performance.now())), + undefined, 512 * 1024, fence) + await fence() + const rows = Array.isArray(response) ? response : single ? [response] : null + if (!rows || rows.length > limit || single && rows.length !== 1 || + rows.some(row => !row || typeof row !== "object" || Array.isArray(row))) return deny("incomplete catalog") + return rows as Row[] + } + async function identityFence() { + const [device] = await read("/system/identity", 1, true) + const [resource] = await read("/system/resource", 1, true) + if (device.name !== identity || resource.version !== version) deny("device identity changed") + } + function requireCatalogIds(rows: Row[]) { + if (rows.some(row => !id(row[".id"])) || new Set(rows.map(row => row[".id"])).size !== rows.length) deny("invalid catalog IDs") + return rows + } + await identityFence() + const [ip] = await read("/ip/settings", 1, true) + if (!yes(ip["ip-forward"]) || (!no(ip["allow-fast-path"]) && !(ownedWg && yes(ip["allow-fast-path"])))) { + deny("forwarding or fast-path state unknown") + } + const policy = requireCatalogIds(await read("/routing/rule", 128)) + if (policy.some(row => !yes(row.disabled))) deny("active routing policy") + const vrfs = requireCatalogIds(await read("/ip/vrf", 64)) + if (vrfs.length !== 1 || vrfs[0].name !== "main" || vrfs[0].interfaces !== "all" || yes(vrfs[0].disabled)) deny("non-default VRF policy") + const interfaces = requireCatalogIds(await read("/interface", 256)) + const addresses = requireCatalogIds(await read("/ip/address", 256)) + const main = requireCatalogIds(await read("/routing/table", 64)).filter(row => row.name === "main") + if (main.length !== 1 || !yes(main[0].fib) || yes(main[0].disabled)) deny("main FIB unconfirmed") + const candidates = requireCatalogIds(await read("/routing/route", 2048)) + for (const row of candidates) { + if (typeof row["routing-table"] !== "string" || !row["routing-table"]) deny("incomplete route catalog") + if (row.afi === "link") { + if (typeof row["dst-address"] !== "string" || !/^[\w.-]{1,64}$/.test(row["dst-address"])) deny("invalid link route") + } else prefix(row["dst-address"]) + } + const bindings = routes.map(pin => { + const interfaceMatches = interfaces.filter(row => row.name === pin.interfaceName) + const iface = interfaceMatches[0] + const assigned = addresses.filter(row => row.address === pin.interfaceCidr) + const local = assigned[0] + if (interfaceMatches.length !== 1 || iface[".id"] !== pin.interfaceId || !isChrWireGuardInterfaceType(iface.type) || + !yes(iface.running) || !no(iface.disabled) || assigned.length !== 1 || local[".id"] !== pin.addressId || + local.interface !== pin.interfaceName || !no(local.disabled) || yes(local.invalid) || + (local["actual-interface"] !== undefined && local["actual-interface"] !== pin.interfaceName)) deny("interface address binding differs") + const exact = candidates.filter(row => row["routing-table"] === "main" && row["dst-address"] === pin.destination + "/32") + const row = exact[0] + if (exact.length !== 1 || row[".id"] !== pin.routeId || !yes(row.active) || !no(row.disabled) || + ![pin.nextHop, pin.nextHop + "%" + pin.interfaceName].includes(String(row.gateway)) || + row["immediate-gw"] !== pin.nextHop + "%" + pin.interfaceName || + yes(row.blackhole) || yes(row.unreachable) || yes(row.prohibit) || yes(row.ecmp)) deny("exact active host route unavailable") + return [pin, canonicalRow(row), canonicalRow(local), iface.type, iface.running, iface.disabled] + }) + const [tracking] = await read("/ip/firewall/connection/tracking", 1, true) + if (!yes(tracking.enabled) && !(ownedWg && tracking.enabled === "auto")) deny("conntrack is not explicitly enabled") + const connections = requireCatalogIds(await read("/ip/firewall/connection", 2048)) + if (ownedWg && tracking.enabled === "auto" && connections.length !== 0) deny("automatic conntrack is not observed empty") + for (const row of connections) { + const protocol = row.protocol + if (protocol === "udp" || protocol === "17") { + const original = endpointsKey(endpoint(row["src-address"]), endpoint(row["dst-address"])) + const reply = endpointsKey(endpoint(row["reply-src-address"]), endpoint(row["reply-dst-address"])) + if (pinnedTuples.has(original) || pinnedTuples.has(reply)) deny("UDP fixture tuple already tracked") + } else if (typeof protocol !== "string" || !/^(tcp|icmp|gre|ipsec-esp|ipsec-ah|sctp|udplite|ospf|ipv6-encap|0|[1-9][0-9]{0,2})$/.test(protocol)) { + deny("unsupported conntrack protocol") + } + } + await identityFence() + const fingerprint = createHash("sha256").update(JSON.stringify([identity, version, profile, ip["allow-fast-path"], tracking.enabled, bindings, + canonicalRow(main[0]), canonicalRow(vrfs[0]), [...pinnedTuples].sort()])).digest("hex") + return { routesVerified: true as const, freshConntrackTuples: true as const, fingerprint, + ...(profile && { profile, ownedInterfaces, + ownedRoutes: routes.map(pin => ({ destination: pin.destination, interfaceName: pin.interfaceName })), + udpPairs: [...pinnedTuples].sort() }) } + } +} diff --git a/backend/src/test/traffic-flow-chr-staged-profile.ts b/backend/src/test/traffic-flow-chr-staged-profile.ts new file mode 100644 index 00000000..409123df --- /dev/null +++ b/backend/src/test/traffic-flow-chr-staged-profile.ts @@ -0,0 +1,9 @@ +import { z } from "zod" +import { CHR_LEGACY_FIXTURE_DEADLINE_MAX_MS, CHR_STAGED_TARGET_CAPTURE_MAX_MS } from "./traffic-flow-chr-target-timing-profile.js" + +export const chrStagedCaptureProfileSchema = z.object({ + kind: z.literal("staged-disabled-target"), + captureMs: z.number().int().min(90000).max(CHR_STAGED_TARGET_CAPTURE_MAX_MS), + requiredCollectorWindowMs: z.number().int().positive().lt(CHR_LEGACY_FIXTURE_DEADLINE_MAX_MS), +}).strict().refine(profile => profile.requiredCollectorWindowMs > profile.captureMs + 210000, + "Staged collector window must include both measured phases and the cleanup reserve") diff --git a/backend/src/test/traffic-flow-chr-target-harness.test.ts b/backend/src/test/traffic-flow-chr-target-harness.test.ts index 6ae2159d..7892493a 100644 --- a/backend/src/test/traffic-flow-chr-target-harness.test.ts +++ b/backend/src/test/traffic-flow-chr-target-harness.test.ts @@ -19,7 +19,7 @@ function fixture() { async put(path: string, body: Record, _timeout?: number, beforeDispatch?: () => Promise): Promise { await beforeDispatch?.() effects.push("PUT " + path) - live.targets.push({ ...body, ".id": "*A", disabled: "false", unknown: "owned" }) + live.targets.push({ ...body, ".id": "*A", disabled: body.disabled ?? "false", unknown: "owned" }) if (extraChange) live.flow.interfaces = "all" if (lostAck) throw Error("private native failure") return { ".id": wrongAck ? "*B" : "*A" } as T @@ -37,6 +37,72 @@ function fixture() { loseDeleteAck: () => { loseDeleteAck = true }, revoke: () => { allowed = false }, onIntent: (fn: () => void) => { afterIntent = fn } } } +function stagedFixture() { + const f = fixture() + f.options.baseline.flow["inactive-flow-timeout"] = f.live.flow["inactive-flow-timeout"] = "15s" + f.options.baseline.targets = [] + f.live.targets = [] + let loseEnableAck = false, mutateForeign = false + const harness = createChrTargetHarness({ + ...f.options, stagedDisabled: true, + verifyContext: async () => { f.effects.push("complete-context") }, + native: { ...f.options.native, + async patch(path: string, body: Record, _timeout?: number, beforeDispatch?: () => Promise): Promise { + await beforeDispatch?.() + assert.equal(path, "/ip/traffic-flow/target/*A") + assert.deepEqual(body, { disabled: "no" }) + f.effects.push("PATCH " + path) + f.live.targets.find(row => row[".id"] === "*A")!.disabled = "false" + if (mutateForeign) f.live.flow.interfaces = "drift" + if (loseEnableAck) throw Error("private acknowledgement failure") + return { ".id": "*A" } as T + }, + }, + }) + return { ...f, harness, loseEnableAck: () => { loseEnableAck = true }, mutateForeign: () => { mutateForeign = true } } +} + +const stagedTimer = { kind: "staged-ipfix-timeout-20s" as const, captureDeadlineAt: 240001, now: () => 1 } + +test("stages a disabled exact target and enables only the same ACK ID inside the separate capture window", async () => { + const f = stagedFixture() + await f.harness.create() + const row = f.live.targets.at(-1)! + assert.equal(row.disabled, "yes") + assert.equal(row["v9-template-timeout"], "20s") + await assert.rejects(f.harness.assertTimerReady(), /Verified owned/) + f.effects.length = 0 + assert.equal((await f.harness.enable(stagedTimer)).id, "*A") + assert.deepEqual(f.effects, [ + "enable:intent:*A", "complete-context", "PATCH /ip/traffic-flow/target/*A", + "enable:acknowledged:*A", "complete-context", "enable:verified:*A", + ]) + await f.harness.assertTimerReady() + await f.harness.remove() + assert.deepEqual(f.live, f.options.baseline) +}) + +test("staged enable refuses expiry, drift and lost acknowledgement without replay", async () => { + const expired = stagedFixture() + await expired.harness.create() + await assert.rejects(expired.harness.enable({ ...stagedTimer, captureDeadlineAt: 240002 }), /timing window/) + assert.equal(expired.effects.some(event => event.startsWith("PATCH ")), false) + const changed = stagedFixture() + await changed.harness.create() + changed.onIntent(() => { changed.live.flow.interfaces = "drift" }) + await assert.rejects(changed.harness.enable(stagedTimer), /inventory changed/) + assert.equal(changed.effects.some(event => event.startsWith("PATCH ")), false) + for (const fault of ["loseEnableAck", "mutateForeign"] as const) { + const f = stagedFixture() + await f.harness.create() + f[fault]() + await assert.rejects(f.harness.enable(stagedTimer), /outcome unknown/) + await assert.rejects(f.harness.enable(stagedTimer), /No verified disabled/) + await assert.rejects(f.harness.remove(), /No verified/) + assert.equal(f.effects.filter(event => event.startsWith("PATCH ")).length, 1) + } +}) + test("creates one exact ACK-owned target and restores only its ID with durable checkpoints", async () => { const f = fixture() assert.deepEqual(await f.harness.create(), { id: "*A", verified: true, automaticReplay: false }) @@ -76,3 +142,50 @@ test("delete ACK loss remains unknown even when the fixture row was removed", as assert.equal(f.effects.filter(value => value.startsWith("DELETE ")).length, 1) assert.equal(f.effects.includes("delete:verified:*A"), false) }) + +test("optional timer writes one reviewed field and verifies its exact normalized readback", async () => { + const f = fixture() + f.options.baseline.flow["inactive-flow-timeout"] = f.live.flow["inactive-flow-timeout"] = "15s" + const originalPut = f.options.native.put + f.options.native.put = async (...args: Parameters) => { + assert.equal(args[1]["v9-template-timeout"], "20s") + assert.equal(Object.hasOwn(args[1], "v9-template-refresh"), false) + const ack = await originalPut(...args) + f.live.targets.at(-1)!["v9-template-timeout"] = "00:00:20" + return ack + } + const harness = createChrTargetHarness({ ...f.options, timerProfile: { kind: "bounded-ipfix-timeout-20s", captureDeadlineAt: 90001, now: () => 1 } }) + const receipt = await harness.create() + assert.equal(receipt.configuredTemplateTimeoutMs, 20000) + assert.equal(receipt.wireRefreshVerified, false) + await harness.assertTimerReady() + await harness.remove() +}) + +test("unsupported timer and elapsed window refuse without replay or guessed fallback", async () => { + for (const readback of [undefined, "20", "30s"]) { + const f = fixture() + f.options.baseline.flow["inactive-flow-timeout"] = f.live.flow["inactive-flow-timeout"] = "00:00:15" + const originalPut = f.options.native.put + f.options.native.put = async (...args: Parameters) => { + const ack = await originalPut(...args) + const row = f.live.targets.at(-1)! + if (readback === undefined) delete row["v9-template-timeout"] + else row["v9-template-timeout"] = readback + return ack + } + const harness = createChrTargetHarness({ ...f.options, timerProfile: { kind: "bounded-ipfix-timeout-20s", captureDeadlineAt: 90001, now: () => 1 } }) + await assert.rejects(harness.create(), /outcome unknown/) + await assert.rejects(harness.assertTimerReady()) + await assert.rejects(harness.remove(), /No verified/) + assert.equal(f.effects.filter(value => value.startsWith("PUT ")).length, 1) + } + const f = fixture() + f.options.baseline.flow["inactive-flow-timeout"] = f.live.flow["inactive-flow-timeout"] = "15s" + let now = 1 + const timerProfile = { kind: "bounded-ipfix-timeout-20s" as const, captureDeadlineAt: 90001, now: () => now } + const harness = createChrTargetHarness({ ...f.options, timerProfile }) + f.onIntent(() => { now = 26002 }) + await assert.rejects(harness.create(), /timing window/) + assert.equal(f.effects.filter(value => value.startsWith("PUT ")).length, 0) +}) diff --git a/backend/src/test/traffic-flow-chr-target-harness.ts b/backend/src/test/traffic-flow-chr-target-harness.ts index 95d2ca79..3d52aab2 100644 --- a/backend/src/test/traffic-flow-chr-target-harness.ts +++ b/backend/src/test/traffic-flow-chr-target-harness.ts @@ -2,29 +2,61 @@ * checkpoint ownership belong to the enclosing native runner. */ import { isIPv4 } from "node:net" import { isDeepStrictEqual } from "node:util" -import { encodeRosId, type MikrotikClient } from "../services/mikrotik.js" +import type { MikrotikClient } from "../services/mikrotik.js" +import { encodeRosId } from "../services/routeros-native-paths.js" + +import { runChrWrite, type ChrWriteAdmission } from "./traffic-flow-chr-write-admission.js" +import { CHR_STAGED_TARGET_CAPTURE_MAX_MS, CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS, CHR_TARGET_TEMPLATE_TIMEOUT_MS } from "./traffic-flow-chr-target-timing-profile.js" +export { CHR_STAGED_TARGET_CAPTURE_MAX_MS, CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS, CHR_TARGET_TEMPLATE_TIMEOUT_MS } from "./traffic-flow-chr-target-timing-profile.js" type Row = Record type Inventory = { deviceId: string; flow: Row; ipfix: Row; targets: Row[] } -type Checkpoint = { action: "create" | "delete"; phase: "intent" | "acknowledged" | "verified"; id?: string } +type Checkpoint = { action: "create" | "enable" | "delete"; phase: "intent" | "acknowledged" | "verified"; id?: string } +export type ChrTargetTimerProfile = { + kind: "bounded-ipfix-timeout-20s" + captureDeadlineAt: number + now?: () => number +} +export type ChrStagedTargetTimerProfile = Omit & { kind: "staged-ipfix-timeout-20s" } +const timerField = "v9-template-timeout" +const timeoutWire = "20s" +const captureMaxMs = 90000 type Options = { baseline: Inventory + relayWrite?: { side: "A" | "B"; admission: ChrWriteAdmission } source: string destination: string port: number - native: Pick read: () => Promise authorize: () => Promise + verifyContext?: () => Promise checkpoint: (entry: Checkpoint) => Promise -} +} & ({ + stagedDisabled?: false + native: Pick + timerProfile?: ChrTargetTimerProfile +} | { + stagedDisabled: true + native: Pick + timerProfile?: never +}) const targetPath = "/ip/traffic-flow/target" const isId = (value: unknown): value is string => typeof value === "string" && /^\*[a-f0-9]{1,16}$/i.test(value) const yes = (value: unknown) => value === true || value === "true" || value === "yes" const no = (value: unknown) => value === false || value === "false" || value === "no" +const ownedInactiveTimeout = (value: unknown) => value === "15s" || value === "00:00:15" const requiredFields = ["bytes", "packets", "src-address", "dst-address", "protocol", "src-port", "dst-port", "in-interface", "out-interface"] function must(value: unknown, message: string): asserts value { if (!value) throw Error(message) } +export function assertChrTargetTimerWindow(timer: ChrTargetTimerProfile | ChrStagedTargetTimerProfile, inactiveTimeout: unknown) { + const remaining = timer.captureDeadlineAt - (timer.now ?? Date.now)() + const max = timer.kind === "staged-ipfix-timeout-20s" ? CHR_STAGED_TARGET_CAPTURE_MAX_MS : captureMaxMs + must((timer.kind === "bounded-ipfix-timeout-20s" || timer.kind === "staged-ipfix-timeout-20s") && + ownedInactiveTimeout(inactiveTimeout) && + Number.isSafeInteger(timer.captureDeadlineAt) && Number.isSafeInteger(remaining) && + remaining >= CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS && remaining <= max, "Owned IPFIX timing window unavailable") +} function snapshot(inventory: Inventory): Inventory { must(typeof inventory.deviceId === "string" && inventory.deviceId.length > 0 && inventory.deviceId.length <= 128 && inventory.flow && inventory.ipfix && Array.isArray(inventory.targets) && inventory.targets.length <= 64 && @@ -37,25 +69,38 @@ function snapshot(inventory: Inventory): Inventory { } export function createChrTargetHarness(options: Options) { - const { read, authorize, checkpoint, native, source, destination, port } = options + const { read, authorize, checkpoint, native, source, destination, port, relayWrite } = options const baseline = snapshot(options.baseline) must(isIPv4(source) && isIPv4(destination) && source !== destination && Number.isInteger(port) && port > 0 && port <= 65535, "Invalid owned target tuple") must(yes(baseline.flow.enabled) && requiredFields.every(key => yes(baseline.ipfix[key])), "Traffic Flow or required IPFIX fields are unavailable") must(!baseline.targets.some(row => row["dst-address"] === destination && String(row.port) === String(port)), "The target tuple already exists") - const fields = Object.freeze({ "dst-address": destination, "src-address": source, port: String(port), version: "ipfix" }) - let phase: "new" | "creating" | "owned" | "deleting" | "done" | "refused" = "new" + const staged = options.stagedDisabled === true + if (staged) { + must(options.timerProfile === undefined && baseline.targets.length === 0 && + ownedInactiveTimeout(baseline.flow["inactive-flow-timeout"]), "Disabled target staging prerequisites unavailable") + } + let timer: ChrTargetTimerProfile | ChrStagedTargetTimerProfile | undefined = options.timerProfile && { ...options.timerProfile } + const assertTimerWindow = () => { if (timer) assertChrTargetTimerWindow(timer, baseline.flow["inactive-flow-timeout"]) } + assertTimerWindow() + const fields = Object.freeze({ "dst-address": destination, "src-address": source, port: String(port), version: "ipfix", + ...(timer || staged ? { [timerField]: timeoutWire } : {}), ...(staged ? { disabled: "yes" } : {}) }) + const fieldsMatch = (row: Row) => Object.entries(fields).every(([key, value]) => key === timerField + ? Object.hasOwn(row, key) && (row[key] === timeoutWire || row[key] === "00:00:20") + : key === "disabled" ? yes(row.disabled) : String(row[key]) === value) + let phase: "new" | "creating" | "staged" | "enabling" | "owned" | "deleting" | "done" | "refused" = "new" let owned: Inventory | undefined let ownedId: string | undefined - const current = async () => { + const current = async (verifyContext = true) => { await authorize() const result = snapshot(await read()) must(result.deviceId === baseline.deviceId, "Target device identity changed") + if (verifyContext) await options.verifyContext?.() await authorize() return result } - const expect = async (value: Inventory) => { - must(isDeepStrictEqual(await current(), value), "Target inventory changed") + const expect = async (value: Inventory, verifyContext = true) => { + must(isDeepStrictEqual(await current(verifyContext), value), "Target inventory changed") } return Object.freeze({ async create() { @@ -65,45 +110,112 @@ export function createChrTargetHarness(options: Options) { try { await expect(baseline) await checkpoint({ action: "create", phase: "intent" }) - const ack = await native.put(targetPath, { ...fields }, 15000, async () => { + const ack = await runChrWrite(relayWrite?.admission, relayWrite && { step: "target:" + relayWrite.side + ":create", + side: relayWrite.side, deviceId: baseline.deviceId, method: "PUT", path: targetPath, body: fields }, + arm => native.put(targetPath, { ...fields }, 15000, async () => { must(!dispatched, "Target creation already dispatched") await expect(baseline) must(!dispatched, "Target creation already dispatched") + assertTimerWindow() dispatched = true - }) + await arm() + })) must(dispatched, "Target creation skipped dispatch fence") const id = ack && typeof ack === "object" ? (ack as Row)[".id"] : undefined must(isId(id) && !baseline.targets.some(row => row[".id"] === id), "Target creation has no exact new ACK ID") await checkpoint({ action: "create", phase: "acknowledged", id }) const result = await current() const row = result.targets.find(item => item[".id"] === id) - must(row && no(row.disabled) && Object.entries(fields).every(([key, value]) => String(row[key]) === value), "ACK target differs on read-back") + must(row && (staged ? yes(row.disabled) : no(row.disabled)) && fieldsMatch(row), "ACK target differs on read-back") must(isDeepStrictEqual({ ...result, targets: result.targets.filter(item => item[".id"] !== id) }, baseline), "Unrelated target inventory changed") await checkpoint({ action: "create", phase: "verified", id }) owned = result ownedId = id - phase = "owned" - return { id, verified: true as const, automaticReplay: false as const } + phase = staged ? "staged" : "owned" + return { id, verified: true as const, automaticReplay: false as const, + ...(timer ? { configuredTemplateTimeoutMs: CHR_TARGET_TEMPLATE_TIMEOUT_MS, wireRefreshVerified: false as const } : {}) } } catch (error) { phase = "refused" if (dispatched) throw Error("Target creation outcome unknown. Automatic replay forbidden.") throw error } }, + async enable(profile: ChrStagedTargetTimerProfile) { + must(options.stagedDisabled && phase === "staged" && owned && ownedId && + profile.kind === "staged-ipfix-timeout-20s", "No verified disabled target to enable") + const expected = owned, id = ownedId, native = options.native + const path = targetPath + "/" + encodeRosId(id) + const body = Object.freeze({ disabled: "no" }) + timer = { ...profile } + phase = "enabling" + let dispatched = false + try { + assertTimerWindow() + // The durable intent follows an exact local read. Complete context + // verification still runs immediately before dispatch and after ACK. + await expect(expected, false) + await checkpoint({ action: "enable", phase: "intent", id }) + const ack = await runChrWrite(relayWrite?.admission, relayWrite && { step: "target:" + relayWrite.side + ":enable", + side: relayWrite.side, deviceId: baseline.deviceId, method: "PATCH", path, body }, + arm => native.patch(path, { ...body }, 15000, async () => { + must(!dispatched, "Target enable already dispatched") + await expect(expected) + must(!dispatched, "Target enable already dispatched") + assertTimerWindow() + dispatched = true + await arm() + })) + must(dispatched && ack && typeof ack === "object" && (ack as Row)[".id"] === id, "Target enable has no exact ACK ID") + await checkpoint({ action: "enable", phase: "acknowledged", id }) + const result = await current() + const reduced = structuredClone(result) + const row = reduced.targets.find(item => item[".id"] === id) + must(row && no(row.disabled), "ACK target is not enabled") + row.disabled = expected.targets.find(item => item[".id"] === id)!.disabled + must(isDeepStrictEqual(reduced, expected), "Target enable changed unrelated inventory") + await checkpoint({ action: "enable", phase: "verified", id }) + owned = result + phase = "owned" + return { id, verified: true as const, automaticReplay: false as const, + configuredTemplateTimeoutMs: CHR_TARGET_TEMPLATE_TIMEOUT_MS, wireRefreshVerified: false as const } + } catch (error) { + phase = "refused" + if (dispatched) throw Error("Target enable outcome unknown. Automatic replay forbidden.") + throw error + } + }, + async assertTimerReady() { + must(timer && phase === "owned" && owned, "Verified owned IPFIX timer required") + assertTimerWindow() + await expect(owned) + assertTimerWindow() + }, + async assertStagedTimerReady() { + must(staged && timer && phase === "owned" && owned, "Verified staged IPFIX timer required") + assertTimerWindow() + // The caller supplies one complete pre-generation topology fence for + // both exact target reads. No native write occurs in this phase. + await expect(owned, false) + assertTimerWindow() + }, async remove() { - must(phase === "owned" && owned && ownedId, "No verified owned target to delete") + must((phase === "owned" || phase === "staged") && owned && ownedId, "No verified owned target to delete") const expected = owned, id = ownedId phase = "deleting" let dispatched = false try { await expect(expected) await checkpoint({ action: "delete", phase: "intent", id }) - await native.delete(targetPath + "/" + encodeRosId(id), 10000, async () => { + const path = targetPath + "/" + encodeRosId(id) + await runChrWrite(relayWrite?.admission, relayWrite && { step: "target:" + relayWrite.side + ":delete", + side: relayWrite.side, deviceId: baseline.deviceId, method: "DELETE", path, body: {} }, + arm => native.delete(path, 10000, async () => { must(!dispatched, "Target deletion already dispatched") await expect(expected) must(!dispatched, "Target deletion already dispatched") dispatched = true - }) + await arm() + })) must(dispatched, "Target deletion skipped dispatch fence") await checkpoint({ action: "delete", phase: "acknowledged", id }) await expect(baseline) diff --git a/backend/src/test/traffic-flow-chr-target-timing-profile.ts b/backend/src/test/traffic-flow-chr-target-timing-profile.ts new file mode 100644 index 00000000..7e143fac --- /dev/null +++ b/backend/src/test/traffic-flow-chr-target-timing-profile.ts @@ -0,0 +1,14 @@ +// Pure protocol limits shared before the disposable database is pinned. +export const CHR_TARGET_TEMPLATE_TIMEOUT_MS = 20000 +export const CHR_STAGED_TARGET_CAPTURE_MAX_MS = 240000 +export const CHR_STAGED_COLLECTOR_APPROVAL_MS = 60000 +export const CHR_COLLECTOR_RESULT_POLL_MS = 45000 +// Two 17-second drains, the template interval, and 10 seconds for exchanges. +export const CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS = 2 * 17000 + CHR_TARGET_TEMPLATE_TIMEOUT_MS + 10000 + +export const CHR_LEGACY_FIXTURE_DEADLINE_MAX_MS = 600000 +export const CHR_STAGED_FIXTURE_DEADLINE_MAX_MS = 3600000 +export type ChrStagedFixtureLease = "staged-disabled-target" +export function chrFixtureDeadlineMaxMs(profile?: ChrStagedFixtureLease) { + return profile === "staged-disabled-target" ? CHR_STAGED_FIXTURE_DEADLINE_MAX_MS : CHR_LEGACY_FIXTURE_DEADLINE_MAX_MS +} diff --git a/backend/src/test/traffic-flow-chr-topology-inventory.test.ts b/backend/src/test/traffic-flow-chr-topology-inventory.test.ts new file mode 100644 index 00000000..b2c99a1c --- /dev/null +++ b/backend/src/test/traffic-flow-chr-topology-inventory.test.ts @@ -0,0 +1,135 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { setImmediate as wait } from "node:timers/promises" +import { createChrTransitTopologyInventory, CHR_TOPOLOGY_CATALOG_PAIR_GETS } from "./traffic-flow-chr-topology-inventory.js" + +const name = "nfa012bcdeea", marker = "netflow-a012bcde" +const wg = { side: "A", menu: "wireguard", step: "edgeA", id: "*10" } as const +const address = { side: "A", menu: "address", step: "address:edgeA", id: "*11" } as const +const ip = { ".id": "*11", address: "198.18.241.2/30", interface: name, comment: marker, dynamic: false, disabled: false, vrf: "main" } +const route = { ".id": "*12", "dst-address": "198.18.241.0/30", gateway: name, "routing-table": "main", dynamic: true, + connect: true, distance: "0", scope: "10", "target-scope": "5" } +const rib = { ...route, afi: "ip", "belongs-to": "connected", "local-address": "198.18.241.2%" + name } +function fixture() { + const tables = { A: {} as Record, B: {} as Record } + const client = (side: "A" | "B") => ({ deviceId: side, client: { async get(path: string, timeout?: number, signal?: AbortSignal, cap?: number) { + assert.equal(timeout, 99); assert(signal instanceof AbortSignal); assert.equal(cap, 1048576) + return structuredClone(tables[side][path] ?? []) as T + } } }) + const reader = createChrTransitTopologyInventory({ runId: "a012bcde", deadlineAt: 100, now: () => 1, authorize: async () => {}, + routers: { A: client("A"), B: client("B") } }) + const snapshot = async (owned: Parameters[0], transition?: Parameters[1]) => { + await reader.assertOutsideScope(owned, transition) + const a = await reader.read("A") + await reader.read("B") + return a.rows + } + async function prepare(interfaceType = "wireguard") { + tables.B["/ip/route"] = [{ ".id": "*22", "dst-address": "192.0.2.0/24", dynamic: true }] + await snapshot([]) + tables.A["/interface"] = [{ ".id": "*10", name, comment: marker, type: interfaceType, disabled: false }] + await snapshot([], { ...wg, action: "create" }) + } + return { tables, reader, snapshot, prepare } +} +test("the generic interface catalogue accepts either WireGuard spelling and refuses foreign types", async () => { + await fixture().prepare("wg") + await assert.rejects(fixture().prepare("ether")) +}) +test("IPv4 DHCP lease countdown is telemetry while gateway and IPv6 countdown remain conserved", async () => { + for (const [path, change, accepted] of [ + ["/ip/dhcp-client", { "expires-after": "4m" }, true], + ["/ip/dhcp-client", { "expires-after": "unexpected" }, false], + ["/ip/dhcp-client", { gateway: "192.0.2.2" }, false], + ["/ipv6/dhcp-client", { "expires-after": "4m" }, false], + ] as const) { + const f = fixture() + const row = { ".id": "*A", interface: "ether1", gateway: "192.0.2.1", "expires-after": "5m" } + f.tables.A[path] = [row] + await f.snapshot([]) + f.tables.A[path] = [{ ...row, ...change }] + if (accepted) await f.snapshot([]) + else await assert.rejects(f.snapshot([]), /Foreign native catalogue drift/) + } +}) +test("single paired capture projects only exact ACK-address connected menu and RIB companions", async () => { + const f = fixture(); await f.prepare() + f.tables.A["/ip/address"] = [ip]; f.tables.A["/ip/route"] = [route]; f.tables.A["/routing/route"] = [rib] + const view = await f.snapshot([wg], { ...address, action: "create" }) + assert.deepEqual(view.address, [ip]); assert.deepEqual(view.route, []) + await assert.rejects(f.reader.read("A"), /snapshot required/) + await f.reader.assertOutsideScope([wg, address]) + await f.reader.read("A") + assert.deepEqual((await f.reader.read("B")).rows.route, f.tables.B["/ip/route"]) + f.tables.A["/ip/address"] = []; f.tables.A["/ip/route"] = []; f.tables.A["/routing/route"] = [] + await f.snapshot([wg, address], { ...address, action: "delete" }) + f.tables.A["/interface"] = [] + await f.snapshot([wg], { ...wg, action: "delete" }) +}) +test("wrong address ACK, conflicting RIB or unrelated peer drift refuses the next admission", async () => { + for (const fault of ["ack", "rib", "peer", "duplicate"]) { + const f = fixture(); await f.prepare() + f.tables.A["/ip/address"] = [{ ...ip, ".id": fault === "ack" ? "*33" : ip[".id"] }] + f.tables.A["/ip/route"] = fault === "duplicate" ? [route, { ...route, ".id": "*14" }] : [route] + f.tables.A["/routing/route"] = [{ ...rib, "local-address": fault === "rib" ? "192.0.2.1" : rib["local-address"] }] + if (fault === "peer") f.tables.B["/ip/firewall/filter"] = [{ ".id": "*22" }] + await assert.rejects(f.snapshot([wg], { ...address, action: "create" })) + await assert.rejects(f.snapshot([wg])) + } +}) +test("previously confirmed companion opaque config cannot change on a later capture", async () => { + const f = fixture(); await f.prepare() + f.tables.A["/ip/address"] = [ip]; f.tables.A["/ip/route"] = [route]; f.tables.A["/routing/route"] = [rib] + await f.snapshot([wg], { ...address, action: "create" }) + f.tables.A["/routing/route"] = [{ ...rib, "unreviewed-field": "different" }] + await assert.rejects(f.snapshot([wg, address]), /companion drift/) +}) + +test("fixed parallel batches keep both authorities fresh and never mix CHR sources", async () => { + let pairFences = 0, requestFences = 0, active = 0, peak = 0 + const requests: string[] = [] + const running = new Set<"A" | "B">() + const client = (side: "A" | "B") => ({ deviceId: side, client: { + async get(path: string, _timeout?: number, _signal?: AbortSignal, cap?: number, before?: () => Promise) { + assert.equal(cap, 1048576) + await before?.() + assert.ok(!running.size || running.has(side)) + running.add(side) + requests.push(side + path) + peak = Math.max(peak, ++active) + await wait() + if (--active === 0) running.clear() + return [] as T + }, + } }) + const reader = createChrTransitTopologyInventory({ runId: "a012bcde", deadlineAt: 100, now: () => 1, parallelism: 2, + authorize: async () => { pairFences++ }, authorizeRequest: async () => { requestFences++ }, + routers: { A: client("A"), B: client("B") } }) + await reader.assertOutsideScope([]) + assert.equal(requests.length, CHR_TOPOLOGY_CATALOG_PAIR_GETS) + assert.equal(new Set(requests).size, requests.length) + assert.equal(peak, 2) + assert.equal(pairFences, 2) + assert.equal(requestFences, 2 * requests.length) + await reader.read("A") + await reader.read("B") +}) + +test("failed concurrent catalogue settles its sibling and never exposes a partial pair", async () => { + let started = 0, completed = 0 + const reader = createChrTransitTopologyInventory({ + runId: "a012bcde", deadlineAt: 100, now: () => 1, parallelism: 2, authorize: async () => {}, + routers: { A: { deviceId: "A", client: { async get() { + if (++started === 1) throw Error("fixed path unavailable") + await wait() + completed++ + return [] as T + } } }, B: { deviceId: "B", client: { async get(): Promise { throw Error("foreign side reached") } } } }, + }) + await assert.rejects(reader.assertOutsideScope([]), /fixed path unavailable/) + assert.equal(started, 2) + assert.equal(completed, 1) + await assert.rejects(reader.read("A"), /snapshot required/) + await assert.rejects(reader.assertOutsideScope([])) + assert.equal(started, 2) +}) diff --git a/backend/src/test/traffic-flow-chr-topology-inventory.ts b/backend/src/test/traffic-flow-chr-topology-inventory.ts new file mode 100644 index 00000000..dd8b2017 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-topology-inventory.ts @@ -0,0 +1,255 @@ +import { isDeepStrictEqual } from "node:util" +import { parseChrVrfPrefix } from "./chr-vrf-companion-address-space.js" +import { stripNativeTelemetry } from "./chr-vrf-companion-telemetry.js" +import { greDependencyPaths, wireguardInterfaceDependencyPaths } from "../services/routeros-native-paths.js" +import type { MikrotikClient } from "../services/mikrotik.js" +import type { createChrTransitTopologyOwner } from "./traffic-flow-chr-topology-owner.js" +import { isChrWireGuardInterfaceType } from "./traffic-flow-chr-transit-fixture.js" + +type OwnerOptions = Parameters[0] +type Fence = OwnerOptions["assertOutsideScope"] +type Side = "A" | "B" +type Row = Record +type Catalogues = Record +const sides = ["A", "B"] as const +const writePaths = { wireguard: "/interface/wireguard", peer: "/interface/wireguard/peers", address: "/ip/address", route: "/ip/route" } as const +const nativePaths = [...new Set([...Object.values(writePaths), "/interface", "/ipv6/address", "/ipv6/route", "/routing/route", + "/ip/vrf", "/routing/table", "/interface/bridge", "/interface/list/member", "/ip/firewall/filter", "/ipv6/firewall/filter", + ...greDependencyPaths, ...wireguardInterfaceDependencyPaths])] +export const CHR_TOPOLOGY_CATALOG_PAIR_GETS = nativePaths.length * 2 +export const CHR_TOPOLOGY_CATALOG_READ_LIMIT = 12000 +const roles = { edgeA: "ea", middleA: "ma", middleB: "mb", edgeB: "eb" } as const +const addressRole = { edgeA: "198.18.241.2/30", middleA: "198.18.241.5/30", middleB: "198.18.241.6/30", edgeB: "198.18.241.9/30" } as const +const id = (row: Row) => row[".id"] +const yes = (value: unknown) => [true, "true", "yes"].includes(value as string | boolean) +const no = (value: unknown) => [false, "false", "no"].includes(value as string | boolean) +const native = (value: unknown) => typeof value === "string" && /^\*[a-f0-9]{1,16}$/i.test(value) +const nativeCanonical = (value: unknown) => String(value).replace(/^\*0+(?=.)/, "*").toLowerCase() +function must(value: unknown, message: string): asserts value { if (!value) throw Error(message) } +function exact(rows: Row[], test: (row: Row) => boolean) { + const matching = rows.filter(test) + must(matching.length <= 1, "Ambiguous topology companion") + return matching[0] +} +const points = (row: Row, name: string) => [row.gateway, row["immediate-gw"], row.interface, row["actual-interface"]].includes(name) || + String(row["local-address"] ?? "").endsWith("%" + name) || row["dst-address"] === "fe80::%" + name + "/64" + +function connected(row: Row, name: string, host: string, network: NonNullable>, rib: boolean) { + const destination = row["dst-address"] === "fe80::%" + name + "/64" ? "fe80::/64" : row["dst-address"] + const value = parseChrVrfPrefix(destination, network.family) + if (!points(row, name) || value?.host !== network.first || value?.prefixLength !== network.prefixLength) return false + if (rib && !(network.family === 4 ? ["ip", "ip4"] : ["ipv6", "ip6"]).includes(String(row.afi))) return false + must(native(id(row)) && row.gateway === name && row["routing-table"] === "main" && yes(row.connect) && + (!Object.hasOwn(row, "connected") || yes(row.connected)) && + (rib ? row["belongs-to"] === "connected" && (!Object.hasOwn(row, "dynamic") || yes(row.dynamic)) : yes(row.dynamic)) && + String(row.distance) === "0" && String(row.scope) === "10" && String(row["target-scope"]) === "5" && + ["static", "bgp", "blackhole", "invalid", "disabled", "ecmp"].every(key => !Object.hasOwn(row, key) || no(row[key])), + "Unproven connected route") + for (const field of ["interface", "immediate-gw"]) must(!Object.hasOwn(row, field) || row[field] === name, "Conflicting connected route interface") + must(!Object.hasOwn(row, "local-address") || row["local-address"] === host + "%" + name, "Conflicting connected route source") + if (rib) must(row["local-address"] === host + "%" + name, "Missing exact connected RIB source") + return true +} + +type Ownership = Parameters[0] +type Transition = Parameters[1] +export function createChrTransitTopologyInventory(options: { + runId: string + deadlineAt: number + authorize(): Promise + authorizeRequest?: () => Promise + parallelism?: 1 | 2 | 4 + routers: Record }> + now?: () => number +}) { + const { runId, deadlineAt, authorize, authorizeRequest = authorize, parallelism = 1, now = Date.now } = options + const routers = { A: { ...options.routers.A }, B: { ...options.routers.B } } + must(/^[a-f0-9]{8}$/.test(runId) && Number.isSafeInteger(deadlineAt) && [1, 2, 4].includes(parallelism) && + routers.A.deviceId && routers.B.deviceId && + routers.A.deviceId !== routers.B.deviceId, "Invalid paired topology reader scope") + const nameFor = (role: keyof typeof roles) => "nf" + runId + roles[role] + const marker = "netflow-" + runId + let baseline: Record | undefined + let cache: Partial> | undefined + let previousCompanions: Record> | undefined + let open = false, refused = false, reads = 0 + const live = () => { + const left = Math.min(8000, Math.floor(deadlineAt - now())) + must(left > 0, "Topology catalogue deadline elapsed") + return left + } + async function readPair() { + const result = {} as Record + if (options.authorizeRequest) { await authorize(); live() } + for (const side of sides) { + const catalogues: Catalogues = {} + let bytes = 0 + for (let offset = 0; offset < nativePaths.length; offset += parallelism) { + const batch = nativePaths.slice(offset, offset + parallelism) + reads += batch.length + must(reads <= CHR_TOPOLOGY_CATALOG_READ_LIMIT, "Topology catalogue read limit exhausted") + const byteLimit = Math.min(1048576, Math.floor((8 * 1048576 - bytes) / batch.length)) + must(byteLimit > 0, "Paired topology catalogue exceeds budget") + const captured = await Promise.allSettled(batch.map(async path => { + await authorizeRequest() + const timeout = live() + const value = await routers[side].client.get(path, timeout, AbortSignal.timeout(timeout), byteLimit, + async () => { await authorizeRequest(); live() }) + live() + must(Array.isArray(value) && value.length <= 1000 && value.every(row => row && typeof row === "object" && !Array.isArray(row)), + "Incomplete native topology catalogue") + const rows = value as Row[] + const size = Buffer.byteLength(JSON.stringify(rows)) + must(size <= byteLimit, "Paired topology catalogue exceeds budget") + must(rows.every(row => native(id(row))) && new Set(rows.map(row => nativeCanonical(id(row)))).size === rows.length, + "Ambiguous native topology catalogue") + return { path, rows: structuredClone(rows), size } + })) + for (const item of captured) { + if (item.status === "rejected") throw item.reason + const { path, rows, size } = item.value + bytes += size + catalogues[path] = rows + } + } + result[side] = catalogues + } + await authorize(); live() + return result + } + function projections(c: Catalogues, entries: Ownership, side: Side) { + const exclusions = new Map>() + const companions = new Map() + const exclude = (path: string, row: Row, step: string) => { + const current = exclusions.get(path) ?? new Set() + must(!current.has(row), "Topology companion has multiple owners") + current.add(row) + companions.set(path + "/" + id(row), { step, row: stripNativeTelemetry(row) }) + exclusions.set(path, current) + } + for (const [role, cidr] of Object.entries(addressRole) as [keyof typeof roles, string][]) { + if (!role.endsWith(side)) continue + const name = nameFor(role) + const wg = entries.find(entry => entry.side === side && entry.menu === "wireguard" && entry.step === role) + const address = entries.find(entry => entry.side === side && entry.menu === "address" && entry.step === "address:" + role) + must(!address || wg, "Owned address lost its WireGuard parent") + if (wg) { + const iface = exact(c["/interface"], row => id(row) === wg.id || row.name === name) + must(iface && id(iface) === wg.id && iface.name === name && isChrWireGuardInterfaceType(iface.type) && iface.comment === marker && + no(iface.disabled) && (!Object.hasOwn(iface, "vrf") || iface.vrf === "main"), "Missing exact WireGuard inventory") + exclude("/interface", iface, role) + const link = exact(c["/routing/route"], row => row.afi === "link" && row["dst-address"] === name) + if (link) { + must(link["belongs-to"] === "interface" && link["routing-table"] === "main" && String(link.distance) === "0", + "Unproven WireGuard link RIB row") + exclude("/routing/route", link, role) + } + const ll = exact(c["/ipv6/address"], row => points(row, name)) + if (ll) { + const range = parseChrVrfPrefix(ll.address, 6) + must(ll.interface === name && range?.prefixLength === 64 && range.first === 0xfe80n << 112n && + yes(ll.dynamic) && yes(ll["auto-link-local"]) && yes(ll["link-local"]) && + (!Object.hasOwn(ll, "actual-interface") || ll["actual-interface"] === name) && + (!Object.hasOwn(ll, "vrf") || ll.vrf === "main") && + (!Object.hasOwn(ll, "disabled") || no(ll.disabled)) && (!Object.hasOwn(ll, "invalid") || no(ll.invalid)) && + (!Object.hasOwn(ll, "comment") || ll.comment === "" || ll.comment === marker), "Unproven WireGuard IPv6 link-local") + exclude("/ipv6/address", ll, role) + const host = String(ll.address).split("/")[0] + const rib = exact(c["/routing/route"], row => connected(row, name, host, range, true)) + must(rib, "Missing exact IPv6 link-local RIB row") + exclude("/routing/route", rib, role) + const kernel = exact(c["/ipv6/route"], row => connected(row, name, host, range, false)) + if (kernel) exclude("/ipv6/route", kernel, role) + } + } + if (address) { + const raw = exact(c["/ip/address"], row => id(row) === address.id || row.address === cidr) + must(raw && id(raw) === address.id && raw.address === cidr && raw.interface === name && raw.comment === marker && + no(raw.dynamic) && no(raw.disabled) && raw.vrf === "main", "Missing exact ACK static address") + const range = parseChrVrfPrefix(cidr, 4)! + const host = cidr.split("/")[0] + for (const path of ["/ip/route", "/routing/route"]) { + const route = exact(c[path], row => connected(row, name, host, range, path === "/routing/route")) + must(route, "Missing exact static-address companion") + exclude(path, route, "address:" + role) + } + } + } + // A static /ip/route object may also appear in the RIB. Only associate a + // unique, native static RIB row with the exact ACK route descriptor. + for (const entry of entries.filter(row => row.side === side && row.menu === "route")) { + const raw = exact(c["/ip/route"], row => id(row) === entry.id) + must(raw && raw.comment === marker && raw["routing-table"] === "main" && no(raw.disabled), "Missing exact ACK host route") + const rib = exact(c["/routing/route"], row => row["dst-address"] === raw["dst-address"] && + row["routing-table"] === raw["routing-table"] && row.gateway === raw.gateway) + if (rib) { + must(rib["belongs-to"] === "static" && yes(rib.static) && ["ip", "ip4"].includes(String(rib.afi)) && + (!Object.hasOwn(rib, "dynamic") || no(rib.dynamic)), "Unproven static RIB route") + exclude("/routing/route", rib, entry.step) + } + } + const projected = Object.fromEntries(nativePaths.map(path => [path, c[path].filter(row => !exclusions.get(path)?.has(row))])) + const outside = Object.fromEntries(nativePaths.filter(path => !Object.values(writePaths).some(writable => writable === path)) + .map(path => { + const rows = projected[path].map(row => { + const stable = stripNativeTelemetry(row) + if (path === "/ip/dhcp-client" && typeof stable["expires-after"] === "string" && + stable["expires-after"].length <= 32 && /^(?=\d)(?:\d+w)?(?:\d+d)?(?:\d+h)?(?:\d+m)?(?:\d+s)?$/.test(stable["expires-after"])) { + delete stable["expires-after"] + } + return stable + }) + if (path === "/routing/route") rows.sort((a, b) => String(id(a)).localeCompare(String(id(b)))) + return [path, rows] + })) + // Keep unrelated dynamic /ip/route entries in the writable projection. + // They may not disappear under a blanket connected/dynamic exclusion. + return { projected, outside, companions } + } + const assertOutsideScope: Fence = async (owned: Ownership, transition?: Transition) => { + must(!open && !refused, "Topology catalogue capture refused") + open = true + cache = undefined + try { + const entries = [...owned] + if (transition) { + const index = entries.findIndex(entry => entry.side === transition.side && entry.menu === transition.menu && entry.step === transition.step) + if (transition.action === "create") { must(index < 0, "Duplicate topology ownership"); entries.push(transition) } + else { must(index >= 0 && entries[index].id === transition.id, "Unknown topology inverse"); entries.splice(index, 1) } + } + const raw = await readPair() + const a = projections(raw.A, entries, "A"), b = projections(raw.B, entries, "B") + const outside = { A: a.outside, B: b.outside } + if (!baseline) { must(!entries.length && !transition, "Topology baseline required before mutation"); baseline = outside } + must(isDeepStrictEqual(outside, baseline), "Foreign native catalogue drift") + const companions = { A: a.companions, B: b.companions } + if (previousCompanions) for (const side of sides) { + const before = previousCompanions[side], after = companions[side] + for (const key of new Set([...before.keys(), ...after.keys()])) { + const old = before.get(key), fresh = after.get(key) + if (isDeepStrictEqual(old, fresh)) continue + must(transition?.side === side && (transition.action === "create" + ? !old && fresh?.step === transition.step : !fresh && old?.step === transition.step), + "Owned topology companion drift") + } + } + previousCompanions = companions + cache = { A: a.projected, B: b.projected } + } catch (error) { refused = true; throw error } + finally { open = false } + } + return Object.freeze({ + assertOutsideScope, + async read(side: Side) { + live() + const captured = cache?.[side] + must(captured && cache, "Paired topology snapshot required") + const rows = { wireguard: captured[writePaths.wireguard], peer: captured[writePaths.peer], + address: captured[writePaths.address], route: captured[writePaths.route] } + // The owner consumes A then B and cannot reuse one capture. + if (side === "A") delete cache.A + else cache = undefined + return { complete: true as const, deviceId: routers[side].deviceId, rows: structuredClone(rows) } + }, + }) +} diff --git a/backend/src/test/traffic-flow-chr-topology-owner.test.ts b/backend/src/test/traffic-flow-chr-topology-owner.test.ts new file mode 100644 index 00000000..065d9258 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-topology-owner.test.ts @@ -0,0 +1,104 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { chrTransitNames } from "./traffic-flow-chr-transit-fixture.js" +import { createChrTransitTopologyOwner } from "./traffic-flow-chr-topology-owner.js" + +import { createChrParentWriteAdmission } from "./traffic-flow-chr-write-admission.js" + +const runId = "a012bcde" +const ports = { linuxA: 50101, edgeA: 50102, middleA: 50103, middleB: 50104, edgeB: 50105, linuxB: 50106, + collector: 50107, calibrationSource: 50112, calibrationDestination: 50113, phase1Source: 50108, phase1Destination: 50109, phase2Source: 50110, phase2Destination: 50111 } +const directions = ["A-to-B", "B-to-A", "export-A", "export-B", "underlay-linuxA-to-A", "underlay-A-to-linuxA", + "underlay-A-to-B", "underlay-B-to-A", "underlay-B-to-linuxB", "underlay-linuxB-to-B"] as const +const isolation = { checkedPorts: Object.values(ports), checkedNames: Object.values(chrTransitNames(runId)), + checkedCidrs: ["198.18.240.1/32", "198.18.240.2/32", "198.18.241.0/30", "198.18.241.4/30", "198.18.241.8/30", "198.18.242.0/29"], + occupiedPorts: [], overlappingCidrs: [], occupiedNames: [], + policies: directions.map(direction => ({ direction, verdict: "allow", nat: "none", fastTrack: "none", ambiguousRules: 0 } as const)) } +type Row = Record +const menus = { "/interface/wireguard": "wireguard", "/interface/wireguard/peers": "peer", "/ip/address": "address", "/ip/route": "route" } as const +function fixture(fault?: string, bridged = false) { + const state = () => ({ wireguard: [] as Row[], peer: [] as Row[], address: [] as Row[], route: [] as Row[] }) + const rows = { A: state(), B: state() }, writes: { method: string; path: string; side: string }[] = [], journal: object[] = [] + let flowRestored = false, tick = 1, pairReads = 0 + const gate = bridged ? createChrParentWriteAdmission({ deadlineAt: 100, now: () => tick, deviceIds: { A: "A", B: "B" }, + assertCurrent: async () => {} }) : undefined + function router(side: "A" | "B", endpoint: string) { + return { deviceId: side, endpoint, read: async () => ({ deviceId: side, complete: true as const, rows: structuredClone(rows[side]) }), + native: { + async put(path: string, fields: Record, _timeout?: number, fence?: () => Promise) { + if (fault !== "skip-fence") await fence?.() + await gate?.authorizeWrite({ side, method: "PUT", path, body: fields }) + writes.push({ method: "PUT", path, side }) + const id = "*" + writes.length.toString(16) + const menu = menus[path as keyof typeof menus] + rows[side][menu].push({ ...fields, ".id": id, ...(menu === "wireguard" ? { "public-key": String.fromCharCode(65 + writes.length).repeat(43) + "=" } : {}) }) + if (fault === "lost-ack") throw Error("private") + if (fault === "foreign") rows.B.address.push({ ".id": "*FF" }) + if (fault === "expire") tick = 100 + return { ".id": id } as T + }, + async delete(path: string, _timeout?: number, fence?: () => Promise) { + await fence?.() + await gate?.authorizeWrite({ side, method: "DELETE", path, body: {} }) + writes.push({ method: "DELETE", path, side }) + const slash = path.lastIndexOf("/"), menu = menus[path.slice(0, slash) as keyof typeof menus] + rows[side][menu] = rows[side][menu].filter(row => row[".id"] !== decodeURIComponent(path.slice(slash + 1))) + }, + } } + } + const owner = createChrTransitTopologyOwner({ scope: [runId, ports, isolation], deadlineAt: 100, now: () => tick, writeAdmission: gate, + routers: { A: router("A", "192.0.2.1"), B: router("B", "192.0.2.2") }, + linux: { endpoint: "192.0.2.3", publicKeys: { linuxA: "X".repeat(43) + "=", linuxB: "Y".repeat(43) + "=" } }, + authorize: async () => {}, checkpoint: async entry => { + journal.push(entry) + if (fault === "drift-after-intent" && entry.phase === "intent") rows.B.address.push({ ".id": "*FF" }) + }, assertOutsideScope: async () => { pairReads++ }, + assertFlowRestored: async () => { assert(flowRestored) } }) + return { owner, rows, writes, journal, gate, flowClean: () => { flowRestored = true }, pairReads: () => pairReads } +} + +test("paired exact ACKs own 18 creates and 18 dependency-ordered inverse writes", async () => { + const f = fixture() + assert.equal((await f.owner.acquire()).attempts, 18) + await f.owner.assertCurrent() + f.flowClean() + assert.equal((await f.owner.restore()).attempts, 36) + assert.deepEqual(f.writes.slice(18).map(row => row.path.slice(0, row.path.lastIndexOf("/"))), + ["/ip/route", "/interface/wireguard/peers", "/ip/address", "/interface/wireguard"].flatMap((path, i) => Array(i ? 4 : 6).fill(path))) + assert.equal(f.journal.length, 108) + assert.equal(f.pairReads(), 75) // 74 write-schedule pairs and the explicit assertCurrent above. + await assert.rejects(f.owner.restore()) +}) + +test("fresh paired comparison immediately before dispatch rejects drift after durable intent", async () => { + const f = fixture("drift-after-intent") + await assert.rejects(f.owner.acquire(), /dispatch refused/) + assert.equal(f.writes.length, 0) + assert.deepEqual(f.journal.map(entry => (entry as { phase: string }).phase), ["intent"]) + assert.equal(f.pairReads(), 2) +}) +test("unknown ACK, lost fence and foreign changes latch before subsequent writes", async () => { + for (const fault of ["lost-ack", "skip-fence", "foreign", "expire"]) { + const f = fixture(fault) + await assert.rejects(f.owner.acquire(), /Topology/) + await assert.rejects(f.owner.acquire()) + await assert.rejects(f.owner.restore()) + assert.equal(f.writes.length, 1) + } +}) +test("target or singleton restoration must precede all inverse attempts", async () => { + const f = fixture() + await f.owner.acquire() + await assert.rejects(f.owner.restore()) + f.flowClean() + await assert.rejects(f.owner.restore()) + assert.equal(f.writes.length, 18) +}) + +test("all topology writes consume the parent relay gate exactly once", async () => { + const f = fixture(undefined, true) + assert.equal((await f.owner.acquire()).attempts, 18) + f.flowClean() + assert.equal((await f.owner.restore()).attempts, 36) + assert.equal(f.gate?.consumed, 36) +}) diff --git a/backend/src/test/traffic-flow-chr-topology-owner.ts b/backend/src/test/traffic-flow-chr-topology-owner.ts new file mode 100644 index 00000000..439d34e0 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-topology-owner.ts @@ -0,0 +1,199 @@ +import { isIPv4 } from "node:net" +import { isDeepStrictEqual } from "node:util" +import type { MikrotikClient } from "../services/mikrotik.js" +import { encodeRosId } from "../services/routeros-native-paths.js" +import { runChrWrite, type ChrWriteAdmission } from "./traffic-flow-chr-write-admission.js" +import { planChrTransitFixture } from "./traffic-flow-chr-transit-fixture.js" + +type Scope = Parameters +type Row = Record +const paths = { wireguard: "/interface/wireguard", peer: "/interface/wireguard/peers", address: "/ip/address", route: "/ip/route" } as const +type Menu = keyof typeof paths +type Side = "A" | "B" +type Role = "edgeA" | "middleA" | "middleB" | "edgeB" +type Inventory = { deviceId: string; complete: true; rows: Record } +type Command = { key: string; side: Side; menu: Menu; fields: Record } +type Checkpoint = { step: string; side: Side; menu: Menu; action: "create" | "delete"; phase: "intent" | "acknowledged" | "verified"; id?: string } +type Options = { + scope: Scope + routers: Record; native: Pick }> + linux: { endpoint: string; publicKeys: { linuxA: string; linuxB: string } } + deadlineAt: number + writeAdmission?: ChrWriteAdmission + authorize(): Promise + checkpoint(entry: Checkpoint): Promise + assertFlowRestored(): Promise + /** Independently verify the complete configuration outside the four writable menus. */ + assertOutsideScope(owned: readonly { step: string; side: Side; menu: Menu; id: string }[], transition?: { step: string; side: Side; menu: Menu; action: Checkpoint["action"]; id: string }): Promise + now?: () => number +} +const roles = ["edgeA", "middleA", "middleB", "edgeB"] as const +const interfaces = { edgeA: "ea", middleA: "ma", middleB: "mb", edgeB: "eb" } as const +const menus = Object.keys(paths) as Menu[] +const roleSide = (role: Role): Side => role.endsWith("A") ? "A" : "B" +const id = (row: Row) => row[".id"] +const nativeId = (value: unknown): value is string => typeof value === "string" && /^\*[a-f0-9]{1,16}$/i.test(value) +const publicKey = (value: unknown): value is string => typeof value === "string" && /^[A-Za-z0-9+/]{43}=$/.test(value) +const off = (value: unknown) => [false, "false", "no"].includes(value as string | boolean) +function must(value: unknown, message: string): asserts value { if (!value) throw Error(message) } +const telemetry = new Set(["running", "active", "inactive", "rx", "tx", "last-handshake", "current-endpoint-address", "current-endpoint-port", "actual-mtu"]) + +/** The pinned reader must provide full configuration rows. Explicitly observed + * dynamic connected routes belong in the independent read-only closure, not + * in the static route projection accepted by this owner. */ +function snapshot(input: Inventory, expectedDevice: string) { + must(input.complete && input.deviceId === expectedDevice && Object.keys(input.rows).sort().join() === [...menus].sort().join() && + Buffer.byteLength(JSON.stringify(input.rows)) <= 2 * 1048576, "Incomplete topology catalogue") + return Object.fromEntries(menus.map(menu => { + const rows = input.rows[menu] + must(Array.isArray(rows) && rows.length <= 1000 && rows.every(row => row && nativeId(id(row))) && + new Set(rows.map(row => String(id(row)).replace(/^\*0+(?=.)/, "*").toLowerCase())).size === rows.length, + "Ambiguous topology catalogue identity") + return [menu, rows.map(row => Object.fromEntries(Object.entries(row).filter(([key]) => !telemetry.has(key)))) + .sort((a, b) => String(id(a)).localeCompare(String(id(b))))] + })) as Record +} +function matches(row: Row, fields: Record) { + return Object.entries(fields).every(([key, value]) => key === "disabled" ? value === "false" && off(row[key]) : String(row[key]) === value) +} + +export function createChrTransitTopologyOwner(options: Options) { + const { authorize, checkpoint, assertFlowRestored, assertOutsideScope, deadlineAt, writeAdmission, now = Date.now } = options + const plan = planChrTransitFixture(...structuredClone(options.scope)), linux = structuredClone(options.linux) + const routers = { A: { ...options.routers.A }, B: { ...options.routers.B } } + must(/^[a-f0-9]{8}$/.test(plan.runId) && Number.isSafeInteger(deadlineAt) && + routers.A.deviceId && routers.B.deviceId && routers.A.deviceId !== routers.B.deviceId && + [routers.A.endpoint, routers.B.endpoint, linux.endpoint].every(isIPv4) && + new Set([routers.A.endpoint, routers.B.endpoint, linux.endpoint]).size === 3 && + publicKey(linux.publicKeys.linuxA) && publicKey(linux.publicKeys.linuxB) && linux.publicKeys.linuxA !== linux.publicKeys.linuxB, + "Invalid pinned transit identities") + const marker = "netflow-" + plan.runId + const name = (role: Role) => "nf" + plan.runId + interfaces[role] + const keys: Record = { ...linux.publicKeys } + const owned = new Map() + let phase: "new" | "creating" | "owned" | "deleting" | "done" | "refused" = "new" + let attempts = 0 + let baseline: Record> + let expected: Record> + const live = () => must(now() < deadlineAt, "Topology authorization expired") + const current = async (transition?: Parameters[1]) => { + live(); await authorize(); await assertOutsideScope([...owned.values()].map(({ command, id }) => ({ step: command.key, side: command.side, menu: command.menu, id })), transition); live() + const a = snapshot(await routers.A.read(), routers.A.deviceId), b = snapshot(await routers.B.read(), routers.B.deviceId) + await authorize(); live() + return { A: a, B: b } + } + const fence = async () => must(isDeepStrictEqual(await current(), expected), "Topology catalogue drift") + const run = async (action: Checkpoint["action"], command: Command, targetId?: string) => { + const { key, side, menu, fields } = command + const entry = { step: key, side, menu, action } + const router = routers[side] + const prior = expected + // The durable intent does not dispatch. Re-read the entire pair at the + // actual native dispatch fence and again on ACK, not before the intent too. + await authorize(); live() + if (action === "delete") { + must(nativeId(targetId) && prior[side][menu].some(row => id(row) === targetId && matches(row, fields)), + "Owned inverse target no longer matches") + } + await checkpoint({ ...entry, phase: "intent", ...(targetId ? { id: targetId } : {}) }) + let sent = false + const beforeSend = async () => { + must(!sent, "Repeated topology dispatch") + await fence() + if (action === "delete") await assertFlowRestored() + await authorize(); live() + must(!sent && attempts < 36, "Topology write budget exhausted") + sent = true; attempts++ + } + try { + let identity = targetId + const path = action === "create" ? paths[menu] : paths[menu] + "/" + encodeRosId(targetId!) + const commandBody = action === "create" ? { ...fields } : {} + const ack = await runChrWrite(writeAdmission, { step: "topology:" + action + ":" + side + ":" + key, + deviceId: router.deviceId, side, method: action === "create" ? "PUT" : "DELETE", path, body: commandBody }, async arm => { + const fence = async () => { await beforeSend(); await arm() } + if (action === "create") return router.native.put(path, commandBody, 15000, fence) + return router.native.delete(path, 15000, fence) + }) + if (action === "create") { + identity = ack && typeof ack === "object" && !Array.isArray(ack) ? (ack as Row)[".id"] as string : undefined + } + must(sent && nativeId(identity), "Native ACK missing exact identity") + await checkpoint({ ...entry, phase: "acknowledged", id: identity }) + const after = await current({ ...entry, id: identity }), reduced = structuredClone(action === "create" ? after : prior) + const row = reduced[side][menu].find(candidate => id(candidate) === identity) + must(row && matches(row, fields), "Native read-back differs from owned descriptor") + reduced[side][menu] = reduced[side][menu].filter(candidate => id(candidate) !== identity) + must(isDeepStrictEqual(reduced, action === "create" ? prior : after), "Topology write changed unrelated configuration") + if (action === "create" && menu === "wireguard") { + must(publicKey(row["public-key"]) && !Object.values(keys).includes(row["public-key"]), "Invalid or reused native WireGuard public key") + keys[key] = row["public-key"] + } + await checkpoint({ ...entry, phase: "verified", id: identity }) + expected = after + if (action === "create") owned.set(key, { command, id: identity }) + else owned.delete(key) + } catch { + throw Error(sent ? "Topology outcome unknown or unverified. Automatic replay forbidden." : "Topology dispatch refused.") + } + } + const interfaceCommands = () => roles.map(role => ({ key: role, side: roleSide(role), menu: "wireguard" as const, + fields: { name: name(role), "listen-port": String(plan.ports[role]), mtu: "1420", disabled: "false", comment: marker } })) + const peerCommands = () => roles.map(role => { + const peer = plan.peers[role], opposite = peer.opposite + const endpoint = opposite.startsWith("linux") ? linux.endpoint : routers[roleSide(opposite as Role)].endpoint + const key = keys[opposite] + must(publicKey(key), "Exact remote peer key unavailable") + return { key: "peer:" + role, side: roleSide(role), menu: "peer" as const, fields: { + interface: name(role), "public-key": key, "allowed-address": peer.allowed.join(","), "endpoint-address": endpoint, + "endpoint-port": String(plan.ports[opposite as keyof typeof plan.ports]), "persistent-keepalive": "10", disabled: "false", comment: marker, + } } + }) + const addressCommands = () => roles.map(role => ({ key: "address:" + role, side: roleSide(role), menu: "address" as const, + fields: { interface: name(role), address: plan.addresses[role], disabled: "false", comment: marker } })) + const routeCommands = () => plan.routes.filter(route => route.node === "A" || route.node === "B").map((route, index) => ({ + key: "route:" + index, side: route.node as Side, menu: "route" as const, fields: { + "dst-address": route.destination, gateway: route.nextHop + "%" + name(route.device as Role), "routing-table": "main", disabled: "false", comment: marker, + }, + })) + return Object.freeze({ + async acquire() { + must(phase === "new", "Topology acquisition already attempted") + phase = "creating" + try { + baseline = expected = await current() + const ifaces = interfaceCommands(), addresses = addressCommands(), routes = routeCommands() + must(ifaces.length === 4 && addresses.length === 4 && routes.length === 6 && + Object.values(baseline).every(rows => menus.every(menu => rows[menu].every(row => row.comment !== marker))) && + ifaces.every(cmd => !baseline[cmd.side].wireguard.some(row => row.name === cmd.fields.name || String(row["listen-port"]) === cmd.fields["listen-port"])) && + addresses.every(cmd => !baseline[cmd.side].address.some(row => row.address === cmd.fields.address)) && + routes.every(cmd => !baseline[cmd.side].route.some(row => row["dst-address"] === cmd.fields["dst-address"])), "Topology preflight collision") + for (const command of [...ifaces, ...addresses]) await run("create", command) + const peers = peerCommands() + must(peers.length === 4, "Topology peer schedule differs") + for (const command of [...peers, ...routes]) await run("create", command) + must(attempts === 18 && owned.size === 18, "Topology create schedule incomplete") + phase = "owned" + return { verified: true as const, attempts, chrPublicKeys: Object.fromEntries(roles.map(role => [role, keys[role]])) } + } catch (error) { phase = "refused"; throw error } + }, + async assertCurrent() { + must(phase === "owned", "Topology is not verified") + try { await fence() } catch (error) { phase = "refused"; throw error } + }, + async restore() { + must(phase === "owned", "Topology inverse has no verified source") + phase = "deleting" + try { + await assertFlowRestored() + for (const menu of ["route", "peer", "address", "wireguard"] as const) + for (const row of [...owned.values()].filter(row => row.command.menu === menu).reverse()) + await run("delete", row.command, row.id) + must(attempts === 36 && owned.size === 0 && isDeepStrictEqual(expected, baseline), "Topology inverse did not restore the baseline") + await fence() + phase = "done" + return { verified: true as const, attempts, automaticReplay: false as const } + } catch (error) { phase = "refused"; throw error } + }, + }) +} diff --git a/backend/src/test/traffic-flow-chr-transit-fixture.test.ts b/backend/src/test/traffic-flow-chr-transit-fixture.test.ts new file mode 100644 index 00000000..9a235741 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-transit-fixture.test.ts @@ -0,0 +1,48 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { chrTransitNames, planChrTransitFixture, verifyChrTransitRoutes } from "./traffic-flow-chr-transit-fixture.js" + +const owner = "a012bcde" +const ports = { linuxA: 50101, edgeA: 50102, middleA: 50103, middleB: 50104, edgeB: 50105, linuxB: 50106, + collector: 50107, calibrationSource: 50112, calibrationDestination: 50113, phase1Source: 50108, phase1Destination: 50109, phase2Source: 50110, phase2Destination: 50111 } +const isolation = { + checkedPorts: Object.values(ports), + checkedNames: Object.values(chrTransitNames(owner)), + checkedCidrs: ["198.18.240.1/32", "198.18.240.2/32", "198.18.241.0/30", "198.18.241.4/30", + "198.18.241.8/30", "198.18.242.0/29"], + occupiedPorts: [], overlappingCidrs: [], occupiedNames: [], + policies: (["A-to-B", "B-to-A", "export-A", "export-B", "underlay-linuxA-to-A", + "underlay-A-to-linuxA", "underlay-A-to-B", "underlay-B-to-A", + "underlay-B-to-linuxB", "underlay-linuxB-to-B"] as const).map(direction => ({ + direction, verdict: "allow", nat: "none", fastTrack: "none", ambiguousRules: 0, + } as const)), +} + +test("plans separate directions through both CHRs and two exporter return routes", () => { + const plan = planChrTransitFixture(owner, ports, isolation) + assert.equal(verifyChrTransitRoutes([...plan.routes].reverse()), true) + assert.deepEqual(plan.routes.filter(row => row.destination === "198.18.240.2/32").map(row => row.device), + ["middleA", "edgeB", "linuxA"]) + assert.deepEqual(plan.routes.filter(row => row.destination === "198.18.240.1/32").map(row => row.device), + ["edgeA", "middleB", "linuxB"]) + assert.equal(plan.routes.filter(row => row.node === "C").length, 2) + assert.ok(plan.peers.edgeA.allowed.includes("198.18.242.3/32")) + assert.ok(plan.peers.edgeB.allowed.includes("198.18.242.3/32")) + assert.deepEqual(plan.topologyWrites.chrA, { wireGuard: 2, peers: 2, addresses: 2, routes: 3 }) + assert.deepEqual(plan.topologyWrites.chrB, plan.topologyWrites.chrA) + assert.equal(plan.actualIngressVerified, false) + assert.throws(() => verifyChrTransitRoutes(plan.routes.map(row => ({ ...row, nextHop: "192.0.2.1" }))), /route selection/) +}) + +test("refuses collisions, incomplete inventory and missing or ambiguous directional policy", () => { + assert.throws(() => planChrTransitFixture(owner, { ...ports, edgeA: ports.linuxA }, isolation), /fixture port/) + assert.throws(() => planChrTransitFixture(owner, { ...ports, calibrationSource: ports.phase1Source }, isolation), /fixture port/) + assert.throws(() => planChrTransitFixture(owner, { ...ports, calibrationDestination: 46000 }, isolation), /fixture port/) + assert.throws(() => planChrTransitFixture(owner, ports, { ...isolation, occupiedPorts: [ports.edgeA] }), /collision/) + assert.throws(() => planChrTransitFixture(owner, ports, { ...isolation, overlappingCidrs: ["198.18.0.0/15"] }), /collision/) + assert.throws(() => planChrTransitFixture(owner, ports, { ...isolation, checkedNames: [] }), /incomplete/) + assert.throws(() => planChrTransitFixture(owner, ports, { ...isolation, policies: isolation.policies.slice(1) }), /FastTrack/) + assert.throws(() => planChrTransitFixture(owner, ports, { + ...isolation, policies: [isolation.policies[1], ...isolation.policies.slice(1)], + }), /FastTrack/) +}) diff --git a/backend/src/test/traffic-flow-chr-transit-fixture.ts b/backend/src/test/traffic-flow-chr-transit-fixture.ts new file mode 100644 index 00000000..c35e06e8 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-transit-fixture.ts @@ -0,0 +1,104 @@ +import { isDeepStrictEqual } from "node:util" + +const address = { + endpointA: "198.18.240.1", endpointB: "198.18.240.2", + linuxA: "198.18.241.1", exportA: "198.18.241.2", + middleA: "198.18.241.5", middleB: "198.18.241.6", + exportB: "198.18.241.9", linuxB: "198.18.241.10", + vethA: "198.18.242.1", vethB: "198.18.242.2", collector: "198.18.242.3", +} as const +const cidrs = ["198.18.240.1/32", "198.18.240.2/32", "198.18.241.0/30", + "198.18.241.4/30", "198.18.241.8/30", "198.18.242.0/29"] as const +const portRoles = ["linuxA", "edgeA", "middleA", "middleB", "edgeB", "linuxB", "collector", + "calibrationSource", "calibrationDestination", "phase1Source", "phase1Destination", "phase2Source", "phase2Destination"] as const +type Ports = Record<(typeof portRoles)[number], number> +type Route = { node: string; destination: string; nextHop: string; device: string } +const host = (ip: string) => ip + "/32" +const addresses = { + edgeA: address.exportA + "/30", middleA: address.middleA + "/30", + middleB: address.middleB + "/30", edgeB: address.exportB + "/30", +} +const peers = { + edgeA: { opposite: "linuxA", allowed: [host(address.endpointA), host(address.collector)] }, + middleA: { opposite: "middleB", allowed: [host(address.endpointB)] }, + middleB: { opposite: "middleA", allowed: [host(address.endpointA)] }, + edgeB: { opposite: "linuxB", allowed: [host(address.endpointB), host(address.collector)] }, + linuxA: { opposite: "edgeA", allowed: [host(address.endpointB), host(address.exportA)] }, + linuxB: { opposite: "edgeB", allowed: [host(address.endpointA), host(address.exportB)] }, +} +const routes: Route[] = [ + { node: "A", destination: host(address.endpointA), nextHop: address.linuxA, device: "edgeA" }, + { node: "A", destination: host(address.endpointB), nextHop: address.middleB, device: "middleA" }, + { node: "A", destination: host(address.collector), nextHop: address.linuxA, device: "edgeA" }, + { node: "B", destination: host(address.endpointB), nextHop: address.linuxB, device: "edgeB" }, + { node: "B", destination: host(address.endpointA), nextHop: address.middleA, device: "middleB" }, + { node: "B", destination: host(address.collector), nextHop: address.linuxB, device: "edgeB" }, + { node: "linuxA", destination: host(address.endpointB), nextHop: address.exportA, device: "linuxA" }, + { node: "linuxB", destination: host(address.endpointA), nextHop: address.exportB, device: "linuxB" }, + { node: "C", destination: host(address.exportA), nextHop: address.vethA, device: "collector-veth" }, + { node: "C", destination: host(address.exportB), nextHop: address.vethB, device: "collector-veth" }, +] +const directions = ["A-to-B", "B-to-A", "export-A", "export-B", + "underlay-linuxA-to-A", "underlay-A-to-linuxA", "underlay-A-to-B", "underlay-B-to-A", + "underlay-B-to-linuxB", "underlay-linuxB-to-B"] as const +type Direction = (typeof directions)[number] +type PolicyCheck = { direction: Direction; verdict: "allow"; nat: "none"; fastTrack: "none"; ambiguousRules: 0 } +type Isolation = { + checkedPorts: readonly number[] + checkedCidrs: readonly string[] + checkedNames: readonly string[] + occupiedPorts: readonly number[] + overlappingCidrs: readonly string[] + occupiedNames: readonly string[] + policies: readonly PolicyCheck[] +} +const must = (condition: unknown, message: string) => { if (!condition) throw Error(message) } +const sameSet = (first: readonly T[], second: readonly T[]) => + isDeepStrictEqual([...first].sort(), [...second].sort()) + +/** Inputs are normalized observations of fixed, independently reviewed scope. + * This planner does not interpret raw RouterOS firewall rules or acquire authority. */ +export function planChrTransitFixture(runId: string, ports: Ports, isolation: Isolation) { + must(/^[a-f0-9]{8}$/.test(runId), "Invalid fixture owner") + must(Object.keys(ports).length === portRoles.length && + portRoles.every(role => Number.isInteger(ports[role]) && ports[role] >= 49152 && ports[role] <= 65535) && + new Set(portRoles.map(role => ports[role])).size === portRoles.length, "Unreserved or duplicate fixture port") + const names = chrTransitNames(runId) + must(sameSet(isolation.checkedPorts, portRoles.map(role => ports[role])) && + sameSet(isolation.checkedCidrs, cidrs) && sameSet(isolation.checkedNames, Object.values(names)), + "Fixture isolation inventory incomplete") + must(isolation.occupiedPorts.length === 0 && isolation.overlappingCidrs.length === 0 && + isolation.occupiedNames.length === 0, "Fixture ownership collision") + must(isolation.policies.length === directions.length && + directions.every(direction => isolation.policies.filter(check => check.direction === direction && + check.verdict === "allow" && check.nat === "none" && check.fastTrack === "none" && + check.ambiguousRules === 0).length === 1), "Firewall, NAT or FastTrack path unavailable") + return structuredClone({ + runId, names, address, cidrs, ports, peers, addresses, routes, + requiredPolicyDirections: directions, + topologyWrites: { chrA: { wireGuard: 2, peers: 2, addresses: 2, routes: 3 }, + chrB: { wireGuard: 2, peers: 2, addresses: 2, routes: 3 }, + linux: { namespaces: 4, wireGuard: 2, vethPairs: 3, bridges: 1 } }, + rollbackOrder: ["netflow-targets", "netflow-singletons", "chr-host-routes", "chr-peers", + "chr-addresses", "chr-wireguard", "linux-owned-processes", "linux-owned-namespaces"], + automaticReplay: false, + actualIngressVerified: false, + }) +} + +export function chrTransitNames(runId: string) { + must(/^[a-f0-9]{8}$/.test(runId), "Invalid fixture owner") + return Object.fromEntries(["na", "nb", "nc", "ns", "la", "lb", "ea", "ma", "mb", "eb", + "av", "as", "bv", "bs", "cv", "cs", "br"].map(role => [role, "nf" + runId + role])) +} + +/** The reviewed route probe returns each selected route with no ambiguity. */ +export function verifyChrTransitRoutes(observed: readonly Route[]) { + must(observed.length === routes.length && + routes.every(route => observed.filter(candidate => isDeepStrictEqual(candidate, route)).length === 1), + "Fixture route selection differs") + return true +} +export function isChrWireGuardInterfaceType(value: unknown) { + return value === "wg" || value === "wireguard" +} diff --git a/backend/src/test/traffic-flow-chr-vps-operator.test.ts b/backend/src/test/traffic-flow-chr-vps-operator.test.ts new file mode 100644 index 00000000..96346e0a --- /dev/null +++ b/backend/src/test/traffic-flow-chr-vps-operator.test.ts @@ -0,0 +1,143 @@ +import assert from "node:assert/strict" +import { createHash, randomUUID } from "node:crypto" +import { mkdtemp, readFile, rm } from "node:fs/promises" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { test } from "node:test" +import { executeDisabledChrVpsOperatorForTests, executeOwnedCollectorForTests } from "./traffic-flow-chr-vps-operator.js" + +const scope = { interfaceName: "owned0", sources: ["192.0.2.1", "192.0.2.2"] as const, destination: "192.0.2.100", port: 4739 } +type Options = Parameters[0] +function capture() { + const bytes = Buffer.alloc(144) + bytes.writeUInt32LE(0xa1b2c3d4); bytes.writeUInt16LE(2, 4); bytes.writeUInt16LE(4, 6) + bytes.writeUInt32LE(4124, 16); bytes.writeUInt32LE(101, 20) + for (const [index, at] of [24, 84].entries()) { + bytes.writeUInt32LE(1700000000, at); bytes.writeUInt32LE(44, at + 8); bytes.writeUInt32LE(44, at + 12) + const ip = bytes.subarray(at + 16, at + 60) + ip[0] = 0x45; ip.writeUInt16BE(44, 2); ip[9] = 17 + Buffer.from([192, 0, 2, index + 1, 192, 0, 2, 100]).copy(ip, 12) + ip.writeUInt16BE(2055, 20); ip.writeUInt16BE(scope.port, 22); ip.writeUInt16BE(24, 24) + ip.writeUInt16BE(10, 28); ip.writeUInt16BE(16, 30) + } + return { bytes, sha256: createHash("sha256").update(bytes).digest("hex"), complete: true, dropped: 0 } +} +function fixture(directory: string) { + const pins = { instance: "owned-instance", pid: 42, invariantProof: "a".repeat(64) } + let enabled = false, foreign = false, lostAck = false + const writes: boolean[] = [] + let drains = 0 + const options: Options = { directory, scope, pins, deadlineAt: Date.now() + 5000, collector: { + authorize: async () => {}, + read: async () => ({ ...pins, listenAddress: scope.destination, listenPort: scope.port, enabled, workerAlive: enabled, + boundAddress: enabled ? scope.destination + ":" + scope.port : null, + udpOwners: enabled || foreign ? [{ pid: foreign ? 999 : pins.pid, address: scope.destination, port: scope.port }] : [] }), + assertDrained: async () => { assert.equal(enabled, false); drains++ }, + applyEnabled: async (desired, fence) => { + await fence() + const contents = await readFile(join(directory, "collector.jsonl"), "utf8") + assert(contents.includes(JSON.stringify({ phase: "intent", enabled: desired }))) + writes.push(desired) + enabled = desired + return { operationId: randomUUID(), acknowledged: !lostAck } + }, + } } + return { options, writes, drains: () => drains, setForeign: () => { foreign = true }, loseAck: () => { lostAck = true } } +} +async function inDirectory(work: (directory: string) => Promise) { + const parent = await mkdtemp(join(fileURLToPath(new URL("../../../tmp/", import.meta.url)), "mm-chr-vps-")) + try { await work(join(parent, "owned")) } finally { await rm(parent, { recursive: true, force: true }) } +} + +test("preserves exact pcap, acknowledges bound owner and verifies clean disabled baseline", async () => { + await inDirectory(async directory => { + const f = fixture(directory), pcap = capture() + await executeOwnedCollectorForTests(f.options, async owner => { + await owner.assertBound() + await owner.retainCapture(pcap) + assert.deepEqual(await readFile(join(directory, "capture.pcap")), pcap.bytes) + await assert.rejects(owner.retainCapture(pcap), { code: "EEXIST" }) + await owner.restore() + }) + assert.deepEqual(f.writes, [true, false]) + assert.equal(f.drains(), 1) + await assert.rejects(executeOwnedCollectorForTests(f.options, async () => {}), { code: "EEXIST" }) + assert.deepEqual(f.writes, [true, false]) + }) +}) + +test("foreign socket and lost enable acknowledgement block ownership and cleanup writes", async () => { + await inDirectory(async directory => { + const f = fixture(directory); f.setForeign() + await assert.rejects(executeOwnedCollectorForTests(f.options, async () => assert.fail()), /UDP owner/) + assert.deepEqual(f.writes, []) + }) + await inDirectory(async directory => { + const f = fixture(directory); f.loseAck() + await assert.rejects(executeOwnedCollectorForTests(f.options, async () => assert.fail()), /lacks acknowledged/) + assert.deepEqual(f.writes, [true]) + }) +}) + +test("an unsupported downstream replay preserves pcap and retires the still-owned collector once", async () => { + await inDirectory(async directory => { + const f = fixture(directory), pcap = capture() + await assert.rejects(executeOwnedCollectorForTests(f.options, async owner => { + await owner.retainCapture(pcap) + throw Error("unsupported downstream replay") + }), /unsupported downstream replay/) + assert.deepEqual(f.writes, [true, false]) + assert.equal(f.drains(), 1) + assert.deepEqual(await readFile(join(directory, "capture.pcap")), pcap.bytes) + }) +}) + +test("cleanup refuses a changed socket owner after an acknowledged startup", async () => { + await inDirectory(async directory => { + const f = fixture(directory) + await assert.rejects(executeOwnedCollectorForTests(f.options, async () => { + f.setForeign() + throw Error("exercise stopped") + }), /UDP owner/) + assert.deepEqual(f.writes, [true]) + assert.equal(f.drains(), 0) + }) +}) + + +test("staged activation spends the credential window only after exact topology preflight", async () => { + for (const denied of ["admission", "preflight", "activation"] as const) { + const stages: string[] = [] + await assert.rejects(executeDisabledChrVpsOperatorForTests({ + directory: "unused", deadlineAt: Date.now() + 1800000, + scope: { ...scope, interfaceName: "nfa012bcdecv", linkType: "EN10MB" }, serverIds: [1, 2], + authorizeFixture: async () => { stages.push("authorize") }, + admitBeforeHardwareWrites: async () => { + stages.push("admit") + if (denied === "admission") throw Error("timing admission refused") + }, + topology: { + acquire: async () => { stages.push("acquire") }, + assertCurrent: async () => { stages.push("topology") }, + preflight: async () => { + stages.push("preflight") + if (denied === "preflight") throw Error("preflight refused") + return { routers: [] as never, expected: [] } + }, + restore: async () => assert.fail("Unadmitted topology inverse"), + }, + requiredCollectorWindowMs: 390000, + activateCollector: async window => { + assert.equal(window, 390000) + stages.push("activation") + throw Error("activation refused") + }, + capture: async () => assert.fail("Unadmitted capture"), + stagedCapture: async () => assert.fail("Unadmitted staged capture"), + generate: async () => assert.fail("Unadmitted generator"), + }), /refused/) + assert.deepEqual(stages, ["authorize", "admit", + ...(denied === "admission" ? [] : ["acquire", "topology", "preflight"]), + ...(denied === "activation" ? ["topology", "activation"] : [])]) + } +}) diff --git a/backend/src/test/traffic-flow-chr-vps-operator.ts b/backend/src/test/traffic-flow-chr-vps-operator.ts new file mode 100644 index 00000000..558f22b3 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-vps-operator.ts @@ -0,0 +1,267 @@ +/** Owning collector lifecycle for an admitted VPS runner. The supplied MCP + * adapter performs the existing settings plan, human approval and apply. */ +import { mkdir, open } from "node:fs/promises" +import { join } from "node:path" +import { isIPv4 } from "node:net" +import { setTimeout as delay } from "node:timers/promises" +import { captureOwnedChrPackets } from "./traffic-flow-chr-capture-process.js" +import { prepareChrReplay } from "./traffic-flow-chr-offline-replay.js" +import { checkChrBaselineIngress, executeOwnedChrCapture } from "./traffic-flow-chr-owned-sequence.js" +import { projectChrWire } from "./traffic-flow-chr-wire.js" +import { executeDisabledChrCapture, type DisabledChrCaptureOptions } from "./traffic-flow-chr-disabled-runner.js" + +type Sequence = Parameters[0] +type Scope = Sequence["scope"] & { interfaceName: string } +type Observation = { + instance: string; pid: number; invariantProof: string + listenAddress: string; listenPort: number; enabled: boolean; workerAlive: boolean; boundAddress: string | null + udpOwners: readonly { pid: number; address: string; port: number }[] +} +type Collector = { + authorize: () => Promise + read: () => Promise + assertDrained: () => Promise + applyEnabled: (enabled: boolean, beforeDispatch: () => Promise) => Promise<{ operationId: string; acknowledged: boolean }> +} +type OwnerOptions = { + directory: string; scope: Scope; deadlineAt: number + pins: { instance: string; pid: number; invariantProof: string } + collector: Collector + mayRestoreAfterFailure?: () => boolean +} +type Capture = { bytes: Buffer; sha256: string; complete: boolean; dropped: number } +function must(value: unknown, reason: string): asserts value { + if (!value) throw Error(reason) +} + +async function ownedCollector(options: OwnerOptions, work: (owner: Sequence["collector"] & { + retainCapture: (capture: Capture) => Promise +}) => Promise) { + const { directory, collector, deadlineAt } = options, scope = structuredClone(options.scope), pins = structuredClone(options.pins) + const live = () => must(Date.now() < deadlineAt, "Owned collector deadline ended") + must(Number.isSafeInteger(deadlineAt) && deadlineAt - Date.now() <= 600000 && deadlineAt > Date.now() && + /^[A-Za-z0-9][A-Za-z0-9_.-]{0,14}$/.test(scope.interfaceName) && scope.interfaceName !== "any" && + scope.sources.length === 2 && new Set([...scope.sources, scope.destination]).size === 3 && + [...scope.sources, scope.destination].every(isIPv4) && Number.isSafeInteger(scope.port) && scope.port > 0 && scope.port <= 65535 && + /^[a-f0-9]{64}$/.test(pins.invariantProof) && Number.isSafeInteger(pins.pid) && pins.pid > 0 && + typeof pins.instance === "string" && pins.instance.length > 0 && pins.instance.length <= 128, "Invalid collector ownership pins") + const authorize = async () => { live(); await collector.authorize(); live() } + const read = async () => { + await authorize() + const current = await collector.read() + await authorize() + must(current.instance === pins.instance && current.pid === pins.pid && current.invariantProof === pins.invariantProof && + current.listenAddress === scope.destination && current.listenPort === scope.port && + Array.isArray(current.udpOwners) && current.udpOwners.length <= 1 && + current.udpOwners.every(row => row.pid === pins.pid && row.address === scope.destination && row.port === scope.port), + "Collector instance, configuration or UDP owner changed") + return current + } + const idle = async () => { + const current = await read() + must(current.enabled === false && current.workerAlive === false && current.boundAddress === null && current.udpOwners.length === 0, + "Collector does not match the disabled baseline") + } + const endpoint = scope.destination + ":" + scope.port + const isBound = (current: Observation) => current.enabled === true && current.workerAlive === true && + current.boundAddress === endpoint && current.udpOwners.length === 1 + await idle() + await mkdir(directory, { mode: 0o700 }) + const syncDirectory = async (path: string) => { + if (process.platform === "win32") return + const handle = await open(path, "r") + try { await handle.sync() } finally { await handle.close() } + } + await syncDirectory(join(directory, "..")) + const journal = await open(join(directory, "collector.jsonl"), "wx", 0o600) + let enableOwned = false, restoreAttempted = false, restored = false + const append = async (record: object) => { + await journal.writeFile(JSON.stringify(record) + "\n") + await journal.sync() + } + const change = async (enabled: boolean, check: () => Promise) => { + await authorize() + await append({ phase: "intent", enabled }) + let dispatched = false + const receipt = await collector.applyEnabled(enabled, async () => { + must(!dispatched, "Collector write already dispatched") + await check() + must(!dispatched, "Collector write already dispatched") + dispatched = true + }) + must(dispatched && receipt.acknowledged === true && /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(receipt.operationId), "Collector write lacks acknowledged operation identity") + await append({ phase: "acknowledged", enabled, operationId: receipt.operationId }) + } + const restore = async () => { + must(enableOwned && !restoreAttempted, "Collector cleanup has no unspent acknowledged owner") + restoreAttempted = true + await change(false, async () => { + const current = await read() + must(current.enabled === true && (current.boundAddress === null || current.boundAddress === endpoint), "Owned collector changed before cleanup") + }) + await authorize() + await collector.assertDrained() + await idle() + await append({ phase: "restored" }) + restored = true + } + try { + await append({ phase: "baseline", pins, scope, enabled: false }) + await syncDirectory(directory) + await change(true, idle) + enableOwned = true + const startupDeadline = Math.min(deadlineAt, Date.now() + 5000) + for (;;) { + if (isBound(await read())) break + must(Date.now() < startupDeadline, "Owned collector did not bind") + await delay(50) + } + await append({ phase: "bound" }) + const result = await work({ + authorize, restore, + assertBound: async () => { must(isBound(await read()), "Owned collector bind is unavailable") }, + retainCapture: async capture => { + await authorize() + const wire = projectChrWire(capture.bytes, { sources: scope.sources, destination: scope.destination, + port: scope.port, ...(scope.linkType && { linkType: scope.linkType }) }) + must(capture.complete === true && capture.dropped === 0 && wire.captureSha256 === capture.sha256, "Capture pin or completeness differs") + const pcap = await open(join(directory, "capture.pcap"), "wx", 0o600) + try { await pcap.writeFile(capture.bytes); await pcap.sync() } finally { await pcap.close() } + await syncDirectory(directory) + await append({ phase: "capture-retained", sha256: capture.sha256, bytes: capture.bytes.length, frames: wire.frames.length }) + }, + }) + must(restored, "Owned exercise did not verify collector cleanup") + return result + } finally { + try { if (enableOwned && !restoreAttempted && (options.mayRestoreAfterFailure?.() ?? true)) await restore() } + finally { await journal.close() } + } +} + +export async function executeChrVpsOperator(options: OwnerOptions & Pick) { + must(process.platform === "linux", "Owned VPS capture requires Linux") + must(options.ingress.existingPort !== options.scope.port && options.serverIds.length === 2 && new Set(options.serverIds).size === 2 && + options.serverIds.every(id => Number.isSafeInteger(id) && id > 0) && options.targets.length === 2 && + options.targets[0].baseline.deviceId !== options.targets[1].baseline.deviceId, "Invalid dedicated two-CHR scope") + checkChrBaselineIngress(options.ingress, options.scope) + return ownedCollector(options, owner => executeOwnedChrCapture({ + ...options, journalPath: join(options.directory, "targets.jsonl"), collector: owner, + capture: async exercise => { + const captured = await captureOwnedChrPackets(options.scope, { + deadlineAt: Math.min(options.deadlineAt, Date.now() + 90000), assertCurrent: owner.assertBound, exercise, + }) + await owner.retainCapture(captured) + return { ...captured, ...prepareChrReplay(captured.wire) } + }, + })) +} + +type CollectorOwner = Pick & Pick +type DisabledOperatorOptions = Omit & + Pick & { + admitBeforeHardwareWrites: () => Promise + capture: (deadlineAt: number, exercise: (signal: AbortSignal) => Promise) => Promise + stagedCapture?: DisabledChrCaptureOptions["capture"] + topology: { + acquire: () => Promise + assertCurrent: () => Promise + preflight: () => Promise> + restore: () => Promise + } +} & ((CollectorOwner & { activateCollector?: never; authorizeFixture?: never }) | { + stagedCapture: DisabledChrCaptureOptions["capture"] + authorizeFixture: () => Promise + requiredCollectorWindowMs: number + activateCollector: (requiredWindowMs: number) => Promise }> +}) + +export async function executeDisabledChrVpsOperator(options: DisabledOperatorOptions) { + must(process.platform === "linux", "Owned VPS capture requires Linux") + return executeDisabledChrVpsOperatorCore(options) +} + +async function executeDisabledChrVpsOperatorCore(options: DisabledOperatorOptions) { + must(options.scope.linkType === "EN10MB" && /^nf[a-f0-9]{8}cv$/.test(options.scope.interfaceName), + "Disabled fixture requires exact Ethernet collector veth") + must(Number.isSafeInteger(options.deadlineAt) && Date.now() < options.deadlineAt && + options.deadlineAt - Date.now() <= (options.activateCollector ? 3600000 : 600000), "Invalid disabled VPS capture deadline") + const { topology } = options + const authorizeFixture = options.activateCollector ? options.authorizeFixture : options.collector.authorize + let acquired = false, collectorEntered = false, collectorRestored = false, collectorRetired = false, routersClean = true + const assertTopology = async () => { + must(Date.now() < options.deadlineAt, "Owned topology deadline ended") + await topology.assertCurrent() + must(Date.now() < options.deadlineAt, "Owned topology deadline ended") + } + try { + await authorizeFixture() + await options.admitBeforeHardwareWrites() + await topology.acquire() + acquired = true + await assertTopology() + const exercise = await topology.preflight() + await assertTopology() + const collectorOwner = options.activateCollector + ? await options.activateCollector(options.requiredCollectorWindowMs) + : options + const activation = "retireCollector" in collectorOwner ? collectorOwner : undefined + const collectorDeadlineAt = activation?.credentialDeadlineAt ?? options.deadlineAt + must(Number.isSafeInteger(collectorDeadlineAt) && Date.now() < collectorDeadlineAt && + collectorDeadlineAt <= options.deadlineAt && collectorDeadlineAt - Date.now() <= 600000 && + (!options.activateCollector || collectorDeadlineAt - Date.now() > options.requiredCollectorWindowMs), "Invalid collector activation deadline") + collectorEntered = true + const result = await ownedCollector({ ...options, ...collectorOwner, deadlineAt: collectorDeadlineAt, mayRestoreAfterFailure: () => routersClean }, async owner => { + let restoreAttempted = false + const restore = async () => { + restoreAttempted = true + await owner.restore() + collectorRestored = true + } + try { + const captureDeadlineAt = Math.min(collectorDeadlineAt, Date.now() + 90000) + const stagedCapture = options.stagedCapture + const { sources, destination, port, linkType } = options.scope + return await executeDisabledChrCapture({ + ...options, ...collectorOwner, ...exercise, deadlineAt: collectorDeadlineAt, scope: { sources, destination, port, linkType }, + ...(stagedCapture ? { stagedTarget: true as const } + : { targetTimerProfile: { kind: "bounded-ipfix-timeout-20s" as const, captureDeadlineAt } }), + journalPath: join(options.directory, "disabled-routers.jsonl"), assertTopology, + authorize: owner.authorize, drainCollector: restore, + reportRouterCleanup: clean => { routersClean = clean }, + capture: async work => { + await owner.assertBound() + const exercise: Parameters[0] = async (signal, timer) => { + await owner.assertBound() + await work(signal, timer) + } + const captured = stagedCapture + ? await stagedCapture(exercise) + : await options.capture(captureDeadlineAt, signal => exercise(signal)) + await owner.assertBound() + await owner.retainCapture(captured) + return captured + }, + }) + } finally { + if (!restoreAttempted && routersClean) await restore() + } + }) + if (activation) { + await activation.retireCollector() + collectorRetired = true + } + return result + } finally { + if (acquired && routersClean && (options.activateCollector ? collectorRetired : !collectorEntered || collectorRestored)) { + await authorizeFixture() + await assertTopology() + await topology.restore() + } + } +} + +export const executeOwnedCollectorForTests = ownedCollector +export const executeDisabledChrVpsOperatorForTests = executeDisabledChrVpsOperatorCore + + diff --git a/backend/src/test/traffic-flow-chr-wire.test.ts b/backend/src/test/traffic-flow-chr-wire.test.ts index 30cf510c..9f04eeca 100644 --- a/backend/src/test/traffic-flow-chr-wire.test.ts +++ b/backend/src/test/traffic-flow-chr-wire.test.ts @@ -1,22 +1,25 @@ import assert from "node:assert/strict" +import { createHash } from "node:crypto" import { test } from "node:test" import { projectChrWire } from "./traffic-flow-chr-wire.js" const scope = { sources: ["192.0.2.1", "192.0.2.2"] as const, destination: "192.0.2.100", port: 4739 } -function capture() { +function capture(ethernet = false, etherType = 0x0800) { const header = Buffer.alloc(24) header.writeUInt32LE(0xa1b2c3d4) header.writeUInt16LE(2, 4) header.writeUInt16LE(4, 6) - header.writeUInt32LE(4124, 16) - header.writeUInt32LE(101, 20) + header.writeUInt32LE(ethernet ? 4138 : 4124, 16) + header.writeUInt32LE(ethernet ? 1 : 101, 20) const frames = [1, 2].map(lastOctet => { - const record = Buffer.alloc(16 + 20 + 8 + 16) + const link = ethernet ? 14 : 0 + const record = Buffer.alloc(16 + link + 20 + 8 + 16) record.writeUInt32LE(1700000000) record.writeUInt32LE(lastOctet, 4) - record.writeUInt32LE(44, 8) - record.writeUInt32LE(44, 12) - const ip = record.subarray(16) + record.writeUInt32LE(44 + link, 8) + record.writeUInt32LE(44 + link, 12) + if (ethernet) record.writeUInt16BE(etherType, 16 + 12) + const ip = record.subarray(16 + link) ip[0] = 0x45 ip.writeUInt16BE(44, 2) ip[9] = 17 @@ -55,3 +58,17 @@ test("refuses wrong link type, incomplete wire bytes, fragments, wrong destinati assert.throws(() => projectChrWire(bytes, scope)) } }) + +test("pins Ethernet framing, preserves original hash, and never infers a link type", () => { + const ethernet = capture(true) + const original = Buffer.from(ethernet) + const ethScope = { ...scope, linkType: "EN10MB" } as const + const result = projectChrWire(ethernet, ethScope) + assert.deepEqual(result.frames, projectChrWire(capture(), { ...scope, linkType: "RAW" }).frames) + assert.equal(result.captureSha256, createHash("sha256").update(original).digest("hex")) + assert.deepEqual(ethernet, original) + assert.throws(() => projectChrWire(ethernet, scope), /pinned link type RAW/) + assert.throws(() => projectChrWire(capture(), ethScope), /pinned link type EN10MB/) + for (const etherType of [0x8100, 0x88a8, 0x86dd, 0x0806]) + assert.throws(() => projectChrWire(capture(true, etherType), ethScope), /untagged Ethernet IPv4/) +}) diff --git a/backend/src/test/traffic-flow-chr-wire.ts b/backend/src/test/traffic-flow-chr-wire.ts index ba99780a..34632ba4 100644 --- a/backend/src/test/traffic-flow-chr-wire.ts +++ b/backend/src/test/traffic-flow-chr-wire.ts @@ -1,4 +1,4 @@ -/** Offline projection of bounded classic RAW-linktype IPv4/UDP pcap files. */ +/** Offline projection of bounded classic IPv4/UDP pcap files. */ import { createHash } from "node:crypto" import { open } from "node:fs/promises" import { isIPv4 } from "node:net" @@ -6,7 +6,15 @@ import { resolve } from "node:path" import { fileURLToPath } from "node:url" const maxCaptureBytes = 512 * 1024 -type Scope = { sources: readonly [string, string]; destination: string; port: number } +export type ChrCaptureLinkType = "RAW" | "EN10MB" +type Scope = { sources: readonly [string, string]; destination: string; port: number; linkType?: ChrCaptureLinkType } +export function chrCaptureLayout(linkType: ChrCaptureLinkType = "RAW") { + switch (linkType) { + case "RAW": return { linkType, pcapType: 101, ipOffset: 0, snapLength: 4124 } + case "EN10MB": return { linkType, pcapType: 1, ipOffset: 14, snapLength: 4138 } + default: throw Error("Unsupported capture link type") + } +} const magic = new Map([["d4c3b2a1", [true, false]], ["a1b2c3d4", [false, false]], ["4d3cb2a1", [true, true]], ["a1b23c4d", [false, true]]]) function must(value: unknown, error: string): asserts value { if (!value) throw Error(error) @@ -15,24 +23,27 @@ function must(value: unknown, error: string): asserts value { export function projectChrWire(bytes: Buffer, scope: Scope) { must(scope.sources.length === 2 && new Set([...scope.sources, scope.destination]).size === 3 && [...scope.sources, scope.destination].every(isIPv4) && Number.isInteger(scope.port) && scope.port > 0 && scope.port <= 65535, "Invalid capture scope") + const layout = chrCaptureLayout(scope.linkType) must(bytes.length >= 24 && bytes.length <= maxCaptureBytes, "Capture outside size bound") const encoding = magic.get(bytes.subarray(0, 4).toString("hex")) must(encoding, "Expected classic pcap") const [little, nanos] = encoding const uint = (at: number) => little ? bytes.readUInt32LE(at) : bytes.readUInt32BE(at) const short = (at: number) => little ? bytes.readUInt16LE(at) : bytes.readUInt16BE(at) - must(short(4) === 2 && short(6) === 4 && uint(20) === 101, "Expected pcap v2.4 with LINKTYPE_RAW") + must(short(4) === 2 && short(6) === 4 && uint(20) === layout.pcapType, "Expected pcap v2.4 with pinned link type " + layout.linkType) const frames = [] let lastTimestamp = -1n for (let at = 24; at < bytes.length;) { must(frames.length < 64 && bytes.length - at >= 16, "Capture has too many frames or an incomplete frame header") const seconds = uint(at), fraction = uint(at + 4), captured = uint(at + 8), original = uint(at + 12) - must(fraction < (nanos ? 1e9 : 1e6) && captured === original && captured >= 28 && captured <= 4124 && + must(fraction < (nanos ? 1e9 : 1e6) && captured === original && captured >= layout.ipOffset + 28 && captured <= layout.snapLength && captured <= uint(16) && bytes.length - at - 16 >= captured, "Incomplete or oversized UDP capture") const timestamp = BigInt(seconds) * 1000000000n + BigInt(fraction) * (nanos ? 1n : 1000n) must(timestamp >= lastTimestamp, "Capture timestamps went backwards") lastTimestamp = timestamp - const packet = bytes.subarray(at + 16, at + 16 + captured) + const frame = bytes.subarray(at + 16, at + 16 + captured) + if (layout.linkType === "EN10MB") must(frame.readUInt16BE(12) === 0x0800, "Expected untagged Ethernet IPv4") + const packet = frame.subarray(layout.ipOffset) at += 16 + captured const header = (packet[0] & 15) * 4 must(packet[0] >>> 4 === 4 && header >= 20 && packet.length >= header + 8 && @@ -56,8 +67,9 @@ export function projectChrWire(bytes: Buffer, scope: Scope) { } async function main() { - const [path, sha, first, second, destination, port, ...rest] = process.argv.slice(2) - must(path && /^[a-f0-9]{64}$/.test(sha ?? "") && first && second && destination && /^\d{1,5}$/.test(port ?? "") && !rest.length, "Expected pcap path, SHA-256 and exact scope") + const [path, sha, first, second, destination, port, linkType, ...rest] = process.argv.slice(2) + must(path && /^[a-f0-9]{64}$/.test(sha ?? "") && first && second && destination && /^\d{1,5}$/.test(port ?? "") && !rest.length && + (linkType === undefined || linkType === "RAW" || linkType === "EN10MB"), "Expected pcap path, SHA-256 and exact scope") const file = await open(path, "r") try { const stat = await file.stat() @@ -70,7 +82,7 @@ async function main() { size += bytesRead } must(size === stat.size, "Capture changed while reading") - const projected = projectChrWire(bytes.subarray(0, size), { sources: [first, second], destination, port: Number(port) }) + const projected = projectChrWire(bytes.subarray(0, size), { sources: [first, second], destination, port: Number(port), ...(linkType === undefined ? {} : { linkType }) }) must(projected.captureSha256 === sha, "Capture SHA-256 differs") console.log(JSON.stringify(projected)) } finally { diff --git a/backend/src/test/traffic-flow-chr-write-admission-ipc.test.ts b/backend/src/test/traffic-flow-chr-write-admission-ipc.test.ts new file mode 100644 index 00000000..5272c69a --- /dev/null +++ b/backend/src/test/traffic-flow-chr-write-admission-ipc.test.ts @@ -0,0 +1,130 @@ +import { spawn } from "node:child_process" +import { once } from "node:events" +import assert from "node:assert/strict" +import { PassThrough } from "node:stream" +import { test } from "node:test" +import { createChrChildAdmissionPipe, serveChrParentAdmissionPipe } from "./traffic-flow-chr-write-admission-ipc.js" +import { createChrParentWriteAdmission, type ChrWriteIntent } from "./traffic-flow-chr-write-admission.js" +import type { ChrStagedFixtureLease } from "./traffic-flow-chr-target-timing-profile.js" + +const first: ChrWriteIntent = { step: "fixture:create", side: "A", deviceId: "a", method: "PUT", + path: "/interface/wireguard", body: { name: "owned" } } +const second: ChrWriteIntent = { step: "fixture:delete", side: "A", deviceId: "a", method: "DELETE", + path: "/interface/wireguard/*A", body: {} } +const wire = ({ side, method, path, body }: ChrWriteIntent) => ({ side, method, path, body }) +function fixture({ failProof = false, deadlineMs = 1000, staged = false, leaseProfile }: { + failProof?: boolean; deadlineMs?: number; staged?: boolean; leaseProfile?: ChrStagedFixtureLease +} = {}) { + const parentIn = new PassThrough(), childIn = new PassThrough(), verified: string[] = [], reserved: string[] = [] + const deadlineAt = Date.now() + deadlineMs + const profile = staged ? "staged-disabled-target" : "direct-target" + const gate = createChrParentWriteAdmission({ deadlineAt, profile, deviceIds: { A: "a", B: "b" }, assertCurrent: async () => {} }) + const stages = [first, second].map(intent => ({ + step: intent.step, + async intent() { + reserved.push(intent.step) + if (intent === second) assert.deepEqual(verified, [first.step]) + return intent + }, + async verify(command: ChrWriteIntent) { + assert.deepEqual(command, intent) + if (failProof) throw Error("Independent proof refused") + verified.push(command.step) + }, + })) + const parent = serveChrParentAdmissionPipe({ input: parentIn, output: childIn }, gate, stages, deadlineAt, profile, leaseProfile) + parent.catch(() => {}) + const child = createChrChildAdmissionPipe({ input: childIn, output: parentIn }, profile) + const send = (intent: ChrWriteIntent) => child.run(intent, async arm => { + await arm() + await gate.authorizeWrite(wire(intent)) + return "native-ack" + }) + return { child, parent, gate, send, verified, reserved, parentIn, childIn } +} +test("the staged write schedule alone cannot extend the admission pipe lease", async () => { + const leaseProfile = "staged-disabled-target" + assert.throws(() => fixture({ deadlineMs: 1200000, staged: true }), /pipe refused/) + assert.throws(() => fixture({ deadlineMs: 1200000, leaseProfile }), /pipe refused/) + assert.throws(() => fixture({ deadlineMs: 3700000, staged: true, leaseProfile }), /pipe refused/) + const f = fixture({ deadlineMs: 1200000, staged: true, leaseProfile }) + assert.equal(await f.send(first), "native-ack") + assert.equal(await f.send(second), "native-ack") + f.child.close() + assert.deepEqual(await f.parent, { completed: 2 }) +}) +test("only the trusted ordered intent can arm and consume the relay, with independent proof before next step", async () => { + const f = fixture() + assert.equal(await f.send(first), "native-ack") + assert.equal(await f.send(second), "native-ack") + f.child.close() + assert.deepEqual(await f.parent, { completed: 2 }) + assert.deepEqual(f.verified, [first.step, second.step]) + assert.equal(f.gate.consumed, 2) +}) +test("out-of-order step, child-chosen body and extra arbitrary intent fields refuse before socket consumption", async () => { + for (const altered of [second, { ...first, body: { name: "foreign" } }, { ...first, deviceId: "b" }]) { + const f = fixture() + await assert.rejects(f.send(altered)) + await assert.rejects(f.parent) + assert.equal(f.gate.consumed, 0) + assert.deepEqual(f.verified, []) + } + const f = fixture() + f.parentIn.end(JSON.stringify({ sequence: 1, op: "reserve", step: first.step, body: { name: "foreign" } }) + "\n") + await assert.rejects(f.parent) + assert.deepEqual(f.reserved, []) +}) +test("child completion cannot substitute for relay consumption or independent parent proof", async () => { + const unconsumed = fixture() + await assert.rejects(unconsumed.child.run(first, async arm => { await arm(); return "invented" })) + await assert.rejects(unconsumed.parent) + assert.equal(unconsumed.gate.consumed, 0) + const unproven = fixture({ failProof: true }) + await assert.rejects(unproven.send(first)) + await assert.rejects(unproven.parent) + assert.deepEqual(unproven.reserved, [first.step]) + assert.equal(unproven.gate.consumed, 1) +}) +test("EOF and deadline retire unused parent grant", async () => { + const eof = fixture() + eof.child.close() + await assert.rejects(eof.parent) + assert.equal(eof.gate.consumed, 0) + const expired = fixture({ deadlineMs: 20 }) + await assert.rejects(expired.parent) + expired.child.close() + assert.equal(expired.gate.consumed, 0) +}) + +test("actual inherited descriptor pair carries only control while parent consumes a separate mock relay event", async () => { + const deadlineAt = Date.now() + 10000 + const gate = createChrParentWriteAdmission({ deadlineAt, deviceIds: { A: "a", B: "b" }, assertCurrent: async () => {} }) + const file = new URL("./traffic-flow-chr-write-admission-ipc.ts", import.meta.url).href + const source = [ + "import { once } from 'node:events';", + "const {openInheritedChrChildAdmission}=await import(" + JSON.stringify(file) + ");", + "const child=openInheritedChrChildAdmission();", + "await child.run(" + JSON.stringify(first) + ",async arm=>{await arm();const ack=once(process.stdin,'data');", + "process.stdout.write('relay-event\\n');await ack;process.stdin.pause();return 'ack';});", + "child.close();", + ].join("") + const child = spawn(process.execPath, ["--import", "tsx", "--input-type=module", "-e", source], + { stdio: ["pipe", "pipe", "ignore", "pipe", "pipe"], windowsHide: true }) + const exited = once(child, "exit") + const event = once(child.stdout!, "data") + const [parentOut, parentIn] = [child.stdio[3], child.stdio[4]] + assert(parentOut?.writable && parentIn?.readable) + const parent = serveChrParentAdmissionPipe({ input: parentIn, output: parentOut }, gate, + [{ step: first.step, intent: async () => first, verify: async () => {} }], deadlineAt) + parent.catch(() => {}) + try { + const [chunk] = await Promise.race([event, exited.then(() => assert.fail("Child exited before mock relay"))]) + assert.match(String(chunk), /^relay-event/) + await gate.authorizeWrite(wire(first)) + child.stdin!.end("ack") + assert.deepEqual(await parent, { completed: 1 }) + assert.deepEqual(await exited, [0, null]) + assert.equal(gate.consumed, 1) + } finally { if (child.exitCode === null && child.signalCode === null) child.kill() } +}) diff --git a/backend/src/test/traffic-flow-chr-write-admission-ipc.ts b/backend/src/test/traffic-flow-chr-write-admission-ipc.ts new file mode 100644 index 00000000..35cad73a --- /dev/null +++ b/backend/src/test/traffic-flow-chr-write-admission-ipc.ts @@ -0,0 +1,180 @@ +import { createHash } from "node:crypto" +import { createReadStream, createWriteStream } from "node:fs" +import type { Readable, Writable } from "node:stream" +import { chrWriteScheduleLimit, type ChrWriteAdmission, type ChrWriteIntent, type ChrWriteScheduleProfile } from "./traffic-flow-chr-write-admission.js" +import { chrFixtureDeadlineMaxMs, type ChrStagedFixtureLease } from "./traffic-flow-chr-target-timing-profile.js" + +type Pipes = { input: Readable; output: Writable } +type Gate = ChrWriteAdmission & { retire(): void } +type Stage = { step: string; intent(): Promise; verify(intent: Readonly): Promise } +type Request = { sequence: number; op: "reserve"; step: string } | + { sequence: number; op: "arm" | "finish" } +type Response = { sequence: number; ok: true; digest?: string } | { sequence: number; ok: false } +const maxFrame = 512 +const safeStep = /^[a-zA-Z0-9:._-]{1,100}$/ +const digestPattern = /^[a-f0-9]{64}$/ +const refused = () => { throw Error("Owned CHR admission pipe refused") } + +function digest(intent: ChrWriteIntent) { + const { step, deviceId, side, method, path, body } = intent + return createHash("sha256").update(JSON.stringify([step, deviceId, side, method, path, + Object.entries(body).sort(([left], [right]) => left.localeCompare(right))])).digest("hex") +} + +async function* frames(input: Readable, maxFrames: number) { + let buffered = Buffer.alloc(0), count = 0 + for await (const incoming of input) { + const bytes = Buffer.isBuffer(incoming) ? incoming : Buffer.from(incoming) + if (bytes.length > maxFrame * maxFrames) refused() + for (let offset = 0; offset < bytes.length;) { + const newline = bytes.indexOf(10, offset) + const end = newline < 0 ? bytes.length : newline + if (buffered.length + end - offset > maxFrame) refused() + buffered = Buffer.concat([buffered, bytes.subarray(offset, end)]) + if (newline < 0) break + if (!buffered.length || ++count > maxFrames) refused() + const value: unknown = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(buffered)) + if (!value || typeof value !== "object" || Array.isArray(value)) refused() + yield value as Record + buffered = Buffer.alloc(0) + offset = newline + 1 + } + } + if (buffered.length) refused() +} + +function write(output: Writable, frame: Request | Response) { + const body = JSON.stringify(frame) + "\n" + if (Buffer.byteLength(body) > maxFrame || output.destroyed) return Promise.reject(Error("Owned CHR admission pipe closed")) + return new Promise((resolve, reject) => output.write(body, error => error ? reject(error) : resolve())) +} + +function defer() { + let resolve!: () => void, reject!: (error: Error) => void + const promise = new Promise((accept, deny) => { resolve = accept; reject = deny }) + promise.catch(() => {}) + return { promise, resolve, reject } +} + +function shape(frame: Record, keys: readonly string[]) { + return Object.keys(frame).length === keys.length && keys.every(key => Object.hasOwn(frame, key)) +} + +/** Parent input/output are the two ends created by the trusted launcher. + * Stage factories run only here and must derive changing IDs from independently + * confirmed state. The child can name the next step, never its native intent. */ +export function serveChrParentAdmissionPipe(pipes: Pipes, gate: Gate, trustedStages: readonly Stage[], deadlineAt: number, + profile: ChrWriteScheduleProfile = "direct-target", leaseProfile?: ChrStagedFixtureLease) { + const maxWrites = chrWriteScheduleLimit(profile) + const stages = trustedStages.map(stage => ({ ...stage })) + if (!stages.length || stages.length > maxWrites || new Set(stages.map(stage => stage.step)).size !== stages.length || + (leaseProfile !== undefined && leaseProfile !== profile) || !Number.isSafeInteger(deadlineAt) || + deadlineAt <= Date.now() || deadlineAt - Date.now() > chrFixtureDeadlineMaxMs(leaseProfile) || + !stages.every(stage => safeStep.test(stage.step) && typeof stage.intent === "function" && typeof stage.verify === "function")) return refused() + pipes.output.on("error", () => {}) + return (async () => { + const timer = setTimeout(() => { gate.retire(); pipes.input.destroy() }, Math.max(1, deadlineAt - Date.now())) + let sequence = 0, cursor = 0 + let slot: { intent: ChrWriteIntent; arm(): Promise; completion: ReturnType; running: Promise; armed: boolean } | undefined + try { + for await (const frame of frames(pipes.input, maxWrites * 3)) { + if (Date.now() >= deadlineAt || frame.sequence !== ++sequence) refused() + if (frame.op === "reserve") { + const stage = stages[cursor] + if (slot || !stage || !shape(frame, ["sequence", "op", "step"]) || frame.step !== stage.step) return refused() + const intent = structuredClone(await stage.intent()) + if (intent.step !== stage.step) refused() + const completion = defer() + let arm: (() => Promise) | undefined + const running = gate.run(intent, callback => { arm = callback; return completion.promise }) + running.catch(() => {}) + if (!arm) { await running; return refused() } + slot = { intent, arm, completion, running, armed: false } + await write(pipes.output, { sequence, ok: true, digest: digest(intent) }) + continue + } + if (!slot || !shape(frame, ["sequence", "op"])) return refused() + if (frame.op === "arm" && !slot.armed) { + await slot.arm() + slot.armed = true + } else if (frame.op === "finish" && slot.armed) { + slot.completion.resolve() + await slot.running + await stages[cursor].verify(structuredClone(slot.intent)) + if (Date.now() >= deadlineAt) refused() + slot = undefined + cursor++ + } else return refused() + await write(pipes.output, { sequence, ok: true }) + } + if (slot || cursor !== stages.length || Date.now() >= deadlineAt) refused() + return { completed: cursor } + } catch (error) { + slot?.completion.reject(Error("Owned CHR admission pipe interrupted")) + await slot?.running.catch(() => {}) + await write(pipes.output, { sequence, ok: false }).catch(() => {}) + throw error + } finally { + clearTimeout(timer) + gate.retire() + pipes.output.end() + if (!pipes.input.destroyed) pipes.input.destroy() + } + })() +} + +/** Child adapter for the existing owner callback. Wire command content is never + * transmitted in the admission pipe. Native HTTP remains a separate channel. */ +export function createChrChildAdmissionPipe(pipes: Pipes, profile: ChrWriteScheduleProfile = "direct-target") { + const maxFrames = chrWriteScheduleLimit(profile) * 3 + pipes.output.on("error", () => {}) + let sequence = 0, busy = false, closed = false + const replies = frames(pipes.input, maxFrames) + async function exchange(request: Omit & { step?: string }, expectDigest = false) { + const current = ++sequence + if (current > maxFrames) return refused() + await write(pipes.output, { ...request, sequence: current } as Request) + const response = await replies.next() + if (response.done || response.value.sequence !== current || response.value.ok !== true || + !shape(response.value, expectDigest ? ["sequence", "ok", "digest"] : ["sequence", "ok"])) return refused() + return response.value + } + const close = () => { + closed = true + pipes.output.end() + if (!pipes.input.destroyed) pipes.input.destroy() + } + return Object.freeze({ + close, + async run(intent: ChrWriteIntent, send: (arm: () => Promise) => Promise) { + if (closed || busy || !safeStep.test(intent.step)) return refused() + busy = true + try { + const expected = digest(structuredClone(intent)) + const response = await exchange({ op: "reserve", step: intent.step }, true) + if (typeof response.digest !== "string" || !digestPattern.test(response.digest) || response.digest !== expected) return refused() + let armed = false + const result = await send(async () => { + if (armed) return refused() + armed = true + await exchange({ op: "arm" }) + }) + if (!armed) return refused() + await exchange({ op: "finish" }) + return result + } catch (error) { + close() + throw error + } finally { busy = false } + }, + }) +} + +/** The launcher grants only these inherited descriptors to the child. */ +export function openInheritedChrChildAdmission() { + return createChrChildAdmissionPipe({ + input: createReadStream("", { fd: 3, autoClose: true }), + output: createWriteStream("", { fd: 4, autoClose: true }), + }) +} + diff --git a/backend/src/test/traffic-flow-chr-write-admission.test.ts b/backend/src/test/traffic-flow-chr-write-admission.test.ts new file mode 100644 index 00000000..529b2398 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-write-admission.test.ts @@ -0,0 +1,81 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { createChrParentWriteAdmission, type ChrWriteIntent } from "./traffic-flow-chr-write-admission.js" + +const first: ChrWriteIntent = { step: "topology:create:A:edgeA", side: "A", deviceId: "device-a", method: "PUT", + path: "/interface/wireguard", body: { name: "fixture", "listen-port": "50101" } } +const second: ChrWriteIntent = { step: "flow:B:flow.enable", side: "B", deviceId: "device-b", method: "POST", + path: "/ip/traffic-flow/set", body: { enabled: "yes", interfaces: "fixture" } } +const wire = ({ side, method, path, body }: ChrWriteIntent) => ({ side, method, path, body }) +function fixture() { + const identities: string[] = [] + let now = 1, revoked = false + const gate = createChrParentWriteAdmission({ deadlineAt: 100, now: () => now, deviceIds: { A: "device-a", B: "device-b" }, + assertCurrent: async (side, identity) => { identities.push(side + ":" + identity); if (revoked) throw Error("revoked") } }) + const send = (intent: ChrWriteIntent) => gate.run(intent, async arm => { await arm(); await gate.authorizeWrite(wire(intent)); return "ACK" }) + return { gate, identities, send, expire: () => { now = 100 }, revoke: () => { revoked = true } } +} +test("relay consumes one exact armed command and distinct ordered intent for both devices", async () => { + const f = fixture() + assert.equal(await f.send(first), "ACK") + assert.equal(await f.send(second), "ACK") + assert.equal(f.gate.consumed, 2) + assert.deepEqual(f.identities, ["A:device-a", "A:device-a", "B:device-b", "B:device-b"]) + await assert.rejects(f.send(first), /admission refused/) + assert.equal(f.gate.consumed, 2) +}) +test("wrong route, method, body, device, unarmed write and duplicate are refused without a new grant", async () => { + for (const altered of [ + { ...wire(first), side: "B" as const }, { ...wire(first), path: "/ip/address" }, + { ...wire(first), method: "PATCH" }, { ...wire(first), body: { ...first.body, "listen-port": "50102" } }, + ]) { + const f = fixture() + await assert.rejects(f.gate.run(first, async arm => { await arm(); await f.gate.authorizeWrite(altered) })) + await assert.rejects(f.send(first)) + assert.equal(f.gate.consumed, 0) + } + const invalid = fixture() + await assert.rejects(invalid.send({ ...first, deviceId: "device-b" })) + const unarmed = fixture() + await assert.rejects(unarmed.gate.run(first, async () => unarmed.gate.authorizeWrite(wire(first)))) + const duplicate = fixture() + await assert.rejects(duplicate.gate.run(first, async arm => { + await arm() + await duplicate.gate.authorizeWrite(wire(first)) + await duplicate.gate.authorizeWrite(wire(first)) + })) + assert.equal(duplicate.gate.consumed, 1) +}) +test("authority drift, deadline and absent socket consumption refuse after local arming", async () => { + for (const kind of ["authority", "deadline", "missing"] as const) { + const f = fixture() + await assert.rejects(f.gate.run(first, async arm => { + await arm() + if (kind === "authority") f.revoke() + if (kind === "deadline") f.expire() + if (kind !== "missing") await f.gate.authorizeWrite(wire(first)) + })) + assert.equal(f.gate.consumed, 0) + await assert.rejects(f.send(second)) + } +}) + +test("staged target PATCH requires explicit profile, exact step, ID path and disabled-only body", async () => { + const intent = { step: "target:A:enable", deviceId: "A", side: "A", method: "PATCH", + path: "/ip/traffic-flow/target/*A", body: { disabled: "no" } } as const + for (const profile of ["direct-target", "staged-disabled-target"] as const) { + const gate = createChrParentWriteAdmission({ profile, deadlineAt: Date.now() + 2000, deviceIds: { A: "A", B: "B" }, assertCurrent: async () => {} }) + const run = gate.run(intent, async arm => { await arm(); return await gate.authorizeWrite(intent) }) + if (profile === "direct-target") await assert.rejects(run) + else assert.deepEqual(await run, intent) + } + for (const bad of [ + { ...intent, step: "target:A:create" }, + { ...intent, path: "/ip/traffic-flow/target/set" }, + { ...intent, body: { disabled: "no", port: "1" } }, + ]) { + const gate = createChrParentWriteAdmission({ profile: "staged-disabled-target", deadlineAt: Date.now() + 2000, + deviceIds: { A: "A", B: "B" }, assertCurrent: async () => {} }) + await assert.rejects(gate.run(bad, async () => assert.fail("Native arm offered to invalid staged target"))) + } +}) diff --git a/backend/src/test/traffic-flow-chr-write-admission.ts b/backend/src/test/traffic-flow-chr-write-admission.ts new file mode 100644 index 00000000..b4c5be80 --- /dev/null +++ b/backend/src/test/traffic-flow-chr-write-admission.ts @@ -0,0 +1,134 @@ +import { isDeepStrictEqual } from "node:util" + +type Side = "A" | "B" +type Method = "PUT" | "PATCH" | "POST" | "DELETE" +type Body = Readonly> +type RelayWrite = { side: Side; method: string; path: string; body: Body } +export type ChrWriteIntent = { step: string; deviceId: string; side: Side; method: Method; path: string; body: Body } +export type ChrWriteAdmission = { + run(intent: ChrWriteIntent, send: (arm: () => Promise) => Promise): Promise +} +type Slot = { intent: ChrWriteIntent; sequence: number; phase: "pending" | "armed" | "authorizing" | "consumed" } + +export type ChrWriteScheduleProfile = "direct-target" | "staged-disabled-target" +export function chrWriteScheduleLimit(profile: ChrWriteScheduleProfile) { + if (profile === "direct-target") return 48 + if (profile === "staged-disabled-target") return 50 + return fail() +} +const fail = () => { throw Error("Exact CHR write admission refused") } +const ownMenus = new Set(["/interface/wireguard", "/interface/wireguard/peers", "/ip/address", "/ip/route", "/ip/traffic-flow/target"]) +const targetMenu = "/ip/traffic-flow/target" +const singletons = new Set(["/ip/traffic-flow", "/ip/traffic-flow/ipfix"]) +const id = /^\*[0-9a-f]{1,16}$/i + +function command(method: string, path: string, body: Body, stagedEnable: boolean) { + if (!body || Object.getPrototypeOf(body) !== Object.prototype || Object.keys(body).length > 24 || + Object.entries(body).some(([key, value]) => !/^[a-z][a-z0-9-]{0,47}$/.test(key) || typeof value !== "string" || value.length > 4096 || + /[\u0000-\u001f]/.test(value))) return fail() + const slash = path.lastIndexOf("/"), base = path.slice(0, slash), token = path.slice(slash + 1) + const count = Object.keys(body).length + if (method === "PUT" && ownMenus.has(path) && count || + method === "DELETE" && ownMenus.has(base) && id.test(token) && count === 0 || + method === "PATCH" && singletons.has(base) && id.test(token) && count || + stagedEnable && method === "PATCH" && base === targetMenu && id.test(token) && count === 1 && body.disabled === "no" || + method === "POST" && singletons.has(base) && token === "set" && count) return + return fail() +} + +/** Keep this owner in the pinned relay parent. Only the trusted workflow + * continuation can install an intent. The relay socket receives consume only. */ +export function createChrParentWriteAdmission(options: { + profile?: ChrWriteScheduleProfile + deadlineAt: number + deviceIds: Readonly> + assertCurrent(side: Side, deviceId: string): Promise + assertIntentCurrent?(intent: Readonly): Promise + now?: () => number +}) { + const { deadlineAt, assertCurrent, assertIntentCurrent, now = Date.now } = options + const profile = options.profile ?? "direct-target" + const maxWrites = chrWriteScheduleLimit(profile) + const isStagedEnable = (intent: ChrWriteIntent) => profile === "staged-disabled-target" && + intent.step === "target:" + intent.side + ":enable" + const devices = { ...options.deviceIds } + if (!devices.A || !devices.B || devices.A === devices.B || !Number.isSafeInteger(deadlineAt)) return fail() + let current: Slot | undefined, refused = false, consumed = 0 + const steps = new Set() + const live = () => { + if (refused || now() >= deadlineAt) return fail() + } + const authority = async (intent: ChrWriteIntent) => { + live() + await assertCurrent(intent.side, intent.deviceId) + await assertIntentCurrent?.(structuredClone(intent)) + live() + } + const same = (slot: Slot) => { + live() + if (current !== slot) return fail() + } + return Object.freeze({ + get consumed() { return consumed }, + retire() { refused = true }, + async run(input: ChrWriteIntent, send: (arm: () => Promise) => Promise): Promise { + let slot: Slot | undefined + try { + live() + if (current || !["A", "B"].includes(input.side) || input.deviceId !== devices[input.side] || + !/^[a-zA-Z0-9:._-]{1,100}$/.test(input.step) || steps.has(input.step) || consumed >= maxWrites) return fail() + command(input.method, input.path, input.body, isStagedEnable(input)) + if (input.step === "target:" + input.side + ":enable" && + (!isStagedEnable(input) || input.method !== "PATCH" || !input.path.startsWith(targetMenu + "/"))) return fail() + const intent = structuredClone(input) + slot = { intent, sequence: consumed + 1, phase: "pending" } + current = slot + const exact = slot + steps.add(intent.step) + const result = await send(async () => { + try { + same(exact) + if (exact.phase !== "pending") return fail() + await authority(intent) + same(exact) + if (exact.phase !== "pending") return fail() + exact.phase = "armed" + } catch (error) { refused = true; throw error } + }) + same(exact) + if (exact.phase !== "consumed" || consumed !== exact.sequence) return fail() + return result + } catch (error) { refused = true; throw error } + finally { if (current === slot) current = undefined } + }, + async authorizeWrite(input: RelayWrite) { + try { + live() + const slot = current + if (!slot || slot.phase !== "armed") return fail() + const intent = slot.intent + command(input.method, input.path, input.body, isStagedEnable(intent)) + if (input.side !== intent.side || input.method !== intent.method || input.path !== intent.path || + !isDeepStrictEqual(input.body, intent.body)) return fail() + slot.phase = "authorizing" + await authority(intent) + same(slot) + if (slot.phase !== "authorizing" || slot.sequence !== consumed + 1) return fail() + slot.phase = "consumed" + consumed++ + return structuredClone(intent) + } catch (error) { refused = true; throw error } + }, + }) +} + +export function runChrWrite(admission: ChrWriteAdmission | undefined, intent: ChrWriteIntent | undefined, + send: (arm: () => Promise) => Promise) { + if (!admission) return send(async () => {}) + if (!intent) return fail() + return admission.run(intent, send) +} + + + +