feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -0,0 +1,12 @@
|
||||
DROP INDEX IF EXISTS idx_job_audit_expired_lease;
|
||||
DROP INDEX IF EXISTS idx_job_audit_claim;
|
||||
ALTER TABLE job_audit DROP COLUMN request_sequence;
|
||||
ALTER TABLE job_audit DROP COLUMN cancel_requested;
|
||||
ALTER TABLE job_audit DROP COLUMN available_at;
|
||||
ALTER TABLE job_audit DROP COLUMN fencing_token;
|
||||
ALTER TABLE job_audit DROP COLUMN attempt;
|
||||
ALTER TABLE job_audit DROP COLUMN lease_until;
|
||||
ALTER TABLE job_audit DROP COLUMN owner_id;
|
||||
ALTER TABLE job_audit DROP COLUMN request_hash;
|
||||
ALTER TABLE job_audit DROP COLUMN request_json;
|
||||
DELETE FROM tenant WHERE id = '00000000-0000-0000-0000-000000000000' AND slug = 'evobgp-system-jobs';
|
||||
@@ -0,0 +1,15 @@
|
||||
ALTER TABLE job_audit ADD COLUMN request_hash TEXT;
|
||||
ALTER TABLE job_audit ADD COLUMN request_sequence BIGINT GENERATED ALWAYS AS IDENTITY;
|
||||
ALTER TABLE job_audit ADD COLUMN request_json JSONB;
|
||||
ALTER TABLE job_audit ADD COLUMN owner_id TEXT;
|
||||
ALTER TABLE job_audit ADD COLUMN lease_until TIMESTAMPTZ;
|
||||
ALTER TABLE job_audit ADD COLUMN attempt INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE job_audit ADD COLUMN fencing_token BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE job_audit ADD COLUMN available_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE job_audit ADD COLUMN cancel_requested BOOLEAN NOT NULL DEFAULT false;
|
||||
INSERT INTO tenant (id, name, slug) VALUES ('00000000-0000-0000-0000-000000000000', 'EvoBGP system jobs', 'evobgp-system-jobs') ON CONFLICT (id) DO NOTHING;
|
||||
CREATE INDEX idx_job_audit_claim ON job_audit (available_at, created_at) WHERE status = 'queued';
|
||||
CREATE INDEX idx_job_audit_expired_lease ON job_audit (lease_until) WHERE status = 'running';
|
||||
-- Give pre-upgrade workers their old 15-minute grace; later claims can recover them.
|
||||
UPDATE job_audit SET owner_id = 'legacy-unfenced', lease_until = now() + interval '15 minutes', attempt = 1
|
||||
WHERE status = 'running' AND lease_until IS NULL;
|
||||
@@ -0,0 +1,7 @@
|
||||
DROP TABLE deployment_speaker;
|
||||
ALTER TABLE bgp_speaker DROP COLUMN deployment_job_id;
|
||||
ALTER TABLE bgp_speaker DROP COLUMN deployment_request_sequence;
|
||||
ALTER TABLE bgp_speaker DROP COLUMN deployment_generation;
|
||||
ALTER TABLE bgp_speaker DROP COLUMN delivery_owner;
|
||||
ALTER TABLE bgp_speaker DROP COLUMN delivery_token;
|
||||
ALTER TABLE bgp_speaker DROP COLUMN delivery_until;
|
||||
@@ -0,0 +1,27 @@
|
||||
ALTER TABLE bgp_speaker ADD COLUMN deployment_generation BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE bgp_speaker ADD COLUMN deployment_request_sequence BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE bgp_speaker ADD COLUMN deployment_job_id UUID;
|
||||
ALTER TABLE bgp_speaker ADD COLUMN delivery_owner TEXT;
|
||||
ALTER TABLE bgp_speaker ADD COLUMN delivery_token BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE bgp_speaker ADD COLUMN delivery_until TIMESTAMPTZ;
|
||||
CREATE TABLE deployment_speaker (
|
||||
job_id UUID NOT NULL REFERENCES job_audit(id) ON DELETE CASCADE,
|
||||
tenant_id UUID NOT NULL REFERENCES tenant(id) ON DELETE CASCADE,
|
||||
speaker_id UUID NOT NULL REFERENCES bgp_speaker(id) ON DELETE CASCADE,
|
||||
revision_id UUID NOT NULL,
|
||||
generation BIGINT NOT NULL,
|
||||
request_sequence BIGINT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
confirmation TEXT NOT NULL DEFAULT '',
|
||||
error_message TEXT NOT NULL DEFAULT '',
|
||||
health_received_at TIMESTAMPTZ,
|
||||
health_observation_token BIGINT NOT NULL DEFAULT 0,
|
||||
health_revision_id TEXT NOT NULL DEFAULT '',
|
||||
health_source_at TEXT NOT NULL DEFAULT '',
|
||||
health_error TEXT NOT NULL DEFAULT '',
|
||||
retry_round INTEGER NOT NULL DEFAULT 0,
|
||||
dispatch_attempts INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY(job_id,speaker_id)
|
||||
);
|
||||
CREATE INDEX deployment_speaker_tenant_job ON deployment_speaker(tenant_id,job_id);
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE telemetry_snapshot;
|
||||
@@ -0,0 +1,21 @@
|
||||
CREATE TABLE telemetry_snapshot (
|
||||
tenant_id UUID NOT NULL REFERENCES tenant(id) ON DELETE CASCADE,
|
||||
target_id TEXT NOT NULL,
|
||||
speaker_id UUID REFERENCES bgp_speaker(id) ON DELETE SET NULL,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('remote_agent','local_bird')),
|
||||
local_authority TEXT NOT NULL DEFAULT '',
|
||||
config_fingerprint TEXT NOT NULL DEFAULT '',
|
||||
bgp_json JSONB NOT NULL DEFAULT '{}',
|
||||
health_json JSONB NOT NULL DEFAULT '{}',
|
||||
next_poll_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
failure_count INTEGER NOT NULL DEFAULT 0,
|
||||
discovery_cursor INTEGER NOT NULL DEFAULT 0,
|
||||
last_refresh_requested_at TIMESTAMPTZ,
|
||||
refresh_pending BOOLEAN NOT NULL DEFAULT false,
|
||||
owner_id TEXT,
|
||||
fencing_token BIGINT NOT NULL DEFAULT 0,
|
||||
lease_until TIMESTAMPTZ,
|
||||
PRIMARY KEY (tenant_id,target_id)
|
||||
);
|
||||
CREATE INDEX telemetry_snapshot_due ON telemetry_snapshot(next_poll_at) WHERE owner_id IS NULL;
|
||||
CREATE INDEX telemetry_snapshot_speaker ON telemetry_snapshot(speaker_id);
|
||||
@@ -0,0 +1,23 @@
|
||||
DROP TRIGGER IF EXISTS render_snapshot_changed ON module_prefix_snapshot;
|
||||
DROP TRIGGER IF EXISTS render_speaker_changed ON bgp_speaker;
|
||||
DROP TRIGGER IF EXISTS render_settings_changed ON global_settings;
|
||||
DROP TRIGGER IF EXISTS render_community_changed ON bgp_community;
|
||||
DROP TRIGGER IF EXISTS render_peer_changed ON bgp_peer;
|
||||
DROP TRIGGER IF EXISTS render_module_doh_changed ON module_doh_profile;
|
||||
DROP TRIGGER IF EXISTS render_doh_profile_changed ON doh_profile;
|
||||
DROP TRIGGER IF EXISTS render_ip_range_entry_changed ON module_ip_range_entry;
|
||||
DROP TRIGGER IF EXISTS render_domain_entry_changed ON module_domain_entry;
|
||||
DROP TRIGGER IF EXISTS render_as_entry_changed ON module_as_entry;
|
||||
DROP TRIGGER IF EXISTS render_cdn_source_changed ON module_cdn_source;
|
||||
DROP TRIGGER IF EXISTS render_module_changed ON module;
|
||||
DROP TRIGGER IF EXISTS module_snapshot_version ON module_prefix_snapshot;
|
||||
DROP TRIGGER IF EXISTS tenant_render_created ON tenant;
|
||||
DROP FUNCTION IF EXISTS render_snapshot_version();
|
||||
DROP FUNCTION IF EXISTS render_semantic_change();
|
||||
DROP FUNCTION IF EXISTS render_tenant_created();
|
||||
DROP FUNCTION IF EXISTS render_bump_module_generation(UUID);
|
||||
DROP FUNCTION IF EXISTS render_bump_tenant_generation(UUID);
|
||||
DROP TABLE IF EXISTS render_revision_manifest;
|
||||
ALTER TABLE module_prefix_snapshot DROP COLUMN IF EXISTS snapshot_version;
|
||||
DROP SEQUENCE IF EXISTS module_snapshot_version_seq;
|
||||
DROP TABLE IF EXISTS tenant_render_generation;
|
||||
@@ -0,0 +1,152 @@
|
||||
-- A tenant clock is the short commit barrier for revision admission. Semantic
|
||||
-- writers update it in their own transaction, after their entity row change.
|
||||
CREATE TABLE tenant_render_generation (
|
||||
tenant_id UUID PRIMARY KEY REFERENCES tenant(id) ON DELETE CASCADE,
|
||||
generation BIGINT NOT NULL DEFAULT 1
|
||||
);
|
||||
INSERT INTO tenant_render_generation(tenant_id, generation)
|
||||
SELECT id, 1 FROM tenant;
|
||||
|
||||
CREATE SEQUENCE module_snapshot_version_seq AS BIGINT;
|
||||
ALTER TABLE module_prefix_snapshot
|
||||
ADD COLUMN snapshot_version BIGINT NOT NULL DEFAULT nextval('module_snapshot_version_seq');
|
||||
|
||||
CREATE TABLE render_revision_manifest (
|
||||
revision_id UUID PRIMARY KEY REFERENCES config_revision(id) ON DELETE CASCADE,
|
||||
tenant_id UUID NOT NULL REFERENCES tenant(id) ON DELETE CASCADE,
|
||||
config_generation BIGINT NOT NULL,
|
||||
manifest_json JSONB NOT NULL
|
||||
);
|
||||
CREATE INDEX render_revision_manifest_tenant ON render_revision_manifest(tenant_id, revision_id);
|
||||
|
||||
CREATE FUNCTION render_bump_tenant_generation(p_tenant UUID) RETURNS void
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF p_tenant IS NULL THEN RETURN; END IF;
|
||||
INSERT INTO tenant_render_generation(tenant_id, generation)
|
||||
VALUES (p_tenant, 2)
|
||||
ON CONFLICT (tenant_id) DO UPDATE
|
||||
SET generation = tenant_render_generation.generation + 1;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE FUNCTION render_bump_module_generation(p_module UUID) RETURNS void
|
||||
LANGUAGE plpgsql AS $$
|
||||
DECLARE v_tenant UUID;
|
||||
BEGIN
|
||||
IF p_module IS NULL THEN RETURN; END IF;
|
||||
SELECT tenant_id INTO v_tenant FROM module WHERE id = p_module;
|
||||
PERFORM render_bump_tenant_generation(v_tenant);
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE FUNCTION render_tenant_created() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
INSERT INTO tenant_render_generation(tenant_id, generation) VALUES(NEW.id, 1);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
CREATE TRIGGER tenant_render_created AFTER INSERT ON tenant
|
||||
FOR EACH ROW EXECUTE FUNCTION render_tenant_created();
|
||||
|
||||
CREATE FUNCTION render_semantic_change() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
DECLARE v_old JSONB; v_new JSONB;
|
||||
v_old_tenant UUID; v_new_tenant UUID;
|
||||
v_old_module UUID; v_new_module UUID;
|
||||
BEGIN
|
||||
IF TG_OP = 'UPDATE' THEN
|
||||
v_old := to_jsonb(OLD);
|
||||
v_new := to_jsonb(NEW);
|
||||
IF TG_TABLE_NAME = 'module' THEN
|
||||
v_old := v_old - ARRAY['input_hash','last_refreshed_at','updated_at'];
|
||||
v_new := v_new - ARRAY['input_hash','last_refreshed_at','updated_at'];
|
||||
ELSIF TG_TABLE_NAME = 'module_cdn_source' THEN
|
||||
v_old := v_old - ARRAY['etag','last_refreshed_at','updated_at'];
|
||||
v_new := v_new - ARRAY['etag','last_refreshed_at','updated_at'];
|
||||
ELSIF TG_TABLE_NAME = 'module_as_entry' THEN
|
||||
v_old := v_old - ARRAY['asn_name','prefix_count','asn_resolved_at','updated_at'];
|
||||
v_new := v_new - ARRAY['asn_name','prefix_count','asn_resolved_at','updated_at'];
|
||||
ELSIF TG_TABLE_NAME = 'module_domain_entry' THEN
|
||||
v_old := v_old - ARRAY['resolve_meta','updated_at'];
|
||||
v_new := v_new - ARRAY['resolve_meta','updated_at'];
|
||||
ELSIF TG_TABLE_NAME = 'bgp_peer' THEN
|
||||
v_old := v_old - ARRAY['meta_json','updated_at'];
|
||||
v_new := v_new - ARRAY['meta_json','updated_at'];
|
||||
ELSIF TG_TABLE_NAME = 'bgp_community' THEN
|
||||
v_old := v_old - ARRAY['title','updated_at'];
|
||||
v_new := v_new - ARRAY['title','updated_at'];
|
||||
ELSIF TG_TABLE_NAME = 'bgp_speaker' THEN
|
||||
v_old := jsonb_build_object('node_ipv4',OLD.meta_json->>'node_ipv4',
|
||||
'bird_bgp_source_ipv4',OLD.meta_json->>'bird_bgp_source_ipv4',
|
||||
'bird_bgp_source_ipv6',OLD.meta_json->>'bird_bgp_source_ipv6');
|
||||
v_new := jsonb_build_object('node_ipv4',NEW.meta_json->>'node_ipv4',
|
||||
'bird_bgp_source_ipv4',NEW.meta_json->>'bird_bgp_source_ipv4',
|
||||
'bird_bgp_source_ipv6',NEW.meta_json->>'bird_bgp_source_ipv6');
|
||||
ELSE
|
||||
v_old := v_old - 'updated_at';
|
||||
v_new := v_new - 'updated_at';
|
||||
END IF;
|
||||
IF v_old = v_new THEN RETURN NEW; END IF;
|
||||
END IF;
|
||||
IF TG_TABLE_NAME IN ('module','doh_profile','bgp_peer','bgp_community','bgp_speaker','global_settings') THEN
|
||||
IF TG_OP = 'INSERT' THEN v_new_tenant := NEW.tenant_id;
|
||||
ELSIF TG_OP = 'DELETE' THEN v_old_tenant := OLD.tenant_id;
|
||||
ELSE v_old_tenant := OLD.tenant_id; v_new_tenant := NEW.tenant_id; END IF;
|
||||
ELSIF TG_TABLE_NAME IN ('module_cdn_source','module_domain_entry','module_as_entry','module_ip_range_entry','module_doh_profile') THEN
|
||||
IF TG_OP = 'INSERT' THEN v_new_module := NEW.module_id;
|
||||
ELSIF TG_OP = 'DELETE' THEN v_old_module := OLD.module_id;
|
||||
ELSE v_old_module := OLD.module_id; v_new_module := NEW.module_id; END IF;
|
||||
ELSIF TG_TABLE_NAME = 'module_prefix_snapshot' THEN
|
||||
IF TG_OP = 'INSERT' THEN v_new_tenant := NEW.tenant_id;
|
||||
ELSIF TG_OP = 'DELETE' THEN v_old_tenant := OLD.tenant_id;
|
||||
ELSE v_old_tenant := OLD.tenant_id; v_new_tenant := NEW.tenant_id; END IF;
|
||||
END IF;
|
||||
IF v_old_module IS NOT NULL THEN PERFORM render_bump_module_generation(v_old_module); END IF;
|
||||
IF v_new_module IS NOT NULL AND v_new_module IS DISTINCT FROM v_old_module THEN
|
||||
PERFORM render_bump_module_generation(v_new_module);
|
||||
END IF;
|
||||
IF v_old_tenant IS NOT NULL THEN PERFORM render_bump_tenant_generation(v_old_tenant); END IF;
|
||||
IF v_new_tenant IS NOT NULL AND v_new_tenant IS DISTINCT FROM v_old_tenant THEN
|
||||
PERFORM render_bump_tenant_generation(v_new_tenant);
|
||||
END IF;
|
||||
IF TG_OP = 'DELETE' THEN RETURN OLD; END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE FUNCTION render_snapshot_version() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
NEW.snapshot_version := nextval('module_snapshot_version_seq');
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
CREATE TRIGGER module_snapshot_version BEFORE INSERT OR UPDATE ON module_prefix_snapshot
|
||||
FOR EACH ROW EXECUTE FUNCTION render_snapshot_version();
|
||||
|
||||
CREATE TRIGGER render_module_changed AFTER INSERT OR UPDATE OR DELETE ON module
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
CREATE TRIGGER render_cdn_source_changed AFTER INSERT OR UPDATE OR DELETE ON module_cdn_source
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
CREATE TRIGGER render_as_entry_changed AFTER INSERT OR UPDATE OR DELETE ON module_as_entry
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
CREATE TRIGGER render_domain_entry_changed AFTER INSERT OR UPDATE OR DELETE ON module_domain_entry
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
CREATE TRIGGER render_ip_range_entry_changed AFTER INSERT OR UPDATE OR DELETE ON module_ip_range_entry
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
CREATE TRIGGER render_doh_profile_changed AFTER INSERT OR UPDATE OR DELETE ON doh_profile
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
CREATE TRIGGER render_module_doh_changed AFTER INSERT OR UPDATE OR DELETE ON module_doh_profile
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
CREATE TRIGGER render_peer_changed AFTER INSERT OR UPDATE OR DELETE ON bgp_peer
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
CREATE TRIGGER render_community_changed AFTER INSERT OR UPDATE OR DELETE ON bgp_community
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
CREATE TRIGGER render_settings_changed AFTER INSERT OR UPDATE OR DELETE ON global_settings
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
CREATE TRIGGER render_speaker_changed AFTER INSERT OR UPDATE OR DELETE ON bgp_speaker
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
CREATE TRIGGER render_snapshot_changed AFTER INSERT OR UPDATE OR DELETE ON module_prefix_snapshot
|
||||
FOR EACH ROW EXECUTE FUNCTION render_semantic_change();
|
||||
Reference in new issue
Block a user