feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-11-02 17:23:59 +07:00
1 parent 924f111e48
commit f8e0520817
@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -61,15 +62,19 @@ func RefreshModuleIngest(ctx context.Context, st store.Backend, hc *http.Client,
 		return fmt.Errorf("pipeline: module disabled")
 	}

-	var prior []store.PrefixRow
-	if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil {
-		prior = snap.Prefixes
-	}
-	rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, prior)
+	snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, moduleID)
 	if err != nil {
 		return err
 	}
-	if err := persistModuleSnapshot(st, tenantID, mod, rows); err != nil {
+	var priorRows []store.PrefixRow
+	if ok && snap != nil {
+		priorRows = snap.Prefixes
+	}
+	rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, modulePrior{provided: true, rows: priorRows})
+	if err != nil {
+		return err
+	}
+	if err := persistModuleSnapshot(ctx, st, tenantID, mod, rows); err != nil {
 		return err
 	}
 	refreshedAt := time.Now().UTC()
@@ -83,6 +88,11 @@ func RenderTenantRevision(ctx context.Context, st store.Backend, hc *http.Client
 	if hc == nil {
 		hc = httpclient.New(httpclient.DefaultTimeout)
 	}
+	if rev, ok, err := unchangedRenderRevision(ctx, st, tenantID); err != nil {
+		return "", err
+	} else if ok {
+		return rev, nil
+	}
 	agg, err := aggregateTenantPrefixRowsAll(ctx, st, hc, tenantID)
 	if err != nil {
 		return "", err
@@ -102,13 +112,71 @@ func RenderTenantRevision(ctx context.Context, st store.Backend, hc *http.Client
 	if err != nil {
 		return "", err
 	}
-	if err := st.CreateRenderRevision(revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil {
+	if err := st.CreateRenderRevision(ctx, revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil {
 		return "", err
 	}
 	applyRevisionRetention(st, tenantID)
 	return revisionID, nil
 }

+// unchangedRenderRevision returns the current revision when generation and snapshot
+// versions still match the latest manifest. A generation change during the read
+// refuses the skip so the caller rebuilds from the snapshots already written.
+func unchangedRenderRevision(ctx context.Context, st store.Backend, tenantID string) (string, bool, error) {
+	if st == nil || strings.TrimSpace(tenantID) == "" {
+		return "", false, nil
+	}
+	gen1, err := st.GetRenderGeneration(ctx, tenantID)
+	if err != nil {
+		if errors.Is(err, store.ErrNotFound) {
+			return "", false, nil
+		}
+		return "", false, err
+	}
+	headers, err := st.ListModuleSnapshotHeaders(ctx, tenantID)
+	if err != nil {
+		return "", false, err
+	}
+	gen2, err := st.GetRenderGeneration(ctx, tenantID)
+	if err != nil {
+		return "", false, err
+	}
+	if gen1 != gen2 {
+		return "", false, nil
+	}
+	man, ok, err := st.LatestRenderManifest(ctx, tenantID)
+	if err != nil || !ok || man == nil {
+		return "", false, err
+	}
+	if man.Generation != gen1 || !sameSnapshotUses(man.Snapshots, headers) {
+		return "", false, nil
+	}
+	gen3, err := st.GetRenderGeneration(ctx, tenantID)
+	if err != nil {
+		return "", false, err
+	}
+	if gen3 != gen1 || strings.TrimSpace(man.RevisionID) == "" {
+		return "", false, nil
+	}
+	return man.RevisionID, true, nil
+}
+
+func sameSnapshotUses(a, b []store.ModuleSnapshotUse) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	aa := append([]store.ModuleSnapshotUse(nil), a...)
+	bb := append([]store.ModuleSnapshotUse(nil), b...)
+	sort.Slice(aa, func(i, j int) bool { return aa[i].ModuleID < aa[j].ModuleID })
+	sort.Slice(bb, func(i, j int) bool { return bb[i].ModuleID < bb[j].ModuleID })
+	for i := range aa {
+		if aa[i].ModuleID != bb[i].ModuleID || aa[i].SnapshotVersion != bb[i].SnapshotVersion {
+			return false
+		}
+	}
+	return true
+}
+
 // RenderTenantRevisionFromPrefixes renders one tenant-wide revision from already materialized prefixes.
 // This is used for fast paths (e.g. peer-only changes) to avoid ingest/external fetches.
 func RenderTenantRevisionFromPrefixes(ctx context.Context, st store.Backend, hc *http.Client, tenantID, triggerModuleID string, rows []store.PrefixRow) (revisionID string, err error) {
@@ -132,7 +200,7 @@ func RenderTenantRevisionFromPrefixes(ctx context.Context, st store.Backend, hc
 	if err != nil {
 		return "", err
 	}
-	if err := st.CreateRenderRevision(revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil {
+	if err := st.CreateRenderRevision(ctx, revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil {
 		return "", err
 	}
 	applyRevisionRetention(st, tenantID)
@@ -148,8 +216,9 @@ func RefreshModule(ctx context.Context, st store.Backend, hc *http.Client, tenan
 }

 // collectModulePrefixRows returns materialized prefix rows for a single module (source of truth from store / ASN resolve / CDN fetch).
-// priorSnapshot is the last stored module snapshot (used to skip CDN fetches when refresh interval has not elapsed).
-func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
+// prior is the last stored module snapshot when the caller already loaded it.
+// CDN collect must not read that snapshot again.
+func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, prior modulePrior) ([]store.PrefixRow, error) {
 	moduleID := mod.ID
 	switch mod.Type {
 	case "IP_RANGES":
@@ -173,13 +242,13 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
 			return nil, err
 		}
 		sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
-		return collectASPrefixRows(ctx, st, hc, tenantID, mod, list, priorSnapshot)
+		return collectASPrefixRows(ctx, st, hc, tenantID, mod, list, prior.rows)
 	case "CDN_CIDRS":
 		sources, err := st.ListCDNSources(tenantID, moduleID)
 		if err != nil {
 			return nil, err
 		}
-		return collectCDNPrefixRows(ctx, st, hc, tenantID, mod, sources, priorSnapshot)
+		return collectCDNPrefixRows(ctx, st, hc, tenantID, mod, sources, prior)
 	case "DOMAINS":
 		entries, err := st.ListDomainEntries(tenantID, moduleID)
 		if err != nil {
@@ -189,7 +258,7 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
 		if err != nil {
 			return nil, err
 		}
-		return collectDomainPrefixRows(ctx, st, hc, mod, profiles, policy, entries, priorSnapshot)
+		return collectDomainPrefixRows(ctx, st, hc, mod, profiles, policy, entries, prior.rows)
 	default:
 		return nil, fmt.Errorf("pipeline: unknown module type %q", mod.Type)
 	}
@@ -260,6 +329,8 @@ func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohPr
 }

 func resolveDomainWithDOHMessage(ctx context.Context, hc *http.Client, baseURL, host string, qtype uint16) ([]netip.Addr, error) {
+	ctx, cancel := dohRequestContext(ctx)
+	defer cancel()
 	msg := new(dns.Msg)
 	msg.SetQuestion(dns.Fqdn(host), qtype)
 	wire, err := msg.Pack()
@@ -289,7 +360,7 @@ func resolveDomainWithDOHMessage(ctx context.Context, hc *http.Client, baseURL,
 		body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
 		return nil, fmt.Errorf("doh dns-message status %s: %s", resp.Status, strings.TrimSpace(string(body)))
 	}
-	raw, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
+	raw, err := httpclient.ReadResponseBody(resp, 2<<20)
 	if err != nil {
 		return nil, err
 	}
@@ -337,6 +408,8 @@ func resolveDomainWithSystemDNS(ctx context.Context, host string) ([]netip.Addr,
 }

 func resolveDomainWithDOHJSON(ctx context.Context, hc *http.Client, baseURL, host, qtype string) ([]netip.Addr, error) {
+	ctx, cancel := dohRequestContext(ctx)
+	defer cancel()
 	u, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, err
@@ -362,7 +435,11 @@ func resolveDomainWithDOHJSON(ctx context.Context, hc *http.Client, baseURL, hos
 		return nil, fmt.Errorf("doh status %s: %s", resp.Status, strings.TrimSpace(string(body)))
 	}
 	var payload dohJSONResponse
-	if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&payload); err != nil {
+	body, err := httpclient.ReadResponseBody(resp, 1<<20)
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(body, &payload); err != nil {
 		return nil, err
 	}
 	var out []netip.Addr
@@ -846,15 +923,6 @@ func intFromSettingsMap(m map[string]any, key string) int {
 	return 0
 }

-func applyRevisionRetention(st store.Backend, tenantID string) {
-	settings, err := st.ListGlobalSettings(tenantID)
-	if err != nil {
-		return
-	}
-	cutoff := RevisionRetentionCutoff(settings)
-	_, _ = st.PruneRevisionsBefore(tenantID, cutoff)
-}
-
 type peerPolicyJSON struct {
 	LocalIPv4 string  `json:"local_ipv4"`
 	LocalIPv6 string  `json:"local_ipv6"`