feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <cursoragent@cursor.com>

internal/pipeline/aggregate.go

@@ -86,20 +86,26 @@ func aggregateTenantPrefixRowsAll(ctx context.Context, st store.Backend, hc *htt
 }
 
 func rowsForModule(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module) ([]store.PrefixRow, error) {
-	if rows, ok, err := moduleRowsFromSnapshot(st, tenantID, mod); err != nil {
-		return nil, err
-	} else if ok {
-		return rows, nil
-	}
-	var prior []store.PrefixRow
-	if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
-		prior = snap.Prefixes
-	}
-	rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, prior)
+	snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID)
 	if err != nil {
 		return nil, err
 	}
-	if err := persistModuleSnapshot(st, tenantID, mod, rows); err != nil {
+	if ok && snap != nil {
+		if rows, matched, err := snapshotRowsIfCurrent(ctx, st, tenantID, mod, snap); err != nil {
+			return nil, err
+		} else if matched {
+			return rows, nil
+		}
+	}
+	var priorRows []store.PrefixRow
+	if ok && snap != nil {
+		priorRows = snap.Prefixes
+	}
+	rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, modulePrior{provided: true, rows: priorRows})
+	if err != nil {
+		return nil, err
+	}
+	if err := persistModuleSnapshot(ctx, st, tenantID, mod, rows); err != nil {
 		return nil, err
 	}
 	return rows, nil

internal/pipeline/cdn_body.go

@@ -0,0 +1,49 @@
+package pipeline
+
+import (
+	"errors"
+	"io"
+)
+
+const cdnBodyLimit = 8 << 20
+
+// ErrCDNBodyTooLarge reports a CDN response larger than the 8 MiB decoded-body limit.
+var ErrCDNBodyTooLarge = errors.New("pipeline: cdn response body exceeds 8 MiB")
+
+type limitedCDNBody struct {
+	io.ReadCloser
+	remaining int64
+}
+
+func (b *limitedCDNBody) Read(p []byte) (int, error) {
+	if len(p) == 0 {
+		return 0, nil
+	}
+	if b.remaining == 0 {
+		var one [1]byte
+		n, err := b.ReadCloser.Read(one[:])
+		if n > 0 {
+			return 0, ErrCDNBodyTooLarge
+		}
+		return 0, err
+	}
+	if int64(len(p)) > b.remaining {
+		p = p[:int(b.remaining)]
+	}
+	n, err := b.ReadCloser.Read(p)
+	b.remaining -= int64(n)
+	return n, err
+}
+
+// ReadCDNBody reads a CDN response and rejects decoded bodies above 8 MiB.
+// The extra byte distinguishes an exact-limit response from a truncated one.
+func ReadCDNBody(body io.Reader) ([]byte, error) {
+	data, err := io.ReadAll(io.LimitReader(body, cdnBodyLimit+1))
+	if err != nil {
+		return nil, err
+	}
+	if len(data) > cdnBodyLimit {
+		return nil, ErrCDNBodyTooLarge
+	}
+	return data, nil
+}

internal/pipeline/cdn_body_test.go

@@ -0,0 +1,79 @@
+package pipeline
+
+import (
+	"compress/gzip"
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"evobgp/internal/store"
+)
+
+func TestReadCDNBodyLimit(t *testing.T) {
+	tests := []struct {
+		name  string
+		extra int64
+		large bool
+	}{
+		{"exact limit", 0, false},
+		{"one byte too large", 1, true},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			reader := io.MultiReader(io.LimitReader(strings.NewReader(strings.Repeat("x", cdnBodyLimit)), cdnBodyLimit), strings.NewReader(strings.Repeat("y", int(tc.extra))))
+			body, err := ReadCDNBody(reader)
+			if tc.large {
+				if !errors.Is(err, ErrCDNBodyTooLarge) || body != nil {
+					t.Fatalf("body=%d err=%v", len(body), err)
+				}
+			} else if err != nil || len(body) != cdnBodyLimit {
+				t.Fatalf("body=%d err=%v", len(body), err)
+			}
+		})
+	}
+}
+
+func TestFetchCDNSourceRowsRejectsOversizedBody(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = io.Copy(w, strings.NewReader(strings.Repeat("x", cdnBodyLimit+1)))
+	}))
+	defer srv.Close()
+	_, err := fetchCDNSourceRows(context.Background(), nil, srv.Client(), "", "", nil,
+		&store.CDNSource{ID: "source", URL: srv.URL, SourceKind: "plain"}, modulePrior{}, time.Now())
+	if !errors.Is(err, ErrCDNBodyTooLarge) {
+		t.Fatalf("oversized CDN source must fail: %v", err)
+	}
+}
+
+func TestCDNTransportBoundsDecodedGzipBody(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Encoding", "gzip")
+		zw := gzip.NewWriter(w)
+		_, _ = io.Copy(zw, io.LimitReader(strings.NewReader(strings.Repeat("x", cdnBodyLimit+1)), cdnBodyLimit+1))
+		_ = zw.Close()
+	}))
+	defer srv.Close()
+
+	client, err := secureCDNClient(srv.Client())
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := client.Get(srv.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if !resp.Uncompressed {
+		t.Fatal("expected net/http gzip decompression")
+	}
+	if _, err := ReadCDNBody(resp.Body); !errors.Is(err, ErrCDNBodyTooLarge) {
+		t.Fatalf("oversized decoded body: %v", err)
+	}
+}

internal/pipeline/cdn_prefetch_test.go

@@ -44,7 +44,7 @@ func TestCollectModulePrefixRows_CDNSendsIfNoneMatch(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, nil)
+	collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, modulePrior{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -89,11 +89,11 @@ func TestCollectModulePrefixRows_CDN304UsesSnapshot(t *testing.T) {
 		Prefix: "203.0.113.0/24",
 		Source: cdnSourceKey(src.ID),
 	}}
-	if err := mergeCDNSourceIntoModuleSnapshot(m, tenant, mod, src.ID, prior); err != nil {
+	if err := mergeCDNSourceIntoModuleSnapshot(context.Background(), m, tenant, mod, src.ID, prior); err != nil {
 		t.Fatal(err)
 	}
 
-	collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, prior)
+	collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, modulePrior{provided: true, rows: prior})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -135,7 +135,7 @@ func TestRefreshModuleIngest_CDN304UsesStoredSnapshot(t *testing.T) {
 		Prefix: "203.0.113.0/24",
 		Source: cdnSourceKey(src.ID),
 	}}
-	if err := mergeCDNSourceIntoModuleSnapshot(m, tenant, mod, src.ID, prior); err != nil {
+	if err := mergeCDNSourceIntoModuleSnapshot(context.Background(), m, tenant, mod, src.ID, prior); err != nil {
 		t.Fatal(err)
 	}
 
@@ -185,7 +185,7 @@ func TestCollectModulePrefixRows_CDN304RetriesWithoutETag(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, nil)
+	collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, modulePrior{})
 	if err != nil {
 		t.Fatal(err)
 	}

internal/pipeline/cdn_snapshot.go

@@ -2,6 +2,7 @@ package pipeline
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,22 +12,86 @@ import (
 	"evobgp/internal/store"
 )
 
+var errCDNSnapshotRead = errors.New("pipeline: CDN snapshot read failed")
+
+// modulePrior is the module snapshot the caller already loaded.
+// provided distinguishes that read from a caller that did not load one.
+type modulePrior struct {
+	provided bool
+	rows     []store.PrefixRow
+}
+
 func cdnSourceKey(sourceID string) string {
 	return "cdn:" + strings.TrimSpace(sourceID)
 }
 
-func cachedCDNPrefixRows(st store.Backend, tenantID, moduleID string, priorSnapshot []store.PrefixRow, sourceKey string) []store.PrefixRow {
-	if cached := prefixRowsForSource(priorSnapshot, sourceKey); len(cached) > 0 {
-		return cached
+// narrowSnapshotReadCheaper reports whether per-source reads move less data than
+// one full module snapshot. One skipped source is filtered in SQL. Several skipped
+// sources share one snapshot read once they are a large share of the module.
+func narrowSnapshotReadCheaper(skipped, total int) bool {
+	if skipped <= 0 || total <= 0 {
+		return false
 	}
-	if st != nil {
-		if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil {
-			if cached := prefixRowsForSource(snap.Prefixes, sourceKey); len(cached) > 0 {
-				return cached
-			}
+	if skipped == 1 {
+		return true
+	}
+	return skipped*2 < total
+}
+
+func listModuleSourceRows(ctx context.Context, st store.Backend, tenantID, moduleID, source string) ([]store.PrefixRow, error) {
+	if st == nil {
+		return nil, nil
+	}
+	rows, err := st.ListModuleSnapshotSourceRows(ctx, tenantID, moduleID, source)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %w", errCDNSnapshotRead, err)
+	}
+	return rows, nil
+}
+
+func cachedCDNPrefixRows(ctx context.Context, st store.Backend, tenantID, moduleID string, prior modulePrior, sourceKey string) ([]store.PrefixRow, error) {
+	if prior.provided {
+		return prefixRowsForSource(prior.rows, sourceKey), nil
+	}
+	return listModuleSourceRows(ctx, st, tenantID, moduleID, sourceKey)
+}
+
+// loadSkippedCDNRows resolves interval-skipped sources without a second full snapshot
+// read when the caller already passed prior rows.
+func loadSkippedCDNRows(ctx context.Context, st store.Backend, tenantID, moduleID string, sources []*store.CDNSource, prior modulePrior, now time.Time) (modulePrior, map[string][]store.PrefixRow, error) {
+	if prior.provided || st == nil {
+		return prior, nil, nil
+	}
+	var skipped []*store.CDNSource
+	for _, src := range sources {
+		if src != nil && shouldSkipCDNSourceFetch(src, now) {
+			skipped = append(skipped, src)
 		}
 	}
-	return nil
+	if len(skipped) == 0 {
+		return prior, nil, nil
+	}
+	if narrowSnapshotReadCheaper(len(skipped), len(sources)) {
+		out := make(map[string][]store.PrefixRow, len(skipped))
+		for _, src := range skipped {
+			key := cdnSourceKey(src.ID)
+			rows, err := listModuleSourceRows(ctx, st, tenantID, moduleID, key)
+			if err != nil {
+				return prior, nil, err
+			}
+			out[key] = rows
+		}
+		return prior, out, nil
+	}
+	snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, moduleID)
+	if err != nil {
+		return prior, nil, fmt.Errorf("%w: %w", errCDNSnapshotRead, err)
+	}
+	rows := []store.PrefixRow(nil)
+	if ok && snap != nil {
+		rows = snap.Prefixes
+	}
+	return modulePrior{provided: true, rows: rows}, nil, nil
 }
 
 func mergeSnapshotDropSource(rows []store.PrefixRow, sourceKey string) []store.PrefixRow {
@@ -69,23 +134,16 @@ func mergeSnapshotKeepSkippedCDN(rows []store.PrefixRow, fetchedSourceIDs, allSo
 	return out
 }
 
-// mergeAllCDNSourcesIntoModuleSnapshot replaces fetched CDN source rows in one write.
-// skipped sources (errors with EVOBGP_CDN_PARTIAL_OK) keep their prior rows.
-func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) error {
-	if st == nil || mod == nil {
-		return nil
+// mergeCDNSnapshotRows builds the full module snapshot after a CDN collect.
+// Fetched sources replace their rows. Skipped sources (partial OK) keep prior rows.
+// Sources absent from allSourceIDs are dropped. Non-CDN rows in prior are kept.
+// The caller persists the result once; this function does not write.
+func mergeCDNSnapshotRows(prior, fetched []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) []store.PrefixRow {
+	kept := mergeSnapshotKeepSkippedCDN(prior, fetchedSourceIDs, allSourceIDs)
+	if len(fetched) == 0 {
+		return kept
 	}
-	unlock := st.LockModuleSnapshot(tenantID, mod.ID)
-	defer unlock()
-	var base []store.PrefixRow
-	if len(priorSnapshot) > 0 {
-		base = priorSnapshot
-	} else if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
-		base = snap.Prefixes
-	}
-	kept := mergeSnapshotKeepSkippedCDN(base, fetchedSourceIDs, allSourceIDs)
-	merged := append(kept, cdnRows...)
-	return persistModuleSnapshot(st, tenantID, mod, merged)
+	return append(kept, fetched...)
 }
 
 func cdnRowsFromParsed(mod *store.Module, src *store.CDNSource, pfxStrings []string) []store.PrefixRow {
@@ -106,19 +164,23 @@ func cdnRowsFromParsed(mod *store.Module, src *store.CDNSource, pfxStrings []str
 }
 
 // mergeCDNSourceIntoModuleSnapshot replaces rows for one CDN source in the module snapshot.
-func mergeCDNSourceIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, sourceID string, newRows []store.PrefixRow) error {
+func mergeCDNSourceIntoModuleSnapshot(ctx context.Context, st store.Backend, tenantID string, mod *store.Module, sourceID string, newRows []store.PrefixRow) error {
 	if st == nil || mod == nil {
 		return nil
 	}
-	unlock := st.LockModuleSnapshot(tenantID, mod.ID)
-	defer unlock()
-	sourceKey := cdnSourceKey(sourceID)
-	var base []store.PrefixRow
-	if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
-		base = mergeSnapshotDropSource(snap.Prefixes, sourceKey)
+	hash, err := moduleIngestInputHash(st, tenantID, mod)
+	if err != nil {
+		return err
 	}
-	merged := append(base, newRows...)
-	return persistModuleSnapshot(st, tenantID, mod, merged)
+	sourceKey := cdnSourceKey(sourceID)
+	return st.UpdateModulePrefixSnapshot(ctx, tenantID, mod.ID, func(current *store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) {
+		var base []store.PrefixRow
+		if current != nil {
+			base = mergeSnapshotDropSource(current.Prefixes, sourceKey)
+		}
+		merged := append(base, newRows...)
+		return &store.ModulePrefixSnapshot{InputHash: hash, Prefixes: merged}, nil
+	})
 }
 
 func parseCDNBody(body string, src *store.CDNSource) ([]string, error) {
@@ -134,7 +196,7 @@ func parseCDNBody(body string, src *store.CDNSource) ([]string, error) {
 }
 
 // fetchCDNSourceRows loads CDN prefixes without persisting the module snapshot (caller merges once).
-func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
+func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, prior modulePrior, now time.Time) ([]store.PrefixRow, error) {
 	u := strings.TrimSpace(src.URL)
 	if u == "" {
 		return nil, nil
@@ -159,7 +221,12 @@ func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client,
 	}
 
 	if resp.StatusCode == http.StatusNotModified {
-		if cached := cachedCDNPrefixRows(st, tenantID, moduleID, priorSnapshot, sourceKey); len(cached) > 0 {
+		cached, cacheErr := cachedCDNPrefixRows(ctx, st, tenantID, moduleID, prior, sourceKey)
+		if cacheErr != nil {
+			_ = resp.Body.Close()
+			return nil, cacheErr
+		}
+		if len(cached) > 0 {
 			_ = resp.Body.Close()
 			return cached, nil
 		}
@@ -183,7 +250,7 @@ func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client,
 		_, _ = io.Copy(io.Discard, resp.Body)
 		return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
 	}
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+	body, err := ReadCDNBody(resp.Body)
 	if err != nil {
 		return nil, err
 	}

internal/pipeline/cdn_snapshot_lock_test.go

@@ -0,0 +1,32 @@
+package pipeline
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"evobgp/internal/store"
+)
+
+type failingSnapshotUpdate struct{ store.Backend }
+
+func (f failingSnapshotUpdate) UpdateModulePrefixSnapshot(context.Context, string, string, store.ModuleSnapshotUpdate) error {
+	return context.DeadlineExceeded
+}
+
+func TestMergeCDNSourceRefusesWriteWhenAtomicUpdateFails(t *testing.T) {
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, _, _ := m.DemoIDs()
+	mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-lock-failure", Enabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = mergeCDNSourceIntoModuleSnapshot(context.Background(), failingSnapshotUpdate{m}, tenant, mod, "source-a", []store.PrefixRow{{Prefix: "192.0.2.0/24", Source: "cdn:source-a"}})
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("expected update failure, got %v", err)
+	}
+	if _, ok, err := m.GetModulePrefixSnapshot(context.Background(), tenant, mod.ID); err != nil || ok {
+		t.Fatalf("snapshot written without lock: ok=%v err=%v", ok, err)
+	}
+}

internal/pipeline/cdn_snapshot_read_test.go

@@ -0,0 +1,141 @@
+package pipeline
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"evobgp/internal/store"
+)
+
+type snapCalls struct {
+	store.Backend
+	full   atomic.Int32
+	narrow atomic.Int32
+}
+
+func (s *snapCalls) GetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) {
+	s.full.Add(1)
+	return s.Backend.GetModulePrefixSnapshot(ctx, tenantID, moduleID)
+}
+
+func (s *snapCalls) ListModuleSnapshotSourceRows(ctx context.Context, tenantID, moduleID, source string) ([]store.PrefixRow, error) {
+	s.narrow.Add(1)
+	return s.Backend.ListModuleSnapshotSourceRows(ctx, tenantID, moduleID, source)
+}
+
+func TestNarrowSnapshotReadCheaper(t *testing.T) {
+	if !narrowSnapshotReadCheaper(1, 8) {
+		t.Fatal("one skipped source should use a narrow read")
+	}
+	if !narrowSnapshotReadCheaper(3, 8) {
+		t.Fatal("a minority of skipped sources should use narrow reads")
+	}
+	if narrowSnapshotReadCheaper(6, 8) {
+		t.Fatal("a majority of skipped sources should share one snapshot read")
+	}
+	if narrowSnapshotReadCheaper(0, 4) {
+		t.Fatal("nothing to read")
+	}
+}
+
+func TestRefreshModuleIngestDoesNotRereadSnapshot(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, _, _ := m.DemoIDs()
+	mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-once", Enabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	interval := 3600
+	fresh := time.Now().UTC().Add(-30 * time.Second)
+	src, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
+		SourceKind:         "txt",
+		URL:                "https://cdn.example/prefixes.txt",
+		RefreshIntervalSec: &interval,
+		LastRefreshedAt:    &fresh,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	rows := []store.PrefixRow{{Prefix: "203.0.113.0/24", Source: cdnSourceKey(src.ID)}}
+	if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash", rows); err != nil {
+		t.Fatal(err)
+	}
+	counted := &snapCalls{Backend: m}
+	if err := RefreshModuleIngest(context.Background(), counted, nil, tenant, mod.ID); err != nil {
+		t.Fatal(err)
+	}
+	if counted.full.Load() != 1 || counted.narrow.Load() != 0 {
+		t.Fatalf("snapshot reads full=%d narrow=%d", counted.full.Load(), counted.narrow.Load())
+	}
+}
+
+func TestSkippedCDNSourceUsesNarrowReadWithoutPrior(t *testing.T) {
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, _, _ := m.DemoIDs()
+	mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-narrow", Enabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	interval := 3600
+	fresh := time.Now().UTC().Add(-30 * time.Second)
+	src := &store.CDNSource{ID: "only", SourceKind: "txt", URL: "https://cdn.example/one", RefreshIntervalSec: &interval, LastRefreshedAt: &fresh}
+	want := []store.PrefixRow{{Prefix: "203.0.113.8/32", Source: "cdn:only"}}
+	if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash", append([]store.PrefixRow{{Prefix: "198.51.100.0/24", Source: "cdn:other"}}, want...)); err != nil {
+		t.Fatal(err)
+	}
+	counted := &snapCalls{Backend: m}
+	got, err := collectCDNPrefixRows(context.Background(), counted, nil, tenant, mod, []*store.CDNSource{src}, modulePrior{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if counted.full.Load() != 0 || counted.narrow.Load() != 1 {
+		t.Fatalf("snapshot reads full=%d narrow=%d", counted.full.Load(), counted.narrow.Load())
+	}
+	if len(got) != 1 || got[0].Prefix != "203.0.113.8/32" || got[0].Source != "cdn:only" {
+		t.Fatalf("rows=%+v", got)
+	}
+}
+
+func TestManySkippedCDNSourcesShareOneSnapshotRead(t *testing.T) {
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, _, _ := m.DemoIDs()
+	mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-shared", Enabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	interval := 3600
+	fresh := time.Now().UTC().Add(-30 * time.Second)
+	sources := []*store.CDNSource{
+		{ID: "a", URL: "https://cdn.example/a", RefreshIntervalSec: &interval, LastRefreshedAt: &fresh},
+		{ID: "b", URL: "https://cdn.example/b", RefreshIntervalSec: &interval, LastRefreshedAt: &fresh},
+	}
+	stored := []store.PrefixRow{
+		{Prefix: "203.0.113.1/32", Source: "cdn:a"},
+		{Prefix: "203.0.113.2/32", Source: "cdn:b"},
+		{Prefix: "198.51.100.0/24", Source: "cdn:gone"},
+	}
+	if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash", stored); err != nil {
+		t.Fatal(err)
+	}
+	counted := &snapCalls{Backend: m}
+	got, err := collectCDNPrefixRows(context.Background(), counted, nil, tenant, mod, sources, modulePrior{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if counted.full.Load() != 1 || counted.narrow.Load() != 0 {
+		t.Fatalf("snapshot reads full=%d narrow=%d", counted.full.Load(), counted.narrow.Load())
+	}
+	seen := map[string]bool{}
+	for _, row := range got {
+		seen[row.Source] = true
+	}
+	if !seen["cdn:a"] || !seen["cdn:b"] || seen["cdn:gone"] {
+		t.Fatalf("rows=%+v", got)
+	}
+}

internal/pipeline/cdn_transport.go

@@ -0,0 +1,137 @@
+package pipeline
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/netip"
+	"time"
+
+	"evobgp/internal/httpclient"
+)
+
+type cdnRoundTripper struct {
+	transport *http.Transport
+}
+
+func (t *cdnRoundTripper) CloseIdleConnections() { t.transport.CloseIdleConnections() }
+
+func (t *cdnRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if _, err := ValidateCDNURL(req.URL.String()); err != nil {
+		return nil, err
+	}
+	resp, err := t.transport.RoundTrip(req)
+	if err != nil {
+		return nil, err
+	}
+	// Transport has already decoded gzip here when it was negotiated by net/http.
+	resp.Body = &limitedCDNBody{ReadCloser: resp.Body, remaining: cdnBodyLimit}
+	return resp, nil
+}
+
+type cdnLookupIP func(context.Context, string) ([]net.IP, error)
+type cdnDialContext func(context.Context, string, string) (net.Conn, error)
+
+// cdnDial resolves every new connection and dials only an address from that checked answer.
+// A mixed public/private DNS answer is rejected in full, regardless of address ordering.
+func cdnDial(ctx context.Context, network, address string, lookup cdnLookupIP, dial cdnDialContext) (net.Conn, error) {
+	host, port, err := net.SplitHostPort(address)
+	if err != nil {
+		return nil, fmt.Errorf("pipeline: cdn dial address: %w", err)
+	}
+	if isBlockedCDNHostname(host) {
+		return nil, fmt.Errorf("pipeline: cdn url blocked host")
+	}
+	var ips []net.IP
+	if addr, err := netip.ParseAddr(host); err == nil {
+		ips = []net.IP{net.IP(addr.AsSlice())}
+	} else {
+		resolveCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+		ips, err = lookup(resolveCtx, host)
+		if err != nil {
+			return nil, fmt.Errorf("pipeline: cdn url dns lookup: %w", err)
+		}
+	}
+	if len(ips) == 0 {
+		return nil, fmt.Errorf("pipeline: cdn url dns lookup: no addresses")
+	}
+	checked := make([]netip.Addr, 0, len(ips))
+	for _, ip := range ips {
+		addr, ok := netip.AddrFromSlice(ip)
+		if !ok || isBlockedCDNIP(addr) {
+			return nil, fmt.Errorf("pipeline: cdn url resolves to blocked address")
+		}
+		checked = append(checked, addr.Unmap())
+	}
+	var dialErr error
+	for _, addr := range checked {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		conn, err := dial(ctx, network, net.JoinHostPort(addr.String(), port))
+		if err == nil {
+			return conn, nil
+		}
+		dialErr = err
+	}
+	return nil, fmt.Errorf("pipeline: cdn dial: %w", dialErr)
+}
+
+func secureCDNClient(hc *http.Client) (*http.Client, error) {
+	if hc == nil {
+		hc = httpclient.New(httpclient.DefaultTimeout)
+	}
+	client := *hc
+	if pools, ok := client.Transport.(*pipelineTransport); ok {
+		client.Transport = pools.cdn
+	}
+	if client.Timeout <= 0 {
+		client.Timeout = httpclient.DefaultTimeout
+	}
+	if _, ok := client.Transport.(*cdnRoundTripper); !ok {
+		base := http.DefaultTransport.(*http.Transport).Clone()
+		switch tr := client.Transport.(type) {
+		case nil:
+		case *http.Transport:
+			// Keep the caller's trust roots for TLS test servers. Other transport
+			// hooks (including custom protocols and dialers) are not inherited.
+			if tr.TLSClientConfig != nil {
+				base.TLSClientConfig = tr.TLSClientConfig.Clone()
+			}
+		default:
+			return nil, fmt.Errorf("pipeline: cdn client requires *http.Transport")
+		}
+		// A proxy connects to the destination outside this process, where its DNS answer
+		// cannot be checked. CDN requests therefore always connect directly.
+		base.Proxy = nil
+		base.DialTLSContext = nil
+		base.DialTLS = nil
+		base.TLSNextProto = nil
+		dialer := &net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}
+		base.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
+			return cdnDial(ctx, network, address, func(ctx context.Context, host string) ([]net.IP, error) {
+				return net.DefaultResolver.LookupIP(ctx, "ip", host)
+			}, dialer.DialContext)
+		}
+		client.Transport = &cdnRoundTripper{transport: base}
+	}
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		if _, err := ValidateCDNURL(req.URL.String()); err != nil {
+			return err
+		}
+		if len(via) >= 10 {
+			return fmt.Errorf("pipeline: too many cdn redirects")
+		}
+		return nil
+	}
+	return &client, nil
+}
+
+// NewCDNHTTPClient returns a reusable HTTPS client that pins each CDN connection
+// to a checked DNS answer. Environment HTTP proxies are intentionally ignored.
+func NewCDNHTTPClient() *http.Client {
+	client, _ := secureCDNClient(httpclient.New(httpclient.DefaultTimeout))
+	return client
+}

internal/pipeline/cdn_transport_test.go

@@ -0,0 +1,197 @@
+package pipeline
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"net/url"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+func TestCDNDialPinsAndChecksEveryAnswer(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
+	answers := [][]net.IP{
+		{net.ParseIP("203.0.113.10")},
+		{net.ParseIP("127.0.0.1")},
+	}
+	lookups := 0
+	var dialed []string
+	lookup := func(context.Context, string) ([]net.IP, error) {
+		answer := answers[lookups]
+		lookups++
+		return answer, nil
+	}
+	dial := func(_ context.Context, _, address string) (net.Conn, error) {
+		dialed = append(dialed, address)
+		return nil, nil
+	}
+	if _, err := cdnDial(context.Background(), "tcp", "cdn.example:443", lookup, dial); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := cdnDial(context.Background(), "tcp", "cdn.example:443", lookup, dial); err == nil {
+		t.Fatal("changed DNS answer must be rejected before dial")
+	}
+	if lookups != 2 || len(dialed) != 1 || dialed[0] != "203.0.113.10:443" {
+		t.Fatalf("lookups=%d dialed=%v", lookups, dialed)
+	}
+}
+
+func TestCDNDialRejectsMixedAndMappedPrivateAnswers(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
+	tests := []struct {
+		name string
+		ips  []net.IP
+	}{
+		{"mixed", []net.IP{net.ParseIP("203.0.113.10"), net.ParseIP("10.0.0.5")}},
+		{"mapped loopback", []net.IP{net.ParseIP("::ffff:127.0.0.1")}},
+		{"mapped private", []net.IP{net.ParseIP("::ffff:10.0.0.1")}},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			called := false
+			_, err := cdnDial(context.Background(), "tcp", "cdn.example:443", func(context.Context, string) ([]net.IP, error) {
+				return tc.ips, nil
+			}, func(context.Context, string, string) (net.Conn, error) {
+				called = true
+				return nil, nil
+			})
+			if err == nil || called {
+				t.Fatalf("want rejection before dial, got err=%v called=%v", err, called)
+			}
+		})
+	}
+	if !isBlockedCDNIP(netip.MustParseAddr("::ffff:169.254.169.254")) {
+		t.Fatal("mapped metadata address must be blocked")
+	}
+}
+
+func TestCDNDialFallsBackOnlyToCheckedIP(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
+	var dialed []string
+	_, err := cdnDial(context.Background(), "tcp", "cdn.example:443", func(context.Context, string) ([]net.IP, error) {
+		return []net.IP{net.ParseIP("2001:db8::1"), net.ParseIP("203.0.113.10")}, nil
+	}, func(_ context.Context, _, address string) (net.Conn, error) {
+		dialed = append(dialed, address)
+		if len(dialed) == 1 {
+			return nil, errors.New("unreachable")
+		}
+		return nil, nil
+	})
+	if err != nil || len(dialed) != 2 || dialed[0] != "[2001:db8::1]:443" || dialed[1] != "203.0.113.10:443" {
+		t.Fatalf("err=%v dialed=%v", err, dialed)
+	}
+}
+
+func TestCDNDialStopsOnCancellation(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	called := 0
+	_, err := cdnDial(ctx, "tcp", "cdn.example:443", func(context.Context, string) ([]net.IP, error) {
+		return []net.IP{net.ParseIP("203.0.113.10"), net.ParseIP("203.0.113.11")}, nil
+	}, func(context.Context, string, string) (net.Conn, error) {
+		called++
+		cancel()
+		return nil, errors.New("unreachable")
+	})
+	if !errors.Is(err, context.Canceled) || called != 1 {
+		t.Fatalf("err=%v dial attempts=%d", err, called)
+	}
+}
+
+func TestCDNClientRejectsUnsafeRedirectsAndProxy(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
+	t.Setenv("HTTPS_PROXY", "http://127.0.0.1:8080")
+	client := NewCDNHTTPClient()
+	tr, ok := client.Transport.(*cdnRoundTripper)
+	if !ok || tr.transport.Proxy != nil {
+		t.Fatal("CDN transport must dial directly")
+	}
+	for _, raw := range []string{
+		"https://127.0.0.1/secret",
+		"https://[::ffff:127.0.0.1]/secret",
+		"http://cdn.example/unsafe",
+		"https://user:password@cdn.example/secret",
+	} {
+		t.Run(raw, func(t *testing.T) {
+			u, err := url.Parse(raw)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := client.CheckRedirect(&http.Request{URL: u}, nil); err == nil {
+				t.Fatal("unsafe redirect accepted")
+			}
+		})
+	}
+}
+
+type cdnTestRoundTripper func(*http.Request) (*http.Response, error)
+
+func (f cdnTestRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
+
+func TestCDNClientRejectsOpaqueTransport(t *testing.T) {
+	_, err := secureCDNClient(&http.Client{Transport: cdnTestRoundTripper(func(*http.Request) (*http.Response, error) {
+		return nil, errors.New("must not be called")
+	})})
+	if err == nil {
+		t.Fatal("opaque transport could bypass checked dial")
+	}
+}
+
+func TestCDNClientRejectsPrivateTLSRedirectBeforeDial(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
+	var hits atomic.Int32
+	var secretHits atomic.Int32
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		if r.URL.Path == "/secret" {
+			secretHits.Add(1)
+		}
+		http.Redirect(w, r, "https://private.example.com/secret", http.StatusFound)
+	}))
+	defer srv.Close()
+
+	client := NewCDNHTTPClient()
+	tr := client.Transport.(*cdnRoundTripper).transport
+	tr.TLSClientConfig = srv.Client().Transport.(*http.Transport).TLSClientConfig.Clone()
+	actual := srv.Listener.Addr().String()
+	var dialed []string
+	var dialedMu sync.Mutex
+	dialer := &net.Dialer{}
+	tr.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
+		return cdnDial(ctx, network, address, func(_ context.Context, host string) ([]net.IP, error) {
+			if host == "private.example.com" {
+				return []net.IP{net.ParseIP("127.0.0.1")}, nil
+			}
+			return []net.IP{net.ParseIP("203.0.113.10")}, nil
+		}, func(ctx context.Context, network, checkedAddress string) (net.Conn, error) {
+			dialedMu.Lock()
+			dialed = append(dialed, checkedAddress)
+			dialedMu.Unlock()
+			return dialer.DialContext(ctx, network, actual)
+		})
+	}
+	_, port, err := net.SplitHostPort(actual)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := client.Get("https://cdn.example.com:" + port + "/redirect")
+	if resp != nil {
+		_ = resp.Body.Close()
+	}
+	if err == nil || !strings.Contains(err.Error(), "blocked address") {
+		t.Fatalf("private TLS redirect was not rejected: %v", err)
+	}
+	dialedMu.Lock()
+	dialedSnapshot := append([]string(nil), dialed...)
+	dialedMu.Unlock()
+	if hits.Load() != 1 || secretHits.Load() != 0 || len(dialedSnapshot) != 1 || dialedSnapshot[0] != "203.0.113.10:"+port {
+		t.Fatalf("hits=%d secretHits=%d dialed=%v", hits.Load(), secretHits.Load(), dialedSnapshot)
+	}
+}

internal/pipeline/cdn_url.go

@@ -20,6 +20,7 @@ func isBlockedCDNIP(ip netip.Addr) bool {
 	if !ip.IsValid() {
 		return true
 	}
+	ip = ip.Unmap()
 	return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsMulticast() ||
 		ip.IsUnspecified() || ip == netip.MustParseAddr("169.254.169.254")
 }

internal/pipeline/collect_parallel.go

@@ -2,6 +2,7 @@ package pipeline
 
 import (
 	"context"
+	"errors"
 	"evobgp/internal/logging"
 	"fmt"
 	"net/http"
@@ -171,7 +172,7 @@ func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client,
 	return out, nil
 }
 
-func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, sources []*store.CDNSource, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
+func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, sources []*store.CDNSource, prior modulePrior) ([]store.PrefixRow, error) {
 	moduleID := mod.ID
 	now := time.Now().UTC()
 
@@ -181,6 +182,10 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
 			valid = append(valid, s)
 		}
 	}
+	prior, skipRows, err := loadSkippedCDNRows(ctx, st, tenantID, moduleID, valid, prior, now)
+	if err != nil {
+		return nil, err
+	}
 	type srcResult struct {
 		rows []store.PrefixRow
 		err  error
@@ -196,23 +201,26 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
 			sem <- struct{}{}
 			defer func() { <-sem }()
 
-			sourceKey := "cdn:" + src.ID
+			sourceKey := cdnSourceKey(src.ID)
 			if shouldSkipCDNSourceFetch(src, now) {
-				if cached := prefixRowsForSource(priorSnapshot, sourceKey); len(cached) > 0 {
+				cached := prefixRowsForSource(prior.rows, sourceKey)
+				if !prior.provided && skipRows != nil {
+					cached = skipRows[sourceKey]
+				}
+				if len(cached) > 0 {
 					results[idx] = srcResult{rows: cached}
 					return
 				}
-				if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil {
-					if cached := prefixRowsForSource(snap.Prefixes, sourceKey); len(cached) > 0 {
-						results[idx] = srcResult{rows: cached}
-						return
-					}
-				}
 			}
-			rows, err := fetchCDNSourceRows(ctx, st, hc, tenantID, moduleID, mod, src, priorSnapshot, now)
+			rows, err := fetchCDNSourceRows(ctx, st, hc, tenantID, moduleID, mod, src, prior, now)
 			if err != nil {
 				if staleOnUpstreamError() {
-					if cached, ok := staleCDNPrefixes(st, tenantID, moduleID, priorSnapshot, src.ID); ok {
+					cached, ok, cacheErr := staleCDNPrefixes(ctx, st, tenantID, moduleID, prior, src.ID)
+					if cacheErr != nil {
+						results[idx] = srcResult{err: cacheErr}
+						return
+					}
+					if ok {
 						logStaleUpstream("cdn", fmt.Sprintf("source %s: %v", src.ID, err))
 						results[idx] = srcResult{rows: cached}
 						return
@@ -235,6 +243,9 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
 	}
 	for i, r := range results {
 		if r.err != nil {
+			if errors.Is(r.err, errCDNSnapshotRead) {
+				return nil, r.err
+			}
 			if cdnPartialOK() {
 				logging.Default().Info(fmt.Sprintf("pipeline: CDN partial skip source error: %v", r.err))
 				skipped++
@@ -248,26 +259,15 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
 	if skipped > 0 && len(fetchedRows) == 0 && len(valid) > 0 {
 		return nil, fmt.Errorf("cdn: all %d source(s) failed (partial ok)", len(valid))
 	}
-	out := fetchedRows
-	if skipped > 0 {
-		base := priorSnapshot
-		if len(base) == 0 {
-			if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil {
-				base = snap.Prefixes
-			}
-		}
-		for _, row := range mergeSnapshotKeepSkippedCDN(base, fetchedIDs, allIDs) {
-			if strings.HasPrefix(strings.TrimSpace(row.Source), "cdn:") {
-				out = append(out, row)
-			}
-		}
-	}
-	if len(valid) > 0 {
-		if err := mergeAllCDNSourcesIntoModuleSnapshot(st, tenantID, mod, priorSnapshot, fetchedRows, fetchedIDs, allIDs); err != nil {
+	base := prior.rows
+	if skipped > 0 && len(base) == 0 && !prior.provided && st != nil {
+		if snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, moduleID); err != nil {
 			return nil, err
+		} else if ok && snap != nil {
+			base = snap.Prefixes
 		}
 	}
-	return out, nil
+	return mergeCDNSnapshotRows(base, fetchedRows, fetchedIDs, allIDs), nil
 }
 
 func collectDomainPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {

internal/pipeline/collect_stale.go

@@ -1,6 +1,7 @@
 package pipeline
 
 import (
+	"context"
 	"evobgp/internal/logging"
 	"fmt"
 	"net/netip"
@@ -63,10 +64,10 @@ func staleDomainPrefixes(priorSnapshot []store.PrefixRow, fqdn string) ([]store.
 	return cached, len(cached) > 0
 }
 
-func staleCDNPrefixes(st store.Backend, tenantID, moduleID string, priorSnapshot []store.PrefixRow, sourceID string) ([]store.PrefixRow, bool) {
+func staleCDNPrefixes(ctx context.Context, st store.Backend, tenantID, moduleID string, prior modulePrior, sourceID string) ([]store.PrefixRow, bool, error) {
 	sourceKey := cdnSourceKey(sourceID)
-	cached := cachedCDNPrefixRows(st, tenantID, moduleID, priorSnapshot, sourceKey)
-	return cached, len(cached) > 0
+	cached, err := cachedCDNPrefixRows(ctx, st, tenantID, moduleID, prior, sourceKey)
+	return cached, len(cached) > 0, err
 }
 
 // asnCacheExpired returns cached ASN prefixes even past TTL (for stale fallback only).

internal/pipeline/collect_stale_test.go

@@ -34,7 +34,7 @@ func TestCollectCDNPrefixRows_StaleOnFetchError(t *testing.T) {
 	prior := []store.PrefixRow{
 		{Prefix: "203.0.113.0/24", Source: "cdn:s1"},
 	}
-	rows, err := collectCDNPrefixRows(context.Background(), st, srv.Client(), tenant, mod, []*store.CDNSource{{ID: "s1", URL: srv.URL, SourceKind: "plain"}}, prior)
+	rows, err := collectCDNPrefixRows(context.Background(), st, srv.Client(), tenant, mod, []*store.CDNSource{{ID: "s1", URL: srv.URL, SourceKind: "plain"}}, modulePrior{provided: true, rows: prior})
 	if err != nil {
 		t.Fatalf("expected stale fallback, got err: %v", err)
 	}
@@ -65,7 +65,7 @@ func TestCollectCDNPrefixRows_FailFastWhenNoStale(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = collectCDNPrefixRows(context.Background(), st, srv.Client(), tenant, mod, []*store.CDNSource{{ID: "s1", URL: srv.URL, SourceKind: "plain"}}, nil)
+	_, err = collectCDNPrefixRows(context.Background(), st, srv.Client(), tenant, mod, []*store.CDNSource{{ID: "s1", URL: srv.URL, SourceKind: "plain"}}, modulePrior{})
 	if err == nil {
 		t.Fatal("expected error when stale disabled and no cache")
 	}

internal/pipeline/doh_limits_test.go

@@ -0,0 +1,147 @@
+package pipeline
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"strings"
+	"testing"
+	"time"
+
+	"evobgp/internal/httpclient"
+	"github.com/miekg/dns"
+)
+
+type dohLimitRoundTrip func(*http.Request) (*http.Response, error)
+
+func (f dohLimitRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+type dohLimitBody struct {
+	io.Reader
+	read   int
+	closed bool
+}
+
+func (b *dohLimitBody) Read(p []byte) (int, error) {
+	n, e := b.Reader.Read(p)
+	b.read += n
+	return n, e
+}
+func (b *dohLimitBody) Close() error { b.closed = true; return nil }
+
+func dohLimitCases(t *testing.T) []struct {
+	name   string
+	limit  int
+	prefix []byte
+	call   func(context.Context, *http.Client, string) ([]netip.Addr, error)
+	pad    byte
+} {
+	t.Helper()
+	msg := new(dns.Msg)
+	msg.Response = true
+	msg.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: "example.test.", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.IPv4(192, 0, 2, 1)}}
+	wire, err := msg.Pack()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return []struct {
+		name   string
+		limit  int
+		prefix []byte
+		call   func(context.Context, *http.Client, string) ([]netip.Addr, error)
+		pad    byte
+	}{
+		{"wire", 2 << 20, wire, func(ctx context.Context, hc *http.Client, url string) ([]netip.Addr, error) {
+			return resolveDomainWithDOHMessage(ctx, hc, url, "example.test", dns.TypeA)
+		}, 0},
+		{"json", 1 << 20, []byte(`{"Answer":[{"type":1,"data":"192.0.2.1"}]}`), func(ctx context.Context, hc *http.Client, url string) ([]netip.Addr, error) {
+			return resolveDomainWithDOHJSON(ctx, hc, url, "example.test", "A")
+		}, ' '},
+	}
+}
+
+func TestDOHCompleteResponseLimits(t *testing.T) {
+	for _, tc := range dohLimitCases(t) {
+		t.Run(tc.name, func(t *testing.T) {
+			for _, known := range []bool{true, false} {
+				for _, extra := range []int{0, 1} {
+					data := append(append([]byte{}, tc.prefix...), bytes.Repeat([]byte{tc.pad}, tc.limit-len(tc.prefix)+extra)...)
+					body := &dohLimitBody{Reader: bytes.NewReader(data)}
+					hc := &http.Client{Transport: dohLimitRoundTrip(func(r *http.Request) (*http.Response, error) {
+						if _, ok := r.Context().Deadline(); !ok {
+							t.Error("missing deadline")
+						}
+						length := int64(-1)
+						if known {
+							length = int64(len(data))
+						}
+						return &http.Response{StatusCode: 200, ContentLength: length, Body: body, Header: make(http.Header)}, nil
+					})}
+					ips, err := tc.call(context.Background(), hc, "https://resolver.example/dns-query")
+					if extra == 0 && (err != nil || len(ips) != 1) {
+						t.Fatalf("exact limit rejected known=%v ips=%v: %v", known, ips, err)
+					}
+					if extra == 1 && !errors.Is(err, httpclient.ErrResponseTooLarge) {
+						t.Fatalf("valid prefix bypassed size cap known=%v: %v", known, err)
+					}
+					if body.read > tc.limit+1 || !body.closed {
+						t.Fatalf("read=%d closed=%v", body.read, body.closed)
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestDOHGzipDecodedLimit(t *testing.T) {
+	for _, tc := range dohLimitCases(t) {
+		t.Run(tc.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Encoding", "gzip")
+				gz := gzip.NewWriter(w)
+				_, _ = gz.Write(tc.prefix)
+				_, _ = gz.Write(bytes.Repeat([]byte{tc.pad}, tc.limit))
+				_ = gz.Close()
+			}))
+			defer server.Close()
+			if _, err := tc.call(context.Background(), server.Client(), server.URL); !errors.Is(err, httpclient.ErrResponseTooLarge) {
+				t.Fatalf("gzip bypass: %v", err)
+			}
+		})
+	}
+}
+
+func TestDOHNon200AndCancellationBounds(t *testing.T) {
+	for _, tc := range dohLimitCases(t) {
+		t.Run(tc.name, func(t *testing.T) {
+			body := &dohLimitBody{Reader: strings.NewReader(strings.Repeat("x", 1<<20))}
+			hc := &http.Client{Transport: dohLimitRoundTrip(func(*http.Request) (*http.Response, error) {
+				return &http.Response{StatusCode: 400, Status: "400 Bad Request", Body: body, Header: make(http.Header)}, nil
+			})}
+			if _, err := tc.call(context.Background(), hc, "https://resolver.example/dns-query"); err == nil || body.read > 1024 || !body.closed {
+				t.Fatalf("non200 read=%d closed=%v error=%v", body.read, body.closed, err)
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+			defer cancel()
+			hc.Transport = dohLimitRoundTrip(func(r *http.Request) (*http.Response, error) { <-r.Context().Done(); return nil, r.Context().Err() })
+			if _, err := tc.call(ctx, hc, "https://resolver.example/dns-query"); !errors.Is(err, context.DeadlineExceeded) {
+				t.Fatalf("caller cancellation lost: %v", err)
+			}
+		})
+	}
+}
+
+func TestDOHJSONRejectsTrailingObject(t *testing.T) {
+	hc := &http.Client{Transport: dohLimitRoundTrip(func(*http.Request) (*http.Response, error) {
+		return &http.Response{StatusCode: 200, ContentLength: -1, Body: io.NopCloser(strings.NewReader(`{"Answer":[]} {"ignored":true}`)), Header: make(http.Header)}, nil
+	})}
+	if _, err := resolveDomainWithDOHJSON(context.Background(), hc, "https://resolver.example/dns-query", "example.test", "A"); err == nil {
+		t.Fatal("trailing JSON was ignored")
+	}
+}

internal/pipeline/doh_resolve.go

@@ -146,3 +146,10 @@ func dohProfileTimeout(profile *store.DohProfile) time.Duration {
 	}
 	return timeout
 }
+
+func dohRequestContext(ctx context.Context) (context.Context, context.CancelFunc) {
+	if _, ok := ctx.Deadline(); ok {
+		return context.WithCancel(ctx)
+	}
+	return context.WithTimeout(ctx, 10*time.Second)
+}

internal/pipeline/http_clients.go

@@ -0,0 +1,35 @@
+package pipeline
+
+import (
+	"net/http"
+
+	"evobgp/internal/httpclient"
+)
+
+// pipelineTransport owns two reusable pools. Ordinary requests retain the
+// explicitly configured DoH/RIPE policy; CDN entry points select the guarded pool.
+type pipelineTransport struct {
+	general http.RoundTripper
+	cdn     http.RoundTripper
+}
+
+func (t *pipelineTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	return t.general.RoundTrip(r)
+}
+
+func (t *pipelineTransport) CloseIdleConnections() {
+	for _, transport := range []http.RoundTripper{t.general, t.cdn} {
+		if closer, ok := transport.(interface{ CloseIdleConnections() }); ok {
+			closer.CloseIdleConnections()
+		}
+	}
+}
+
+// NewPipelineHTTPClient preserves configured DoH/RIPE access while giving CDN
+// fetches a separate reusable guarded transport without an unbounded client cache.
+func NewPipelineHTTPClient() *http.Client {
+	client := httpclient.New(httpclient.DefaultTimeout)
+	cdn := NewCDNHTTPClient()
+	client.Transport = &pipelineTransport{general: client.Transport, cdn: cdn.Transport}
+	return client
+}

internal/pipeline/http_clients_test.go

@@ -0,0 +1,66 @@
+package pipeline
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+)
+
+func TestPipelineClientReusesSeparateCDNPool(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
+	var connections atomic.Int32
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "192.0.2.0/24\n") }))
+	server.Config.ConnState = func(_ net.Conn, state http.ConnState) {
+		if state == http.StateNew {
+			connections.Add(1)
+		}
+	}
+	server.StartTLS()
+	defer server.Close()
+	client := NewPipelineHTTPClient()
+	defer client.CloseIdleConnections()
+	// Configure only test trust roots; the normal dial/redirect guards remain.
+	pools := client.Transport.(*pipelineTransport)
+	pools.cdn.(*cdnRoundTripper).transport.TLSClientConfig = server.Client().Transport.(*http.Transport).TLSClientConfig.Clone()
+	for range 3 {
+		req, _ := http.NewRequest(http.MethodGet, server.URL, nil)
+		response, err := UpstreamHTTPDo(context.Background(), client, req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = ReadCDNBody(response.Body)
+		_ = response.Body.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	if connections.Load() != 1 {
+		t.Fatalf("CDN requests created %d connections", connections.Load())
+	}
+}
+
+func TestPipelineClientSeparatesPrivateProfileAndCDN(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
+	var calls atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { calls.Add(1); _, _ = io.WriteString(w, "ok") }))
+	defer server.Close()
+	client := NewPipelineHTTPClient()
+	defer client.CloseIdleConnections()
+	response, err := client.Get(server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, _ = io.Copy(io.Discard, response.Body)
+	_ = response.Body.Close()
+	req, _ := http.NewRequest(http.MethodGet, server.URL, nil)
+	if _, err = UpstreamHTTPDo(context.Background(), client, req); err == nil {
+		t.Fatal("CDN used trusted profile pool")
+	}
+	if calls.Load() != 1 {
+		t.Fatalf("CDN reached private profile server: %d", calls.Load())
+	}
+}

internal/pipeline/module_hash.go

@@ -1,6 +1,7 @@
 package pipeline
 
 import (
+	"context"
 	"strings"
 
 	"evobgp/internal/store"
@@ -11,7 +12,7 @@ func moduleIngestInputHash(st store.Backend, tenantID string, mod *store.Module)
 	return store.ComputeModuleInputHash(st, tenantID, mod)
 }
 
-func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, rows []store.PrefixRow) error {
+func persistModuleSnapshot(ctx context.Context, st store.Backend, tenantID string, mod *store.Module, rows []store.PrefixRow) error {
 	if st == nil || mod == nil {
 		return nil
 	}
@@ -19,17 +20,14 @@ func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module,
 	if err != nil {
 		return err
 	}
-	if err := st.SetModulePrefixSnapshot(tenantID, mod.ID, hash, rows); err != nil {
-		return err
-	}
-	_ = st.SetModuleInputHash(tenantID, mod.ID, hash)
-	return nil
+	return st.SetModulePrefixSnapshot(ctx, tenantID, mod.ID, hash, rows)
 }
 
-func moduleRowsFromSnapshot(st store.Backend, tenantID string, mod *store.Module) ([]store.PrefixRow, bool, error) {
-	snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID)
-	if err != nil || !ok || snap == nil {
-		return nil, false, err
+// snapshotRowsIfCurrent returns rows when snap still matches the module input hash.
+// The caller already loaded snap, so this does not read the snapshot again.
+func snapshotRowsIfCurrent(ctx context.Context, st store.Backend, tenantID string, mod *store.Module, snap *store.ModulePrefixSnapshot) ([]store.PrefixRow, bool, error) {
+	if snap == nil {
+		return nil, false, nil
 	}
 	if h := strings.TrimSpace(mod.InputHash); h != "" {
 		if snap.InputHash != h {
@@ -45,6 +43,8 @@ func moduleRowsFromSnapshot(st store.Backend, tenantID string, mod *store.Module
 	if snap.InputHash != hash {
 		return nil, false, nil
 	}
-	_ = st.SetModuleInputHash(tenantID, mod.ID, hash)
+	if err := st.SetModuleInputHash(ctx, tenantID, mod.ID, hash); err != nil {
+		return nil, false, err
+	}
 	return append([]store.PrefixRow(nil), snap.Prefixes...), true, nil
 }

internal/pipeline/opt_collect_test.go

@@ -98,7 +98,7 @@ func TestCollectCDNPrefixRows_PartialSkipKeepsPrior(t *testing.T) {
 		{ID: "good", URL: good.URL, SourceKind: "plain"},
 		{ID: "bad", URL: bad.URL, SourceKind: "plain"},
 	}
-	rows, err := collectCDNPrefixRows(context.Background(), st, good.Client(), tenant, mod, sources, prior)
+	rows, err := collectCDNPrefixRows(context.Background(), st, good.Client(), tenant, mod, sources, modulePrior{provided: true, rows: prior})
 	if err != nil {
 		t.Fatalf("partial skip should succeed: %v", err)
 	}
@@ -126,18 +126,18 @@ func TestMergeCDNSource_ParallelNoLostUpdate(t *testing.T) {
 	wg.Add(2)
 	go func() {
 		defer wg.Done()
-		_ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s1", []store.PrefixRow{
+		_ = mergeCDNSourceIntoModuleSnapshot(context.Background(), st, tenant, mod, "s1", []store.PrefixRow{
 			{Prefix: "1.0.0.0/24", Source: "cdn:s1"},
 		})
 	}()
 	go func() {
 		defer wg.Done()
-		_ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s2", []store.PrefixRow{
+		_ = mergeCDNSourceIntoModuleSnapshot(context.Background(), st, tenant, mod, "s2", []store.PrefixRow{
 			{Prefix: "2.0.0.0/24", Source: "cdn:s2"},
 		})
 	}()
 	wg.Wait()
-	snap, ok, err := st.GetModulePrefixSnapshot(tenant, mod.ID)
+	snap, ok, err := st.GetModulePrefixSnapshot(context.Background(), tenant, mod.ID)
 	if err != nil || !ok || snap == nil {
 		t.Fatalf("snapshot missing: ok=%v err=%v", ok, err)
 	}
@@ -173,7 +173,7 @@ func TestInputHashInvalidatedOnEntryCRUD(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := st.SetModuleInputHash(tenant, mod.ID, "pre"); err != nil {
+	if err := st.SetModuleInputHash(context.Background(), tenant, mod.ID, "pre"); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := st.CreateIPRangeEntry(tenant, mod.ID, &store.IPRangeEntry{Prefix: "10.0.0.0/8"}); err != nil {

internal/pipeline/prefetch.go

@@ -2,6 +2,7 @@ package pipeline
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"strings"
 	"sync"
@@ -57,6 +58,7 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
 		return nil
 	}
 	sem := make(chan struct{}, collectConcurrency())
+	errs := make(chan error, len(tasks))
 	var wg sync.WaitGroup
 	for _, task := range tasks {
 		wg.Add(1)
@@ -64,23 +66,32 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
 			defer wg.Done()
 			sem <- struct{}{}
 			defer func() { <-sem }()
-			prefetchOneCDNSource(ctx, st, hc, t)
+			if err := prefetchOneCDNSource(ctx, st, hc, t); err != nil {
+				errs <- err
+			}
 		}(task)
 	}
 	wg.Wait()
+	close(errs)
+	for err := range errs {
+		return err
+	}
 	return nil
 }
 
-func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) {
+func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) error {
 	now := time.Now().UTC()
 	tid, mod, src := t.tenantID, t.mod, t.src
 	omod, err := st.GetModule(tid, mod.ID)
 	if err != nil || omod == nil {
-		return
+		return nil
 	}
-	rows, err := fetchCDNSourceRows(ctx, st, hc, tid, mod.ID, omod, src, nil, now)
+	rows, err := fetchCDNSourceRows(ctx, st, hc, tid, mod.ID, omod, src, modulePrior{}, now)
 	if err != nil {
-		return
+		if errors.Is(err, errCDNSnapshotRead) {
+			return err
+		}
+		return nil
 	}
-	_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
+	return mergeCDNSourceIntoModuleSnapshot(ctx, st, tid, omod, src.ID, rows)
 }

internal/pipeline/refresh.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -61,15 +62,19 @@ func RefreshModuleIngest(ctx context.Context, st store.Backend, hc *http.Client,
 		return fmt.Errorf("pipeline: module disabled")
 	}
 
-	var prior []store.PrefixRow
-	if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil {
-		prior = snap.Prefixes
-	}
-	rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, prior)
+	snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, moduleID)
 	if err != nil {
 		return err
 	}
-	if err := persistModuleSnapshot(st, tenantID, mod, rows); err != nil {
+	var priorRows []store.PrefixRow
+	if ok && snap != nil {
+		priorRows = snap.Prefixes
+	}
+	rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, modulePrior{provided: true, rows: priorRows})
+	if err != nil {
+		return err
+	}
+	if err := persistModuleSnapshot(ctx, st, tenantID, mod, rows); err != nil {
 		return err
 	}
 	refreshedAt := time.Now().UTC()
@@ -83,6 +88,11 @@ func RenderTenantRevision(ctx context.Context, st store.Backend, hc *http.Client
 	if hc == nil {
 		hc = httpclient.New(httpclient.DefaultTimeout)
 	}
+	if rev, ok, err := unchangedRenderRevision(ctx, st, tenantID); err != nil {
+		return "", err
+	} else if ok {
+		return rev, nil
+	}
 	agg, err := aggregateTenantPrefixRowsAll(ctx, st, hc, tenantID)
 	if err != nil {
 		return "", err
@@ -102,13 +112,71 @@ func RenderTenantRevision(ctx context.Context, st store.Backend, hc *http.Client
 	if err != nil {
 		return "", err
 	}
-	if err := st.CreateRenderRevision(revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil {
+	if err := st.CreateRenderRevision(ctx, revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil {
 		return "", err
 	}
 	applyRevisionRetention(st, tenantID)
 	return revisionID, nil
 }
 
+// unchangedRenderRevision returns the current revision when generation and snapshot
+// versions still match the latest manifest. A generation change during the read
+// refuses the skip so the caller rebuilds from the snapshots already written.
+func unchangedRenderRevision(ctx context.Context, st store.Backend, tenantID string) (string, bool, error) {
+	if st == nil || strings.TrimSpace(tenantID) == "" {
+		return "", false, nil
+	}
+	gen1, err := st.GetRenderGeneration(ctx, tenantID)
+	if err != nil {
+		if errors.Is(err, store.ErrNotFound) {
+			return "", false, nil
+		}
+		return "", false, err
+	}
+	headers, err := st.ListModuleSnapshotHeaders(ctx, tenantID)
+	if err != nil {
+		return "", false, err
+	}
+	gen2, err := st.GetRenderGeneration(ctx, tenantID)
+	if err != nil {
+		return "", false, err
+	}
+	if gen1 != gen2 {
+		return "", false, nil
+	}
+	man, ok, err := st.LatestRenderManifest(ctx, tenantID)
+	if err != nil || !ok || man == nil {
+		return "", false, err
+	}
+	if man.Generation != gen1 || !sameSnapshotUses(man.Snapshots, headers) {
+		return "", false, nil
+	}
+	gen3, err := st.GetRenderGeneration(ctx, tenantID)
+	if err != nil {
+		return "", false, err
+	}
+	if gen3 != gen1 || strings.TrimSpace(man.RevisionID) == "" {
+		return "", false, nil
+	}
+	return man.RevisionID, true, nil
+}
+
+func sameSnapshotUses(a, b []store.ModuleSnapshotUse) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	aa := append([]store.ModuleSnapshotUse(nil), a...)
+	bb := append([]store.ModuleSnapshotUse(nil), b...)
+	sort.Slice(aa, func(i, j int) bool { return aa[i].ModuleID < aa[j].ModuleID })
+	sort.Slice(bb, func(i, j int) bool { return bb[i].ModuleID < bb[j].ModuleID })
+	for i := range aa {
+		if aa[i].ModuleID != bb[i].ModuleID || aa[i].SnapshotVersion != bb[i].SnapshotVersion {
+			return false
+		}
+	}
+	return true
+}
+
 // RenderTenantRevisionFromPrefixes renders one tenant-wide revision from already materialized prefixes.
 // This is used for fast paths (e.g. peer-only changes) to avoid ingest/external fetches.
 func RenderTenantRevisionFromPrefixes(ctx context.Context, st store.Backend, hc *http.Client, tenantID, triggerModuleID string, rows []store.PrefixRow) (revisionID string, err error) {
@@ -132,7 +200,7 @@ func RenderTenantRevisionFromPrefixes(ctx context.Context, st store.Backend, hc
 	if err != nil {
 		return "", err
 	}
-	if err := st.CreateRenderRevision(revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil {
+	if err := st.CreateRenderRevision(ctx, revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil {
 		return "", err
 	}
 	applyRevisionRetention(st, tenantID)
@@ -148,8 +216,9 @@ func RefreshModule(ctx context.Context, st store.Backend, hc *http.Client, tenan
 }
 
 // collectModulePrefixRows returns materialized prefix rows for a single module (source of truth from store / ASN resolve / CDN fetch).
-// priorSnapshot is the last stored module snapshot (used to skip CDN fetches when refresh interval has not elapsed).
-func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
+// prior is the last stored module snapshot when the caller already loaded it.
+// CDN collect must not read that snapshot again.
+func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, prior modulePrior) ([]store.PrefixRow, error) {
 	moduleID := mod.ID
 	switch mod.Type {
 	case "IP_RANGES":
@@ -173,13 +242,13 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
 			return nil, err
 		}
 		sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
-		return collectASPrefixRows(ctx, st, hc, tenantID, mod, list, priorSnapshot)
+		return collectASPrefixRows(ctx, st, hc, tenantID, mod, list, prior.rows)
 	case "CDN_CIDRS":
 		sources, err := st.ListCDNSources(tenantID, moduleID)
 		if err != nil {
 			return nil, err
 		}
-		return collectCDNPrefixRows(ctx, st, hc, tenantID, mod, sources, priorSnapshot)
+		return collectCDNPrefixRows(ctx, st, hc, tenantID, mod, sources, prior)
 	case "DOMAINS":
 		entries, err := st.ListDomainEntries(tenantID, moduleID)
 		if err != nil {
@@ -189,7 +258,7 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
 		if err != nil {
 			return nil, err
 		}
-		return collectDomainPrefixRows(ctx, st, hc, mod, profiles, policy, entries, priorSnapshot)
+		return collectDomainPrefixRows(ctx, st, hc, mod, profiles, policy, entries, prior.rows)
 	default:
 		return nil, fmt.Errorf("pipeline: unknown module type %q", mod.Type)
 	}
@@ -260,6 +329,8 @@ func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohPr
 }
 
 func resolveDomainWithDOHMessage(ctx context.Context, hc *http.Client, baseURL, host string, qtype uint16) ([]netip.Addr, error) {
+	ctx, cancel := dohRequestContext(ctx)
+	defer cancel()
 	msg := new(dns.Msg)
 	msg.SetQuestion(dns.Fqdn(host), qtype)
 	wire, err := msg.Pack()
@@ -289,7 +360,7 @@ func resolveDomainWithDOHMessage(ctx context.Context, hc *http.Client, baseURL,
 		body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
 		return nil, fmt.Errorf("doh dns-message status %s: %s", resp.Status, strings.TrimSpace(string(body)))
 	}
-	raw, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
+	raw, err := httpclient.ReadResponseBody(resp, 2<<20)
 	if err != nil {
 		return nil, err
 	}
@@ -337,6 +408,8 @@ func resolveDomainWithSystemDNS(ctx context.Context, host string) ([]netip.Addr,
 }
 
 func resolveDomainWithDOHJSON(ctx context.Context, hc *http.Client, baseURL, host, qtype string) ([]netip.Addr, error) {
+	ctx, cancel := dohRequestContext(ctx)
+	defer cancel()
 	u, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, err
@@ -362,7 +435,11 @@ func resolveDomainWithDOHJSON(ctx context.Context, hc *http.Client, baseURL, hos
 		return nil, fmt.Errorf("doh status %s: %s", resp.Status, strings.TrimSpace(string(body)))
 	}
 	var payload dohJSONResponse
-	if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&payload); err != nil {
+	body, err := httpclient.ReadResponseBody(resp, 1<<20)
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(body, &payload); err != nil {
 		return nil, err
 	}
 	var out []netip.Addr
@@ -846,15 +923,6 @@ func intFromSettingsMap(m map[string]any, key string) int {
 	return 0
 }
 
-func applyRevisionRetention(st store.Backend, tenantID string) {
-	settings, err := st.ListGlobalSettings(tenantID)
-	if err != nil {
-		return
-	}
-	cutoff := RevisionRetentionCutoff(settings)
-	_, _ = st.PruneRevisionsBefore(tenantID, cutoff)
-}
-
 type peerPolicyJSON struct {
 	LocalIPv4 string  `json:"local_ipv4"`
 	LocalIPv6 string  `json:"local_ipv6"`

internal/pipeline/refresh_snapshot_order_test.go

@@ -0,0 +1,121 @@
+package pipeline
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"evobgp/internal/store"
+)
+
+func TestCDNRefreshSameSetDoesNotRewriteOrRevise(t *testing.T) {
+	t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, _, _ := m.DemoIDs()
+	disabled := false
+	for _, mod := range m.ListModules(tenant) {
+		if mod == nil {
+			continue
+		}
+		if _, err := m.UpdateModule(tenant, mod.ID, &store.ModulePatch{Enabled: &disabled}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	var body atomic.Value
+	body.Store("203.0.113.0/24\n198.51.100.0/24\n")
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(body.Load().(string)))
+	}))
+	defer srv.Close()
+
+	mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-order", Enabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{SourceKind: "plain", URL: srv.URL}); err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	if err := RefreshModuleIngest(ctx, m, srv.Client(), tenant, mod.ID); err != nil {
+		t.Fatal(err)
+	}
+	snap1, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, mod.ID)
+	if err != nil || !ok || snap1 == nil {
+		t.Fatalf("snapshot: ok=%v err=%v", ok, err)
+	}
+	gen1, err := m.GetRenderGeneration(ctx, tenant)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := RefreshModuleIngest(ctx, m, srv.Client(), tenant, mod.ID); err != nil {
+		t.Fatal(err)
+	}
+	snapSame, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, mod.ID)
+	if err != nil || !ok || snapSame == nil {
+		t.Fatal(err)
+	}
+	genSame, err := m.GetRenderGeneration(ctx, tenant)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if snapSame.Version != snap1.Version || !snapSame.CollectedAt.Equal(snap1.CollectedAt) || genSame != gen1 {
+		t.Fatalf("unchanged CDN rewritten: version %d->%d gen %d->%d", snap1.Version, snapSame.Version, gen1, genSame)
+	}
+
+	body.Store("198.51.100.0/24\n203.0.113.0/24\n")
+	if err := RefreshModuleIngest(ctx, m, srv.Client(), tenant, mod.ID); err != nil {
+		t.Fatal(err)
+	}
+	snap2, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, mod.ID)
+	if err != nil || !ok || snap2 == nil {
+		t.Fatal(err)
+	}
+	gen2, err := m.GetRenderGeneration(ctx, tenant)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if snap2.Version != snap1.Version || !snap2.CollectedAt.Equal(snap1.CollectedAt) || gen2 != gen1 {
+		t.Fatalf("reordered CDN rewritten: version %d->%d gen %d->%d", snap1.Version, snap2.Version, gen1, gen2)
+	}
+	if len(snap2.Prefixes) != len(snap1.Prefixes) || snap2.Prefixes[0].Prefix != snap1.Prefixes[0].Prefix {
+		t.Fatalf("row order changed without a content change: %+v", snap2.Prefixes)
+	}
+
+	before, _, _ := m.ListRevisions(tenant, "", "", 50)
+	rev1, err := RenderTenantRevision(ctx, m, srv.Client(), tenant, mod.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rev2, err := RenderTenantRevision(ctx, m, srv.Client(), tenant, mod.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rev2 != rev1 {
+		t.Fatalf("order-stable inputs created another revision: %s vs %s", rev1, rev2)
+	}
+	mid, _, _ := m.ListRevisions(tenant, "", "", 50)
+	if len(mid) != len(before)+1 {
+		t.Fatalf("revisions after unchanged renders: before=%d after=%d", len(before), len(mid))
+	}
+
+	body.Store("203.0.113.0/24\n192.0.2.0/24\n")
+	if err := RefreshModuleIngest(ctx, m, srv.Client(), tenant, mod.ID); err != nil {
+		t.Fatal(err)
+	}
+	rev3, err := RenderTenantRevision(ctx, m, srv.Client(), tenant, mod.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rev3 == rev1 {
+		t.Fatal("prefix change did not create a revision")
+	}
+	after, _, _ := m.ListRevisions(tenant, "", "", 50)
+	if len(after) != len(mid)+1 {
+		t.Fatalf("prefix change revisions: before=%d after=%d", len(mid), len(after))
+	}
+}

internal/pipeline/revision_prune.go

@@ -0,0 +1,108 @@
+package pipeline
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+
+	"evobgp/internal/store"
+)
+
+const (
+	revisionPruneMinInterval = 5 * time.Minute
+	revisionPruneTimeout     = 2 * time.Minute
+)
+
+// revisionPruneGate runs at most one prune per tenant per interval and drops
+// overlapping attempts so renders do not stack prune transactions.
+type revisionPruneGate struct {
+	mu       sync.Mutex
+	last     map[string]time.Time
+	inflight map[string]chan struct{}
+	now      func() time.Time
+}
+
+var revisionPrunes revisionPruneGate
+
+func (g *revisionPruneGate) start(tenantID string) bool {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	now := time.Now()
+	if g.now != nil {
+		now = g.now()
+	}
+	if g.last == nil {
+		g.last = make(map[string]time.Time)
+		g.inflight = make(map[string]chan struct{})
+	}
+	if _, running := g.inflight[tenantID]; running {
+		return false
+	}
+	if last, ok := g.last[tenantID]; ok && now.Sub(last) < revisionPruneMinInterval {
+		return false
+	}
+	g.last[tenantID] = now
+	g.inflight[tenantID] = make(chan struct{})
+	return true
+}
+
+func (g *revisionPruneGate) done(tenantID string) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	ch, ok := g.inflight[tenantID]
+	if !ok {
+		return
+	}
+	close(ch)
+	delete(g.inflight, tenantID)
+}
+
+// applyRevisionRetention schedules retention for tenantID and returns without
+// waiting. Prune uses its own short context, at most once per tenant per five
+// minutes, and overlapping renders do not start another transaction.
+func applyRevisionRetention(st store.Backend, tenantID string) {
+	tenantID = strings.TrimSpace(tenantID)
+	if st == nil || tenantID == "" {
+		return
+	}
+	if !revisionPrunes.start(tenantID) {
+		return
+	}
+	go func() {
+		defer revisionPrunes.done(tenantID)
+		ctx, cancel := context.WithTimeout(context.Background(), revisionPruneTimeout)
+		defer cancel()
+		pruneTenantRevisions(ctx, st, tenantID)
+	}()
+}
+
+// pruneTenantRevisions applies the tenant retention cutoff immediately.
+func pruneTenantRevisions(ctx context.Context, st store.Backend, tenantID string) {
+	if st == nil || ctx == nil {
+		return
+	}
+	if err := ctx.Err(); err != nil {
+		return
+	}
+	settings, err := st.ListGlobalSettings(tenantID)
+	if err != nil {
+		return
+	}
+	_, _ = st.PruneRevisionsBefore(ctx, tenantID, RevisionRetentionCutoff(settings))
+}
+
+func waitRevisionPrune(ctx context.Context, tenantID string) error {
+	revisionPrunes.mu.Lock()
+	ch := revisionPrunes.inflight[strings.TrimSpace(tenantID)]
+	revisionPrunes.mu.Unlock()
+	if ch == nil {
+		return nil
+	}
+	select {
+	case <-ch:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

internal/pipeline/revision_prune_async_test.go

@@ -0,0 +1,120 @@
+package pipeline
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"evobgp/internal/store"
+)
+
+type blockingPrune struct {
+	store.Backend
+	calls   atomic.Int32
+	started chan struct{}
+	release chan struct{}
+	once    sync.Once
+}
+
+func (b *blockingPrune) PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (int, error) {
+	n := b.calls.Add(1)
+	if n == 1 {
+		b.once.Do(func() { close(b.started) })
+		select {
+		case <-b.release:
+		case <-ctx.Done():
+			return 0, ctx.Err()
+		}
+	}
+	return b.Backend.PruneRevisionsBefore(ctx, tenantID, cutoff)
+}
+
+func TestRenderDoesNotWaitForRevisionPrune(t *testing.T) {
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, modIP, _, _ := m.DemoIDs()
+	release := make(chan struct{})
+	var releaseOnce sync.Once
+	closeRelease := func() { releaseOnce.Do(func() { close(release) }) }
+	t.Cleanup(closeRelease)
+	st := &blockingPrune{Backend: m, started: make(chan struct{}), release: release}
+	var shift atomic.Int64
+	revisionPrunes.mu.Lock()
+	prevNow := revisionPrunes.now
+	revisionPrunes.now = func() time.Time {
+		return time.Now().Add(time.Duration(shift.Load()) * time.Minute)
+	}
+	revisionPrunes.mu.Unlock()
+	t.Cleanup(func() {
+		revisionPrunes.mu.Lock()
+		revisionPrunes.now = prevNow
+		revisionPrunes.mu.Unlock()
+	})
+
+	rows := []store.PrefixRow{{Prefix: "203.0.113.55/32", Source: "ip_range"}}
+	renderDone := make(chan error, 1)
+	go func() {
+		_, err := RenderTenantRevisionFromPrefixes(context.Background(), st, nil, tenant, modIP, rows)
+		renderDone <- err
+	}()
+	select {
+	case err := <-renderDone:
+		if err != nil {
+			t.Fatal(err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("render waited for revision prune")
+	}
+	select {
+	case <-st.started:
+	case <-time.After(5 * time.Second):
+		t.Fatal("prune did not start")
+	}
+	if st.calls.Load() != 1 {
+		t.Fatalf("prune calls=%d", st.calls.Load())
+	}
+	applyRevisionRetention(st, tenant)
+	if st.calls.Load() != 1 {
+		t.Fatalf("overlapping render stacked prune calls=%d", st.calls.Load())
+	}
+	closeRelease()
+	waitCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	if err := waitRevisionPrune(waitCtx, tenant); err != nil {
+		t.Fatal(err)
+	}
+	shift.Store(10)
+	applyRevisionRetention(st, tenant)
+	waitCtx2, cancel2 := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel2()
+	if err := waitRevisionPrune(waitCtx2, tenant); err != nil {
+		t.Fatal(err)
+	}
+	if st.calls.Load() != 2 {
+		t.Fatalf("prune after interval calls=%d", st.calls.Load())
+	}
+}
+
+func TestPruneTenantRevisionsDirect(t *testing.T) {
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, _, _ := m.DemoIDs()
+	var calls atomic.Int32
+	st := pruneCountBackend{Backend: m, calls: &calls}
+	pruneTenantRevisions(context.Background(), st, tenant)
+	if calls.Load() != 1 {
+		t.Fatalf("calls=%d", calls.Load())
+	}
+}
+
+type pruneCountBackend struct {
+	store.Backend
+	calls *atomic.Int32
+}
+
+func (p pruneCountBackend) PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (int, error) {
+	p.calls.Add(1)
+	return p.Backend.PruneRevisionsBefore(ctx, tenantID, cutoff)
+}

internal/pipeline/scheduler_due.go

@@ -8,7 +8,8 @@ import (
 )
 
 // SchedulerTickSec matches the scheduler ticker interval (internal/scheduler).
-const SchedulerTickSec = 30
+// The module refresh window stays at least 60s; a shorter tick only notices the boundary sooner.
+const SchedulerTickSec = 10
 
 // ModuleDueForScheduler reports whether a module's refresh interval bucket rolled since the last scheduler tick.
 // A stable per-module offset (fnv32 of ID) spreads bucket boundaries so modules with the same interval

internal/pipeline/upstream_http.go

@@ -14,8 +14,9 @@ func UpstreamHTTPDo(ctx context.Context, hc *http.Client, req *http.Request) (*h
 }
 
 func upstreamHTTPDo(ctx context.Context, hc *http.Client, req *http.Request) (*http.Response, error) {
-	if hc == nil {
-		hc = httpclient.New(httpclient.DefaultTimeout)
+	hc, err := secureCDNClient(hc)
+	if err != nil {
+		return nil, err
 	}
 	resp, err := httpclient.DoWithBreaker(ctx, hc, req, 3)
 	if err != nil {