EvoBGP/internal/pipeline/prefetch.go

package pipeline

import (
	"context"
	"io"
	"net/http"
	"strings"
	"time"

	"evobgp/internal/store"
)

// PrefetchCDNSourceETags performs conditional GETs for CDN sources; on 200 parses CIDRs into module_prefix_snapshot.
func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Client) error {
	if hc == nil {
		hc = http.DefaultClient
	}
	tenants, err := st.ListTenantIDs()
	if err != nil {
		return err
	}
	now := time.Now().UTC()
	for _, tid := range tenants {
		for _, mod := range st.ListModules(tid) {
			if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" {
				continue
			}
			omod, err := st.GetModule(tid, mod.ID)
			if err != nil {
				continue
			}
			sources, err := st.ListCDNSources(tid, mod.ID)
			if err != nil {
				continue
			}
			var prior []store.PrefixRow
			if snap, ok, _ := st.GetModulePrefixSnapshot(tid, mod.ID); ok && snap != nil {
				prior = snap.Prefixes
			}
			for _, src := range sources {
				if src == nil || strings.TrimSpace(src.URL) == "" {
					continue
				}
				req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(src.URL), nil)
				if err != nil {
					continue
				}
				if etag := strings.TrimSpace(src.Etag); etag != "" {
					req.Header.Set("If-None-Match", etag)
				}
				resp, err := hc.Do(req)
				if err != nil {
					continue
				}
				if resp.StatusCode == http.StatusNotModified {
					_ = resp.Body.Close()
					continue
				}
				if resp.StatusCode != http.StatusOK {
					_, _ = io.Copy(io.Discard, resp.Body)
					_ = resp.Body.Close()
					continue
				}
				body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
				_ = resp.Body.Close()
				if err != nil {
					continue
				}
				prefixStrs, err := parseCDNBody(string(body), src)
				if err != nil {
					continue
				}
				newEtag := strings.TrimSpace(resp.Header.Get("ETag"))
				patch := &store.CDNSourcePatch{LastRefreshedAt: &now}
				if newEtag != "" && newEtag != strings.TrimSpace(src.Etag) {
					e := newEtag
					patch.Etag = &e
				}
				_, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, patch)
				rows := cdnRowsFromParsed(omod, src, prefixStrs)
				_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
				_ = prior // prior may be stale after merge; refresh for next source in loop
				if snap, ok, _ := st.GetModulePrefixSnapshot(tid, mod.ID); ok && snap != nil {
					prior = snap.Prefixes
				}
			}
		}
	}
	return nil
}