EvoBGP/internal/pipeline/refresh.go

package pipeline

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io"
	"net/netip"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	"evobgp/internal/asnresolve"
	"evobgp/internal/birdfmt"
	"evobgp/internal/store"

	"github.com/google/uuid"
)

const (
	birdFilterNameV4    = "evobgp_export_v4"
	birdFilterNameV6    = "evobgp_export_v6"
	auxBirdFullExpanded = "_bird_full_expanded.conf"
)

// MaterializedASPrefixKey returns the revision snapshot key for an AS-only entry (not a CIDR).
func MaterializedASPrefixKey(asn int64) string {
	return fmt.Sprintf("as:%d", asn)
}

// RefreshModule runs ingest (where applicable) for one module, then renders a new revision whose
// BIRD materialization includes prefixes from all enabled modules of the tenant (others via live collect).
// If the tenant-wide materialized prefix set is unchanged from the latest revision, returns that
// revision id and does not insert a duplicate config_revision.
func RefreshModule(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string) (revisionID string, err error) {
	if hc == nil {
		hc = http.DefaultClient
	}
	mod, err := st.GetModule(tenantID, moduleID)
	if err != nil {
		return "", err
	}
	if !mod.Enabled {
		return "", fmt.Errorf("module disabled")
	}

	rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod)
	if err != nil {
		return "", err
	}

	agg, err := aggregateTenantPrefixRows(ctx, st, hc, tenantID, moduleID, rows)
	if err != nil {
		return "", err
	}
	hash := hashAggregatedMaterialization(tenantID, agg)
	if prev := latestTenantRevision(st, tenantID); prev != nil && prev.ContentHash == hash {
		return prev.ID, nil
	}

	revisionID = uuid.NewString()
	parent := parentRevision(st, tenantID, moduleID)
	preview, err := buildPreviewFragments(st, tenantID, moduleID, revisionID, agg)
	if err != nil {
		return "", err
	}
	if err := st.CreateRenderRevision(revisionID, tenantID, moduleID, parent, hash, preview, agg); err != nil {
		return "", err
	}
	return revisionID, nil
}

// collectModulePrefixRows returns materialized prefix rows for a single module (source of truth from store / ASN resolve / CDN fetch).
func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module) ([]store.PrefixRow, error) {
	moduleID := mod.ID
	switch mod.Type {
	case "IP_RANGES":
		list, err := st.ListIPRangeEntries(tenantID, moduleID)
		if err != nil {
			return nil, err
		}
		var rows []store.PrefixRow
		for _, e := range list {
			comm := e.CommunityID
			if comm == nil && mod.DefaultCommunityID != nil {
				c := *mod.DefaultCommunityID
				comm = &c
			}
			rows = append(rows, store.PrefixRow{Prefix: e.Prefix, CommunityID: comm, Source: "ip_range"})
		}
		return rows, nil
	case "AS_PREFIXES":
		list, err := st.ListASEntries(tenantID, moduleID)
		if err != nil {
			return nil, err
		}
		sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
		legacy := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE")) == "0"
		seenPfx := make(map[string]struct{})
		var rows []store.PrefixRow
		for i, e := range list {
			if !store.ValidASN(e.ASN) {
				continue
			}
			comm := e.CommunityID
			if comm == nil && mod.DefaultCommunityID != nil {
				c := *mod.DefaultCommunityID
				comm = &c
			}
			if legacy {
				rows = append(rows, store.PrefixRow{Prefix: MaterializedASPrefixKey(e.ASN), CommunityID: comm, Source: "as_entry"})
				continue
			}
			if i > 0 {
				asnresolve.PolitePause()
			}
			pfxs, err := asnresolve.AnnouncedPrefixes(ctx, hc, e.ASN)
			if err != nil {
				return nil, fmt.Errorf("resolve AS%d: %w", e.ASN, err)
			}
			holder := ""
			asnresolve.PolitePause()
			if h, err := asnresolve.ASHolderName(ctx, hc, e.ASN); err == nil {
				holder = h
			}
			now := time.Now().UTC()
			if err := st.UpdateASEntryResolveMeta(tenantID, moduleID, e.ID, holder, int64(len(pfxs)), now); err != nil {
				return nil, fmt.Errorf("as entry meta AS%d: %w", e.ASN, err)
			}
			src := fmt.Sprintf("as:%d", e.ASN)
			for _, pfx := range pfxs {
				k := pfx.String()
				if _, ok := seenPfx[k]; ok {
					continue
				}
				seenPfx[k] = struct{}{}
				rows = append(rows, store.PrefixRow{Prefix: k, CommunityID: comm, Source: src})
			}
		}
		return rows, nil
	case "CDN_CIDRS":
		sources, err := st.ListCDNSources(tenantID, moduleID)
		if err != nil {
			return nil, err
		}
		var rows []store.PrefixRow
		for _, src := range sources {
			u := strings.TrimSpace(src.URL)
			if u == "" {
				continue
			}
			req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
			if err != nil {
				return nil, err
			}
			if strings.TrimSpace(src.Etag) != "" {
				req.Header.Set("If-None-Match", strings.TrimSpace(src.Etag))
			}
			resp, err := hc.Do(req)
			if err != nil {
				return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
			}
			if resp.StatusCode == http.StatusNotModified {
				_ = resp.Body.Close()
				continue
			}
			if resp.StatusCode != http.StatusOK {
				_, _ = io.Copy(io.Discard, resp.Body)
				_ = resp.Body.Close()
				return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
			}
			body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
			_ = resp.Body.Close()
			if err != nil {
				return nil, err
			}
			etag := strings.TrimSpace(resp.Header.Get("ETag"))
			if etag != "" && etag != strings.TrimSpace(src.Etag) {
				e := etag
				_, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, &store.CDNSourcePatch{Etag: &e})
			}
			pfxs, err := ExtractCIDRs(string(body), src.SourceKind, src.PrefixPath)
			if err != nil {
				return nil, fmt.Errorf("cdn parse %s: %w", u, err)
			}
			for _, pfx := range pfxs {
				comm := src.CommunityID
				if comm == nil && mod.DefaultCommunityID != nil {
					c := *mod.DefaultCommunityID
					comm = &c
				}
				rows = append(rows, store.PrefixRow{Prefix: pfx.String(), CommunityID: comm, Source: "cdn:" + src.ID})
			}
		}
		return rows, nil
	case "DOMAINS":
		entries, err := st.ListDomainEntries(tenantID, moduleID)
		if err != nil {
			return nil, err
		}
		var profile *store.DohProfile
		if mod.DohProfileID != nil && strings.TrimSpace(*mod.DohProfileID) != "" {
			profile, err = st.GetDohProfile(tenantID, strings.TrimSpace(*mod.DohProfileID))
			if err != nil {
				return nil, fmt.Errorf("get doh profile: %w", err)
			}
		}
		var rows []store.PrefixRow
		seen := make(map[string]struct{})
		for _, e := range entries {
			if e == nil {
				continue
			}
			comm := e.CommunityID
			if comm == nil && mod.DefaultCommunityID != nil {
				c := *mod.DefaultCommunityID
				comm = &c
			}
			addrs, err := resolveDomainIPs(ctx, hc, profile, e.FQDN)
			if err != nil {
				return nil, fmt.Errorf("resolve domain %q: %w", e.FQDN, err)
			}
			src := "domain:" + strings.TrimSpace(e.FQDN)
			for _, ip := range addrs {
				cidr := ipToHostPrefix(ip)
				if cidr == "" {
					continue
				}
				key := cidr + "|" + src
				if _, ok := seen[key]; ok {
					continue
				}
				seen[key] = struct{}{}
				rows = append(rows, store.PrefixRow{
					Prefix:      cidr,
					CommunityID: comm,
					Source:      src,
				})
			}
		}
		return rows, nil
	default:
		return nil, fmt.Errorf("unknown module type %q", mod.Type)
	}
}

type dohJSONAnswer struct {
	Type int    `json:"type"`
	Data string `json:"data"`
}

type dohJSONResponse struct {
	Answer []dohJSONAnswer `json:"Answer"`
}

func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohProfile, fqdn string) ([]netip.Addr, error) {
	host := strings.TrimSpace(strings.TrimSuffix(fqdn, "."))
	if host == "" {
		return nil, nil
	}
	if profile == nil || strings.TrimSpace(profile.URL) == "" {
		return nil, nil
	}

	timeout := 10 * time.Second
	if profile.TimeoutMs != nil && *profile.TimeoutMs > 0 {
		timeout = time.Duration(*profile.TimeoutMs) * time.Millisecond
	}
	dctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	// RFC8484 endpoint with JSON mode: ?name=<fqdn>&type=A/AAAA
	v4, err4 := resolveDomainWithDOHJSON(dctx, hc, strings.TrimSpace(profile.URL), host, "A")
	v6, err6 := resolveDomainWithDOHJSON(dctx, hc, strings.TrimSpace(profile.URL), host, "AAAA")
	if err4 != nil && err6 != nil {
		return nil, fmt.Errorf("doh failed for A and AAAA: %v; %v", err4, err6)
	}
	return uniqAddrs(append(v4, v6...)), nil
}

func resolveDomainWithDOHJSON(ctx context.Context, hc *http.Client, baseURL, host, qtype string) ([]netip.Addr, error) {
	u, err := url.Parse(baseURL)
	if err != nil {
		return nil, err
	}
	q := u.Query()
	q.Set("name", host)
	q.Set("type", qtype)
	u.RawQuery = q.Encode()

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/dns-json")

	resp, err := hc.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
		return nil, fmt.Errorf("doh status %s: %s", resp.Status, strings.TrimSpace(string(body)))
	}
	var payload dohJSONResponse
	if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&payload); err != nil {
		return nil, err
	}
	var out []netip.Addr
	for _, ans := range payload.Answer {
		if (qtype == "A" && ans.Type != 1) || (qtype == "AAAA" && ans.Type != 28) {
			continue
		}
		ip, err := netip.ParseAddr(strings.TrimSpace(ans.Data))
		if err != nil {
			continue
		}
		out = append(out, ip.Unmap())
	}
	return uniqAddrs(out), nil
}

func uniqAddrs(in []netip.Addr) []netip.Addr {
	seen := make(map[string]struct{}, len(in))
	out := make([]netip.Addr, 0, len(in))
	for _, a := range in {
		if !a.IsValid() {
			continue
		}
		k := a.String()
		if _, ok := seen[k]; ok {
			continue
		}
		seen[k] = struct{}{}
		out = append(out, a)
	}
	return out
}

func ipToHostPrefix(ip netip.Addr) string {
	if !ip.IsValid() {
		return ""
	}
	bits := 128
	if ip.Is4() {
		bits = 32
	}
	return netip.PrefixFrom(ip, bits).Masked().String()
}

// aggregateTenantPrefixRows builds the union of materialized prefixes for all enabled modules.
// The module that triggered refresh contributes freshRows; every other module is collected live from the store
// (same logic as refresh). We do not reuse other modules' saved revisions as prefix sources, because each revision
// already stores the full tenant-wide aggregate — mixing them with freshRows would duplicate prefixes.
func aggregateTenantPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, changedModuleID string, freshRows []store.PrefixRow) ([]store.PrefixRow, error) {
	mods := st.ListModules(tenantID)
	var out []store.PrefixRow
	for _, m := range mods {
		if m == nil || !m.Enabled {
			continue
		}
		if m.ID == changedModuleID {
			out = append(out, freshRows...)
			continue
		}
		omod, err := st.GetModule(tenantID, m.ID)
		if err != nil {
			return nil, err
		}
		rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, omod)
		if err != nil {
			return nil, fmt.Errorf("module %s: %w", m.ID, err)
		}
		out = append(out, rows...)
	}
	return out, nil
}

func parentRevision(st store.Backend, tenantID, moduleID string) *string {
	items, _, _ := st.ListRevisions(tenantID, moduleID, "", 1)
	if len(items) == 0 {
		return nil
	}
	id := items[0].ID
	return &id
}

// latestTenantRevision is the newest config_revision for the tenant (any module), or nil.
func latestTenantRevision(st store.Backend, tenantID string) *store.Revision {
	items, _, _ := st.ListRevisions(tenantID, "", "", 1)
	if len(items) == 0 {
		return nil
	}
	return items[0]
}

// hashAggregatedMaterialization hashes the full tenant-wide prefix set used for BIRD (all enabled modules).
func hashAggregatedMaterialization(tenantID string, rows []store.PrefixRow) string {
	type line struct{ p, c, s string }
	var lines []line
	for _, r := range rows {
		c := ""
		if r.CommunityID != nil {
			c = *r.CommunityID
		}
		lines = append(lines, line{r.Prefix, c, r.Source})
	}
	sort.Slice(lines, func(i, j int) bool {
		if lines[i].p != lines[j].p {
			return lines[i].p < lines[j].p
		}
		if lines[i].c != lines[j].c {
			return lines[i].c < lines[j].c
		}
		return lines[i].s < lines[j].s
	})
	h := sha256.New()
	h.Write([]byte(strings.TrimSpace(tenantID)))
	h.Write([]byte{0})
	for _, l := range lines {
		h.Write([]byte(l.p))
		h.Write([]byte{1})
		h.Write([]byte(l.c))
		h.Write([]byte{1})
		h.Write([]byte(l.s))
		h.Write([]byte{0})
	}
	return fmt.Sprintf("sha256:%x", h.Sum(nil))
}

func buildPreviewFragments(st store.Backend, tenantID, moduleID, revisionID string, rows []store.PrefixRow) (map[string]string, error) {
	v4, v6, pathASNs, sr4, sr6, err := materializeRowsForBird(st, tenantID, rows)
	if err != nil {
		return nil, err
	}
	f4, err := birdfmt.RenderExportFilterIPv4(birdFilterNameV4, v4, pathASNs)
	if err != nil {
		return nil, err
	}
	f6, err := birdfmt.RenderExportFilterIPv6(birdFilterNameV6, v6, pathASNs)
	if err != nil {
		return nil, err
	}
	staticV4 := birdfmt.RenderStaticIPv4Routes("evobgp_prefixes_v4", sr4)
	staticV6 := birdfmt.RenderStaticIPv6Routes("evobgp_prefixes_v6", sr6)

	locals := birdLocalsFromStore(st, tenantID)
	tplBody, err := birdfmt.RenderBGPTemplates(birdfmt.BGPTemplatesOptions{
		LocalASN:       locals.localASN,
		ExportFilterV4: birdFilterNameV4,
		ExportFilterV6: birdFilterNameV6,
	})
	if err != nil {
		return nil, err
	}
	peersBody, err := renderPeersBirdFragment(st, tenantID, locals)
	if err != nil {
		return nil, err
	}

	main, err := birdfmt.RenderMainBirdConf(birdfmt.MainBirdConfOptions{
		RouterID: locals.routerID,
		Includes: birdfmt.StandardIncludeFragments(),
		Preamble: fmt.Sprintf("EvoBGP tenant aggregate config (trigger module %s) revision %s", moduleID, revisionID),
	})
	if err != nil {
		return nil, err
	}

	p4 := birdfmt.FragmentIncludePath(birdfmt.FragmentFiltersV4)
	p6 := birdfmt.FragmentIncludePath(birdfmt.FragmentFiltersV6)
	pTpl := birdfmt.FragmentIncludePath(birdfmt.FragmentBGPTemplate)
	px4 := birdfmt.FragmentIncludePath(birdfmt.FragmentPrefixesV4)
	px6 := birdfmt.FragmentIncludePath(birdfmt.FragmentPrefixesV6)
	pPeers := birdfmt.FragmentIncludePath(birdfmt.FragmentPeers)

	out := map[string]string{
		"bird.conf": main,
		p4:          birdfmt.JoinFragments(birdfmt.ManagedBanner(revisionID), f4),
		p6:          birdfmt.JoinFragments(birdfmt.ManagedBanner(revisionID), f6),
		pTpl:        birdfmt.JoinFragments(birdfmt.ManagedBanner(revisionID), tplBody),
		px4:         birdfmt.JoinFragments(birdfmt.ManagedBanner(revisionID), staticV4),
		px6:         birdfmt.JoinFragments(birdfmt.ManagedBanner(revisionID), staticV6),
		pPeers:      peersBody,
	}
	out[auxBirdFullExpanded] = buildExpandedBirdText(main, out)
	return out, nil
}

type birdLocals struct {
	routerID string
	localV4  string
	localV6  string
	localASN uint32
}

func birdLocalsFromStore(st store.Backend, tenantID string) birdLocals {
	def := birdLocals{
		routerID: "192.0.2.1",
		localV4:  "192.0.2.1",
		localV6:  "2001:db8::1",
		localASN: 65001,
	}
	settings, err := st.ListGlobalSettings(tenantID)
	if err != nil {
		return def
	}
	loc := def
	if s := stringFromSettingsMap(settings, "bird_router_id"); s != "" {
		loc.routerID = s
	}
	if s := stringFromSettingsMap(settings, "bird_local_ipv4"); s != "" {
		loc.localV4 = s
	}
	if s := stringFromSettingsMap(settings, "bird_local_ipv6"); s != "" {
		loc.localV6 = s
	}
	if n := uint32FromSettingsMap(settings, "bird_local_asn"); n != 0 {
		loc.localASN = n
	}
	if s := stringFromSettingsMap(settings, "bird_bgp_source_ipv4"); s != "" {
		// BIRD router id must be an IPv4 address; historically aligned with optional BGP source setting.
		loc.routerID = strings.TrimSpace(s)
	}
	return loc
}

func stringFromSettingsMap(m map[string]any, key string) string {
	v, ok := m[key]
	if !ok || v == nil {
		return ""
	}
	s, ok := v.(string)
	if !ok {
		return ""
	}
	return strings.TrimSpace(s)
}

func uint32FromSettingsMap(m map[string]any, key string) uint32 {
	v, ok := m[key]
	if !ok || v == nil {
		return 0
	}
	switch x := v.(type) {
	case float64:
		if x >= 1 && x <= 4294967295 {
			return uint32(x)
		}
	case int:
		if x >= 1 && x <= 4294967295 {
			return uint32(x)
		}
	case int64:
		if x >= 1 && x <= 4294967295 {
			return uint32(x)
		}
	case string:
		if n, err := strconv.ParseUint(strings.TrimSpace(x), 10, 32); err == nil && n >= 1 {
			return uint32(n)
		}
	}
	return 0
}

type peerPolicyJSON struct {
	LocalIPv4 string  `json:"local_ipv4"`
	LocalIPv6 string  `json:"local_ipv6"`
	LocalASN  float64 `json:"local_asn"`
}

func effectivePeerLocals(loc birdLocals, pol peerPolicyJSON) (v4, v6 string, asn uint32) {
	v4 = strings.TrimSpace(loc.localV4)
	v6 = strings.TrimSpace(loc.localV6)
	if s := strings.TrimSpace(pol.LocalIPv4); s != "" {
		v4 = s
	}
	if s := strings.TrimSpace(pol.LocalIPv6); s != "" {
		v6 = s
	}
	asn = loc.localASN
	if pol.LocalASN >= 1 && pol.LocalASN <= 4294967295 {
		asn = uint32(pol.LocalASN)
	}
	return v4, v6, asn
}

// peerNeedsLocalOverride is true when the peer's effective local IP or ASN should override template "local as …" (add explicit "local <addr> as …" on the peer).
func peerNeedsLocalOverride(loc birdLocals, effLocal string, effASN uint32, ipv4 bool) bool {
	if ipv4 {
		return strings.TrimSpace(effLocal) != strings.TrimSpace(loc.localV4) || effASN != loc.localASN
	}
	return strings.TrimSpace(effLocal) != strings.TrimSpace(loc.localV6) || effASN != loc.localASN
}

func renderPeersBirdFragment(st store.Backend, tenantID string, loc birdLocals) (string, error) {
	peers := st.ListPeers(tenantID)
	var parts []string
	parts = append(parts, birdfmt.ManagedBanner("peers"))
	for _, p := range peers {
		if p == nil || !p.Enabled {
			continue
		}
		addr, ok := store.ParsePeerNeighbor(p.Neighbor)
		if !ok {
			continue
		}
		if !store.ValidASN(p.RemoteASN) {
			continue
		}
		pol := parsePeerPolicies(p.PoliciesJSON)
		lv4, lv6, asn := effectivePeerLocals(loc, pol)
		proto := peerProtocolName(p.ID)
		ra := uint32(p.RemoteASN)
		if addr.Is4() {
			opts := birdfmt.BGPPeerFromTemplateOptions{
				ProtocolName: proto,
				TemplateName: birdfmt.BGPTemplateNameV4,
				NeighborIP:   addr.String(),
				NeighborASN:  ra,
			}
			if peerNeedsLocalOverride(loc, lv4, asn, true) {
				opts.OverrideLocalIP = lv4
				opts.OverrideLocalASN = asn
			}
			s, err := birdfmt.RenderProtocolBGPFromTemplate(opts)
			if err != nil {
				return "", err
			}
			parts = append(parts, s)
			continue
		}
		if addr.Is6() {
			opts := birdfmt.BGPPeerFromTemplateOptions{
				ProtocolName: proto,
				TemplateName: birdfmt.BGPTemplateNameV6,
				NeighborIP:   addr.String(),
				NeighborASN:  ra,
			}
			if peerNeedsLocalOverride(loc, lv6, asn, false) {
				opts.OverrideLocalIP = lv6
				opts.OverrideLocalASN = asn
			}
			s, err := birdfmt.RenderProtocolBGPFromTemplate(opts)
			if err != nil {
				return "", err
			}
			parts = append(parts, s)
		}
	}
	if len(parts) == 1 {
		parts = append(parts, "# (no enabled BGP peers with valid neighbor addresses)\n")
	}
	return birdfmt.JoinFragments(parts...), nil
}

func parsePeerPolicies(raw string) peerPolicyJSON {
	raw = strings.TrimSpace(raw)
	if raw == "" || raw == "{}" {
		return peerPolicyJSON{}
	}
	var pol peerPolicyJSON
	_ = json.Unmarshal([]byte(raw), &pol)
	return pol
}

func peerProtocolName(peerID string) string {
	s := strings.ReplaceAll(strings.TrimSpace(peerID), "-", "")
	if len(s) > 16 {
		s = s[:16]
	}
	if s == "" {
		s = "x"
	}
	return "evobgp_p_" + s
}

// buildExpandedBirdText concatenates bird.conf and the contents of each standard include (for UI / preview).
func buildExpandedBirdText(main string, frags map[string]string) string {
	var b strings.Builder
	b.WriteString(strings.TrimSpace(main))
	b.WriteString("\n")
	for _, inc := range birdfmt.StandardIncludeFragments() {
		b.WriteString("\n# ---------- include \"")
		b.WriteString(inc)
		b.WriteString("\" ----------\n")
		body := strings.TrimSpace(frags[inc])
		if body == "" {
			b.WriteString("# (empty)\n")
			continue
		}
		b.WriteString(body)
		if !strings.HasSuffix(body, "\n") {
			b.WriteByte('\n')
		}
	}
	return b.String()
}