feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <cursoragent@cursor.com>

internal/observability/metrics.go

@@ -20,10 +20,12 @@ import (
 const namespace = "evobgp"
 
 var (
-	metricsStore          atomic.Value // store.Backend
+	metricsStore          atomic.Value // storeBackendHolder
 	registerCollectorOnce sync.Once
 )
 
+type storeBackendHolder struct{ backend store.Backend }
+
 var (
 	httpRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
@@ -117,7 +119,7 @@ var (
 	upstreamBreakerOpen = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Name:      "upstream_breaker_open",
-		Help:      "1 if per-host HTTP circuit breaker is open.",
+		Help:      "Last request-time observation of the per-host circuit breaker (1=open, 0=closed); idle host series expire with the bounded cache.",
 	}, []string{"host"})
 )
 
@@ -173,6 +175,9 @@ func RecordJobTerminal(kind, status string) {
 
 // RecordJobAuditDepth updates durable job_audit depth gauges.
 func RecordJobAuditDepth(counts map[string]int64) {
+	for _, status := range []string{"queued", "running", "succeeded", "failed", "cancelled"} {
+		jobAuditDepth.WithLabelValues(status).Set(float64(counts[status]))
+	}
 	for status, n := range counts {
 		if status == "" {
 			status = "unknown"
@@ -193,6 +198,15 @@ func SetUpstreamBreakerOpen(host string, open bool) {
 	upstreamBreakerOpen.WithLabelValues(host).Set(v)
 }
 
+// RemoveUpstreamBreaker removes an expired host series together with its cache entry.
+// Callers must retain active entries until their final metric update completes.
+func RemoveUpstreamBreaker(host string) {
+	if host == "" {
+		host = "_"
+	}
+	upstreamBreakerOpen.DeleteLabelValues(host)
+}
+
 // SetBuildInfo sets evobgp_build_info gauge (idempotent labels).
 func SetBuildInfo(version, gitSHA string) {
 	if version == "" {
@@ -205,34 +219,49 @@ func SetBuildInfo(version, gitSHA string) {
 }
 
 type memoryStoreCollector struct {
-	prefixMaxDesc *prometheus.Desc
-	prefixSumDesc *prometheus.Desc
-	peersDesc     *prometheus.Desc
-	peerStateDesc *prometheus.Desc
+	prefixMaxDesc         *prometheus.Desc
+	prefixSumDesc         *prometheus.Desc
+	peersDesc             *prometheus.Desc
+	peerStateDesc         *prometheus.Desc
+	storedMaxDesc         *prometheus.Desc
+	storedSumDesc         *prometheus.Desc
+	inventoryDesc         *prometheus.Desc
+	storedStateDesc       *prometheus.Desc
+	collectionSuccessDesc *prometheus.Desc
 }
 
 func newMemoryStoreCollector() *memoryStoreCollector {
 	return &memoryStoreCollector{
 		prefixMaxDesc: prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "", "materialized_prefixes_max"),
-			"Maximum materialized_prefix_count among all revisions in the store.",
+			"Deprecated: maximum materialized_prefix_count among all stored revisions; use stored_revision_materialized_prefixes_max.",
 			nil, nil,
 		),
 		prefixSumDesc: prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "", "materialized_prefixes_sum"),
-			"Sum of materialized_prefix_count over revisions (development aggregate).",
+			"Deprecated: sum of materialized_prefix_count over all stored revisions; use stored_revision_materialized_prefixes_sum.",
 			nil, nil,
 		),
 		peersDesc: prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "", "bgp_peers_configured_total"),
-			"BGP peers configured in the control-plane store.",
+			"Deprecated: BGP peers configured in the control-plane store; use inventory_bgp_peers.",
 			nil, nil,
 		),
 		peerStateDesc: prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "", "bgp_peer_sessions"),
-			"Configured BGP peers in the store by session_state (intent / last known, not live BIRD).",
+			"Deprecated: configured BGP peers by persisted session_state, not live BIRD; use stored_bgp_peer_session_state.",
 			[]string{"state"}, nil,
 		),
+		storedMaxDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_max"),
+			"Maximum materialized prefix count across all stored revisions; historical, not current routes.", nil, nil),
+		storedSumDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_sum"),
+			"Sum of materialized prefix counts across all stored revisions; historical, not current routes.", nil, nil),
+		inventoryDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "inventory_bgp_peers"),
+			"Configured BGP peer inventory in the control-plane store, regardless of live state.", nil, nil),
+		storedStateDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_bgp_peer_session_state"),
+			"BGP peer inventory grouped by persisted session_state; not a live observation.", []string{"state"}, nil),
+		collectionSuccessDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "store_metrics_collection_success"),
+			"1 if the latest store metrics collection succeeded, 0 on database aggregation error; other store metrics are omitted on error.", nil, nil),
 	}
 }
 
@@ -241,6 +270,11 @@ func (c *memoryStoreCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- c.prefixSumDesc
 	ch <- c.peersDesc
 	ch <- c.peerStateDesc
+	ch <- c.storedMaxDesc
+	ch <- c.storedSumDesc
+	ch <- c.inventoryDesc
+	ch <- c.storedStateDesc
+	ch <- c.collectionSuccessDesc
 }
 
 func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
@@ -248,19 +282,44 @@ func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
 	if v == nil {
 		return
 	}
-	b, ok := v.(store.Backend)
-	if !ok || b == nil {
+	holder, ok := v.(storeBackendHolder)
+	if !ok || holder.backend == nil {
 		return
 	}
-	maxN, sumN := b.MaterializedPrefixStats()
+	b := holder.backend
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	maxN, sumN, peers, states, err := b.ObserveStoreMetrics(ctx)
+	cancel()
+	if err != nil {
+		ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 0)
+		return
+	}
+	ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 1)
 	ch <- prometheus.MustNewConstMetric(c.prefixMaxDesc, prometheus.GaugeValue, float64(maxN))
 	ch <- prometheus.MustNewConstMetric(c.prefixSumDesc, prometheus.GaugeValue, float64(sumN))
-	ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(b.PeerCount()))
-	for state, n := range b.PeerSessionCountsByState() {
+	ch <- prometheus.MustNewConstMetric(c.storedMaxDesc, prometheus.GaugeValue, float64(maxN))
+	ch <- prometheus.MustNewConstMetric(c.storedSumDesc, prometheus.GaugeValue, float64(sumN))
+	ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(peers))
+	ch <- prometheus.MustNewConstMetric(c.inventoryDesc, prometheus.GaugeValue, float64(peers))
+	canonicalStates := make(map[string]int)
+	for state, n := range states {
 		if state == "" {
 			state = "unknown"
 		}
 		ch <- prometheus.MustNewConstMetric(c.peerStateDesc, prometheus.GaugeValue, float64(n), state)
+		canonicalStates[canonicalPeerState(state)] += n
+	}
+	for state, n := range canonicalStates {
+		ch <- prometheus.MustNewConstMetric(c.storedStateDesc, prometheus.GaugeValue, float64(n), state)
+	}
+}
+
+func canonicalPeerState(state string) string {
+	switch strings.ToLower(strings.TrimSpace(state)) {
+	case "idle", "connect", "active", "opensent", "openconfirm", "established", "unknown":
+		return strings.ToLower(strings.TrimSpace(state))
+	default:
+		return "other"
 	}
 }
 
@@ -269,7 +328,7 @@ func RegisterStoreBackend(b store.Backend) {
 	if b == nil {
 		return
 	}
-	metricsStore.Store(b)
+	metricsStore.Store(storeBackendHolder{backend: b})
 	registerCollectorOnce.Do(func() {
 		prometheus.DefaultRegisterer.MustRegister(newMemoryStoreCollector())
 	})

internal/observability/metrics_store_test.go

@@ -0,0 +1,116 @@
+package observability
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"evobgp/internal/store"
+)
+
+type failingMetricsBackend struct{ store.Backend }
+
+func (failingMetricsBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
+	return 0, 0, 0, nil, errors.New("database unavailable")
+}
+
+type storedMetricsBackend struct{ store.Backend }
+
+func (storedMetricsBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
+	return 12, 24, 1, map[string]int{"established": 1}, nil
+}
+
+type legacyStateBackend struct{ store.Backend }
+
+func (legacyStateBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
+	return 0, 0, 2, map[string]int{" Established ": 1, "established": 1}, nil
+}
+
+func TestStoreMetricsCollectionErrorOmitsAggregates(t *testing.T) {
+	metricsStore.Store(storeBackendHolder{backend: failingMetricsBackend{}})
+	registry := prometheus.NewRegistry()
+	if err := registry.Register(newMemoryStoreCollector()); err != nil {
+		t.Fatal(err)
+	}
+	families, err := registry.Gather()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(families) != 1 || families[0].GetName() != "evobgp_store_metrics_collection_success" {
+		t.Fatalf("expected only collection status on error, got %v", families)
+	}
+	if got := families[0].Metric[0].Gauge.GetValue(); got != 0 {
+		t.Fatalf("collection status = %v, want 0", got)
+	}
+}
+
+func TestStoredRevisionMetricsDoNotClaimCurrentRoutes(t *testing.T) {
+	metricsStore.Store(storeBackendHolder{backend: storedMetricsBackend{}})
+	registry := prometheus.NewRegistry()
+	if err := registry.Register(newMemoryStoreCollector()); err != nil {
+		t.Fatal(err)
+	}
+	families, err := registry.Gather()
+	if err != nil {
+		t.Fatal(err)
+	}
+	values := make(map[string]float64)
+	for _, family := range families {
+		if len(family.Metric) > 0 && family.Metric[0].Gauge != nil {
+			values[family.GetName()] = family.Metric[0].Gauge.GetValue()
+		}
+	}
+	if values["evobgp_stored_revision_materialized_prefixes_sum"] != 24 || values["evobgp_materialized_prefixes_sum"] != 24 {
+		t.Fatalf("stored revision and legacy sums must match: %v", values)
+	}
+	if values["evobgp_store_metrics_collection_success"] != 1 {
+		t.Fatalf("successful collection not reported: %v", values)
+	}
+	if _, ok := values["evobgp_current_routes"]; ok {
+		t.Fatal("historical revision sum must not be published as current routes")
+	}
+}
+
+func TestCanonicalPeerStateBoundsNewLabelValues(t *testing.T) {
+	if got := canonicalPeerState("Established"); got != "established" {
+		t.Fatalf("state = %q", got)
+	}
+	if got := canonicalPeerState("unexpected peer text"); got != "other" {
+		t.Fatalf("state = %q", got)
+	}
+}
+
+func TestLegacyPeerStateLabelPreservedWhileNewLabelIsCanonical(t *testing.T) {
+	metricsStore.Store(storeBackendHolder{backend: legacyStateBackend{}})
+	registry := prometheus.NewRegistry()
+	if err := registry.Register(newMemoryStoreCollector()); err != nil {
+		t.Fatal(err)
+	}
+	families, err := registry.Gather()
+	if err != nil {
+		t.Fatal(err)
+	}
+	legacyRaw, newCanonical := false, false
+	for _, family := range families {
+		for _, metric := range family.Metric {
+			if len(metric.Label) != 1 {
+				continue
+			}
+			switch family.GetName() {
+			case "evobgp_bgp_peer_sessions":
+				if metric.Label[0].GetValue() == " Established " && metric.Gauge.GetValue() == 1 {
+					legacyRaw = true
+				}
+			case "evobgp_stored_bgp_peer_session_state":
+				if metric.Label[0].GetValue() == "established" && metric.Gauge.GetValue() == 2 {
+					newCanonical = true
+				}
+			}
+		}
+	}
+	if !legacyRaw || !newCanonical {
+		t.Fatalf("legacy raw state preserved=%v, new canonical state aggregated=%v", legacyRaw, newCanonical)
+	}
+}