feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -20,10 +20,12 @@ import (
|
||||
const namespace = "evobgp"
|
||||
|
||||
var (
|
||||
metricsStore atomic.Value // store.Backend
|
||||
metricsStore atomic.Value // storeBackendHolder
|
||||
registerCollectorOnce sync.Once
|
||||
)
|
||||
|
||||
type storeBackendHolder struct{ backend store.Backend }
|
||||
|
||||
var (
|
||||
httpRequests = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
@@ -117,7 +119,7 @@ var (
|
||||
upstreamBreakerOpen = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "upstream_breaker_open",
|
||||
Help: "1 if per-host HTTP circuit breaker is open.",
|
||||
Help: "Last request-time observation of the per-host circuit breaker (1=open, 0=closed); idle host series expire with the bounded cache.",
|
||||
}, []string{"host"})
|
||||
)
|
||||
|
||||
@@ -173,6 +175,9 @@ func RecordJobTerminal(kind, status string) {
|
||||
|
||||
// RecordJobAuditDepth updates durable job_audit depth gauges.
|
||||
func RecordJobAuditDepth(counts map[string]int64) {
|
||||
for _, status := range []string{"queued", "running", "succeeded", "failed", "cancelled"} {
|
||||
jobAuditDepth.WithLabelValues(status).Set(float64(counts[status]))
|
||||
}
|
||||
for status, n := range counts {
|
||||
if status == "" {
|
||||
status = "unknown"
|
||||
@@ -193,6 +198,15 @@ func SetUpstreamBreakerOpen(host string, open bool) {
|
||||
upstreamBreakerOpen.WithLabelValues(host).Set(v)
|
||||
}
|
||||
|
||||
// RemoveUpstreamBreaker removes an expired host series together with its cache entry.
|
||||
// Callers must retain active entries until their final metric update completes.
|
||||
func RemoveUpstreamBreaker(host string) {
|
||||
if host == "" {
|
||||
host = "_"
|
||||
}
|
||||
upstreamBreakerOpen.DeleteLabelValues(host)
|
||||
}
|
||||
|
||||
// SetBuildInfo sets evobgp_build_info gauge (idempotent labels).
|
||||
func SetBuildInfo(version, gitSHA string) {
|
||||
if version == "" {
|
||||
@@ -205,34 +219,49 @@ func SetBuildInfo(version, gitSHA string) {
|
||||
}
|
||||
|
||||
type memoryStoreCollector struct {
|
||||
prefixMaxDesc *prometheus.Desc
|
||||
prefixSumDesc *prometheus.Desc
|
||||
peersDesc *prometheus.Desc
|
||||
peerStateDesc *prometheus.Desc
|
||||
prefixMaxDesc *prometheus.Desc
|
||||
prefixSumDesc *prometheus.Desc
|
||||
peersDesc *prometheus.Desc
|
||||
peerStateDesc *prometheus.Desc
|
||||
storedMaxDesc *prometheus.Desc
|
||||
storedSumDesc *prometheus.Desc
|
||||
inventoryDesc *prometheus.Desc
|
||||
storedStateDesc *prometheus.Desc
|
||||
collectionSuccessDesc *prometheus.Desc
|
||||
}
|
||||
|
||||
func newMemoryStoreCollector() *memoryStoreCollector {
|
||||
return &memoryStoreCollector{
|
||||
prefixMaxDesc: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "materialized_prefixes_max"),
|
||||
"Maximum materialized_prefix_count among all revisions in the store.",
|
||||
"Deprecated: maximum materialized_prefix_count among all stored revisions; use stored_revision_materialized_prefixes_max.",
|
||||
nil, nil,
|
||||
),
|
||||
prefixSumDesc: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "materialized_prefixes_sum"),
|
||||
"Sum of materialized_prefix_count over revisions (development aggregate).",
|
||||
"Deprecated: sum of materialized_prefix_count over all stored revisions; use stored_revision_materialized_prefixes_sum.",
|
||||
nil, nil,
|
||||
),
|
||||
peersDesc: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "bgp_peers_configured_total"),
|
||||
"BGP peers configured in the control-plane store.",
|
||||
"Deprecated: BGP peers configured in the control-plane store; use inventory_bgp_peers.",
|
||||
nil, nil,
|
||||
),
|
||||
peerStateDesc: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "bgp_peer_sessions"),
|
||||
"Configured BGP peers in the store by session_state (intent / last known, not live BIRD).",
|
||||
"Deprecated: configured BGP peers by persisted session_state, not live BIRD; use stored_bgp_peer_session_state.",
|
||||
[]string{"state"}, nil,
|
||||
),
|
||||
storedMaxDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_max"),
|
||||
"Maximum materialized prefix count across all stored revisions; historical, not current routes.", nil, nil),
|
||||
storedSumDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_sum"),
|
||||
"Sum of materialized prefix counts across all stored revisions; historical, not current routes.", nil, nil),
|
||||
inventoryDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "inventory_bgp_peers"),
|
||||
"Configured BGP peer inventory in the control-plane store, regardless of live state.", nil, nil),
|
||||
storedStateDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_bgp_peer_session_state"),
|
||||
"BGP peer inventory grouped by persisted session_state; not a live observation.", []string{"state"}, nil),
|
||||
collectionSuccessDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "store_metrics_collection_success"),
|
||||
"1 if the latest store metrics collection succeeded, 0 on database aggregation error; other store metrics are omitted on error.", nil, nil),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,6 +270,11 @@ func (c *memoryStoreCollector) Describe(ch chan<- *prometheus.Desc) {
|
||||
ch <- c.prefixSumDesc
|
||||
ch <- c.peersDesc
|
||||
ch <- c.peerStateDesc
|
||||
ch <- c.storedMaxDesc
|
||||
ch <- c.storedSumDesc
|
||||
ch <- c.inventoryDesc
|
||||
ch <- c.storedStateDesc
|
||||
ch <- c.collectionSuccessDesc
|
||||
}
|
||||
|
||||
func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
@@ -248,19 +282,44 @@ func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
if v == nil {
|
||||
return
|
||||
}
|
||||
b, ok := v.(store.Backend)
|
||||
if !ok || b == nil {
|
||||
holder, ok := v.(storeBackendHolder)
|
||||
if !ok || holder.backend == nil {
|
||||
return
|
||||
}
|
||||
maxN, sumN := b.MaterializedPrefixStats()
|
||||
b := holder.backend
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
maxN, sumN, peers, states, err := b.ObserveStoreMetrics(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 0)
|
||||
return
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 1)
|
||||
ch <- prometheus.MustNewConstMetric(c.prefixMaxDesc, prometheus.GaugeValue, float64(maxN))
|
||||
ch <- prometheus.MustNewConstMetric(c.prefixSumDesc, prometheus.GaugeValue, float64(sumN))
|
||||
ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(b.PeerCount()))
|
||||
for state, n := range b.PeerSessionCountsByState() {
|
||||
ch <- prometheus.MustNewConstMetric(c.storedMaxDesc, prometheus.GaugeValue, float64(maxN))
|
||||
ch <- prometheus.MustNewConstMetric(c.storedSumDesc, prometheus.GaugeValue, float64(sumN))
|
||||
ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(peers))
|
||||
ch <- prometheus.MustNewConstMetric(c.inventoryDesc, prometheus.GaugeValue, float64(peers))
|
||||
canonicalStates := make(map[string]int)
|
||||
for state, n := range states {
|
||||
if state == "" {
|
||||
state = "unknown"
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.peerStateDesc, prometheus.GaugeValue, float64(n), state)
|
||||
canonicalStates[canonicalPeerState(state)] += n
|
||||
}
|
||||
for state, n := range canonicalStates {
|
||||
ch <- prometheus.MustNewConstMetric(c.storedStateDesc, prometheus.GaugeValue, float64(n), state)
|
||||
}
|
||||
}
|
||||
|
||||
func canonicalPeerState(state string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(state)) {
|
||||
case "idle", "connect", "active", "opensent", "openconfirm", "established", "unknown":
|
||||
return strings.ToLower(strings.TrimSpace(state))
|
||||
default:
|
||||
return "other"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,7 +328,7 @@ func RegisterStoreBackend(b store.Backend) {
|
||||
if b == nil {
|
||||
return
|
||||
}
|
||||
metricsStore.Store(b)
|
||||
metricsStore.Store(storeBackendHolder{backend: b})
|
||||
registerCollectorOnce.Do(func() {
|
||||
prometheus.DefaultRegisterer.MustRegister(newMemoryStoreCollector())
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user