Files
EvoBGP/internal/httpapi/telemetry.go
T
DenozordecandCursor f8e0520817 feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-11-02 17:23:59 +07:00

155 lines
5.3 KiB
Go

package httpapi
import (
"context"
"net/http"
"time"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
"evobgp/internal/telemetry"
)
func (s *Server) loadTelemetry(ctx context.Context, tenant string, speakers []*store.Speaker) ([]speakerBGPLive, map[string]map[string]any, error) {
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
snapshots, err := s.store.ListTelemetry(ctx, tenant)
if err != nil {
return nil, nil, err
}
views, statuses := hydrateTelemetry(speakers, snapshots, time.Now(), telemetry.ConfigFromEnv().StaleAfter)
return views, statuses, nil
}
// snapshotObservation uses control-plane receipt time, never the remote clock.
func snapshotObservation(o store.TelemetryObservation, now time.Time, staleAfter time.Duration, invalid string) liveObservation {
v := liveObservation{ObservedAt: liveTime(o.ObservedAt), ReceivedAt: liveTime(o.ReceivedAt), LastSuccessAt: liveTime(o.LastSuccessAt), Source: o.Source, Quality: "unknown"}
if !o.ReceivedAt.IsZero() {
v.ExpiresAt = liveTime(o.ReceivedAt.Add(staleAfter))
}
if v.Source == "" {
v.Source = "none"
}
switch {
case invalid != "":
v.ErrorCode = invalid
case o.ReceivedAt.IsZero():
v.ErrorCode = "not_polled"
case o.ReceivedAt.After(now) || now.Sub(o.ReceivedAt) > staleAfter:
v.Quality, v.ErrorCode = "stale", "snapshot_expired"
case o.Error != "":
v.Quality, v.ErrorCode = "unreachable", liveErrorCode(o.Error)
if !o.LastSuccessAt.IsZero() {
v.Quality = "stale"
}
default:
v.Quality = "ok"
}
return v
}
func hydrateTelemetry(speakers []*store.Speaker, snapshots []store.TelemetrySnapshot, now time.Time, staleAfter time.Duration) ([]speakerBGPLive, map[string]map[string]any) {
byID := make(map[string]*store.Speaker, len(speakers))
for _, sp := range speakers {
if sp != nil {
byID[sp.ID] = sp
}
}
grouped := make(map[string][]store.TelemetrySnapshot)
for _, snapshot := range snapshots {
if snapshot.SpeakerID != "" && byID[snapshot.SpeakerID] == nil {
continue
}
if snapshot.SpeakerID == "" && snapshot.Kind != "local_bird" {
continue
}
grouped[snapshot.SpeakerID] = append(grouped[snapshot.SpeakerID], snapshot)
}
views := make([]speakerBGPLive, 0, len(speakers)+1)
statuses := make(map[string]map[string]any, len(speakers))
appendView := func(id string, sp *store.Speaker) {
view := speakerBGPLive{SpeakerID: id, Label: speakerDisplayLabel(sp)}
if sp == nil {
view.Label = "CP (local)"
}
var health *agentHealthPoll
rows := grouped[id]
// A restarted local collector may leave an expired authority behind.
// Only simultaneously current authorities conflict; never prefer one
// of two current BIRD nodes arbitrarily.
if len(rows) > 1 {
current := make([]store.TelemetrySnapshot, 0, len(rows))
for _, row := range rows {
if row.Kind == "local_bird" && row.ConfigFingerprint == store.TelemetrySpeakerFingerprint(sp) && !row.BGP.ReceivedAt.IsZero() && !row.BGP.ReceivedAt.After(now) && now.Sub(row.BGP.ReceivedAt) <= staleAfter {
current = append(current, row)
}
}
if len(current) > 0 {
rows = current
}
}
if len(rows) > 1 {
view.Error = "ambiguous telemetry authorities"
view.ObservationOverride = &liveObservation{Source: "none", Quality: "unknown", ErrorCode: "ambiguous_authority"}
} else if len(rows) == 1 {
row := rows[0]
invalid := ""
if row.ConfigFingerprint != store.TelemetrySpeakerFingerprint(sp) {
invalid = "configuration_changed"
}
bgp := snapshotObservation(row.BGP.TelemetryObservation, now, staleAfter, invalid)
view.ObservationOverride = &bgp
view.Polled = !row.BGP.ReceivedAt.IsZero()
view.ObservedAt, view.ReceivedAt, view.LastSuccessAt, view.Source = row.BGP.ObservedAt, row.BGP.ReceivedAt, row.BGP.LastSuccessAt, row.BGP.Source
if bgp.Quality == "ok" {
view.Sessions = row.BGP.Sessions
} else {
view.Error = bgp.ErrorCode
}
if row.Kind == "remote_agent" {
h := snapshotObservation(row.Health.TelemetryObservation, now, staleAfter, invalid)
result := nodedispatch.AgentHealthResult{OK: row.Health.OK && h.Quality == "ok", Error: row.Health.Error, LastSyncAt: row.Health.LastSyncAt, LastAppliedRevisionID: row.Health.LastAppliedRevisionID}
if h.Quality != "ok" {
result.Error = h.ErrorCode
}
if h.Quality == "ok" && !result.OK {
h.Quality, h.ErrorCode = "unreachable", "reported_unhealthy"
}
health = &agentHealthPoll{result: result, observationOverride: &h}
}
}
views = append(views, view)
if sp != nil {
statuses[id] = speakerLiveStatusWithPoll(sp, view, health)
}
}
for _, sp := range speakers {
if sp != nil {
appendView(sp.ID, sp)
}
}
if len(grouped[""]) > 0 {
appendView("", nil)
}
return views, statuses
}
func (s *Server) handleTelemetryRefresh(w http.ResponseWriter, r *http.Request) {
a, ok := authFromContext(r.Context())
if !ok {
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
return
}
if !s.requirePerm(w, a, "bgp:network:read") {
return
}
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
defer cancel()
requested, err := s.store.RequestTelemetryRefresh(ctx, a.TenantID, 10*time.Second)
if err != nil {
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry refresh unavailable")
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"requested": requested})
}