Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
155 lines
5.3 KiB
Go
155 lines
5.3 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"time"
|
|
|
|
"evobgp/internal/nodedispatch"
|
|
"evobgp/internal/store"
|
|
"evobgp/internal/telemetry"
|
|
)
|
|
|
|
func (s *Server) loadTelemetry(ctx context.Context, tenant string, speakers []*store.Speaker) ([]speakerBGPLive, map[string]map[string]any, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
|
defer cancel()
|
|
snapshots, err := s.store.ListTelemetry(ctx, tenant)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
views, statuses := hydrateTelemetry(speakers, snapshots, time.Now(), telemetry.ConfigFromEnv().StaleAfter)
|
|
return views, statuses, nil
|
|
}
|
|
|
|
// snapshotObservation uses control-plane receipt time, never the remote clock.
|
|
func snapshotObservation(o store.TelemetryObservation, now time.Time, staleAfter time.Duration, invalid string) liveObservation {
|
|
v := liveObservation{ObservedAt: liveTime(o.ObservedAt), ReceivedAt: liveTime(o.ReceivedAt), LastSuccessAt: liveTime(o.LastSuccessAt), Source: o.Source, Quality: "unknown"}
|
|
if !o.ReceivedAt.IsZero() {
|
|
v.ExpiresAt = liveTime(o.ReceivedAt.Add(staleAfter))
|
|
}
|
|
if v.Source == "" {
|
|
v.Source = "none"
|
|
}
|
|
switch {
|
|
case invalid != "":
|
|
v.ErrorCode = invalid
|
|
case o.ReceivedAt.IsZero():
|
|
v.ErrorCode = "not_polled"
|
|
case o.ReceivedAt.After(now) || now.Sub(o.ReceivedAt) > staleAfter:
|
|
v.Quality, v.ErrorCode = "stale", "snapshot_expired"
|
|
case o.Error != "":
|
|
v.Quality, v.ErrorCode = "unreachable", liveErrorCode(o.Error)
|
|
if !o.LastSuccessAt.IsZero() {
|
|
v.Quality = "stale"
|
|
}
|
|
default:
|
|
v.Quality = "ok"
|
|
}
|
|
return v
|
|
}
|
|
|
|
func hydrateTelemetry(speakers []*store.Speaker, snapshots []store.TelemetrySnapshot, now time.Time, staleAfter time.Duration) ([]speakerBGPLive, map[string]map[string]any) {
|
|
byID := make(map[string]*store.Speaker, len(speakers))
|
|
for _, sp := range speakers {
|
|
if sp != nil {
|
|
byID[sp.ID] = sp
|
|
}
|
|
}
|
|
grouped := make(map[string][]store.TelemetrySnapshot)
|
|
for _, snapshot := range snapshots {
|
|
if snapshot.SpeakerID != "" && byID[snapshot.SpeakerID] == nil {
|
|
continue
|
|
}
|
|
if snapshot.SpeakerID == "" && snapshot.Kind != "local_bird" {
|
|
continue
|
|
}
|
|
grouped[snapshot.SpeakerID] = append(grouped[snapshot.SpeakerID], snapshot)
|
|
}
|
|
views := make([]speakerBGPLive, 0, len(speakers)+1)
|
|
statuses := make(map[string]map[string]any, len(speakers))
|
|
appendView := func(id string, sp *store.Speaker) {
|
|
view := speakerBGPLive{SpeakerID: id, Label: speakerDisplayLabel(sp)}
|
|
if sp == nil {
|
|
view.Label = "CP (local)"
|
|
}
|
|
var health *agentHealthPoll
|
|
rows := grouped[id]
|
|
// A restarted local collector may leave an expired authority behind.
|
|
// Only simultaneously current authorities conflict; never prefer one
|
|
// of two current BIRD nodes arbitrarily.
|
|
if len(rows) > 1 {
|
|
current := make([]store.TelemetrySnapshot, 0, len(rows))
|
|
for _, row := range rows {
|
|
if row.Kind == "local_bird" && row.ConfigFingerprint == store.TelemetrySpeakerFingerprint(sp) && !row.BGP.ReceivedAt.IsZero() && !row.BGP.ReceivedAt.After(now) && now.Sub(row.BGP.ReceivedAt) <= staleAfter {
|
|
current = append(current, row)
|
|
}
|
|
}
|
|
if len(current) > 0 {
|
|
rows = current
|
|
}
|
|
}
|
|
if len(rows) > 1 {
|
|
view.Error = "ambiguous telemetry authorities"
|
|
view.ObservationOverride = &liveObservation{Source: "none", Quality: "unknown", ErrorCode: "ambiguous_authority"}
|
|
} else if len(rows) == 1 {
|
|
row := rows[0]
|
|
invalid := ""
|
|
if row.ConfigFingerprint != store.TelemetrySpeakerFingerprint(sp) {
|
|
invalid = "configuration_changed"
|
|
}
|
|
bgp := snapshotObservation(row.BGP.TelemetryObservation, now, staleAfter, invalid)
|
|
view.ObservationOverride = &bgp
|
|
view.Polled = !row.BGP.ReceivedAt.IsZero()
|
|
view.ObservedAt, view.ReceivedAt, view.LastSuccessAt, view.Source = row.BGP.ObservedAt, row.BGP.ReceivedAt, row.BGP.LastSuccessAt, row.BGP.Source
|
|
if bgp.Quality == "ok" {
|
|
view.Sessions = row.BGP.Sessions
|
|
} else {
|
|
view.Error = bgp.ErrorCode
|
|
}
|
|
if row.Kind == "remote_agent" {
|
|
h := snapshotObservation(row.Health.TelemetryObservation, now, staleAfter, invalid)
|
|
result := nodedispatch.AgentHealthResult{OK: row.Health.OK && h.Quality == "ok", Error: row.Health.Error, LastSyncAt: row.Health.LastSyncAt, LastAppliedRevisionID: row.Health.LastAppliedRevisionID}
|
|
if h.Quality != "ok" {
|
|
result.Error = h.ErrorCode
|
|
}
|
|
if h.Quality == "ok" && !result.OK {
|
|
h.Quality, h.ErrorCode = "unreachable", "reported_unhealthy"
|
|
}
|
|
health = &agentHealthPoll{result: result, observationOverride: &h}
|
|
}
|
|
}
|
|
views = append(views, view)
|
|
if sp != nil {
|
|
statuses[id] = speakerLiveStatusWithPoll(sp, view, health)
|
|
}
|
|
}
|
|
for _, sp := range speakers {
|
|
if sp != nil {
|
|
appendView(sp.ID, sp)
|
|
}
|
|
}
|
|
if len(grouped[""]) > 0 {
|
|
appendView("", nil)
|
|
}
|
|
return views, statuses
|
|
}
|
|
|
|
func (s *Server) handleTelemetryRefresh(w http.ResponseWriter, r *http.Request) {
|
|
a, ok := authFromContext(r.Context())
|
|
if !ok {
|
|
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
|
return
|
|
}
|
|
if !s.requirePerm(w, a, "bgp:network:read") {
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
|
|
defer cancel()
|
|
requested, err := s.store.RequestTelemetryRefresh(ctx, a.TenantID, 10*time.Second)
|
|
if err != nil {
|
|
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry refresh unavailable")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusAccepted, map[string]any{"requested": requested})
|
|
}
|