feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
+185
-116
@@ -1,47 +1,98 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
const peerLiveCacheTTL = 15 * time.Second
|
||||
|
||||
type speakerBGPLive struct {
|
||||
SpeakerID string
|
||||
Label string
|
||||
Sessions []birdfmt.BGPSession
|
||||
Error string
|
||||
SpeakerID string
|
||||
Label string
|
||||
Sessions []birdfmt.BGPSession
|
||||
Error string
|
||||
Polled bool
|
||||
ObservedAt time.Time
|
||||
ReceivedAt time.Time
|
||||
LastSuccessAt time.Time
|
||||
Source string
|
||||
ObservationOverride *liveObservation
|
||||
}
|
||||
|
||||
type peerLiveCacheEntry struct {
|
||||
at time.Time
|
||||
views []speakerBGPLive
|
||||
type liveObservation struct {
|
||||
ObservedAt *string `json:"observed_at"`
|
||||
ReceivedAt *string `json:"received_at"`
|
||||
LastSuccessAt *string `json:"last_success_at"`
|
||||
ExpiresAt *string `json:"expires_at,omitempty"`
|
||||
Source string `json:"source"`
|
||||
Quality string `json:"quality"`
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
}
|
||||
|
||||
var peerLiveCache sync.Map // tenantID -> peerLiveCacheEntry
|
||||
func liveTime(t time.Time) *string {
|
||||
if t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
v := t.UTC().Format(time.RFC3339Nano)
|
||||
return &v
|
||||
}
|
||||
|
||||
func (v speakerBGPLive) observed() bool {
|
||||
return v.Polled || v.Error != "" || len(v.Sessions) > 0
|
||||
}
|
||||
|
||||
func (v speakerBGPLive) successful() bool {
|
||||
return v.observed() && v.Error == ""
|
||||
}
|
||||
|
||||
func (v speakerBGPLive) observation() liveObservation {
|
||||
if v.ObservationOverride != nil {
|
||||
return *v.ObservationOverride
|
||||
}
|
||||
o := liveObservation{ObservedAt: liveTime(v.ObservedAt), ReceivedAt: liveTime(v.ReceivedAt), LastSuccessAt: liveTime(v.LastSuccessAt), Source: v.Source, Quality: "unknown"}
|
||||
if o.Source == "" {
|
||||
o.Source = "none"
|
||||
}
|
||||
switch {
|
||||
case v.successful():
|
||||
o.Quality = "ok"
|
||||
case v.Error != "" && !v.LastSuccessAt.IsZero():
|
||||
o.Quality = "stale"
|
||||
o.ErrorCode = liveErrorCode(v.Error)
|
||||
case v.Error != "":
|
||||
o.Quality = "unreachable"
|
||||
o.ErrorCode = liveErrorCode(v.Error)
|
||||
default:
|
||||
o.ErrorCode = "not_polled"
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
func liveErrorCode(err string) string {
|
||||
if strings.Contains(strings.ToLower(err), "not configured") || strings.Contains(strings.ToLower(err), "missing") {
|
||||
return "not_configured"
|
||||
}
|
||||
return "poll_failed"
|
||||
}
|
||||
|
||||
type peerSessionOnSpeaker struct {
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
State string `json:"state"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
State string `json:"state"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
Observation liveObservation `json:"observation"`
|
||||
}
|
||||
|
||||
type liveSpeakerPoll struct {
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
OK bool `json:"ok"`
|
||||
SessionCount int `json:"session_count"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
OK bool `json:"ok"`
|
||||
SessionCount *int `json:"session_count"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
Observation liveObservation `json:"observation"`
|
||||
}
|
||||
|
||||
func speakerDisplayLabel(sp *store.Speaker) string {
|
||||
@@ -65,95 +116,21 @@ func speakerDisplayLabel(sp *store.Speaker) string {
|
||||
return sp.ID
|
||||
}
|
||||
|
||||
func masterSpeakerID(speakers []*store.Speaker) string {
|
||||
for _, sp := range speakers {
|
||||
if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
|
||||
return sp.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Server) collectSpeakerBGPLive(ctx context.Context, tenantID string, fresh bool) []speakerBGPLive {
|
||||
if !fresh {
|
||||
if v, ok := peerLiveCache.Load(tenantID); ok {
|
||||
ent := v.(peerLiveCacheEntry)
|
||||
if time.Since(ent.at) < peerLiveCacheTTL {
|
||||
return ent.views
|
||||
}
|
||||
}
|
||||
}
|
||||
speakers := s.store.ListSpeakersForTenant(tenantID)
|
||||
views := make([]speakerBGPLive, 0, len(speakers)+1)
|
||||
|
||||
if sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")); sock != "" {
|
||||
v := speakerBGPLive{Label: "CP (local BIRD)"}
|
||||
if mid := masterSpeakerID(speakers); mid != "" {
|
||||
v.SpeakerID = mid
|
||||
for _, sp := range speakers {
|
||||
if sp != nil && sp.ID == mid {
|
||||
v.Label = speakerDisplayLabel(sp)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")))
|
||||
if err != nil {
|
||||
v.Error = err.Error()
|
||||
} else {
|
||||
v.Sessions = birdfmt.ParseBGPSessions(out)
|
||||
}
|
||||
views = append(views, v)
|
||||
}
|
||||
|
||||
opts := nodedispatch.Options{Timeout: 8 * time.Second}
|
||||
type resWrap struct {
|
||||
sp *store.Speaker
|
||||
res nodedispatch.BirdProtocolsResult
|
||||
}
|
||||
ch := make(chan resWrap, len(speakers))
|
||||
var wg sync.WaitGroup
|
||||
for _, sp := range speakers {
|
||||
if sp == nil {
|
||||
continue
|
||||
}
|
||||
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
||||
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(speaker *store.Speaker) {
|
||||
defer wg.Done()
|
||||
ch <- resWrap{
|
||||
sp: speaker,
|
||||
res: nodedispatch.FetchBirdProtocols(ctx, speaker, opts),
|
||||
}
|
||||
}(sp)
|
||||
}
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
for rw := range ch {
|
||||
views = append(views, speakerBGPLive{
|
||||
SpeakerID: rw.sp.ID,
|
||||
Label: speakerDisplayLabel(rw.sp),
|
||||
Sessions: rw.res.Sessions,
|
||||
Error: rw.res.Error,
|
||||
})
|
||||
}
|
||||
|
||||
peerLiveCache.Store(tenantID, peerLiveCacheEntry{at: time.Now(), views: views})
|
||||
return views
|
||||
}
|
||||
|
||||
func liveSpeakerPollJSON(views []speakerBGPLive) []liveSpeakerPoll {
|
||||
out := make([]liveSpeakerPoll, 0, len(views))
|
||||
for _, v := range views {
|
||||
var sessionCount *int
|
||||
if v.successful() {
|
||||
count := len(v.Sessions)
|
||||
sessionCount = &count
|
||||
}
|
||||
out = append(out, liveSpeakerPoll{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
OK: v.Error == "",
|
||||
SessionCount: len(v.Sessions),
|
||||
OK: v.successful(),
|
||||
SessionCount: sessionCount,
|
||||
PollError: v.Error,
|
||||
Observation: v.observation(),
|
||||
})
|
||||
}
|
||||
return out
|
||||
@@ -183,27 +160,30 @@ func matchPeerOnSpeakers(peer *store.BGPPeer, views []speakerBGPLive) (
|
||||
protoName := birdfmt.PeerProtocolName(peer.ID)
|
||||
|
||||
for _, v := range views {
|
||||
if v.Error != "" && len(v.Sessions) == 0 {
|
||||
if !v.successful() {
|
||||
on = append(on, peerSessionOnSpeaker{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
PollError: v.Error,
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
PollError: v.Error,
|
||||
Observation: v.observation(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
sess := findPeerSession(v.Sessions, protoName, neighbor, hasNeighbor)
|
||||
if sess == nil {
|
||||
on = append(on, peerSessionOnSpeaker{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: "absent",
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: "absent",
|
||||
Observation: v.observation(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
hit := peerSessionOnSpeaker{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: sess.State,
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: sess.State,
|
||||
Observation: v.observation(),
|
||||
}
|
||||
on = append(on, hit)
|
||||
if strings.EqualFold(strings.TrimSpace(sess.State), "Established") {
|
||||
@@ -273,20 +253,109 @@ func sessionStateRank(state string) int {
|
||||
}
|
||||
|
||||
func applyPeerLiveFields(row map[string]any, peer *store.BGPPeer, views []speakerBGPLive) {
|
||||
state, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views)
|
||||
expected := expectedPeerViews(peer, views)
|
||||
state, _, _, _, _, _ := matchPeerOnSpeakers(peer, expected)
|
||||
_, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views)
|
||||
row["session_on_speakers"] = on
|
||||
row["established_on_speakers"] = establishedOn
|
||||
row["session_conflict"] = false
|
||||
row["session_mismatch"] = mismatch
|
||||
row["last_known_session_state"] = row["session_state"]
|
||||
row["session_state"] = nil
|
||||
if state != "" {
|
||||
row["session_state"] = state
|
||||
}
|
||||
if state == "" && len(expected) > 0 && allSpeakerPollsSuccessful(expected) {
|
||||
row["session_state"] = "absent"
|
||||
}
|
||||
row["live_observation"] = peerLiveObservation(expected)
|
||||
if connLabel != "" {
|
||||
row["connected_speaker_label"] = connLabel
|
||||
}
|
||||
row["connected_speaker_id"] = peerLiveSpeakerIDOrNull(connID)
|
||||
}
|
||||
|
||||
func expectedPeerViews(peer *store.BGPPeer, views []speakerBGPLive) []speakerBGPLive {
|
||||
if peer == nil || peer.SpeakerID == nil || strings.TrimSpace(*peer.SpeakerID) == "" {
|
||||
return views
|
||||
}
|
||||
want := strings.TrimSpace(*peer.SpeakerID)
|
||||
expected := make([]speakerBGPLive, 0, 1)
|
||||
for _, v := range views {
|
||||
if strings.EqualFold(v.SpeakerID, want) {
|
||||
expected = append(expected, v)
|
||||
}
|
||||
}
|
||||
return expected
|
||||
}
|
||||
|
||||
func allSpeakerPollsSuccessful(views []speakerBGPLive) bool {
|
||||
for _, v := range views {
|
||||
if !v.successful() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func peerLiveObservation(views []speakerBGPLive) liveObservation {
|
||||
if len(views) == 0 {
|
||||
return speakerBGPLive{}.observation()
|
||||
}
|
||||
result := liveObservation{Source: views[0].Source, Quality: "unknown"}
|
||||
if result.Source == "" {
|
||||
result.Source = "none"
|
||||
}
|
||||
var latestObserved, latestReceived, latestSuccess time.Time
|
||||
allOK, allFailed, anyStale := true, true, false
|
||||
for _, v := range views {
|
||||
o := v.observation()
|
||||
if v.ObservedAt.After(latestObserved) {
|
||||
latestObserved = v.ObservedAt
|
||||
}
|
||||
if v.ReceivedAt.After(latestReceived) {
|
||||
latestReceived = v.ReceivedAt
|
||||
}
|
||||
if v.LastSuccessAt.After(latestSuccess) {
|
||||
latestSuccess = v.LastSuccessAt
|
||||
}
|
||||
allOK = allOK && v.successful()
|
||||
allFailed = allFailed && o.Quality == "unreachable"
|
||||
anyStale = anyStale || o.Quality == "stale"
|
||||
if result.ErrorCode == "" && o.ErrorCode != "" {
|
||||
result.ErrorCode = o.ErrorCode
|
||||
}
|
||||
if o.ExpiresAt != nil {
|
||||
expires, err := time.Parse(time.RFC3339Nano, *o.ExpiresAt)
|
||||
if err == nil {
|
||||
previous := time.Time{}
|
||||
if result.ExpiresAt != nil {
|
||||
previous, _ = time.Parse(time.RFC3339Nano, *result.ExpiresAt)
|
||||
}
|
||||
if previous.IsZero() || expires.Before(previous) {
|
||||
result.ExpiresAt = o.ExpiresAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
result.ObservedAt = liveTime(latestObserved)
|
||||
result.ReceivedAt = liveTime(latestReceived)
|
||||
result.LastSuccessAt = liveTime(latestSuccess)
|
||||
if len(views) > 1 {
|
||||
result.Source = "mixed"
|
||||
}
|
||||
switch {
|
||||
case allOK:
|
||||
result.Quality = "ok"
|
||||
result.ErrorCode = ""
|
||||
case anyStale:
|
||||
result.Quality = "stale"
|
||||
case allFailed:
|
||||
result.Quality = "unreachable"
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func peerLiveSpeakerIDOrNull(id string) any {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return nil
|
||||
|
||||
Reference in new issue
Block a user