feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -0,0 +1,39 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
// These regressions ensure unobserved live state cannot inherit persisted health.
|
||||
func TestStage0ReproPeerWithoutObservationIsNotEstablished(t *testing.T) {
|
||||
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"}
|
||||
cases := []struct {
|
||||
name string
|
||||
views []speakerBGPLive
|
||||
}{
|
||||
{name: "no_collector"},
|
||||
{name: "all_timeout", views: []speakerBGPLive{{SpeakerID: "replica", Error: "timeout"}}},
|
||||
{name: "session_absent", views: []speakerBGPLive{{SpeakerID: "replica", Label: "replica"}}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
row := map[string]any{"session_state": "Established"}
|
||||
applyPeerLiveFields(row, peer, tc.views)
|
||||
if row["session_state"] == "Established" {
|
||||
t.Fatal("S01: persisted Established is exposed without a current session observation")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStage0ReproUnpolledMasterIsNotHealthy(t *testing.T) {
|
||||
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||
// collectSpeakerLiveStatus constructs this view when no poll result exists.
|
||||
view := speakerBGPLive{SpeakerID: sp.ID, Label: "master"}
|
||||
row := speakerLiveStatusJSON(sp, view, nil)
|
||||
if row["bgp_poll_ok"] == true || row["agent_ok"] == true {
|
||||
t.Fatal("S01: absent poll is exposed as successful BGP/agent health")
|
||||
}
|
||||
}
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
"time"
|
||||
|
||||
"evobgp/internal/db"
|
||||
"evobgp/internal/httpclient"
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/observability"
|
||||
"evobgp/internal/pipeline"
|
||||
"evobgp/internal/repository"
|
||||
"evobgp/internal/store"
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
|
||||
// NewCDNHTTPClient returns the shared HTTP client for CDN and preview fetches (PERF-02 / ERR-03).
|
||||
func NewCDNHTTPClient() *http.Client {
|
||||
return httpclient.New(httpclient.DefaultTimeout)
|
||||
return pipeline.NewCDNHTTPClient()
|
||||
}
|
||||
|
||||
// BootstrapWorkers opens the same store.Backend and jobs.Registry as New (without HTTP or bundle keys).
|
||||
@@ -50,46 +50,15 @@ func BootstrapWorkers(ctx context.Context, opts Options) (store.Backend, *jobs.R
|
||||
backend = mem
|
||||
}
|
||||
|
||||
cdnHTTP := NewCDNHTTPClient()
|
||||
wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: cdnHTTP}
|
||||
// Configured DoH profiles and RIPE service overrides retain the previous
|
||||
// general HTTP/HTTPS transport, including explicitly configured private
|
||||
// endpoints. CDN calls independently secure this client in UpstreamHTTPDo.
|
||||
workerHTTP := pipeline.NewPipelineHTTPClient()
|
||||
wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: workerHTTP}
|
||||
reg := jobs.NewRegistry(wk.Process)
|
||||
wk.Registry = reg
|
||||
if pool != nil {
|
||||
audit := repository.NewJobAuditWriter(pool)
|
||||
jobMeta := func(j *jobs.Job) map[string]any {
|
||||
if j == nil {
|
||||
return nil
|
||||
}
|
||||
st := j.Snapshot()
|
||||
meta, _ := st["meta"].(map[string]any)
|
||||
return meta
|
||||
}
|
||||
reg.SetPersistHooks(
|
||||
func(j *jobs.Job) {
|
||||
if j == nil {
|
||||
return
|
||||
}
|
||||
audit.UpsertQueued(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, j.ModuleID, jobMeta(j))
|
||||
},
|
||||
func(j *jobs.Job) {
|
||||
if j == nil {
|
||||
return
|
||||
}
|
||||
audit.UpsertRunning(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, jobMeta(j))
|
||||
},
|
||||
func(j *jobs.Job) {
|
||||
if j == nil {
|
||||
return
|
||||
}
|
||||
st := j.Snapshot()
|
||||
status, _ := st["status"].(string)
|
||||
var errMsg *string
|
||||
if e, ok := st["error"].(string); ok && e != "" {
|
||||
errMsg = &e
|
||||
}
|
||||
audit.MarkTerminal(context.Background(), j.TenantID, j.ID, status, errMsg, time.Now().UTC())
|
||||
},
|
||||
)
|
||||
reg.UseDurableQueue(backend)
|
||||
}
|
||||
observability.RegisterStoreBackend(backend)
|
||||
return backend, reg, pool, nil
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/httpclient"
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/pipeline"
|
||||
"evobgp/internal/store"
|
||||
"github.com/miekg/dns"
|
||||
)
|
||||
|
||||
func TestBootstrapWorkerPreservesExplicitPrivateDOH(t *testing.T) {
|
||||
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
|
||||
t.Setenv("EVOBGP_BIRDC_SOCKET", "")
|
||||
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
|
||||
var calls atomic.Int32
|
||||
resolver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
raw, err := base64.RawURLEncoding.DecodeString(r.URL.Query().Get("dns"))
|
||||
if err != nil {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
query := new(dns.Msg)
|
||||
if err = query.Unpack(raw); err != nil || len(query.Question) == 0 {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
answer := new(dns.Msg)
|
||||
answer.SetReply(query)
|
||||
if query.Question[0].Qtype == dns.TypeA {
|
||||
answer.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: query.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.IPv4(192, 0, 2, 20)}}
|
||||
}
|
||||
body, err := answer.Pack()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/dns-message")
|
||||
_, _ = w.Write(body)
|
||||
}))
|
||||
defer resolver.Close()
|
||||
st, reg, _, err := BootstrapWorkers(context.Background(), Options{SeedDemo: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
reg.RequestCancelAll()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if e := reg.Drain(ctx); e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
}()
|
||||
tenant, _, _, _, _ := st.DemoIDs()
|
||||
profile, err := st.CreateDohProfile(tenant, &store.DohProfile{Name: "explicit private resolver", URL: resolver.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mod, err := st.CreateModule(tenant, &store.Module{Name: "private-doh", Type: "DOMAINS", Enabled: true, DohProfileIDs: []string{profile.ID}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = st.CreateDomainEntry(tenant, mod.ID, &store.DomainEntry{FQDN: "private-resolver.example.test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j, _, err := reg.Enqueue(tenant, jobs.KindModuleRefresh, nil, &mod.ID, map[string]any{"module_id": mod.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.After(5 * time.Second)
|
||||
ticker := time.NewTicker(5 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
snap := j.Snapshot()
|
||||
status := snap["status"]
|
||||
if status == jobs.StatusSucceeded {
|
||||
break
|
||||
}
|
||||
if status == jobs.StatusFailed || status == jobs.StatusCancelled {
|
||||
t.Fatalf("worker refresh failed: %+v", snap)
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("worker did not finish")
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
if calls.Load() < 2 {
|
||||
t.Fatalf("bootstrap worker blocked private DoH: %d requests", calls.Load())
|
||||
}
|
||||
// The same general-client choice must not remove CDN's independent ACL.
|
||||
cdn := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("private CDN reached") }))
|
||||
defer cdn.Close()
|
||||
req, err := http.NewRequest(http.MethodGet, cdn.URL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = pipeline.UpstreamHTTPDo(context.Background(), httpclient.New(httpclient.DefaultTimeout), req); err == nil {
|
||||
t.Fatal("private CDN accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestPreviewCDNSourceRejectsOversizedBody(t *testing.T) {
|
||||
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = io.Copy(w, strings.NewReader(strings.Repeat("x", (8<<20)+1)))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
st := store.NewMemory()
|
||||
st.SeedDemo()
|
||||
tenant, _, _, _, _ := st.DemoIDs()
|
||||
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "oversized-preview", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := &Server{store: st, cdnHTTP: srv.Client()}
|
||||
body, err := json.Marshal(map[string]string{"url": srv.URL, "source_kind": "plain"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+mod.ID+"/cdn-sources/preview", strings.NewReader(string(body)))
|
||||
req.SetPathValue("module_id", mod.ID)
|
||||
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"}))
|
||||
recorder := httptest.NewRecorder()
|
||||
s.handlePreviewCDNSource(recorder, req)
|
||||
if recorder.Code != http.StatusBadGateway {
|
||||
t.Fatalf("oversized preview status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewCDNErrorBodyBoundAndQuotaBeforeFetch(t *testing.T) {
|
||||
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||
t.Setenv("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", "1")
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, module, _, _, _ := m.DemoIDs()
|
||||
var calls atomic.Int32
|
||||
upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = io.WriteString(w, strings.Repeat("x", 64<<10))
|
||||
w.(http.Flusher).Flush()
|
||||
// No EOF: an unbounded drain would wait until the caller deadline.
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer upstream.Close()
|
||||
s := &Server{store: m, cdnHTTP: upstream.Client()}
|
||||
request := func() *http.Request {
|
||||
r := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/cdn-sources/preview", strings.NewReader(`{"url":"`+upstream.URL+`","source_kind":"plain"}`))
|
||||
r.SetPathValue("module_id", module)
|
||||
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"}))
|
||||
}
|
||||
first := httptest.NewRecorder()
|
||||
firstRequest := request()
|
||||
ctx, cancel := context.WithTimeout(firstRequest.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
s.handlePreviewCDNSource(first, firstRequest.WithContext(ctx))
|
||||
if first.Code != http.StatusBadGateway || calls.Load() != 1 || ctx.Err() != nil {
|
||||
t.Fatalf("unbounded drain status=%d calls=%d ctx=%v", first.Code, calls.Load(), ctx.Err())
|
||||
}
|
||||
second := httptest.NewRecorder()
|
||||
s.handlePreviewCDNSource(second, request())
|
||||
if second.Code != http.StatusTooManyRequests || calls.Load() != 1 || second.Header().Get("Retry-After") == "" {
|
||||
t.Fatalf("quota fetch bypass status=%d calls=%d", second.Code, calls.Load())
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,7 @@ func (s *Server) withCORS(h http.Handler) http.Handler {
|
||||
}
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key, Accept, X-Tenant-Id")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PATCH, DELETE, OPTIONS")
|
||||
w.Header().Set("Access-Control-Expose-Headers", "Location, X-EvoBGP-Job-Status, Retry-After")
|
||||
if r.Method == http.MethodOptions {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func (s *Server) replayJobResponse(w http.ResponseWriter, r *http.Request, in store.JobEnqueue) bool {
|
||||
j, found, err := s.jobs.ReplayContext(r.Context(), in)
|
||||
if err != nil {
|
||||
writeJobEnqueueError(w, err)
|
||||
return true
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
snap := j.Snapshot()
|
||||
body := map[string]any{"job_id": j.ID, "status": snap["status"]}
|
||||
if j.Kind == jobs.KindTenantRefresh {
|
||||
ids := snap["meta"].(map[string]any)["module_ids"]
|
||||
n := 0
|
||||
switch v := ids.(type) {
|
||||
case []any:
|
||||
n = len(v)
|
||||
case []string:
|
||||
n = len(v)
|
||||
}
|
||||
body["module_ids"] = ids
|
||||
body["modules_due"] = n
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, body)
|
||||
return true
|
||||
}
|
||||
|
||||
func writeJobEnqueueError(w http.ResponseWriter, err error) {
|
||||
switch {
|
||||
case errors.Is(err, store.ErrJobConflict):
|
||||
writeProblem(w, http.StatusConflict, "Conflict", "idempotency key belongs to a different job request")
|
||||
case errors.Is(err, store.ErrJobQueueFull):
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "job queue is full")
|
||||
default:
|
||||
writeInternalError(w, "job_enqueue", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeJobReadError(w http.ResponseWriter, err error) {
|
||||
if errors.Is(err, jobs.ErrNotFound) || errors.Is(err, store.ErrJobNotFound) {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
||||
return
|
||||
}
|
||||
writeInternalError(w, "job_read", err)
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type unavailableJobQueue struct{ store.JobQueue }
|
||||
|
||||
func (q unavailableJobQueue) EnqueueJob(context.Context, store.JobEnqueue) (store.JobRecord, bool, error) {
|
||||
return store.JobRecord{}, false, errors.New("database unavailable")
|
||||
}
|
||||
|
||||
func TestJobEnqueueFailureNeverAccepts(t *testing.T) {
|
||||
r := jobs.NewRegistry(nil)
|
||||
r.UseDurableQueue(unavailableJobQueue{})
|
||||
s := &Server{jobs: r}
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/bird/reload", nil)
|
||||
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "operator"}))
|
||||
w := httptest.NewRecorder()
|
||||
s.handleBirdReload(w, req)
|
||||
if w.Code == http.StatusAccepted || w.Code < 500 {
|
||||
t.Fatalf("DB failure returned %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutomaticEnqueueFailureIsVisible(t *testing.T) {
|
||||
r := jobs.NewRegistry(nil)
|
||||
r.UseDurableQueue(unavailableJobQueue{})
|
||||
s := &Server{jobs: r}
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/peers", nil)
|
||||
s.enqueuePeerReconcile(w, req, "tenant", "peer_create")
|
||||
if got := w.Header().Get("X-EvoBGP-Job-Status"); got != "enqueue-failed" {
|
||||
t.Fatalf("scheduling failure hidden: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
type selectorBackend struct {
|
||||
store.Backend
|
||||
modules []*store.Module
|
||||
reads int
|
||||
}
|
||||
|
||||
func (b *selectorBackend) ListModules(string) []*store.Module { b.reads++; return b.modules }
|
||||
|
||||
func TestTenantRefreshReplayBeforeDynamicSelection(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
b := &selectorBackend{Backend: m, modules: []*store.Module{{ID: "module-a", Enabled: true, RefreshIntervalSec: 1}}}
|
||||
r := jobs.NewRegistry(nil)
|
||||
r.UseDurableQueue(m)
|
||||
s := &Server{store: b, jobs: r}
|
||||
request := func() *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/tenant/refresh", strings.NewReader(`{}`))
|
||||
req.Header.Set("Idempotency-Key", "same-request")
|
||||
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "editor"}))
|
||||
w := httptest.NewRecorder()
|
||||
s.handleTenantRefresh(w, req)
|
||||
return w
|
||||
}
|
||||
first := request()
|
||||
if first.Code != http.StatusAccepted {
|
||||
t.Fatalf("first=%d %s", first.Code, first.Body.String())
|
||||
}
|
||||
var accepted map[string]any
|
||||
if err := json.Unmarshal(first.Body.Bytes(), &accepted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err := m.ClaimJobs(context.Background(), "worker", 1, time.Minute)
|
||||
if err != nil || len(claimed) != 1 {
|
||||
t.Fatalf("claim %+v %v", claimed, err)
|
||||
}
|
||||
claimed[0].Status = "succeeded"
|
||||
if err = m.FinishJob(context.Background(), claimed[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.modules = nil
|
||||
reads := b.reads
|
||||
replay := request()
|
||||
if replay.Code != http.StatusAccepted {
|
||||
t.Fatalf("replay=%d %s", replay.Code, replay.Body.String())
|
||||
}
|
||||
var got map[string]any
|
||||
if err = json.Unmarshal(replay.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["job_id"] != accepted["job_id"] || got["status"] != "succeeded" || b.reads != reads {
|
||||
t.Fatalf("replay re-resolved selector: %+v reads %d->%d", got, reads, b.reads)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestJobReportDurableProgressAndTenantScope(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, revision, speaker := m.DemoIDs()
|
||||
j, _, err := m.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: jobs.KindDeployApply})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
|
||||
if err != nil || len(claimed) != 1 {
|
||||
t.Fatalf("claim: %v %v", claimed, err)
|
||||
}
|
||||
f := store.JobFence{JobID: j.ID, TenantID: tenant, OwnerID: "worker", Token: claimed[0].FencingToken}
|
||||
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.RecordDeploymentSpeaker(ctx, f, d, "unknown", "unconfirmed", "response lost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A fresh registry has no in-process job or progress metadata.
|
||||
r := jobs.NewRegistry(nil)
|
||||
r.UseDurableQueue(m)
|
||||
s := &Server{store: m, jobs: r}
|
||||
request := func(tenantID, jobID string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/jobs/"+jobID+"/report", nil)
|
||||
req.SetPathValue("job_id", jobID)
|
||||
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenantID, Role: "operator"}))
|
||||
w := httptest.NewRecorder()
|
||||
s.handleGetJobReport(w, req)
|
||||
return w
|
||||
}
|
||||
check := func(id string) {
|
||||
t.Helper()
|
||||
w := request(tenant, id)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("report: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
var body struct {
|
||||
Progress []struct {
|
||||
State, Confirmation string
|
||||
JobID string `json:"job_id"`
|
||||
} `json:"deployment_progress"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(body.Progress) != 1 || body.Progress[0].JobID != j.ID || body.Progress[0].State != "unknown" || body.Progress[0].Confirmation != "unconfirmed" {
|
||||
t.Fatalf("lost persisted progress: %s", w.Body.String())
|
||||
}
|
||||
}
|
||||
check(j.ID)
|
||||
recovery, _, err := r.EnqueueContext(ctx, tenant, jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check(recovery.ID)
|
||||
// A later explicit completion releases the delivery reservation. Once recovery
|
||||
// has its own persisted row, its report must not show the parent's old state.
|
||||
if err := m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recoveryRows, err := m.ClaimJobs(ctx, "recovery-worker", 1, time.Minute)
|
||||
if err != nil || len(recoveryRows) != 1 || recoveryRows[0].ID != recovery.ID {
|
||||
t.Fatalf("recovery claim: %v %v", recoveryRows, err)
|
||||
}
|
||||
rf := store.JobFence{JobID: recovery.ID, TenantID: tenant, OwnerID: "recovery-worker", Token: recoveryRows[0].FencingToken}
|
||||
rd, err := m.BeginDeploymentSpeaker(ctx, rf, speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.RecordDeploymentSpeaker(ctx, rf, rd, "applied", "agent_reported", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := request(tenant, recovery.ID)
|
||||
var report struct {
|
||||
Progress []struct {
|
||||
State string
|
||||
JobID string `json:"job_id"`
|
||||
} `json:"deployment_progress"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w.Code != http.StatusOK || len(report.Progress) != 1 || report.Progress[0].JobID != recovery.ID || report.Progress[0].State != "applied" {
|
||||
t.Fatalf("recovery own progress hidden: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
if w := request("other-tenant", j.ID); w.Code != http.StatusNotFound {
|
||||
t.Fatalf("cross-tenant report: %d", w.Code)
|
||||
}
|
||||
foreign, _, err := r.EnqueueContext(ctx, "other-tenant", jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w := request("other-tenant", foreign.ID); w.Code != http.StatusNotFound {
|
||||
t.Fatalf("cross-tenant source: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"net/http/httptest"
|
||||
|
||||
"evobgp/internal/bundle"
|
||||
)
|
||||
|
||||
// TestLegacyNodeBinaryCurrentControlPlane exercises an independently built old
|
||||
// CLI against today's HTTP handlers. It never calls apply or touches host BIRD.
|
||||
func TestLegacyNodeBinaryCurrentControlPlane(t *testing.T) {
|
||||
binary := os.Getenv("EVOBGP_TEST_LEGACY_NODE_BIN")
|
||||
if binary == "" {
|
||||
t.Skip("EVOBGP_TEST_LEGACY_NODE_BIN not set")
|
||||
}
|
||||
binary, err := filepath.Abs(binary)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(binary); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer srv.Close()
|
||||
tenant, _, _, revision, speaker := srv.Store().DemoIDs()
|
||||
mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,other-node|other-tenant|node")
|
||||
if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server := httptest.NewServer(srv.Handler())
|
||||
defer server.Close()
|
||||
run := func(wantSuccess bool, args ...string) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
output, err := exec.CommandContext(ctx, binary, args...).CombinedOutput()
|
||||
if ctx.Err() != nil {
|
||||
t.Fatalf("legacy CLI timed out: %v", ctx.Err())
|
||||
}
|
||||
if wantSuccess && err != nil {
|
||||
t.Fatalf("legacy CLI failed: %v: %s", err, output)
|
||||
}
|
||||
if !wantSuccess && err == nil {
|
||||
t.Fatalf("legacy CLI unexpectedly accepted invalid request: %s", output)
|
||||
}
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "current-bundle.tar.gz")
|
||||
run(true, "pull-bundle", "-base-url", server.URL, "-token", "legacy-node", "-speaker-id", speaker, "-o", path)
|
||||
run(true, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64())
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub))
|
||||
if err != nil || verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker {
|
||||
t.Fatalf("legacy CLI fetched the wrong revision/speaker or an invalid bundle: %v", err)
|
||||
}
|
||||
// The archive is valid, but verification with an unrelated key must fail.
|
||||
run(false, "verify-bundle", "-f", path, "-pubkey-hex", strings.Repeat("00", ed25519.PublicKeySize))
|
||||
for _, token := range []string{"invalid-node", "other-node"} {
|
||||
denied := filepath.Join(t.TempDir(), "denied.tar.gz")
|
||||
run(false, "pull-bundle", "-base-url", server.URL, "-token", token, "-speaker-id", speaker, "-o", denied)
|
||||
if _, err := os.Stat(denied); !os.IsNotExist(err) {
|
||||
t.Fatalf("unauthorized CLI wrote a bundle: %v", err)
|
||||
}
|
||||
}
|
||||
// A damaged payload must not pass the old verifier with the correct key.
|
||||
if err := os.WriteFile(path, []byte("not a signed bundle"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run(false, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64())
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"evobgp/internal/bundle"
|
||||
"evobgp/internal/pipeline"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
// nodeBundleCacheMaxEntries is the maximum number of finished node bundles retained.
|
||||
nodeBundleCacheMaxEntries = 32
|
||||
// nodeBundleCacheMaxBytes is the maximum total size of cached bundle bodies (64 MiB).
|
||||
nodeBundleCacheMaxBytes = 64 << 20
|
||||
)
|
||||
|
||||
// errNodeBundleRevisionMissing is returned when the revision cannot be loaded.
|
||||
// It is not cached: the next pull tries again.
|
||||
var errNodeBundleRevisionMissing = errors.New("httpapi: revision not found")
|
||||
|
||||
// nodeBundleOpError carries the writeInternalError operation name for a failed build.
|
||||
type nodeBundleOpError struct {
|
||||
op string
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *nodeBundleOpError) Error() string {
|
||||
if e == nil || e.err == nil {
|
||||
return "httpapi: node bundle"
|
||||
}
|
||||
return fmt.Sprintf("httpapi: %s: %v", e.op, e.err)
|
||||
}
|
||||
|
||||
func (e *nodeBundleOpError) Unwrap() error { return e.err }
|
||||
|
||||
// nodeBundleBuilder produces the gzip body served to evobgp-node.
|
||||
// The default implementation is bundle.BuildGzippedTar, whose archive contains
|
||||
// manifest.json and bundle.sig. Tests replace it to count rebuilds.
|
||||
type nodeBundleBuilder func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error)
|
||||
|
||||
// nodeBundleCache stores finished gzip bodies. A hit returns the bytes of the
|
||||
// build that filled the entry, signature included. Errors are not stored.
|
||||
// Concurrent loads of one key share a single build.
|
||||
type nodeBundleCache struct {
|
||||
mu sync.Mutex
|
||||
maxN int
|
||||
maxBytes int
|
||||
total int
|
||||
items map[string]*nodeBundleEntry
|
||||
lru []string // index 0 is the oldest idle entry
|
||||
flights map[string]*nodeBundleFlight
|
||||
}
|
||||
|
||||
type nodeBundleEntry struct {
|
||||
body []byte
|
||||
}
|
||||
|
||||
type nodeBundleFlight struct {
|
||||
done chan struct{}
|
||||
body []byte
|
||||
err error
|
||||
joined int
|
||||
}
|
||||
|
||||
func newNodeBundleCache(maxEntries, maxBytes int) *nodeBundleCache {
|
||||
return &nodeBundleCache{
|
||||
maxN: maxEntries,
|
||||
maxBytes: maxBytes,
|
||||
items: make(map[string]*nodeBundleEntry),
|
||||
flights: make(map[string]*nodeBundleFlight),
|
||||
}
|
||||
}
|
||||
|
||||
// nodeBundleCacheKey identifies one servable bundle.
|
||||
// overlayFP is a digest of the speaker overlay output (router id / source addresses,
|
||||
// node IPv4 fallback, peer and discovery render). Revision content itself is immutable,
|
||||
// so the revision id stands in for the shared prefix files.
|
||||
func nodeBundleCacheKey(tenantID, revisionID, speakerID, overlayFP string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(tenantID) + len(revisionID) + len(speakerID) + len(overlayFP) + 3)
|
||||
b.WriteString(tenantID)
|
||||
b.WriteByte(0)
|
||||
b.WriteString(revisionID)
|
||||
b.WriteByte(0)
|
||||
b.WriteString(speakerID)
|
||||
b.WriteByte(0)
|
||||
b.WriteString(overlayFP)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// nodeBundleOverlayFingerprint hashes the speaker overlay that handleNodeBundle
|
||||
// would apply on top of an immutable revision. Empty base fragments are enough:
|
||||
// bird.conf and the peers fragment are fully re-rendered from tenant settings,
|
||||
// speaker source addresses, and the live peer set. Revision file bytes are not
|
||||
// part of this digest.
|
||||
func nodeBundleOverlayFingerprint(st store.Backend, tenantID, speakerID, revisionID string) (string, error) {
|
||||
frags, err := pipeline.OverlayFragmentsForSpeaker(st, tenantID, speakerID, revisionID, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
|
||||
}
|
||||
keys := make([]string, 0, len(frags))
|
||||
for k := range frags {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
h := sha256.New()
|
||||
for _, k := range keys {
|
||||
if err := hashWriteString(h, k); err != nil {
|
||||
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
|
||||
}
|
||||
if err := hashWriteString(h, frags[k]); err != nil {
|
||||
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
|
||||
}
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func hashWriteString(h interface{ Write([]byte) (int, error) }, s string) error {
|
||||
var buf [8]byte
|
||||
binary.BigEndian.PutUint64(buf[:], uint64(len(s)))
|
||||
if _, err := h.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := h.Write([]byte(s))
|
||||
return err
|
||||
}
|
||||
|
||||
// do returns the cached body or runs build once for key.
|
||||
// build's bool is false when the result must not be stored (failure is also not stored).
|
||||
// Callers must not mutate the returned slice.
|
||||
func (c *nodeBundleCache) do(ctx context.Context, key string, build func(context.Context) ([]byte, bool, error)) (body []byte, err error) {
|
||||
if c == nil {
|
||||
return nil, errors.New("httpapi: nil node bundle cache")
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
c.mu.Lock()
|
||||
if e, ok := c.items[key]; ok {
|
||||
c.touchLocked(key)
|
||||
body = e.body
|
||||
c.mu.Unlock()
|
||||
return body, nil
|
||||
}
|
||||
if f, ok := c.flights[key]; ok {
|
||||
f.joined++
|
||||
c.mu.Unlock()
|
||||
return waitNodeBundleFlight(ctx, f)
|
||||
}
|
||||
if err = ctx.Err(); err != nil {
|
||||
c.mu.Unlock()
|
||||
return nil, err
|
||||
}
|
||||
f := &nodeBundleFlight{done: make(chan struct{})}
|
||||
c.flights[key] = f
|
||||
c.mu.Unlock()
|
||||
|
||||
var cacheOK bool
|
||||
var finished bool
|
||||
finish := func() {
|
||||
if finished {
|
||||
return
|
||||
}
|
||||
finished = true
|
||||
out := body
|
||||
c.mu.Lock()
|
||||
delete(c.flights, key)
|
||||
if err == nil && cacheOK {
|
||||
if stored, ok := c.storeLocked(key, out); ok {
|
||||
out = stored
|
||||
}
|
||||
}
|
||||
f.body = out
|
||||
f.err = err
|
||||
close(f.done)
|
||||
c.mu.Unlock()
|
||||
body = out
|
||||
}
|
||||
defer func() {
|
||||
if rec := recover(); rec != nil {
|
||||
body = nil
|
||||
cacheOK = false
|
||||
err = fmt.Errorf("httpapi: node bundle build: %v", rec)
|
||||
}
|
||||
finish()
|
||||
}()
|
||||
|
||||
// Detach from the leader's cancel so one disconnected agent does not fail the shared build.
|
||||
body, cacheOK, err = build(context.WithoutCancel(ctx))
|
||||
return
|
||||
}
|
||||
|
||||
func waitNodeBundleFlight(ctx context.Context, f *nodeBundleFlight) ([]byte, error) {
|
||||
select {
|
||||
case <-f.done:
|
||||
return f.body, f.err
|
||||
case <-ctx.Done():
|
||||
select {
|
||||
case <-f.done:
|
||||
return f.body, f.err
|
||||
default:
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// storeLocked retains body when it fits under both caps, evicting the oldest idle
|
||||
// entries first. body is copied. The stored slice is returned.
|
||||
func (c *nodeBundleCache) storeLocked(key string, body []byte) ([]byte, bool) {
|
||||
n := len(body)
|
||||
if n == 0 || c.maxN <= 0 || c.maxBytes <= 0 || n > c.maxBytes {
|
||||
return nil, false
|
||||
}
|
||||
if e, ok := c.items[key]; ok {
|
||||
c.total -= len(e.body)
|
||||
delete(c.items, key)
|
||||
c.removeLRULocked(key)
|
||||
}
|
||||
for len(c.items) > 0 && (len(c.items) >= c.maxN || c.total+n > c.maxBytes) {
|
||||
if len(c.lru) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
c.deleteOldestLocked()
|
||||
}
|
||||
if len(c.items) >= c.maxN || c.total+n > c.maxBytes {
|
||||
return nil, false
|
||||
}
|
||||
stored := make([]byte, n)
|
||||
copy(stored, body)
|
||||
c.items[key] = &nodeBundleEntry{body: stored}
|
||||
c.total += n
|
||||
c.lru = append(c.lru, key)
|
||||
return stored, true
|
||||
}
|
||||
|
||||
func (c *nodeBundleCache) touchLocked(key string) {
|
||||
c.removeLRULocked(key)
|
||||
c.lru = append(c.lru, key)
|
||||
}
|
||||
|
||||
func (c *nodeBundleCache) deleteOldestLocked() {
|
||||
if len(c.lru) == 0 {
|
||||
return
|
||||
}
|
||||
key := c.lru[0]
|
||||
c.lru = c.lru[1:]
|
||||
if e, ok := c.items[key]; ok {
|
||||
c.total -= len(e.body)
|
||||
delete(c.items, key)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *nodeBundleCache) removeLRULocked(key string) {
|
||||
for i, k := range c.lru {
|
||||
if k != key {
|
||||
continue
|
||||
}
|
||||
copy(c.lru[i:], c.lru[i+1:])
|
||||
c.lru = c.lru[:len(c.lru)-1]
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) buildNodeBundle(revisionID, speakerID string, fragments map[string]string) ([]byte, error) {
|
||||
if s != nil && s.nodeBundleBuild != nil {
|
||||
return s.nodeBundleBuild(revisionID, speakerID, fragments, s.bundlePriv)
|
||||
}
|
||||
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, s.bundlePriv)
|
||||
}
|
||||
|
||||
// loadNodeBundle reads the revision, applies the speaker overlay, and signs the gzip.
|
||||
// The bool is false when overlay inputs moved during the load, so the caller must not
|
||||
// cache the body under the key captured before the load. Failed loads are not cacheable.
|
||||
func (s *Server) loadNodeBundle(ctx context.Context, tenantID, speakerID, revisionID, overlayFP string) ([]byte, bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
fpNow, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID)
|
||||
if err != nil {
|
||||
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
|
||||
}
|
||||
rev, err := s.store.GetRevision(tenantID, revisionID)
|
||||
if err != nil {
|
||||
return nil, false, errNodeBundleRevisionMissing
|
||||
}
|
||||
overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, tenantID, speakerID, revisionID, rev.PreviewFragments)
|
||||
if err != nil {
|
||||
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
|
||||
}
|
||||
fpAfter, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID)
|
||||
if err != nil {
|
||||
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
|
||||
}
|
||||
body, err := s.buildNodeBundle(revisionID, speakerID, overlaid)
|
||||
if err != nil {
|
||||
return nil, false, &nodeBundleOpError{op: "internal", err: err}
|
||||
}
|
||||
cacheable := fpNow == overlayFP && fpAfter == fpNow && len(body) > 0
|
||||
return body, cacheable, nil
|
||||
}
|
||||
@@ -0,0 +1,791 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/bundle"
|
||||
"evobgp/internal/signing"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestNodeBundleCacheKeySeparatesTenantsRevisionsSpeakers(t *testing.T) {
|
||||
a := nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay")
|
||||
if a == nodeBundleCacheKey("tenant-b", "rev", "spk", "overlay") {
|
||||
t.Fatal("tenants collided")
|
||||
}
|
||||
if a == nodeBundleCacheKey("tenant-a", "rev-2", "spk", "overlay") {
|
||||
t.Fatal("revisions collided")
|
||||
}
|
||||
if a == nodeBundleCacheKey("tenant-a", "rev", "spk-2", "overlay") {
|
||||
t.Fatal("speakers collided")
|
||||
}
|
||||
if a == nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay-2") {
|
||||
t.Fatal("overlay inputs collided")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeBundleCacheHitSingleflightAndErrors(t *testing.T) {
|
||||
c := newNodeBundleCache(8, 1<<20)
|
||||
var builds atomic.Int32
|
||||
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
return []byte("signed-body"), true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
return []byte("other"), true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds.Load() != 1 {
|
||||
t.Fatalf("second pull rebuilt: builds=%d", builds.Load())
|
||||
}
|
||||
if !bytes.Equal(got, []byte("signed-body")) {
|
||||
t.Fatalf("hit body = %q", got)
|
||||
}
|
||||
|
||||
t.Run("singleflight", func(t *testing.T) {
|
||||
c := newNodeBundleCache(8, 1<<20)
|
||||
var builds atomic.Int32
|
||||
release := make(chan struct{})
|
||||
const n = 8
|
||||
var wg sync.WaitGroup
|
||||
bodies := make([][]byte, n)
|
||||
errs := make([]error, n)
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
bodies[i], errs[i] = c.do(context.Background(), "same", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
<-release
|
||||
return []byte("signed-bundle"), true, nil
|
||||
})
|
||||
}(i)
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
c.mu.Lock()
|
||||
joined := 0
|
||||
if f := c.flights["same"]; f != nil {
|
||||
joined = f.joined
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if builds.Load() > 1 {
|
||||
close(release)
|
||||
wg.Wait()
|
||||
t.Fatalf("builds = %d before followers joined", builds.Load())
|
||||
}
|
||||
if builds.Load() == 1 && joined == n-1 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
close(release)
|
||||
wg.Wait()
|
||||
t.Fatalf("builds=%d joined=%d", builds.Load(), joined)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
close(release)
|
||||
wg.Wait()
|
||||
if builds.Load() != 1 {
|
||||
t.Fatalf("builds = %d, want 1", builds.Load())
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if errs[i] != nil {
|
||||
t.Fatalf("waiter %d: %v", i, errs[i])
|
||||
}
|
||||
if !bytes.Equal(bodies[i], []byte("signed-bundle")) {
|
||||
t.Fatalf("waiter %d body = %q", i, bodies[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("canceled waiter does not drop the build", func(t *testing.T) {
|
||||
c := newNodeBundleCache(4, 1024)
|
||||
release := make(chan struct{})
|
||||
var builds atomic.Int32
|
||||
leaderErr := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
<-release
|
||||
return []byte("body"), true, nil
|
||||
})
|
||||
leaderErr <- err
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
c.mu.Lock()
|
||||
_, ok := c.flights["k"]
|
||||
c.mu.Unlock()
|
||||
if ok || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err := c.do(ctx, "k", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
return []byte("nope"), true, nil
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("follower err = %v", err)
|
||||
}
|
||||
close(release)
|
||||
if err := <-leaderErr; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds.Load() != 1 {
|
||||
t.Fatalf("builds = %d", builds.Load())
|
||||
}
|
||||
_, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
return []byte("other"), true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds.Load() != 1 {
|
||||
t.Fatal("successful build was not cached after a canceled waiter")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("errors are not cached", func(t *testing.T) {
|
||||
c := newNodeBundleCache(4, 1024)
|
||||
calls := 0
|
||||
boom := errors.New("httpapi: bundle build failed")
|
||||
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return nil, true, boom
|
||||
})
|
||||
if !errors.Is(err, boom) {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
_, err = c.do(context.Background(), "miss", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return nil, false, errNodeBundleRevisionMissing
|
||||
})
|
||||
if !errors.Is(err, errNodeBundleRevisionMissing) {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return []byte("ok"), true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "ok" || calls != 3 {
|
||||
t.Fatalf("calls=%d body=%q", calls, got)
|
||||
}
|
||||
_, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return []byte("other"), true, nil
|
||||
})
|
||||
if err != nil || calls != 3 {
|
||||
t.Fatalf("cached error path rebuilt: calls=%d err=%v", calls, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unstorable success is not cached", func(t *testing.T) {
|
||||
c := newNodeBundleCache(4, 1024)
|
||||
calls := 0
|
||||
for i := 0; i < 2; i++ {
|
||||
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return []byte("fresh"), false, nil
|
||||
})
|
||||
if err != nil || string(got) != "fresh" {
|
||||
t.Fatalf("i=%d err=%v body=%q", i, err, got)
|
||||
}
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("calls = %d, want 2", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNodeBundleCacheEvictsIdleAndRespectsCaps(t *testing.T) {
|
||||
t.Run("entry cap evicts oldest idle", func(t *testing.T) {
|
||||
c := newNodeBundleCache(2, 100)
|
||||
put := func(key, body string) {
|
||||
t.Helper()
|
||||
got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) {
|
||||
return []byte(body), true, nil
|
||||
})
|
||||
if err != nil || string(got) != body {
|
||||
t.Fatalf("put %s: %v %q", key, err, got)
|
||||
}
|
||||
}
|
||||
put("a", "A")
|
||||
put("b", "B")
|
||||
if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) {
|
||||
t.Fatal("recent entry was rebuilt")
|
||||
return nil, false, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put("c", "C")
|
||||
assertBundleCacheAccounting(t, c)
|
||||
if _, ok := c.items["b"]; ok {
|
||||
t.Fatal("idle entry b was kept")
|
||||
}
|
||||
if _, ok := c.items["a"]; !ok || c.items["c"] == nil {
|
||||
t.Fatalf("kept %v", cacheKeys(c))
|
||||
}
|
||||
if len(c.items) > 2 {
|
||||
t.Fatalf("entry cap exceeded: %d", len(c.items))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("byte cap", func(t *testing.T) {
|
||||
c := newNodeBundleCache(8, 10)
|
||||
put := func(key string, n int) {
|
||||
t.Helper()
|
||||
body := bytes.Repeat([]byte("x"), n)
|
||||
got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) {
|
||||
return body, true, nil
|
||||
})
|
||||
if err != nil || !bytes.Equal(got, body) {
|
||||
t.Fatalf("put %s: %v len=%d", key, err, len(got))
|
||||
}
|
||||
assertBundleCacheAccounting(t, c)
|
||||
}
|
||||
put("a", 4)
|
||||
put("b", 4)
|
||||
if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) {
|
||||
t.Fatal("rebuilt a")
|
||||
return nil, false, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put("c", 4)
|
||||
assertBundleCacheAccounting(t, c)
|
||||
if _, ok := c.items["b"]; ok {
|
||||
t.Fatal("byte cap kept idle entry b")
|
||||
}
|
||||
if len(c.items) != 2 || c.total != 8 {
|
||||
t.Fatalf("n=%d total=%d", len(c.items), c.total)
|
||||
}
|
||||
put("d", 6)
|
||||
put("e", 6)
|
||||
assertBundleCacheAccounting(t, c)
|
||||
if len(c.items) != 1 || c.total != 6 {
|
||||
t.Fatalf("after oversized pair n=%d total=%d keys=%v", len(c.items), c.total, cacheKeys(c))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("entry larger than the byte cap is not stored", func(t *testing.T) {
|
||||
c := newNodeBundleCache(4, 8)
|
||||
calls := 0
|
||||
big := bytes.Repeat([]byte("y"), 9)
|
||||
for i := 0; i < 2; i++ {
|
||||
got, err := c.do(context.Background(), "big", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return big, true, nil
|
||||
})
|
||||
if err != nil || !bytes.Equal(got, big) {
|
||||
t.Fatalf("i=%d err=%v", i, err)
|
||||
}
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("oversize body was cached, calls=%d", calls)
|
||||
}
|
||||
if len(c.items) != 0 || c.total != 0 {
|
||||
t.Fatalf("stored oversize n=%d total=%d", len(c.items), c.total)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNodeBundleOverlayFingerprintInputs(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, rev, speaker := m.DemoIDs()
|
||||
|
||||
base, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if base != again {
|
||||
t.Fatal("fingerprint changed without an overlay input change")
|
||||
}
|
||||
|
||||
sp, err := m.GetSpeaker(tenant, speaker)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dispatch := store.MergeSpeakerMetaJSON(sp.MetaJSON, store.SpeakerMeta{
|
||||
LastDispatchAt: "2026-01-01T00:00:00Z",
|
||||
LastDispatchStatus: "ok",
|
||||
SyncStatus: "synced",
|
||||
})
|
||||
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &dispatch}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base {
|
||||
t.Fatalf("dispatch meta changed fingerprint: %v %s", err, got)
|
||||
}
|
||||
|
||||
peers := m.ListPeers(tenant)
|
||||
if len(peers) == 0 {
|
||||
t.Fatal("expected demo peers")
|
||||
}
|
||||
state := "Active"
|
||||
if _, err := m.UpdatePeer(tenant, peers[0].ID, &store.PeerPatch{SessionState: &state}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base {
|
||||
t.Fatalf("session state changed fingerprint: %v %s", err, got)
|
||||
}
|
||||
|
||||
if err := m.PatchGlobalSettings(tenant, map[string]any{"bird_router_id": "203.0.113.7"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
routed, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if routed == base {
|
||||
t.Fatal("router id setting did not change the overlay fingerprint")
|
||||
}
|
||||
|
||||
nodeMeta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"})
|
||||
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeMeta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nodeFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if nodeFP == routed {
|
||||
t.Fatal("node ipv4 did not change the overlay fingerprint")
|
||||
}
|
||||
|
||||
sourceMeta := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||
NodeIPv4: "198.51.100.9",
|
||||
BirdBgpSourceIPv4: "203.0.113.40",
|
||||
})
|
||||
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &sourceMeta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sourceFP == nodeFP {
|
||||
t.Fatal("source ipv4 did not change the overlay fingerprint")
|
||||
}
|
||||
|
||||
nodeOnly := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||
NodeIPv4: "198.51.100.8",
|
||||
BirdBgpSourceIPv4: "203.0.113.40",
|
||||
})
|
||||
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeOnly}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != sourceFP {
|
||||
t.Fatalf("unused node ipv4 changed fingerprint: %v %s vs %s", err, got, sourceFP)
|
||||
}
|
||||
|
||||
v6 := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||
NodeIPv4: "198.51.100.8",
|
||||
BirdBgpSourceIPv4: "203.0.113.40",
|
||||
BirdBgpSourceIPv6: "2001:db8::40",
|
||||
})
|
||||
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &v6}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v6FP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Source v6 changes local v6. When no peer policy overrides it, the rendered
|
||||
// peers fragment can stay the same. A difference means the overlay emitted it;
|
||||
// equality means a rebuild would not change the bundle.
|
||||
_ = v6FP
|
||||
|
||||
if _, err := m.CreatePeer(tenant, &store.BGPPeer{
|
||||
Name: "cache-extra",
|
||||
Neighbor: "203.0.113.50",
|
||||
RemoteASN: 64512,
|
||||
Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
withPeer, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if withPeer == v6FP {
|
||||
t.Fatal("added peer did not change the overlay fingerprint")
|
||||
}
|
||||
|
||||
sp2, err := m.CreateSpeaker(tenant, &store.Speaker{
|
||||
Role: "replica",
|
||||
Endpoint: "https://203.0.113.2",
|
||||
MetaJSON: v6,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, err := nodeBundleOverlayFingerprint(m, tenant, sp2.ID, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if other == withPeer {
|
||||
t.Fatal("speakers with the same addresses shared an overlay fingerprint")
|
||||
}
|
||||
|
||||
otherRev, err := m.CreateRollbackRevision(tenant, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
revFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, otherRev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if revFP == withPeer {
|
||||
t.Fatal("revision id did not change the overlay fingerprint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleNodeBundleCache(t *testing.T) {
|
||||
srv, ts, ids := startNodeBundleServer(t)
|
||||
if srv.bundleCache.maxN != nodeBundleCacheMaxEntries || srv.bundleCache.maxBytes != nodeBundleCacheMaxBytes {
|
||||
t.Fatalf("caps n=%d bytes=%d", srv.bundleCache.maxN, srv.bundleCache.maxBytes)
|
||||
}
|
||||
var builds atomic.Int32
|
||||
srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
|
||||
builds.Add(1)
|
||||
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv)
|
||||
}
|
||||
client := ts.Client()
|
||||
|
||||
status, raw := pullNodeBundle(t, client, ts.URL, ids.speaker, "00000000-0000-0000-0000-000000000099")
|
||||
if status != http.StatusNotFound {
|
||||
t.Fatalf("missing revision status %d: %s", status, raw)
|
||||
}
|
||||
if builds.Load() != 0 {
|
||||
t.Fatal("missing revision built a bundle")
|
||||
}
|
||||
|
||||
body1 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
body2 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 1 {
|
||||
t.Fatalf("second pull rebuilt: builds=%d", builds.Load())
|
||||
}
|
||||
if !bytes.Equal(body1, body2) {
|
||||
t.Fatal("cache hit was not byte-for-byte identical")
|
||||
}
|
||||
verifyNodeBundle(t, srv, body2)
|
||||
|
||||
sp2, err := srv.Store().CreateSpeaker(ids.tenant, &store.Speaker{
|
||||
Role: "replica",
|
||||
Endpoint: "https://203.0.113.2",
|
||||
MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "203.0.113.2"}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bodyB := mustPullNodeBundle(t, client, ts.URL, sp2.ID, ids.rev)
|
||||
if builds.Load() != 2 {
|
||||
t.Fatalf("second speaker builds=%d", builds.Load())
|
||||
}
|
||||
if bytes.Equal(body1, bodyB) {
|
||||
t.Fatal("different speakers returned the same bundle")
|
||||
}
|
||||
if !strings.Contains(bundleFile(t, bodyB, "bird.conf"), "203.0.113.2") {
|
||||
t.Fatal("speaker B bundle missing its router id")
|
||||
}
|
||||
body3 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 2 || !bytes.Equal(body1, body3) {
|
||||
t.Fatalf("speaker A collided with B: builds=%d equal=%v", builds.Load(), bytes.Equal(body1, body3))
|
||||
}
|
||||
|
||||
meta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"})
|
||||
if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body4 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 3 {
|
||||
t.Fatalf("changed node ipv4 builds=%d, want 3", builds.Load())
|
||||
}
|
||||
if bytes.Equal(body1, body4) {
|
||||
t.Fatal("changed overlay input reused the previous bundle")
|
||||
}
|
||||
conf := bundleFile(t, body4, "bird.conf")
|
||||
if !strings.Contains(conf, "198.51.100.9") {
|
||||
t.Fatalf("rebuilt bird.conf missing node ipv4: %s", conf)
|
||||
}
|
||||
verifyNodeBundle(t, srv, body4)
|
||||
body5 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 3 || !bytes.Equal(body4, body5) {
|
||||
t.Fatal("rebuilt bundle was not cached")
|
||||
}
|
||||
|
||||
source := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||
NodeIPv4: "198.51.100.9",
|
||||
BirdBgpSourceIPv4: "203.0.113.40",
|
||||
})
|
||||
if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &source}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body6 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 4 {
|
||||
t.Fatalf("changed source ipv4 builds=%d, want 4", builds.Load())
|
||||
}
|
||||
conf = bundleFile(t, body6, "bird.conf")
|
||||
if !strings.Contains(conf, "203.0.113.40") {
|
||||
t.Fatalf("source ipv4 not applied: %s", conf)
|
||||
}
|
||||
verifyNodeBundle(t, srv, body6)
|
||||
|
||||
otherRev, err := srv.Store().CreateRollbackRevision(ids.tenant, ids.rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bodyRev := mustPullNodeBundle(t, client, ts.URL, ids.speaker, otherRev)
|
||||
if builds.Load() != 5 {
|
||||
t.Fatalf("other revision builds=%d", builds.Load())
|
||||
}
|
||||
if bytes.Equal(body6, bodyRev) {
|
||||
t.Fatal("revisions shared a bundle body")
|
||||
}
|
||||
cached := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 5 || !bytes.Equal(body6, cached) {
|
||||
t.Fatal("original revision was evicted or rebuilt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleNodeBundleSingleflight(t *testing.T) {
|
||||
srv, ts, ids := startNodeBundleServer(t)
|
||||
var builds atomic.Int32
|
||||
release := make(chan struct{})
|
||||
srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
|
||||
builds.Add(1)
|
||||
<-release
|
||||
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv)
|
||||
}
|
||||
client := ts.Client()
|
||||
const n = 2
|
||||
bodies := make([][]byte, n)
|
||||
errs := make([]error, n)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
bodies[i], errs[i] = pullNodeBundleOK(client, ts.URL, ids.speaker, ids.rev)
|
||||
}(i)
|
||||
}
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
joined := 0
|
||||
for {
|
||||
if builds.Load() > 1 {
|
||||
close(release)
|
||||
wg.Wait()
|
||||
t.Fatalf("concurrent pulls built %d times", builds.Load())
|
||||
}
|
||||
srv.bundleCache.mu.Lock()
|
||||
joined = 0
|
||||
for _, f := range srv.bundleCache.flights {
|
||||
joined += f.joined
|
||||
}
|
||||
srv.bundleCache.mu.Unlock()
|
||||
if builds.Load() == 1 && joined >= 1 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
close(release)
|
||||
wg.Wait()
|
||||
t.Fatalf("timed out builds=%d joined=%d", builds.Load(), joined)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
close(release)
|
||||
wg.Wait()
|
||||
if builds.Load() != 1 {
|
||||
t.Fatalf("builds = %d, want 1", builds.Load())
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if errs[i] != nil {
|
||||
t.Fatal(errs[i])
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(bodies[0], bodies[1]) {
|
||||
t.Fatal("singleflight returned different bodies")
|
||||
}
|
||||
verifyNodeBundle(t, srv, bodies[0])
|
||||
}
|
||||
|
||||
type nodeBundleIDs struct {
|
||||
tenant, rev, speaker string
|
||||
}
|
||||
|
||||
func startNodeBundleServer(t *testing.T) (*Server, *httptest.Server, nodeBundleIDs) {
|
||||
t.Helper()
|
||||
srv, err := New(Options{
|
||||
InsecureDev: true,
|
||||
SeedDemo: true,
|
||||
BundleSeedHex: testBundleSeed,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(srv.Close)
|
||||
tenant, _, _, rev, speaker := srv.Store().DemoIDs()
|
||||
mustSetTestAPIKeys(t, srv, "nodekey|"+tenant+"|node")
|
||||
ts := httptest.NewServer(srv.Handler())
|
||||
t.Cleanup(ts.Close)
|
||||
return srv, ts, nodeBundleIDs{tenant: tenant, rev: rev, speaker: speaker}
|
||||
}
|
||||
|
||||
func mustPullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) []byte {
|
||||
t.Helper()
|
||||
status, body := pullNodeBundle(t, client, base, speaker, rev)
|
||||
if status != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", status, body)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func pullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) (int, []byte) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer nodekey")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return resp.StatusCode, body
|
||||
}
|
||||
|
||||
func pullNodeBundleOK(client *http.Client, base, speaker, rev string) ([]byte, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer nodekey")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, errors.New(resp.Status + ": " + string(body))
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func verifyNodeBundle(t *testing.T, srv *Server, raw []byte) {
|
||||
t.Helper()
|
||||
pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := signing.VerifyGzippedTar(raw, ed25519.PublicKey(pub)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func bundleFile(t *testing.T, gz []byte, name string) string {
|
||||
t.Helper()
|
||||
gr, err := gzip.NewReader(bytes.NewReader(gz))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = gr.Close() }()
|
||||
tr := tar.NewReader(gr)
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hdr.Name != name {
|
||||
continue
|
||||
}
|
||||
var b strings.Builder
|
||||
if _, err := io.Copy(&b, tr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
t.Fatalf("archive missing %s", name)
|
||||
return ""
|
||||
}
|
||||
|
||||
func assertBundleCacheAccounting(t *testing.T, c *nodeBundleCache) {
|
||||
t.Helper()
|
||||
if len(c.items) != len(c.lru) {
|
||||
t.Fatalf("items %d lru %d", len(c.items), len(c.lru))
|
||||
}
|
||||
sum := 0
|
||||
seen := make(map[string]bool, len(c.lru))
|
||||
for _, k := range c.lru {
|
||||
e, ok := c.items[k]
|
||||
if !ok {
|
||||
t.Fatalf("lru key %q missing from items", k)
|
||||
}
|
||||
if seen[k] {
|
||||
t.Fatalf("duplicate lru key %q", k)
|
||||
}
|
||||
seen[k] = true
|
||||
sum += len(e.body)
|
||||
}
|
||||
if sum != c.total {
|
||||
t.Fatalf("total=%d sum=%d", c.total, sum)
|
||||
}
|
||||
if c.maxN > 0 && len(c.items) > c.maxN {
|
||||
t.Fatalf("entries %d over cap %d", len(c.items), c.maxN)
|
||||
}
|
||||
if c.maxBytes > 0 && c.total > c.maxBytes {
|
||||
t.Fatalf("bytes %d over cap %d", c.total, c.maxBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func cacheKeys(c *nodeBundleCache) []string {
|
||||
keys := make([]string, 0, len(c.items))
|
||||
for k := range c.items {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/bundle"
|
||||
)
|
||||
|
||||
func TestLegacyCompatibilityNodePullAndAuth(t *testing.T) {
|
||||
srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer srv.Close()
|
||||
tenant, _, _, revision, speaker := srv.Store().DemoIDs()
|
||||
mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,operator|"+tenant+"|operator,other-node|other-tenant|node")
|
||||
if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := srv.Handler()
|
||||
call := func(path, token string) (int, []byte, string) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
handler.ServeHTTP(w, req)
|
||||
resp := w.Result()
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return resp.StatusCode, body, resp.Header.Get("Content-Type")
|
||||
}
|
||||
latestPath := "/v1/speakers/" + speaker + "/revisions/latest"
|
||||
bundlePath := "/v1/speakers/" + speaker + "/bundle/" + revision
|
||||
for _, path := range []string{latestPath, bundlePath} {
|
||||
for _, tc := range []struct {
|
||||
token string
|
||||
want int
|
||||
}{{"", http.StatusUnauthorized}, {"operator", http.StatusForbidden}, {"other-node", http.StatusForbidden}} {
|
||||
status, _, _ := call(path, tc.token)
|
||||
if status != tc.want {
|
||||
t.Fatalf("%s token %q: status %d, want %d", path, tc.token, status, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
status, raw, _ := call(latestPath, "legacy-node")
|
||||
if status != http.StatusOK {
|
||||
t.Fatalf("legacy latest: status %d: %s", status, raw)
|
||||
}
|
||||
var latest struct {
|
||||
RevisionID string `json:"revision_id"`
|
||||
PublishedAt string `json:"published_at"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &latest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if latest.RevisionID != revision || latest.PublishedAt == "" {
|
||||
t.Fatalf("legacy latest response: %+v", latest)
|
||||
}
|
||||
status, raw, contentType := call("/v1/bundle/signing-public-key", "operator")
|
||||
if status != http.StatusOK || contentType != "application/json" {
|
||||
t.Fatalf("public key status %d, type %q: %s", status, contentType, raw)
|
||||
}
|
||||
var key struct {
|
||||
PublicKeyBase64 string `json:"public_key_base64"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub, err := base64.StdEncoding.DecodeString(key.PublicKeyBase64)
|
||||
if err != nil || len(pub) != ed25519.PublicKeySize {
|
||||
t.Fatalf("legacy public key: len=%d err=%v", len(pub), err)
|
||||
}
|
||||
status, raw, contentType = call(bundlePath, "legacy-node")
|
||||
if status != http.StatusOK || contentType != "application/gzip" {
|
||||
t.Fatalf("legacy bundle: status %d, type %q: %s", status, contentType, raw)
|
||||
}
|
||||
verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub))
|
||||
if err != nil {
|
||||
t.Fatalf("legacy signed bundle verification: %v", err)
|
||||
}
|
||||
if verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker || len(verified.Files) == 0 {
|
||||
t.Fatalf("legacy bundle manifest: %+v", verified.Manifest)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type expensiveOperation int
|
||||
|
||||
const (
|
||||
lookupOperation expensiveOperation = iota
|
||||
cdnPreviewOperation
|
||||
revisionPreviewOperation
|
||||
)
|
||||
|
||||
type tenantWork struct {
|
||||
start time.Time
|
||||
counts [3]int
|
||||
active int
|
||||
}
|
||||
|
||||
// operationLimiter is deliberately local to one API process. It never queues
|
||||
// waiters or evicts active tenants to admit new identities.
|
||||
type operationLimiter struct {
|
||||
mu sync.Mutex
|
||||
tenants map[string]*tenantWork
|
||||
active, maxActive, maxTenantActive, maxTenants int
|
||||
rates [3]int
|
||||
nextSweep time.Time
|
||||
}
|
||||
|
||||
func newOperationLimiter() *operationLimiter {
|
||||
return &operationLimiter{
|
||||
tenants: make(map[string]*tenantWork), maxTenants: 4096,
|
||||
maxActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_MAX_CONCURRENT", 4, 64),
|
||||
maxTenantActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", 2, 64),
|
||||
rates: [3]int{operationEnvLimit("EVOBGP_API_LOOKUP_PER_MINUTE", 120, 100000), operationEnvLimit("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", 30, 100000), operationEnvLimit("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", 30, 100000)},
|
||||
}
|
||||
}
|
||||
|
||||
func operationEnvLimit(name string, fallback, maximum int) int {
|
||||
n, err := strconv.Atoi(os.Getenv(name))
|
||||
if err != nil || n < 1 || n > maximum {
|
||||
return fallback
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (l *operationLimiter) acquire(tenant string, op expensiveOperation, now time.Time) (func(), int) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if !now.Before(l.nextSweep) {
|
||||
for key, entry := range l.tenants {
|
||||
if entry.active == 0 && !now.Before(entry.start.Add(time.Minute)) {
|
||||
delete(l.tenants, key)
|
||||
}
|
||||
}
|
||||
l.nextSweep = now.Add(time.Second)
|
||||
}
|
||||
if l.active >= l.maxActive {
|
||||
return nil, 1
|
||||
}
|
||||
entry := l.tenants[tenant]
|
||||
if entry == nil {
|
||||
if len(l.tenants) >= l.maxTenants {
|
||||
return nil, 1
|
||||
}
|
||||
entry = &tenantWork{start: now}
|
||||
l.tenants[tenant] = entry
|
||||
}
|
||||
if !now.Before(entry.start.Add(time.Minute)) {
|
||||
entry.start = now
|
||||
entry.counts = [3]int{}
|
||||
}
|
||||
if entry.active >= l.maxTenantActive {
|
||||
return nil, 1
|
||||
}
|
||||
if entry.counts[op] >= l.rates[op] {
|
||||
wait := entry.start.Add(time.Minute).Sub(now)
|
||||
return nil, max(1, int((wait+time.Second-1)/time.Second))
|
||||
}
|
||||
entry.counts[op]++
|
||||
entry.active++
|
||||
l.active++
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { l.mu.Lock(); defer l.mu.Unlock(); entry.active--; l.active-- }) }, 0
|
||||
}
|
||||
|
||||
func (s *Server) acquireOperation(w http.ResponseWriter, r *http.Request, tenant string, op expensiveOperation) (func(), bool) {
|
||||
if r.Context().Err() != nil {
|
||||
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request cancelled")
|
||||
return nil, false
|
||||
}
|
||||
s.operationOnce.Do(func() { s.operationLimiter = newOperationLimiter() })
|
||||
release, retry := s.operationLimiter.acquire(tenant, op, time.Now())
|
||||
if release == nil {
|
||||
w.Header().Set("Retry-After", strconv.Itoa(retry))
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "preview/lookup quota exceeded; retry later")
|
||||
return nil, false
|
||||
}
|
||||
return release, true
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestOperationLimitsSharedCapacityAndWindows(t *testing.T) {
|
||||
l := newOperationLimiter()
|
||||
l.maxActive = 2
|
||||
l.maxTenantActive = 1
|
||||
l.rates = [3]int{1, 2, 2}
|
||||
now := time.Now()
|
||||
a, _ := l.acquire("a", lookupOperation, now)
|
||||
if a == nil {
|
||||
t.Fatal("first denied")
|
||||
}
|
||||
if r, _ := l.acquire("a", cdnPreviewOperation, now); r != nil {
|
||||
t.Fatal("tenant concurrency bypass")
|
||||
}
|
||||
b, _ := l.acquire("b", revisionPreviewOperation, now)
|
||||
if b == nil {
|
||||
t.Fatal("other tenant denied")
|
||||
}
|
||||
if r, _ := l.acquire("c", lookupOperation, now); r != nil {
|
||||
t.Fatal("global concurrency bypass")
|
||||
}
|
||||
a()
|
||||
a()
|
||||
b()
|
||||
if l.active != 0 {
|
||||
t.Fatal("release not idempotent")
|
||||
}
|
||||
if r, retry := l.acquire("a", lookupOperation, now.Add(time.Second)); r != nil || retry != 59 {
|
||||
t.Fatalf("rate bypass/retry admitted=%t seconds=%d", r != nil, retry)
|
||||
}
|
||||
c, _ := l.acquire("a", cdnPreviewOperation, now.Add(time.Second))
|
||||
if c == nil {
|
||||
t.Fatal("independent operation denied")
|
||||
}
|
||||
c()
|
||||
d, _ := l.acquire("a", lookupOperation, now.Add(time.Minute))
|
||||
if d == nil {
|
||||
t.Fatal("expired window not reset")
|
||||
}
|
||||
d()
|
||||
}
|
||||
|
||||
func TestOperationTenantMapBoundWithoutActiveEviction(t *testing.T) {
|
||||
l := newOperationLimiter()
|
||||
l.maxTenants = 1
|
||||
now := time.Now()
|
||||
a, _ := l.acquire("a", lookupOperation, now)
|
||||
if r, _ := l.acquire("b", lookupOperation, now.Add(2*time.Minute)); r != nil {
|
||||
t.Fatal("active entry evicted")
|
||||
}
|
||||
a()
|
||||
b, _ := l.acquire("b", lookupOperation, now.Add(3*time.Minute))
|
||||
if b == nil || len(l.tenants) != 1 {
|
||||
t.Fatal("idle expiry did not free bounded state")
|
||||
}
|
||||
b()
|
||||
}
|
||||
|
||||
func TestOperationConcurrentAdmissionBound(t *testing.T) {
|
||||
l := newOperationLimiter()
|
||||
l.maxActive = 4
|
||||
l.maxTenantActive = 4
|
||||
l.rates = [3]int{100, 100, 100}
|
||||
results := make(chan func(), 32)
|
||||
var wg sync.WaitGroup
|
||||
for range 32 {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); r, _ := l.acquire("tenant", lookupOperation, time.Now()); results <- r }()
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
accepted := 0
|
||||
for r := range results {
|
||||
if r != nil {
|
||||
accepted++
|
||||
r()
|
||||
}
|
||||
}
|
||||
if accepted != 4 || l.active != 0 {
|
||||
t.Fatalf("accepted=%d active=%d", accepted, l.active)
|
||||
}
|
||||
}
|
||||
|
||||
type waitingRevisionStore struct {
|
||||
store.Backend
|
||||
entered chan struct{}
|
||||
unblock chan struct{}
|
||||
}
|
||||
|
||||
func (s waitingRevisionStore) GetRevision(tenant, id string) (*store.Revision, error) {
|
||||
s.entered <- struct{}{}
|
||||
<-s.unblock
|
||||
return s.Backend.GetRevision(tenant, id)
|
||||
}
|
||||
|
||||
func TestRevisionQuotaHeldUntilWorkReturns(t *testing.T) {
|
||||
t.Setenv("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", "1")
|
||||
t.Setenv("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", "30")
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, revision, _ := m.DemoIDs()
|
||||
entered := make(chan struct{}, 2)
|
||||
unblock := make(chan struct{})
|
||||
s := &Server{store: waitingRevisionStore{Backend: m, entered: entered, unblock: unblock}}
|
||||
request := func() *http.Request {
|
||||
r := httptest.NewRequest(http.MethodGet, "/v1/revisions/"+revision+"/preview", nil)
|
||||
r.SetPathValue("revision_id", revision)
|
||||
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"}))
|
||||
}
|
||||
ctx, cancel := context.WithCancel(request().Context())
|
||||
defer cancel()
|
||||
first := httptest.NewRecorder()
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); s.handleRevisionPreview(first, request().WithContext(ctx)) }()
|
||||
<-entered
|
||||
cancel() // Backend deliberately ignores cancellation: capacity stays reserved.
|
||||
second := httptest.NewRecorder()
|
||||
s.handleRevisionPreview(second, request())
|
||||
if second.Code != http.StatusTooManyRequests || second.Header().Get("Retry-After") == "" {
|
||||
close(unblock)
|
||||
<-done
|
||||
t.Fatalf("quota: %d %s", second.Code, second.Body.String())
|
||||
}
|
||||
close(unblock)
|
||||
<-done
|
||||
third := httptest.NewRecorder()
|
||||
s.handleRevisionPreview(third, request())
|
||||
if third.Code != http.StatusOK {
|
||||
t.Fatalf("release failed: %d %s", third.Code, third.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
@@ -44,46 +43,6 @@ func peerDiscoveryJSON(d *store.BGPPeerDiscovery) map[string]any {
|
||||
return m
|
||||
}
|
||||
|
||||
// syncPeerDiscoveriesFromLive upserts pending discoveries from dynamic BGP sessions.
|
||||
func (s *Server) syncPeerDiscoveriesFromLive(tenantID string, views []speakerBGPLive) {
|
||||
if s.store == nil || tenantID == "" {
|
||||
return
|
||||
}
|
||||
knownNeighbors := map[string]struct{}{}
|
||||
for _, p := range s.store.ListPeers(tenantID) {
|
||||
if p == nil {
|
||||
continue
|
||||
}
|
||||
if n, ok := store.NormalizePeerNeighborString(p.Neighbor); ok {
|
||||
knownNeighbors[n] = struct{}{}
|
||||
}
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, v := range views {
|
||||
for _, sess := range v.Sessions {
|
||||
if !birdfmt.IsDynamicDiscoverySession(sess.Name) {
|
||||
continue
|
||||
}
|
||||
neighbor, ok := store.NormalizePeerNeighborString(sess.Neighbor)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, known := knownNeighbors[neighbor]; known {
|
||||
continue
|
||||
}
|
||||
_, _ = s.store.UpsertPeerDiscovery(tenantID, &store.PeerDiscoveryUpsert{
|
||||
SpeakerID: v.SpeakerID,
|
||||
NeighborID: strings.TrimSpace(sess.NeighborID),
|
||||
Neighbor: neighbor,
|
||||
RemoteASN: sess.NeighborAS,
|
||||
ProtocolName: sess.Name,
|
||||
SessionState: sess.State,
|
||||
SeenAt: now,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok {
|
||||
@@ -97,11 +56,6 @@ func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Reques
|
||||
if status == "" {
|
||||
status = store.PeerDiscoveryPending
|
||||
}
|
||||
// Refresh live discovery when listing pending.
|
||||
if status == store.PeerDiscoveryPending || status == "all" {
|
||||
views := s.collectSpeakerBGPLive(r.Context(), a.TenantID, true)
|
||||
s.syncPeerDiscoveriesFromLive(a.TenantID, views)
|
||||
}
|
||||
listStatus := status
|
||||
if status == "all" {
|
||||
listStatus = ""
|
||||
@@ -138,7 +92,7 @@ func (s *Server) handleApprovePeerDiscovery(w http.ResponseWriter, r *http.Reque
|
||||
writeStoreErr(w, err)
|
||||
return
|
||||
}
|
||||
s.enqueuePeerReconcile(a.TenantID, "peer_discovery_approve")
|
||||
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_approve")
|
||||
s.recordCRUDAudit(r, a, "bgp.peer.discovery.approve", "Approved discovered peer "+peer.Neighbor, peer.ID, map[string]any{
|
||||
"peer_id": peer.ID,
|
||||
"discovery_id": disc.ID,
|
||||
|
||||
+185
-116
@@ -1,47 +1,98 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
const peerLiveCacheTTL = 15 * time.Second
|
||||
|
||||
type speakerBGPLive struct {
|
||||
SpeakerID string
|
||||
Label string
|
||||
Sessions []birdfmt.BGPSession
|
||||
Error string
|
||||
SpeakerID string
|
||||
Label string
|
||||
Sessions []birdfmt.BGPSession
|
||||
Error string
|
||||
Polled bool
|
||||
ObservedAt time.Time
|
||||
ReceivedAt time.Time
|
||||
LastSuccessAt time.Time
|
||||
Source string
|
||||
ObservationOverride *liveObservation
|
||||
}
|
||||
|
||||
type peerLiveCacheEntry struct {
|
||||
at time.Time
|
||||
views []speakerBGPLive
|
||||
type liveObservation struct {
|
||||
ObservedAt *string `json:"observed_at"`
|
||||
ReceivedAt *string `json:"received_at"`
|
||||
LastSuccessAt *string `json:"last_success_at"`
|
||||
ExpiresAt *string `json:"expires_at,omitempty"`
|
||||
Source string `json:"source"`
|
||||
Quality string `json:"quality"`
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
}
|
||||
|
||||
var peerLiveCache sync.Map // tenantID -> peerLiveCacheEntry
|
||||
func liveTime(t time.Time) *string {
|
||||
if t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
v := t.UTC().Format(time.RFC3339Nano)
|
||||
return &v
|
||||
}
|
||||
|
||||
func (v speakerBGPLive) observed() bool {
|
||||
return v.Polled || v.Error != "" || len(v.Sessions) > 0
|
||||
}
|
||||
|
||||
func (v speakerBGPLive) successful() bool {
|
||||
return v.observed() && v.Error == ""
|
||||
}
|
||||
|
||||
func (v speakerBGPLive) observation() liveObservation {
|
||||
if v.ObservationOverride != nil {
|
||||
return *v.ObservationOverride
|
||||
}
|
||||
o := liveObservation{ObservedAt: liveTime(v.ObservedAt), ReceivedAt: liveTime(v.ReceivedAt), LastSuccessAt: liveTime(v.LastSuccessAt), Source: v.Source, Quality: "unknown"}
|
||||
if o.Source == "" {
|
||||
o.Source = "none"
|
||||
}
|
||||
switch {
|
||||
case v.successful():
|
||||
o.Quality = "ok"
|
||||
case v.Error != "" && !v.LastSuccessAt.IsZero():
|
||||
o.Quality = "stale"
|
||||
o.ErrorCode = liveErrorCode(v.Error)
|
||||
case v.Error != "":
|
||||
o.Quality = "unreachable"
|
||||
o.ErrorCode = liveErrorCode(v.Error)
|
||||
default:
|
||||
o.ErrorCode = "not_polled"
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
func liveErrorCode(err string) string {
|
||||
if strings.Contains(strings.ToLower(err), "not configured") || strings.Contains(strings.ToLower(err), "missing") {
|
||||
return "not_configured"
|
||||
}
|
||||
return "poll_failed"
|
||||
}
|
||||
|
||||
type peerSessionOnSpeaker struct {
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
State string `json:"state"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
State string `json:"state"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
Observation liveObservation `json:"observation"`
|
||||
}
|
||||
|
||||
type liveSpeakerPoll struct {
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
OK bool `json:"ok"`
|
||||
SessionCount int `json:"session_count"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
OK bool `json:"ok"`
|
||||
SessionCount *int `json:"session_count"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
Observation liveObservation `json:"observation"`
|
||||
}
|
||||
|
||||
func speakerDisplayLabel(sp *store.Speaker) string {
|
||||
@@ -65,95 +116,21 @@ func speakerDisplayLabel(sp *store.Speaker) string {
|
||||
return sp.ID
|
||||
}
|
||||
|
||||
func masterSpeakerID(speakers []*store.Speaker) string {
|
||||
for _, sp := range speakers {
|
||||
if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
|
||||
return sp.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Server) collectSpeakerBGPLive(ctx context.Context, tenantID string, fresh bool) []speakerBGPLive {
|
||||
if !fresh {
|
||||
if v, ok := peerLiveCache.Load(tenantID); ok {
|
||||
ent := v.(peerLiveCacheEntry)
|
||||
if time.Since(ent.at) < peerLiveCacheTTL {
|
||||
return ent.views
|
||||
}
|
||||
}
|
||||
}
|
||||
speakers := s.store.ListSpeakersForTenant(tenantID)
|
||||
views := make([]speakerBGPLive, 0, len(speakers)+1)
|
||||
|
||||
if sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")); sock != "" {
|
||||
v := speakerBGPLive{Label: "CP (local BIRD)"}
|
||||
if mid := masterSpeakerID(speakers); mid != "" {
|
||||
v.SpeakerID = mid
|
||||
for _, sp := range speakers {
|
||||
if sp != nil && sp.ID == mid {
|
||||
v.Label = speakerDisplayLabel(sp)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")))
|
||||
if err != nil {
|
||||
v.Error = err.Error()
|
||||
} else {
|
||||
v.Sessions = birdfmt.ParseBGPSessions(out)
|
||||
}
|
||||
views = append(views, v)
|
||||
}
|
||||
|
||||
opts := nodedispatch.Options{Timeout: 8 * time.Second}
|
||||
type resWrap struct {
|
||||
sp *store.Speaker
|
||||
res nodedispatch.BirdProtocolsResult
|
||||
}
|
||||
ch := make(chan resWrap, len(speakers))
|
||||
var wg sync.WaitGroup
|
||||
for _, sp := range speakers {
|
||||
if sp == nil {
|
||||
continue
|
||||
}
|
||||
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
||||
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(speaker *store.Speaker) {
|
||||
defer wg.Done()
|
||||
ch <- resWrap{
|
||||
sp: speaker,
|
||||
res: nodedispatch.FetchBirdProtocols(ctx, speaker, opts),
|
||||
}
|
||||
}(sp)
|
||||
}
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
for rw := range ch {
|
||||
views = append(views, speakerBGPLive{
|
||||
SpeakerID: rw.sp.ID,
|
||||
Label: speakerDisplayLabel(rw.sp),
|
||||
Sessions: rw.res.Sessions,
|
||||
Error: rw.res.Error,
|
||||
})
|
||||
}
|
||||
|
||||
peerLiveCache.Store(tenantID, peerLiveCacheEntry{at: time.Now(), views: views})
|
||||
return views
|
||||
}
|
||||
|
||||
func liveSpeakerPollJSON(views []speakerBGPLive) []liveSpeakerPoll {
|
||||
out := make([]liveSpeakerPoll, 0, len(views))
|
||||
for _, v := range views {
|
||||
var sessionCount *int
|
||||
if v.successful() {
|
||||
count := len(v.Sessions)
|
||||
sessionCount = &count
|
||||
}
|
||||
out = append(out, liveSpeakerPoll{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
OK: v.Error == "",
|
||||
SessionCount: len(v.Sessions),
|
||||
OK: v.successful(),
|
||||
SessionCount: sessionCount,
|
||||
PollError: v.Error,
|
||||
Observation: v.observation(),
|
||||
})
|
||||
}
|
||||
return out
|
||||
@@ -183,27 +160,30 @@ func matchPeerOnSpeakers(peer *store.BGPPeer, views []speakerBGPLive) (
|
||||
protoName := birdfmt.PeerProtocolName(peer.ID)
|
||||
|
||||
for _, v := range views {
|
||||
if v.Error != "" && len(v.Sessions) == 0 {
|
||||
if !v.successful() {
|
||||
on = append(on, peerSessionOnSpeaker{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
PollError: v.Error,
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
PollError: v.Error,
|
||||
Observation: v.observation(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
sess := findPeerSession(v.Sessions, protoName, neighbor, hasNeighbor)
|
||||
if sess == nil {
|
||||
on = append(on, peerSessionOnSpeaker{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: "absent",
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: "absent",
|
||||
Observation: v.observation(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
hit := peerSessionOnSpeaker{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: sess.State,
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: sess.State,
|
||||
Observation: v.observation(),
|
||||
}
|
||||
on = append(on, hit)
|
||||
if strings.EqualFold(strings.TrimSpace(sess.State), "Established") {
|
||||
@@ -273,20 +253,109 @@ func sessionStateRank(state string) int {
|
||||
}
|
||||
|
||||
func applyPeerLiveFields(row map[string]any, peer *store.BGPPeer, views []speakerBGPLive) {
|
||||
state, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views)
|
||||
expected := expectedPeerViews(peer, views)
|
||||
state, _, _, _, _, _ := matchPeerOnSpeakers(peer, expected)
|
||||
_, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views)
|
||||
row["session_on_speakers"] = on
|
||||
row["established_on_speakers"] = establishedOn
|
||||
row["session_conflict"] = false
|
||||
row["session_mismatch"] = mismatch
|
||||
row["last_known_session_state"] = row["session_state"]
|
||||
row["session_state"] = nil
|
||||
if state != "" {
|
||||
row["session_state"] = state
|
||||
}
|
||||
if state == "" && len(expected) > 0 && allSpeakerPollsSuccessful(expected) {
|
||||
row["session_state"] = "absent"
|
||||
}
|
||||
row["live_observation"] = peerLiveObservation(expected)
|
||||
if connLabel != "" {
|
||||
row["connected_speaker_label"] = connLabel
|
||||
}
|
||||
row["connected_speaker_id"] = peerLiveSpeakerIDOrNull(connID)
|
||||
}
|
||||
|
||||
func expectedPeerViews(peer *store.BGPPeer, views []speakerBGPLive) []speakerBGPLive {
|
||||
if peer == nil || peer.SpeakerID == nil || strings.TrimSpace(*peer.SpeakerID) == "" {
|
||||
return views
|
||||
}
|
||||
want := strings.TrimSpace(*peer.SpeakerID)
|
||||
expected := make([]speakerBGPLive, 0, 1)
|
||||
for _, v := range views {
|
||||
if strings.EqualFold(v.SpeakerID, want) {
|
||||
expected = append(expected, v)
|
||||
}
|
||||
}
|
||||
return expected
|
||||
}
|
||||
|
||||
func allSpeakerPollsSuccessful(views []speakerBGPLive) bool {
|
||||
for _, v := range views {
|
||||
if !v.successful() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func peerLiveObservation(views []speakerBGPLive) liveObservation {
|
||||
if len(views) == 0 {
|
||||
return speakerBGPLive{}.observation()
|
||||
}
|
||||
result := liveObservation{Source: views[0].Source, Quality: "unknown"}
|
||||
if result.Source == "" {
|
||||
result.Source = "none"
|
||||
}
|
||||
var latestObserved, latestReceived, latestSuccess time.Time
|
||||
allOK, allFailed, anyStale := true, true, false
|
||||
for _, v := range views {
|
||||
o := v.observation()
|
||||
if v.ObservedAt.After(latestObserved) {
|
||||
latestObserved = v.ObservedAt
|
||||
}
|
||||
if v.ReceivedAt.After(latestReceived) {
|
||||
latestReceived = v.ReceivedAt
|
||||
}
|
||||
if v.LastSuccessAt.After(latestSuccess) {
|
||||
latestSuccess = v.LastSuccessAt
|
||||
}
|
||||
allOK = allOK && v.successful()
|
||||
allFailed = allFailed && o.Quality == "unreachable"
|
||||
anyStale = anyStale || o.Quality == "stale"
|
||||
if result.ErrorCode == "" && o.ErrorCode != "" {
|
||||
result.ErrorCode = o.ErrorCode
|
||||
}
|
||||
if o.ExpiresAt != nil {
|
||||
expires, err := time.Parse(time.RFC3339Nano, *o.ExpiresAt)
|
||||
if err == nil {
|
||||
previous := time.Time{}
|
||||
if result.ExpiresAt != nil {
|
||||
previous, _ = time.Parse(time.RFC3339Nano, *result.ExpiresAt)
|
||||
}
|
||||
if previous.IsZero() || expires.Before(previous) {
|
||||
result.ExpiresAt = o.ExpiresAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
result.ObservedAt = liveTime(latestObserved)
|
||||
result.ReceivedAt = liveTime(latestReceived)
|
||||
result.LastSuccessAt = liveTime(latestSuccess)
|
||||
if len(views) > 1 {
|
||||
result.Source = "mixed"
|
||||
}
|
||||
switch {
|
||||
case allOK:
|
||||
result.Quality = "ok"
|
||||
result.ErrorCode = ""
|
||||
case anyStale:
|
||||
result.Quality = "stale"
|
||||
case allFailed:
|
||||
result.Quality = "unreachable"
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func peerLiveSpeakerIDOrNull(id string) any {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return nil
|
||||
|
||||
@@ -2,11 +2,84 @@ package httpapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestApplyPeerLiveFieldsObservationStates(t *testing.T) {
|
||||
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"}
|
||||
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
|
||||
cases := []struct {
|
||||
name string
|
||||
views []speakerBGPLive
|
||||
wantState any
|
||||
wantQuality string
|
||||
}{
|
||||
{"no collector", nil, nil, "unknown"},
|
||||
{"successful empty poll", []speakerBGPLive{{Polled: true, Source: "agent_bird_protocols", ObservedAt: now, ReceivedAt: now, LastSuccessAt: now}}, "absent", "ok"},
|
||||
{"first failure", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now}}, nil, "unreachable"},
|
||||
{"failure after success", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now, LastSuccessAt: now.Add(-time.Minute)}}, nil, "stale"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
row := map[string]any{"session_state": "Established"}
|
||||
applyPeerLiveFields(row, peer, tc.views)
|
||||
if row["session_state"] != tc.wantState || row["last_known_session_state"] != "Established" {
|
||||
t.Fatalf("session_state=%v last_known=%v", row["session_state"], row["last_known_session_state"])
|
||||
}
|
||||
observation := row["live_observation"].(liveObservation)
|
||||
if observation.Quality != tc.wantQuality {
|
||||
t.Fatalf("observation=%+v", observation)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPeerLiveFieldsExpectedSpeakerCoverage(t *testing.T) {
|
||||
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
|
||||
assigned := "speaker-a"
|
||||
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2", SpeakerID: &assigned}
|
||||
viewAError := speakerBGPLive{SpeakerID: assigned, Polled: true, Error: "timeout", ObservedAt: now, Source: "agent_bird_protocols"}
|
||||
viewAEmpty := speakerBGPLive{SpeakerID: assigned, Polled: true, ObservedAt: now, LastSuccessAt: now, Source: "agent_bird_protocols"}
|
||||
viewBEmpty := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, ObservedAt: now.Add(time.Minute), LastSuccessAt: now.Add(time.Minute), Source: "agent_bird_protocols"}
|
||||
viewBError := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, Error: "timeout", ObservedAt: now.Add(time.Minute), Source: "agent_bird_protocols"}
|
||||
cases := []struct {
|
||||
name string
|
||||
assigned bool
|
||||
views []speakerBGPLive
|
||||
wantState any
|
||||
wantQuality string
|
||||
wantObserved string
|
||||
}{
|
||||
{"assigned timeout unrelated success", true, []speakerBGPLive{viewAError, viewBEmpty}, nil, "unreachable", now.Format(time.RFC3339Nano)},
|
||||
{"assigned success unrelated failure", true, []speakerBGPLive{viewAEmpty, viewBError}, "absent", "ok", now.Format(time.RFC3339Nano)},
|
||||
{"global mixed outcome", false, []speakerBGPLive{viewAEmpty, viewBError}, nil, "unknown", now.Add(time.Minute).Format(time.RFC3339Nano)},
|
||||
{"assigned speaker not observed", true, []speakerBGPLive{viewBEmpty}, nil, "unknown", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
candidate := *peer
|
||||
if !tc.assigned {
|
||||
candidate.SpeakerID = nil
|
||||
}
|
||||
row := map[string]any{"session_state": "Established"}
|
||||
applyPeerLiveFields(row, &candidate, tc.views)
|
||||
obs := row["live_observation"].(liveObservation)
|
||||
if row["session_state"] != tc.wantState || obs.Quality != tc.wantQuality {
|
||||
t.Fatalf("state=%v observation=%+v", row["session_state"], obs)
|
||||
}
|
||||
if tc.wantObserved == "" && obs.ObservedAt != nil {
|
||||
t.Fatalf("unexpected timestamp %v", *obs.ObservedAt)
|
||||
}
|
||||
if tc.wantObserved != "" && (obs.ObservedAt == nil || *obs.ObservedAt != tc.wantObserved) {
|
||||
t.Fatalf("observed_at=%v want=%q", obs.ObservedAt, tc.wantObserved)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchPeerOnSpeakers_establishedOnReplica(t *testing.T) {
|
||||
peer := &store.BGPPeer{
|
||||
ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
|
||||
|
||||
@@ -9,23 +9,33 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// authRateLimiter is a simple per-IP token bucket for public auth-ish endpoints.
|
||||
const maxAuthRateLimitClients = 4096
|
||||
|
||||
type authRateHit struct {
|
||||
start time.Time
|
||||
count int
|
||||
}
|
||||
|
||||
// authRateLimiter keeps a bounded fixed-window counter per client IP.
|
||||
type authRateLimiter struct {
|
||||
mu sync.Mutex
|
||||
hits map[string][]time.Time
|
||||
limit int
|
||||
window time.Duration
|
||||
mu sync.Mutex
|
||||
hits map[string]authRateHit
|
||||
limit int
|
||||
window time.Duration
|
||||
maxClients int
|
||||
nextSweep time.Time
|
||||
}
|
||||
|
||||
func newAuthRateLimiter() *authRateLimiter {
|
||||
limit := 60
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 {
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 && n <= 1000 {
|
||||
limit = n
|
||||
}
|
||||
return &authRateLimiter{
|
||||
hits: make(map[string][]time.Time),
|
||||
limit: limit,
|
||||
window: time.Minute,
|
||||
hits: make(map[string]authRateHit),
|
||||
limit: limit,
|
||||
window: time.Minute,
|
||||
maxClients: maxAuthRateLimitClients,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,23 +43,40 @@ func (l *authRateLimiter) allow(ip string) bool {
|
||||
if l == nil {
|
||||
return true
|
||||
}
|
||||
now := time.Now()
|
||||
return l.allowAt(ip, time.Now())
|
||||
}
|
||||
|
||||
func (l *authRateLimiter) allowAt(ip string, now time.Time) bool {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
cut := now.Add(-l.window)
|
||||
arr := l.hits[ip]
|
||||
kept := arr[:0]
|
||||
for _, t := range arr {
|
||||
if t.After(cut) {
|
||||
kept = append(kept, t)
|
||||
if !l.nextSweep.IsZero() && !now.Before(l.nextSweep) {
|
||||
l.nextSweep = time.Time{}
|
||||
for key, hit := range l.hits {
|
||||
expiry := hit.start.Add(l.window)
|
||||
if !now.Before(expiry) {
|
||||
delete(l.hits, key)
|
||||
} else if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) {
|
||||
l.nextSweep = expiry
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(kept) >= l.limit {
|
||||
l.hits[ip] = kept
|
||||
hit, exists := l.hits[ip]
|
||||
if !exists && len(l.hits) >= l.maxClients {
|
||||
// Never evict an active counter: a new identity must wait for expiry.
|
||||
return false
|
||||
}
|
||||
kept = append(kept, now)
|
||||
l.hits[ip] = kept
|
||||
if exists && hit.count >= l.limit {
|
||||
return false
|
||||
}
|
||||
if !exists {
|
||||
hit.start = now
|
||||
expiry := now.Add(l.window)
|
||||
if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) {
|
||||
l.nextSweep = expiry
|
||||
}
|
||||
}
|
||||
hit.count++
|
||||
l.hits[ip] = hit
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const requestBodyLimit int64 = 8 << 20
|
||||
|
||||
// withRequestBodyLimit must run inside authMiddleware. Reading the complete
|
||||
// bounded body before dispatch prevents a valid JSON prefix from committing a
|
||||
// mutation before a later size/read error is discovered. Content-Type is not a
|
||||
// security boundary; CSV retains its additional row/field checks in its handler.
|
||||
func withRequestBodyLimit(next http.Handler) http.Handler {
|
||||
return withBodyLimitAndConcurrency(requestBodyLimit, requestBodyConcurrency(), next)
|
||||
}
|
||||
|
||||
func withBodyLimit(limit int64, next http.Handler) http.Handler {
|
||||
return withBodyLimitAndConcurrency(limit, 4, next)
|
||||
}
|
||||
|
||||
func requestBodyConcurrency() int {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_API_BODY_MAX_CONCURRENT")))
|
||||
if err == nil && n >= 1 && n <= 64 {
|
||||
return n
|
||||
}
|
||||
return 4
|
||||
}
|
||||
|
||||
func withBodyLimitAndConcurrency(limit int64, concurrency int, next http.Handler) http.Handler {
|
||||
slots := make(chan struct{}, concurrency)
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
original := r.Body
|
||||
if original != nil {
|
||||
defer func() { _ = original.Close() }()
|
||||
}
|
||||
if r.ContentLength > limit {
|
||||
writeRequestBodyTooLarge(w)
|
||||
return
|
||||
}
|
||||
if r.Context().Err() != nil {
|
||||
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled")
|
||||
return
|
||||
}
|
||||
if r.Body == nil || r.Body == http.NoBody {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
defer func() { <-slots }()
|
||||
default:
|
||||
w.Header().Set("Retry-After", "1")
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "request body processing capacity is full")
|
||||
return
|
||||
}
|
||||
bounded := http.MaxBytesReader(w, r.Body, limit)
|
||||
data, err := io.ReadAll(requestContextReader{ctx: r.Context(), reader: bounded})
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
writeRequestBodyTooLarge(w)
|
||||
return
|
||||
}
|
||||
var timeout net.Error
|
||||
if r.Context().Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timeout) && timeout.Timeout()) {
|
||||
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusBadRequest, "Bad Request", "could not read request body")
|
||||
return
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewReader(data))
|
||||
r.ContentLength = int64(len(data))
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func writeRequestBodyTooLarge(w http.ResponseWriter) {
|
||||
writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "request body exceeds the 8 MiB limit")
|
||||
}
|
||||
|
||||
type requestContextReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (r requestContextReader) Read(p []byte) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.reader.Read(p)
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRequestBodyKnownAndUnknownBoundsBeforeMutation(t *testing.T) {
|
||||
const limit = 32
|
||||
for _, unknown := range []bool{false, true} {
|
||||
for _, contentType := range []string{"", "application/json", "text/plain"} {
|
||||
for _, extra := range []int{0, 1} {
|
||||
label := "known"
|
||||
if unknown {
|
||||
label = "unknown"
|
||||
}
|
||||
t.Run(label+"/"+contentType+"/"+string(rune('0'+extra)), func(t *testing.T) {
|
||||
mutations := 0
|
||||
h := withBodyLimit(limit, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var input map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
mutations++
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}))
|
||||
body := `{"x":1}` + strings.Repeat(" ", limit-len(`{"x":1}`)+extra)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
|
||||
if unknown {
|
||||
req.ContentLength = -1
|
||||
}
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if extra == 0 {
|
||||
if w.Code != http.StatusCreated || mutations != 1 {
|
||||
t.Fatalf("exact limit rejected: %d mutations=%d", w.Code, mutations)
|
||||
}
|
||||
} else if w.Code != http.StatusRequestEntityTooLarge || mutations != 0 {
|
||||
t.Fatalf("oversized tail mutated: %d mutations=%d", w.Code, mutations)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type checkedRequestBody struct {
|
||||
reads int
|
||||
closed bool
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (b *checkedRequestBody) Read(p []byte) (int, error) { b.reads++; return b.reader.Read(p) }
|
||||
func (b *checkedRequestBody) Close() error { b.closed = true; return nil }
|
||||
|
||||
type failingRequestReader struct{}
|
||||
|
||||
func (failingRequestReader) Read([]byte) (int, error) {
|
||||
return 0, errors.New("untrusted detailed read failure")
|
||||
}
|
||||
|
||||
func TestRequestBodyErrorsDoNotMutate(t *testing.T) {
|
||||
for _, cancelled := range []bool{false, true} {
|
||||
calls := 0
|
||||
b := &checkedRequestBody{reader: io.MultiReader(strings.NewReader(`{"x":1}`), failingRequestReader{})}
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
|
||||
req.Body = b
|
||||
req.ContentLength = -1
|
||||
if cancelled {
|
||||
ctx, cancel := context.WithCancel(req.Context())
|
||||
cancel()
|
||||
req = req.WithContext(ctx)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { calls++ })).ServeHTTP(w, req)
|
||||
want := http.StatusBadRequest
|
||||
if cancelled {
|
||||
want = http.StatusRequestTimeout
|
||||
}
|
||||
if w.Code != want || calls != 0 || !b.closed || strings.Contains(w.Body.String(), "untrusted detailed") {
|
||||
t.Fatalf("error handling status=%d calls=%d body=%s", w.Code, calls, w.Body.String())
|
||||
}
|
||||
if cancelled && b.reads != 0 {
|
||||
t.Fatal("read body after cancellation")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAuthenticationRunsFirst(t *testing.T) {
|
||||
b := &checkedRequestBody{reader: failingRequestReader{}}
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
|
||||
req.Body = b
|
||||
req.ContentLength = requestBodyLimit + 1
|
||||
s := &Server{}
|
||||
called := false
|
||||
h := s.authMiddleware(withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusUnauthorized || b.reads != 0 || called {
|
||||
t.Fatalf("body processed before auth: %d reads=%d", w.Code, b.reads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAppliesToGETAndIgnoresClaimedZeroLength(t *testing.T) {
|
||||
for _, length := range []int64{-1, 0, requestBodyLimit + 1} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/jobs", strings.NewReader(strings.Repeat("x", int(requestBodyLimit)+1)))
|
||||
req.ContentLength = length
|
||||
called := false
|
||||
w := httptest.NewRecorder()
|
||||
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })).ServeHTTP(w, req)
|
||||
if w.Code != http.StatusRequestEntityTooLarge || called {
|
||||
t.Fatalf("GET size bypass length=%d code=%d", length, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyHandlerWiringAndNoMutation(t *testing.T) {
|
||||
s, err := New(Options{SeedDemo: true, InsecureDev: true, BundleSeedHex: testBundleSeed})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
tenant, _, _, _, _ := s.store.DemoIDs()
|
||||
before := len(s.store.ListModules(tenant))
|
||||
h := s.Handler()
|
||||
prefix := `{"type":"IP_RANGES","name":"bounded-create","enabled":false}`
|
||||
for _, extra := range []int{1, 0} {
|
||||
body := prefix + strings.Repeat(" ", int(requestBodyLimit)-len(prefix)+extra)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
|
||||
req.ContentLength = -1
|
||||
req.Header.Set("Authorization", "Bearer dev")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if extra == 1 {
|
||||
if w.Code != http.StatusRequestEntityTooLarge || len(s.store.ListModules(tenant)) != before {
|
||||
t.Fatalf("oversized JSON committed: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
} else if w.Code != http.StatusCreated || len(s.store.ListModules(tenant)) != before+1 {
|
||||
t.Fatalf("8MiB body rejected: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyLeavesStreamingResponseUnbuffered(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/runtime-logs/stream", nil)
|
||||
withRequestBodyLimit(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
t.Fatal("response lost Flusher")
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = io.WriteString(w, "data: ready\n\n")
|
||||
f.Flush()
|
||||
})).ServeHTTP(w, req)
|
||||
if !w.Flushed || w.Body.String() != "data: ready\n\n" {
|
||||
t.Fatal("stream response buffered")
|
||||
}
|
||||
}
|
||||
|
||||
type blockedAdmissionBody struct {
|
||||
entered chan struct{}
|
||||
resume chan struct{}
|
||||
sent bool
|
||||
}
|
||||
|
||||
func (b *blockedAdmissionBody) Read(p []byte) (int, error) {
|
||||
if b.sent {
|
||||
return 0, io.EOF
|
||||
}
|
||||
b.sent = true
|
||||
close(b.entered)
|
||||
<-b.resume
|
||||
p[0] = 'x'
|
||||
return 1, nil
|
||||
}
|
||||
func (*blockedAdmissionBody) Close() error { return nil }
|
||||
|
||||
func TestRequestBodyAdmissionRejectsWithoutWaitingAndHoldsThroughHandler(t *testing.T) {
|
||||
readEntered, readResume := make(chan struct{}), make(chan struct{})
|
||||
handlerEntered, handlerResume := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/empty" {
|
||||
once.Do(func() { close(handlerEntered) })
|
||||
<-handlerResume
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
first := httptest.NewRequest(http.MethodPost, "/first", nil)
|
||||
first.Body = &blockedAdmissionBody{entered: readEntered, resume: readResume}
|
||||
first.ContentLength = 0 // A claimed zero must not bypass body admission.
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); h.ServeHTTP(httptest.NewRecorder(), first) }()
|
||||
select {
|
||||
case <-readEntered:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("first body did not start")
|
||||
}
|
||||
reject := func() {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/second", strings.NewReader("x"))
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusTooManyRequests || w.Header().Get("Retry-After") != "1" {
|
||||
t.Fatalf("admission did not reject: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
reject()
|
||||
// An empty legacy GET remains available while all body slots are occupied.
|
||||
empty := httptest.NewRecorder()
|
||||
h.ServeHTTP(empty, httptest.NewRequest(http.MethodGet, "/empty", nil))
|
||||
if empty.Code != http.StatusNoContent {
|
||||
t.Fatal("empty GET consumed body slot")
|
||||
}
|
||||
close(readResume)
|
||||
select {
|
||||
case <-handlerEntered:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("handler did not start")
|
||||
}
|
||||
reject() // The fully read buffer is still retained by the running handler.
|
||||
close(handlerResume)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("first request did not finish")
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/third", strings.NewReader("x")))
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Fatalf("body slot not released: %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAdmissionReleasesOnReadError(t *testing.T) {
|
||||
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("read error reached handler") }))
|
||||
for range 2 {
|
||||
req := httptest.NewRequest(http.MethodPost, "/bad", nil)
|
||||
req.Body = io.NopCloser(failingRequestReader{})
|
||||
req.ContentLength = -1
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("read failure retained slot: %d", w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAdmissionEnvironment(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
raw string
|
||||
want int
|
||||
}{{"", 4}, {"1", 1}, {"64", 64}, {"65", 4}, {"0", 4}, {"-1", 4}, {"invalid", 4}} {
|
||||
t.Run(tc.raw, func(t *testing.T) {
|
||||
t.Setenv("EVOBGP_API_BODY_MAX_CONCURRENT", tc.raw)
|
||||
if got := requestBodyConcurrency(); got != tc.want {
|
||||
t.Fatalf("concurrency=%d want%d", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+147
-79
@@ -8,14 +8,12 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/bundle"
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/observability"
|
||||
"evobgp/internal/pipeline"
|
||||
@@ -39,7 +37,7 @@ func (s *Server) Handler() http.Handler {
|
||||
// Firewall subsystem moved to the standalone EvoFirewall service; see docs/firewall.md.
|
||||
// Registered on the public mux so it wins over the "/v1/" subtree below regardless of auth.
|
||||
s.mux.HandleFunc("/v1/firewall/", s.handleFirewallGone)
|
||||
s.mux.Handle("/v1/", s.authMiddleware(wrappedV1))
|
||||
s.mux.Handle("/v1/", s.authMiddleware(withRequestBodyLimit(wrappedV1)))
|
||||
return s.withCORS(observability.HTTPMiddleware(s.mux))
|
||||
}
|
||||
|
||||
@@ -60,6 +58,7 @@ func (s *Server) registerV1(m *http.ServeMux) {
|
||||
m.HandleFunc("GET /modules/{module_id}", s.handleGetModule)
|
||||
m.HandleFunc("GET /peers", s.handleListPeers)
|
||||
m.HandleFunc("GET /speakers", s.handleListSpeakers)
|
||||
m.HandleFunc("POST /telemetry/refresh", s.handleTelemetryRefresh)
|
||||
m.HandleFunc("GET /bundle/signing-public-key", s.handleBundleSigningPublicKey)
|
||||
m.HandleFunc("POST /modules/{module_id}/refresh", s.handleModuleRefresh)
|
||||
m.HandleFunc("POST /tenant/refresh", s.handleTenantRefresh)
|
||||
@@ -79,6 +78,7 @@ func (s *Server) registerV1(m *http.ServeMux) {
|
||||
m.HandleFunc("GET /jobs/{job_id}", s.handleGetJob)
|
||||
m.HandleFunc("GET /jobs/{job_id}/report", s.handleGetJobReport)
|
||||
m.HandleFunc("POST /jobs/{job_id}/cancel", s.handleCancelJob)
|
||||
m.HandleFunc("POST /jobs/{job_id}/retry", s.handleRetryJob)
|
||||
m.HandleFunc("GET /speakers/{speaker_id}/revisions/latest", s.handleNodeLatestRevision)
|
||||
m.HandleFunc("GET /speakers/{speaker_id}/bundle/{revision_id}", s.handleNodeBundle)
|
||||
m.HandleFunc("POST /nodes/enroll", s.handleNodeEnroll)
|
||||
@@ -295,12 +295,10 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) {
|
||||
allPeers := s.store.ListPeers(a.TenantID)
|
||||
allPeers = store.FilterOwned(allPeers, func(p *store.BGPPeer) string { return p.CreatedByUserID }, a.Kind, a.IsAdmin, a.UserID)
|
||||
page, next, more := store.PaginateOffset(allPeers, r.URL.Query().Get("cursor"), parseListLimit(r))
|
||||
fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1")
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second)
|
||||
defer cancel()
|
||||
liveViews := s.collectSpeakerBGPLive(ctx, a.TenantID, fresh)
|
||||
if fresh {
|
||||
s.syncPeerDiscoveriesFromLive(a.TenantID, liveViews)
|
||||
liveViews, _, err := s.loadTelemetry(r.Context(), a.TenantID, s.store.ListSpeakersForTenant(a.TenantID))
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable")
|
||||
return
|
||||
}
|
||||
items := make([]map[string]any, 0, len(page))
|
||||
for _, p := range page {
|
||||
@@ -316,30 +314,6 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) liveBGPProtocolStates(r *http.Request) map[string]string {
|
||||
if r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1") {
|
||||
return s.liveBGPProtocolStatesFresh(r.Context())
|
||||
}
|
||||
if cached, ok := observability.CachedBirdProtocolStates(90 * time.Second); ok {
|
||||
return cached
|
||||
}
|
||||
return s.liveBGPProtocolStatesFresh(r.Context())
|
||||
}
|
||||
|
||||
func (s *Server) liveBGPProtocolStatesFresh(ctx context.Context) map[string]string {
|
||||
sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET"))
|
||||
if sock == "" {
|
||||
return map[string]string{}
|
||||
}
|
||||
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")))
|
||||
if err != nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
states := birdfmt.ParseBGPProtocolStates(out)
|
||||
observability.SetBirdProtocolStates(states)
|
||||
return states
|
||||
}
|
||||
|
||||
// parseBGPProtocolStates parses `birdc show protocols all` summary rows into protocol_name -> state.
|
||||
func parseBGPProtocolStates(output string) map[string]string {
|
||||
out := make(map[string]string)
|
||||
@@ -403,9 +377,12 @@ func (s *Server) handleListSpeakers(w http.ResponseWriter, r *http.Request) {
|
||||
fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1")
|
||||
var liveByID map[string]map[string]any
|
||||
if fresh {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second)
|
||||
defer cancel()
|
||||
liveByID = s.collectSpeakerLiveStatus(ctx, a.TenantID, true, speakers)
|
||||
var err error
|
||||
_, liveByID, err = s.loadTelemetry(r.Context(), a.TenantID, speakers)
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable")
|
||||
return
|
||||
}
|
||||
}
|
||||
items := make([]map[string]any, 0, len(speakers))
|
||||
for _, sp := range speakers {
|
||||
@@ -448,12 +425,12 @@ func (s *Server) handleModuleRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
idemPtr = &idem
|
||||
}
|
||||
mid := mod.ID
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{
|
||||
"module_id": moduleID,
|
||||
"trigger": "api",
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -480,6 +457,17 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
moduleIDs := body.ModuleIDs
|
||||
sort.Strings(body.ModuleIDs)
|
||||
requestIDs := append([]string{}, body.ModuleIDs...)
|
||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||
var idemPtr *string
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
request := map[string]any{"module_ids": requestIDs, "trigger": "api"}
|
||||
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindTenantRefresh, IdempotencyKey: idemPtr, Request: request}) {
|
||||
return
|
||||
}
|
||||
if len(moduleIDs) == 0 {
|
||||
now := time.Now().UTC()
|
||||
for _, mod := range s.store.ListModules(a.TenantID) {
|
||||
@@ -488,21 +476,16 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(moduleIDs) == 0 {
|
||||
if len(moduleIDs) == 0 && idem == "" {
|
||||
writeJSON(w, http.StatusNoContent, map[string]any{"message": "no modules due for refresh"})
|
||||
return
|
||||
}
|
||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||
var idemPtr *string
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueRequestContext(r.Context(), a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{
|
||||
"module_ids": moduleIDs,
|
||||
"trigger": "api",
|
||||
})
|
||||
}, request)
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -510,7 +493,7 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"job_id": snap["job_id"],
|
||||
"status": snap["status"],
|
||||
"module_ids": moduleIDs,
|
||||
"module_ids": j.Snapshot()["meta"].(map[string]any)["module_ids"],
|
||||
"modules_due": len(moduleIDs),
|
||||
})
|
||||
}
|
||||
@@ -560,8 +543,8 @@ func revisionJSON(rev *store.Revision) map[string]any {
|
||||
return m
|
||||
}
|
||||
|
||||
// enqueueModuleRefreshIfEnabled queues module_refresh when the module exists and is enabled (best-effort, no HTTP error).
|
||||
func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger string) {
|
||||
// enqueueModuleRefreshIfEnabled reports scheduling failure after a committed CRUD mutation.
|
||||
func (s *Server) enqueueModuleRefreshIfEnabled(w http.ResponseWriter, r *http.Request, tenantID, moduleID, trigger string) {
|
||||
if s.jobs == nil {
|
||||
return
|
||||
}
|
||||
@@ -570,22 +553,35 @@ func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger strin
|
||||
return
|
||||
}
|
||||
mid := moduleID
|
||||
key := "module_refresh:" + moduleID
|
||||
_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindModuleRefresh, &key, &mid, map[string]any{
|
||||
// Each mutation is a new logical refresh. Reusing a module-wide key would
|
||||
// incorrectly replay an older terminal job after later configuration changes.
|
||||
_, _, err = s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindModuleRefresh, nil, &mid, map[string]any{
|
||||
"module_id": moduleID,
|
||||
"trigger": trigger,
|
||||
})
|
||||
s.reportAutomaticEnqueue(w, r, trigger, moduleID, err)
|
||||
}
|
||||
|
||||
// enqueuePeerReconcile queues fast peer-only reconcile/render (best-effort, no HTTP error).
|
||||
func (s *Server) enqueuePeerReconcile(tenantID, trigger string) {
|
||||
func (s *Server) enqueuePeerReconcile(w http.ResponseWriter, r *http.Request, tenantID, trigger string) {
|
||||
if s.jobs == nil {
|
||||
return
|
||||
}
|
||||
_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{
|
||||
_, _, err := s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{
|
||||
"trigger": trigger,
|
||||
"job_title": "Обновление BGP пиров",
|
||||
})
|
||||
s.reportAutomaticEnqueue(w, r, trigger, "", err)
|
||||
}
|
||||
|
||||
func (s *Server) reportAutomaticEnqueue(w http.ResponseWriter, r *http.Request, trigger, target string, err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-EvoBGP-Job-Status", "enqueue-failed")
|
||||
if a, ok := authFromContext(r.Context()); ok {
|
||||
s.recordCRUDAudit(r, a, "bgp.job.enqueue_failed", "Configuration saved; automatic job was not queued", target, map[string]any{"trigger": trigger, "error": err.Error()})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleGetRevision(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -614,6 +610,11 @@ func (s *Server) handleRevisionPreview(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.requirePerm(w, a, "bgp:operations:read") {
|
||||
return
|
||||
}
|
||||
release, ok := s.acquireOperation(w, r, a.TenantID, revisionPreviewOperation)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
rev, err := s.store.GetRevision(a.TenantID, r.PathValue("revision_id"))
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
||||
@@ -706,11 +707,11 @@ func (s *Server) handleRevisionRollback(w http.ResponseWriter, r *http.Request)
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{
|
||||
"source_revision_id": revID,
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -751,13 +752,13 @@ func (s *Server) handleApply(w http.ResponseWriter, r *http.Request) {
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
||||
"revision_id": revID,
|
||||
"strategy": body.Strategy,
|
||||
"trigger": "api",
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -802,13 +803,13 @@ func (s *Server) handleSpeakerApply(w http.ResponseWriter, r *http.Request) {
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
||||
"revision_id": revID,
|
||||
"speaker_id": spkID,
|
||||
"trigger": "api",
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -830,9 +831,9 @@ func (s *Server) handleBirdReload(w http.ResponseWriter, r *http.Request) {
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil)
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil)
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -867,7 +868,11 @@ func (s *Server) handleListJobs(w http.ResponseWriter, r *http.Request) {
|
||||
status := r.URL.Query().Get("status")
|
||||
kind := r.URL.Query().Get("kind")
|
||||
cursor := r.URL.Query().Get("cursor")
|
||||
list, next, more := s.jobs.List(a.TenantID, status, kind, cursor, limit)
|
||||
list, next, more, err := s.jobs.ListContext(r.Context(), a.TenantID, status, kind, cursor, limit)
|
||||
if err != nil {
|
||||
writeInternalError(w, "jobs_list", err)
|
||||
return
|
||||
}
|
||||
items := make([]map[string]any, 0, len(list))
|
||||
for _, j := range list {
|
||||
items = append(items, j.Snapshot())
|
||||
@@ -886,9 +891,9 @@ func (s *Server) handleGetJob(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.requirePerm(w, a, "bgp:operations:read") {
|
||||
return
|
||||
}
|
||||
j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id"))
|
||||
j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
||||
writeJobReadError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, j.Snapshot())
|
||||
@@ -903,9 +908,9 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.requirePerm(w, a, "bgp:operations:read") {
|
||||
return
|
||||
}
|
||||
j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id"))
|
||||
j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
||||
writeJobReadError(w, err)
|
||||
return
|
||||
}
|
||||
snap := j.Snapshot()
|
||||
@@ -929,6 +934,35 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) {
|
||||
out["revision_id"] = v
|
||||
}
|
||||
}
|
||||
sourceJobID := ""
|
||||
if source, ok := meta["source_deployment_job_id"].(string); ok && source != "" {
|
||||
// Resolve through the tenant-scoped registry before exposing linked progress.
|
||||
if _, err := s.jobs.GetContext(r.Context(), a.TenantID, source); err != nil {
|
||||
writeJobReadError(w, err)
|
||||
return
|
||||
}
|
||||
sourceJobID = source
|
||||
out["source_deployment_job_id"] = source
|
||||
}
|
||||
rows, err := s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, j.ID)
|
||||
if err == nil && len(rows) == 0 && sourceJobID != "" {
|
||||
rows, err = s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, sourceJobID)
|
||||
}
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "deployment progress unavailable")
|
||||
return
|
||||
}
|
||||
progress := make([]map[string]any, 0, len(rows))
|
||||
for _, d := range rows {
|
||||
progress = append(progress, map[string]any{
|
||||
"job_id": d.JobID, "speaker_id": d.SpeakerID, "revision_id": d.RevisionID,
|
||||
"generation": d.Generation, "state": d.State, "confirmation": d.Confirmation,
|
||||
"error": d.Error, "dispatch_attempts": d.DispatchAttempts, "retry_round": d.RetryRound,
|
||||
"health_received_at": d.HealthReceivedAt, "health_revision_id": d.HealthRevisionID,
|
||||
"health_source_at": d.HealthSourceAt, "health_error": d.HealthError,
|
||||
})
|
||||
}
|
||||
out["deployment_progress"] = progress
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
@@ -941,9 +975,34 @@ func (s *Server) handleCancelJob(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.requirePerm(w, a, "bgp:operations:admin") {
|
||||
return
|
||||
}
|
||||
j, err := s.jobs.RequestCancel(a.TenantID, r.PathValue("job_id"))
|
||||
j, err := s.jobs.RequestCancelContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
||||
writeJobReadError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, j.Snapshot())
|
||||
}
|
||||
|
||||
func (s *Server) handleRetryJob(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok {
|
||||
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
||||
return
|
||||
}
|
||||
if !s.requirePerm(w, a, "bgp:operations:admin") {
|
||||
return
|
||||
}
|
||||
j, err := s.jobs.RetryContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrJobQueueFull) {
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrJobConflict) {
|
||||
writeProblem(w, http.StatusConflict, "Conflict", "job is not failed")
|
||||
return
|
||||
}
|
||||
writeJobReadError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, j.Snapshot())
|
||||
@@ -998,21 +1057,30 @@ func (s *Server) handleNodeBundle(w http.ResponseWriter, r *http.Request) {
|
||||
writeProblem(w, http.StatusForbidden, "Forbidden", "speaker not in tenant scope")
|
||||
return
|
||||
}
|
||||
rev, err := s.store.GetRevision(a.TenantID, rid)
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
||||
return
|
||||
}
|
||||
frags := rev.PreviewFragments
|
||||
overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, a.TenantID, sid, rid, frags)
|
||||
overlayFP, err := nodeBundleOverlayFingerprint(s.store, a.TenantID, sid, rid)
|
||||
if err != nil {
|
||||
writeInternalError(w, "bundle overlay", err)
|
||||
return
|
||||
}
|
||||
frags = overlaid
|
||||
tgz, err := bundle.BuildGzippedTar(rid, sid, frags, s.bundlePriv)
|
||||
// Finished gzip, including manifest and bundle.sig. Revision bytes are immutable;
|
||||
// the overlay fingerprint keeps speaker-specific renders from colliding.
|
||||
tgz, err := s.bundleCache.do(r.Context(), nodeBundleCacheKey(a.TenantID, rid, sid, overlayFP), func(ctx context.Context) ([]byte, bool, error) {
|
||||
return s.loadNodeBundle(ctx, a.TenantID, sid, rid, overlayFP)
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
if errors.Is(err, errNodeBundleRevisionMissing) {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return
|
||||
}
|
||||
op := "internal"
|
||||
var opErr *nodeBundleOpError
|
||||
if errors.As(err, &opErr) && opErr.op != "" {
|
||||
op = opErr.op
|
||||
}
|
||||
writeInternalError(w, op, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
|
||||
@@ -3,7 +3,6 @@ package httpapi
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -132,23 +131,7 @@ func (s *Server) pushAuditToPortal(entry *store.AuditEntry) {
|
||||
}
|
||||
|
||||
func clientIP(r *http.Request) string {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
if xff := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); xff != "" {
|
||||
parts := strings.Split(xff, ",")
|
||||
if len(parts) > 0 {
|
||||
return strings.TrimSpace(parts[0])
|
||||
}
|
||||
}
|
||||
if xrip := strings.TrimSpace(r.Header.Get("X-Real-IP")); xrip != "" {
|
||||
return xrip
|
||||
}
|
||||
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
|
||||
if err != nil {
|
||||
return strings.TrimSpace(r.RemoteAddr)
|
||||
}
|
||||
return host
|
||||
return clientIPWithTrustedProxies(r, trustedProxyPrefixes())
|
||||
}
|
||||
|
||||
// initAuditPusher wires portal push when URL and secret are configured.
|
||||
|
||||
@@ -296,10 +296,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||
return
|
||||
}
|
||||
if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil {
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||
release, ok := s.acquireOperation(w, r, a.TenantID, cdnPreviewOperation)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id"))
|
||||
if err != nil {
|
||||
writeStoreErr(w, err)
|
||||
@@ -309,6 +310,10 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "module type must be CDN_CIDRS")
|
||||
return
|
||||
}
|
||||
if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil {
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||
return
|
||||
}
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "invalid url")
|
||||
@@ -321,11 +326,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
|
||||
writeBadGateway(w, "cdn preview fetch", fmt.Errorf("upstream status: %s", resp.Status))
|
||||
return
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
raw, err := pipeline.ReadCDNBody(resp.Body)
|
||||
if err != nil {
|
||||
writeBadGateway(w, "cdn preview read body", err)
|
||||
return
|
||||
@@ -379,7 +384,7 @@ func (s *Server) handlePostCDNSource(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.cdn_source.create", "Created CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID, "url": x.URL})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_create")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_create")
|
||||
writeJSON(w, http.StatusCreated, cdnSourceJSON(x))
|
||||
}
|
||||
|
||||
@@ -410,7 +415,7 @@ func (s *Server) handlePatchCDNSource(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.cdn_source.update", "Updated CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_patch")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_patch")
|
||||
writeJSON(w, http.StatusOK, cdnSourceJSON(x))
|
||||
}
|
||||
|
||||
@@ -426,7 +431,7 @@ func (s *Server) handleDeleteCDNSource(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.cdn_source.delete", "Deleted CDN source", sourceID, map[string]any{"module_id": mid, "source_id": sourceID})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_delete")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_delete")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -485,7 +490,7 @@ func (s *Server) handlePostAS(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.as_entry.create", "Created AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_create")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_create")
|
||||
writeJSON(w, http.StatusCreated, asEntryJSON(x))
|
||||
}
|
||||
|
||||
@@ -506,7 +511,7 @@ func (s *Server) handlePatchAS(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.as_entry.update", "Updated AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_patch")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_patch")
|
||||
writeJSON(w, http.StatusOK, asEntryJSON(x))
|
||||
}
|
||||
|
||||
@@ -522,7 +527,7 @@ func (s *Server) handleDeleteAS(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.as_entry.delete", "Deleted AS entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_delete")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_delete")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -566,7 +571,7 @@ func (s *Server) handlePostDomain(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.domain_entry.create", "Created domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_create")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_create")
|
||||
writeJSON(w, http.StatusCreated, domainEntryJSON(x))
|
||||
}
|
||||
|
||||
@@ -587,7 +592,7 @@ func (s *Server) handlePatchDomain(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.domain_entry.update", "Updated domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_patch")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_patch")
|
||||
writeJSON(w, http.StatusOK, domainEntryJSON(x))
|
||||
}
|
||||
|
||||
@@ -603,7 +608,7 @@ func (s *Server) handleDeleteDomain(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.domain_entry.delete", "Deleted domain entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_delete")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_delete")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -647,7 +652,7 @@ func (s *Server) handlePostIPRange(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.ip_range.create", "Created IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_create")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_create")
|
||||
writeJSON(w, http.StatusCreated, ipRangeJSON(x))
|
||||
}
|
||||
|
||||
@@ -668,7 +673,7 @@ func (s *Server) handlePatchIPRange(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.ip_range.update", "Updated IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_patch")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_patch")
|
||||
writeJSON(w, http.StatusOK, ipRangeJSON(x))
|
||||
}
|
||||
|
||||
@@ -684,7 +689,7 @@ func (s *Server) handleDeleteIPRange(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.ip_range.delete", "Deleted IP range entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_delete")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_delete")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -777,8 +782,12 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
moduleID := r.PathValue("module_id")
|
||||
res, err := importer.ImportModuleEntriesCSV(s.store, a.TenantID, moduleID, r.Body)
|
||||
res, err := importer.ImportModuleEntriesCSV(r.Context(), s.store, a.TenantID, moduleID, r.Body)
|
||||
if err != nil {
|
||||
if errors.Is(err, importer.ErrCSVTooLarge) {
|
||||
writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "csv exceeds 8 MiB, 10000 data rows or 64 fields per record; split the import into smaller requests")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrInvalidInput) {
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv is empty")
|
||||
return
|
||||
@@ -802,11 +811,11 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req
|
||||
if res.Imported > 0 {
|
||||
switch res.ModuleType {
|
||||
case "AS_PREFIXES":
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "as_entry_import_csv")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "as_entry_import_csv")
|
||||
case "DOMAINS":
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "domain_entry_import_csv")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "domain_entry_import_csv")
|
||||
case "IP_RANGES":
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "ip_range_import_csv")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "ip_range_import_csv")
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
@@ -1055,7 +1064,7 @@ func (s *Server) handlePostPeer(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.peer.create", "Created BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
|
||||
s.enqueuePeerReconcile(a.TenantID, "peer_create")
|
||||
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_create")
|
||||
writeJSON(w, http.StatusCreated, peerJSON(x))
|
||||
}
|
||||
|
||||
@@ -1099,7 +1108,7 @@ func (s *Server) handlePatchPeer(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.peer.update", "Updated BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
|
||||
s.enqueuePeerReconcile(a.TenantID, "peer_patch")
|
||||
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_patch")
|
||||
writeJSON(w, http.StatusOK, peerJSON(x))
|
||||
}
|
||||
|
||||
@@ -1120,7 +1129,7 @@ func (s *Server) handleDeletePeer(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.peer.delete", "Deleted BGP peer", peerID, map[string]any{"peer_id": peerID})
|
||||
s.enqueuePeerReconcile(a.TenantID, "peer_delete")
|
||||
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_delete")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -1295,7 +1304,7 @@ func (s *Server) handlePatchSettings(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
if peerDiscoverySettingsChanged(body) {
|
||||
s.enqueuePeerReconcile(a.TenantID, "peer_discovery_settings")
|
||||
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_settings")
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.settings.update", "Updated tenant settings", a.TenantID, map[string]any{"keys": settingsAuditKeys(body)})
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"io"
|
||||
@@ -8,8 +9,60 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/importer"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestCSVImportLimitsBeforeMutations(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
mod, err := m.CreateModule(tenant, &store.Module{Type: "AS_PREFIXES", Name: "csv-limits"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
module := mod.ID
|
||||
s := &Server{store: m}
|
||||
before, err := m.ListASEntries(tenant, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
header := "asn,community\n"
|
||||
validPrefix := header + "64512,\n"
|
||||
oversized := validPrefix + strings.Repeat("\n", importer.MaxCSVBytes-len(validPrefix)) + "64513,\n"
|
||||
for _, tc := range []struct {
|
||||
name, body string
|
||||
unknownLength bool
|
||||
status int
|
||||
}{
|
||||
{"exact", header + strings.Repeat("\n", importer.MaxCSVBytes-len(header)), false, http.StatusOK},
|
||||
{"oversize", oversized, false, http.StatusRequestEntityTooLarge},
|
||||
{"unknown_length", oversized, true, http.StatusRequestEntityTooLarge},
|
||||
{"rows", header + strings.Repeat("64512,\n", importer.MaxCSVRows+1), true, http.StatusRequestEntityTooLarge},
|
||||
{"fields", header + "64512," + strings.Repeat(",", importer.MaxCSVFields), false, http.StatusRequestEntityTooLarge},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/entries.csv", strings.NewReader(tc.body))
|
||||
if tc.unknownLength {
|
||||
req.ContentLength = -1
|
||||
req.TransferEncoding = []string{"chunked"}
|
||||
}
|
||||
req.SetPathValue("module_id", module)
|
||||
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"}))
|
||||
w := httptest.NewRecorder()
|
||||
s.handleImportModuleEntriesCSV(w, req)
|
||||
if w.Code != tc.status {
|
||||
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
after, e := m.ListASEntries(tenant, module)
|
||||
if e != nil || len(after) != len(before) {
|
||||
t.Fatalf("mutations on rejected/header-only request: count=%d err=%v", len(after), e)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleEntriesCSVImportExportIPRanges(t *testing.T) {
|
||||
srv, err := New(Options{
|
||||
InsecureDev: true,
|
||||
|
||||
@@ -24,6 +24,11 @@ func (s *Server) handleLookup(w http.ResponseWriter, r *http.Request) {
|
||||
writeProblem(w, http.StatusBadRequest, "Bad Request", "query parameter q is required")
|
||||
return
|
||||
}
|
||||
release, ok := s.acquireOperation(w, r, a.TenantID, lookupOperation)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
res, err := lookup.Lookup(r.Context(), s.store, a.TenantID, q)
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrInvalidInput) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -31,7 +32,7 @@ func TestLookupMembershipHTTP(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := srv.Store().SetModulePrefixSnapshot(tenant, modIP, "t", []store.PrefixRow{
|
||||
if err := srv.Store().SetModulePrefixSnapshot(context.Background(), tenant, modIP, "t", []store.PrefixRow{
|
||||
{Prefix: "198.51.100.0/24", CommunityID: &cid, Source: "ip_range"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -242,10 +242,6 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
kind := "maintenance_policy_run"
|
||||
if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) {
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
||||
return
|
||||
}
|
||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||
var idemPtr *string
|
||||
if idem != "" {
|
||||
@@ -255,11 +251,18 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request
|
||||
if dryRun {
|
||||
title = "Maintenance policy dry-run"
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
||||
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindMaintenancePolicyRun, IdempotencyKey: idemPtr, Meta: map[string]any{"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title}}) {
|
||||
return
|
||||
}
|
||||
if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) {
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
||||
return
|
||||
}
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
||||
"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title,
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "maintenance_policy_enqueue", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/pgmonitor"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -68,19 +69,22 @@ func (s *Server) enqueuePostgresMaint(w http.ResponseWriter, r *http.Request, a
|
||||
if !s.requirePostgres(w) || !s.requireOperatorStrict(w, a) {
|
||||
return
|
||||
}
|
||||
if !s.checkPgMaintRateLimit(a.TenantID, kind) {
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
||||
return
|
||||
}
|
||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||
var idemPtr *string
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
meta["actor_prefix"] = actorPrefix(a)
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, kind, idemPtr, nil, meta)
|
||||
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: kind, IdempotencyKey: idemPtr, Meta: meta}) {
|
||||
return
|
||||
}
|
||||
if !s.checkPgMaintRateLimit(a.TenantID, kind) {
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
||||
return
|
||||
}
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, kind, idemPtr, nil, meta)
|
||||
if err != nil {
|
||||
writeInternalError(w, "postgres_maint_enqueue", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -184,12 +188,12 @@ func (s *Server) handlePostgresCleanup(w http.ResponseWriter, r *http.Request) {
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
||||
"policy_id": policyID, "dry_run": body.DryRun, "actor_prefix": actorPrefix(a),
|
||||
"job_title": "PostgreSQL cleanup (deprecated path)",
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "postgres_maint_enqueue", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
|
||||
@@ -8,15 +8,17 @@ import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/audit"
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/logging"
|
||||
"evobgp/internal/maintenance"
|
||||
"evobgp/internal/pgmonitor"
|
||||
"evobgp/internal/repository"
|
||||
"evobgp/internal/runtimelogs"
|
||||
"evobgp/internal/store"
|
||||
"evobgp/internal/telemetry"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
@@ -30,7 +32,11 @@ type Server struct {
|
||||
maintStats *maintenance.DBStatsProvider
|
||||
jobs *jobs.Registry
|
||||
authLimiter *authRateLimiter
|
||||
operationOnce sync.Once
|
||||
operationLimiter *operationLimiter
|
||||
bundlePriv ed25519.PrivateKey
|
||||
bundleCache *nodeBundleCache
|
||||
nodeBundleBuild nodeBundleBuilder
|
||||
keyResolver *apiKeyResolver
|
||||
firewallResolver *firewallTokenResolver
|
||||
bundleSeedHex string
|
||||
@@ -39,6 +45,11 @@ type Server struct {
|
||||
runtimeLogs *runtimelogs.Service
|
||||
runtimeLogsPolicyTenant string
|
||||
mux *http.ServeMux
|
||||
backgroundMu sync.Mutex
|
||||
backgroundStarted bool
|
||||
backgroundClosed bool
|
||||
backgroundCancel context.CancelFunc
|
||||
telemetryCollector *telemetry.Collector
|
||||
|
||||
// Portal / dual-auth (JWT) configuration.
|
||||
jwtSecret string
|
||||
@@ -118,6 +129,7 @@ func New(opts Options) (*Server, error) {
|
||||
maintStats: maintStats,
|
||||
jobs: reg,
|
||||
bundlePriv: priv,
|
||||
bundleCache: newNodeBundleCache(nodeBundleCacheMaxEntries, nodeBundleCacheMaxBytes),
|
||||
keyResolver: resolver,
|
||||
firewallResolver: fwResolver,
|
||||
bundleSeedHex: strings.TrimSpace(opts.BundleSeedHex),
|
||||
@@ -143,6 +155,29 @@ func New(opts Options) (*Server, error) {
|
||||
|
||||
// Close releases database resources.
|
||||
func (s *Server) Close() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.backgroundMu.Lock()
|
||||
if s.backgroundClosed {
|
||||
s.backgroundMu.Unlock()
|
||||
return
|
||||
}
|
||||
s.backgroundClosed = true
|
||||
cancel := s.backgroundCancel
|
||||
collector := s.telemetryCollector
|
||||
s.backgroundMu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
if collector != nil {
|
||||
_ = collector.Close(context.Background())
|
||||
}
|
||||
// Do not close the pool while a worker is still persisting its final state.
|
||||
if s.jobs != nil {
|
||||
s.jobs.RequestCancelAll()
|
||||
_ = s.jobs.Drain(context.Background())
|
||||
}
|
||||
if s.pgPool != nil {
|
||||
s.pgPool.Close()
|
||||
}
|
||||
@@ -154,26 +189,44 @@ func (s *Server) Store() store.Backend { return s.store }
|
||||
// Jobs exposes the in-process async job registry (for scheduler / evobgp-all).
|
||||
func (s *Server) Jobs() *jobs.Registry { return s.jobs }
|
||||
|
||||
// StartBackground starts PostgreSQL monitoring and maintenance schedulers until ctx is cancelled.
|
||||
// StartBackground starts shared schedulers and telemetry polling once until ctx is cancelled.
|
||||
func (s *Server) StartBackground(ctx context.Context) {
|
||||
if s != nil && s.pgPool != nil {
|
||||
pgmonitor.StartScheduler(ctx, s.pgPool)
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.backgroundMu.Lock()
|
||||
defer s.backgroundMu.Unlock()
|
||||
if s.backgroundClosed || s.backgroundStarted {
|
||||
return
|
||||
}
|
||||
s.backgroundStarted = true
|
||||
backgroundCtx, cancel := context.WithCancel(ctx)
|
||||
s.backgroundCancel = cancel
|
||||
if s.store != nil {
|
||||
s.telemetryCollector = telemetry.Start(backgroundCtx, s.store, telemetry.ConfigFromEnv())
|
||||
}
|
||||
if s.pgPool != nil {
|
||||
pgmonitor.StartScheduler(backgroundCtx, s.pgPool)
|
||||
if s.jobs != nil {
|
||||
jobs.StartDurableQueueLoop(ctx, s.jobs, repository.NewJobAuditWriter(s.pgPool))
|
||||
jobs.StartDurableQueueLoop(backgroundCtx, s.jobs)
|
||||
jobs.StartDeploymentReconciliationLoop(backgroundCtx, s.jobs, s.store)
|
||||
}
|
||||
}
|
||||
if s != nil && s.maintConfig != nil && s.jobs != nil {
|
||||
maintenance.StartScheduler(ctx, s.maintConfig, func(policyID string, dryRun bool, idem string) {
|
||||
maintenance.StartScheduler(backgroundCtx, s.maintConfig, func(policyID string, dryRun bool, idem string) {
|
||||
key := idem
|
||||
_, _, _ = s.jobs.Enqueue("", jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{
|
||||
_, _, err := s.jobs.EnqueueContext(backgroundCtx, store.SystemJobTenantID, jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{
|
||||
"policy_id": policyID,
|
||||
"dry_run": dryRun,
|
||||
"trigger": "scheduler",
|
||||
})
|
||||
if err != nil {
|
||||
logging.With("component", "maintenance.scheduler").Error("enqueue failed", "policy_id", policyID, "err", err)
|
||||
}
|
||||
}, 30*time.Second)
|
||||
}
|
||||
if s != nil && s.runtimeLogs != nil && s.store != nil {
|
||||
runtimelogs.StartAutoCleanupScheduler(ctx, runtimelogs.SchedulerDeps{
|
||||
runtimelogs.StartAutoCleanupScheduler(backgroundCtx, runtimelogs.SchedulerDeps{
|
||||
Service: s.runtimeLogs,
|
||||
Store: s.store,
|
||||
PolicyTenant: s.runtimeLogsPolicyTenant,
|
||||
|
||||
@@ -6,7 +6,11 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -165,24 +169,65 @@ func publicControlPlaneURL(r *http.Request, override string) string {
|
||||
if s := strings.TrimSpace(override); s != "" {
|
||||
return strings.TrimRight(s, "/")
|
||||
}
|
||||
if origin := strings.TrimSpace(r.Header.Get("Origin")); strings.HasPrefix(origin, "http://") || strings.HasPrefix(origin, "https://") {
|
||||
return strings.TrimRight(origin, "/")
|
||||
}
|
||||
proto := strings.TrimSpace(r.Header.Get("X-Forwarded-Proto"))
|
||||
if proto == "" {
|
||||
proto = "https"
|
||||
}
|
||||
host := strings.TrimSpace(r.Header.Get("X-Forwarded-Host"))
|
||||
if i := strings.Index(host, ","); i >= 0 {
|
||||
host = strings.TrimSpace(host[:i])
|
||||
}
|
||||
if host == "" {
|
||||
host = strings.TrimSpace(r.Host)
|
||||
}
|
||||
if host == "" {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
return proto + "://" + host
|
||||
if peer, ok := peerAddress(r); ok && isTrustedProxy(peer, trustedProxyPrefixes()) {
|
||||
protos := r.Header.Values("X-Forwarded-Proto")
|
||||
hosts := r.Header.Values("X-Forwarded-Host")
|
||||
if len(protos) == 1 && len(hosts) == 1 {
|
||||
proto := strings.TrimSpace(protos[0])
|
||||
host := strings.TrimSpace(hosts[0])
|
||||
if (proto == "http" || proto == "https") && validControlPlaneHost(host) {
|
||||
return proto + "://" + host
|
||||
}
|
||||
}
|
||||
}
|
||||
if !validControlPlaneHost(r.Host) {
|
||||
return ""
|
||||
}
|
||||
proto := "http"
|
||||
if r.TLS != nil {
|
||||
proto = "https"
|
||||
}
|
||||
return proto + "://" + r.Host
|
||||
}
|
||||
|
||||
func validControlPlaneHost(host string) bool {
|
||||
if host == "" || strings.TrimSpace(host) != host || strings.HasSuffix(host, ":") || strings.ContainsAny(host, " ,/@?#\\\t\r\n") {
|
||||
return false
|
||||
}
|
||||
u, err := url.Parse("http://" + host)
|
||||
if err != nil || u.Host != host || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" {
|
||||
return false
|
||||
}
|
||||
name := u.Hostname()
|
||||
if name == "" {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(host, "[") {
|
||||
if _, err := netip.ParseAddr(name); err != nil {
|
||||
return false
|
||||
}
|
||||
} else if net.ParseIP(name) == nil {
|
||||
for _, label := range strings.Split(name, ".") {
|
||||
if label == "" || label[0] == '-' || label[len(label)-1] == '-' {
|
||||
return false
|
||||
}
|
||||
for _, c := range label {
|
||||
if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if port := u.Port(); port != "" {
|
||||
n, err := strconv.Atoi(port)
|
||||
if err != nil || n < 1 || n > 65535 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Server) attachReplicaInstall(resp map[string]any, sp *store.Speaker, nodeToken string, req speakerCreateRequest, r *http.Request) {
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
@@ -11,6 +9,34 @@ import (
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type agentHealthPoll struct {
|
||||
result nodedispatch.AgentHealthResult
|
||||
at time.Time
|
||||
lastSuccessAt time.Time
|
||||
observationOverride *liveObservation
|
||||
}
|
||||
|
||||
func agentHealthObservation(poll *agentHealthPoll) liveObservation {
|
||||
if poll == nil {
|
||||
return liveObservation{Source: "none", Quality: "unknown", ErrorCode: "not_polled"}
|
||||
}
|
||||
if poll.observationOverride != nil {
|
||||
return *poll.observationOverride
|
||||
}
|
||||
o := liveObservation{ObservedAt: liveTime(poll.at), ReceivedAt: liveTime(poll.at), LastSuccessAt: liveTime(poll.lastSuccessAt), Source: "agent_health", Quality: "ok"}
|
||||
if !poll.result.OK {
|
||||
o.Quality = "unreachable"
|
||||
if !poll.lastSuccessAt.IsZero() {
|
||||
o.Quality = "stale"
|
||||
}
|
||||
o.ErrorCode = liveErrorCode(poll.result.Error)
|
||||
if poll.result.Error == "agent reported ok=false" {
|
||||
o.ErrorCode = "reported_unhealthy"
|
||||
}
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) {
|
||||
total = len(sessions)
|
||||
for _, s := range sessions {
|
||||
@@ -22,29 +48,48 @@ func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) {
|
||||
}
|
||||
|
||||
func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *nodedispatch.AgentHealthResult) map[string]any {
|
||||
var poll *agentHealthPoll
|
||||
if health != nil {
|
||||
poll = &agentHealthPoll{result: *health}
|
||||
}
|
||||
return speakerLiveStatusWithPoll(sp, view, poll)
|
||||
}
|
||||
|
||||
func speakerLiveStatusWithPoll(sp *store.Speaker, view speakerBGPLive, health *agentHealthPoll) map[string]any {
|
||||
total, established := countBGPSessions(view.Sessions)
|
||||
bgpObservation := view.observation()
|
||||
agentObservation := agentHealthObservation(health)
|
||||
if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") && health == nil {
|
||||
agentObservation = bgpObservation
|
||||
}
|
||||
m := map[string]any{
|
||||
"label": view.Label,
|
||||
"bgp_poll_ok": view.Error == "",
|
||||
"bgp_sessions_total": total,
|
||||
"bgp_established": established,
|
||||
"bgp_poll_ok": view.successful(),
|
||||
"bgp_sessions_total": nil,
|
||||
"bgp_established": nil,
|
||||
"bgp_observation": bgpObservation,
|
||||
"agent_observation": agentObservation,
|
||||
}
|
||||
if view.successful() {
|
||||
m["bgp_sessions_total"] = total
|
||||
m["bgp_established"] = established
|
||||
}
|
||||
if view.Error != "" {
|
||||
m["bgp_poll_error"] = view.Error
|
||||
}
|
||||
if health != nil {
|
||||
m["agent_ok"] = health.OK
|
||||
if health.Error != "" {
|
||||
m["agent_error"] = health.Error
|
||||
m["agent_ok"] = health.result.OK
|
||||
if health.result.Error != "" {
|
||||
m["agent_error"] = health.result.Error
|
||||
}
|
||||
if health.LastSyncAt != "" {
|
||||
m["agent_last_sync_at"] = health.LastSyncAt
|
||||
if health.result.LastSyncAt != "" {
|
||||
m["agent_last_sync_at"] = health.result.LastSyncAt
|
||||
}
|
||||
if health.LastAppliedRevisionID != "" {
|
||||
m["agent_last_applied_revision_id"] = health.LastAppliedRevisionID
|
||||
if health.result.LastAppliedRevisionID != "" {
|
||||
m["agent_last_applied_revision_id"] = health.result.LastAppliedRevisionID
|
||||
}
|
||||
} else if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
|
||||
m["agent_ok"] = view.Error == ""
|
||||
m["agent_ok"] = view.successful()
|
||||
if view.Error != "" {
|
||||
m["agent_error"] = view.Error
|
||||
}
|
||||
@@ -52,7 +97,7 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded
|
||||
m["agent_ok"] = false
|
||||
m["agent_error"] = "agent health not polled"
|
||||
}
|
||||
if len(view.Sessions) > 0 {
|
||||
if view.successful() {
|
||||
sess := make([]map[string]any, 0, len(view.Sessions))
|
||||
for _, s := range view.Sessions {
|
||||
row := map[string]any{
|
||||
@@ -68,62 +113,3 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func (s *Server) collectSpeakerLiveStatus(ctx context.Context, tenantID string, fresh bool, speakers []*store.Speaker) map[string]map[string]any {
|
||||
views := s.collectSpeakerBGPLive(ctx, tenantID, fresh)
|
||||
viewByID := make(map[string]speakerBGPLive, len(views))
|
||||
for _, v := range views {
|
||||
if v.SpeakerID != "" {
|
||||
viewByID[v.SpeakerID] = v
|
||||
}
|
||||
}
|
||||
|
||||
opts := nodedispatch.Options{Timeout: 8 * time.Second}
|
||||
type healthWrap struct {
|
||||
id string
|
||||
h nodedispatch.AgentHealthResult
|
||||
}
|
||||
healthCh := make(chan healthWrap, len(speakers))
|
||||
var wg sync.WaitGroup
|
||||
for _, sp := range speakers {
|
||||
if sp == nil {
|
||||
continue
|
||||
}
|
||||
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
||||
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(speaker *store.Speaker) {
|
||||
defer wg.Done()
|
||||
healthCh <- healthWrap{
|
||||
id: speaker.ID,
|
||||
h: nodedispatch.FetchAgentHealth(ctx, speaker, opts),
|
||||
}
|
||||
}(sp)
|
||||
}
|
||||
wg.Wait()
|
||||
close(healthCh)
|
||||
healthByID := make(map[string]nodedispatch.AgentHealthResult, len(speakers))
|
||||
for hw := range healthCh {
|
||||
healthByID[hw.id] = hw.h
|
||||
}
|
||||
|
||||
out := make(map[string]map[string]any, len(speakers))
|
||||
for _, sp := range speakers {
|
||||
if sp == nil {
|
||||
continue
|
||||
}
|
||||
view, ok := viewByID[sp.ID]
|
||||
if !ok {
|
||||
view = speakerBGPLive{SpeakerID: sp.ID, Label: speakerDisplayLabel(sp)}
|
||||
}
|
||||
var hp *nodedispatch.AgentHealthResult
|
||||
if h, ok := healthByID[sp.ID]; ok {
|
||||
hCopy := h
|
||||
hp = &hCopy
|
||||
}
|
||||
out[sp.ID] = speakerLiveStatusJSON(sp, view, hp)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -2,12 +2,29 @@ package httpapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestSpeakerLiveStatusJSONEmptyAndUnpolled(t *testing.T) {
|
||||
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||
unpolled := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID}, nil)
|
||||
if unpolled["agent_ok"] != false || unpolled["bgp_poll_ok"] != false || unpolled["bgp_sessions_total"] != nil {
|
||||
t.Fatalf("unpolled=%#v", unpolled)
|
||||
}
|
||||
if unpolled["bgp_observation"].(liveObservation).Quality != "unknown" {
|
||||
t.Fatalf("unpolled observation=%#v", unpolled["bgp_observation"])
|
||||
}
|
||||
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
|
||||
empty := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID, Polled: true, ObservedAt: now, ReceivedAt: now, LastSuccessAt: now, Source: "cp_birdc"}, nil)
|
||||
if empty["agent_ok"] != true || empty["bgp_poll_ok"] != true || empty["bgp_sessions_total"] != 0 {
|
||||
t.Fatalf("empty=%#v", empty)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountBGPSessions(t *testing.T) {
|
||||
total, est := countBGPSessions([]birdfmt.BGPSession{
|
||||
{Name: "p1", State: "Established"},
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
"evobgp/internal/telemetry"
|
||||
)
|
||||
|
||||
func (s *Server) loadTelemetry(ctx context.Context, tenant string, speakers []*store.Speaker) ([]speakerBGPLive, map[string]map[string]any, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
snapshots, err := s.store.ListTelemetry(ctx, tenant)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
views, statuses := hydrateTelemetry(speakers, snapshots, time.Now(), telemetry.ConfigFromEnv().StaleAfter)
|
||||
return views, statuses, nil
|
||||
}
|
||||
|
||||
// snapshotObservation uses control-plane receipt time, never the remote clock.
|
||||
func snapshotObservation(o store.TelemetryObservation, now time.Time, staleAfter time.Duration, invalid string) liveObservation {
|
||||
v := liveObservation{ObservedAt: liveTime(o.ObservedAt), ReceivedAt: liveTime(o.ReceivedAt), LastSuccessAt: liveTime(o.LastSuccessAt), Source: o.Source, Quality: "unknown"}
|
||||
if !o.ReceivedAt.IsZero() {
|
||||
v.ExpiresAt = liveTime(o.ReceivedAt.Add(staleAfter))
|
||||
}
|
||||
if v.Source == "" {
|
||||
v.Source = "none"
|
||||
}
|
||||
switch {
|
||||
case invalid != "":
|
||||
v.ErrorCode = invalid
|
||||
case o.ReceivedAt.IsZero():
|
||||
v.ErrorCode = "not_polled"
|
||||
case o.ReceivedAt.After(now) || now.Sub(o.ReceivedAt) > staleAfter:
|
||||
v.Quality, v.ErrorCode = "stale", "snapshot_expired"
|
||||
case o.Error != "":
|
||||
v.Quality, v.ErrorCode = "unreachable", liveErrorCode(o.Error)
|
||||
if !o.LastSuccessAt.IsZero() {
|
||||
v.Quality = "stale"
|
||||
}
|
||||
default:
|
||||
v.Quality = "ok"
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func hydrateTelemetry(speakers []*store.Speaker, snapshots []store.TelemetrySnapshot, now time.Time, staleAfter time.Duration) ([]speakerBGPLive, map[string]map[string]any) {
|
||||
byID := make(map[string]*store.Speaker, len(speakers))
|
||||
for _, sp := range speakers {
|
||||
if sp != nil {
|
||||
byID[sp.ID] = sp
|
||||
}
|
||||
}
|
||||
grouped := make(map[string][]store.TelemetrySnapshot)
|
||||
for _, snapshot := range snapshots {
|
||||
if snapshot.SpeakerID != "" && byID[snapshot.SpeakerID] == nil {
|
||||
continue
|
||||
}
|
||||
if snapshot.SpeakerID == "" && snapshot.Kind != "local_bird" {
|
||||
continue
|
||||
}
|
||||
grouped[snapshot.SpeakerID] = append(grouped[snapshot.SpeakerID], snapshot)
|
||||
}
|
||||
views := make([]speakerBGPLive, 0, len(speakers)+1)
|
||||
statuses := make(map[string]map[string]any, len(speakers))
|
||||
appendView := func(id string, sp *store.Speaker) {
|
||||
view := speakerBGPLive{SpeakerID: id, Label: speakerDisplayLabel(sp)}
|
||||
if sp == nil {
|
||||
view.Label = "CP (local)"
|
||||
}
|
||||
var health *agentHealthPoll
|
||||
rows := grouped[id]
|
||||
// A restarted local collector may leave an expired authority behind.
|
||||
// Only simultaneously current authorities conflict; never prefer one
|
||||
// of two current BIRD nodes arbitrarily.
|
||||
if len(rows) > 1 {
|
||||
current := make([]store.TelemetrySnapshot, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if row.Kind == "local_bird" && row.ConfigFingerprint == store.TelemetrySpeakerFingerprint(sp) && !row.BGP.ReceivedAt.IsZero() && !row.BGP.ReceivedAt.After(now) && now.Sub(row.BGP.ReceivedAt) <= staleAfter {
|
||||
current = append(current, row)
|
||||
}
|
||||
}
|
||||
if len(current) > 0 {
|
||||
rows = current
|
||||
}
|
||||
}
|
||||
if len(rows) > 1 {
|
||||
view.Error = "ambiguous telemetry authorities"
|
||||
view.ObservationOverride = &liveObservation{Source: "none", Quality: "unknown", ErrorCode: "ambiguous_authority"}
|
||||
} else if len(rows) == 1 {
|
||||
row := rows[0]
|
||||
invalid := ""
|
||||
if row.ConfigFingerprint != store.TelemetrySpeakerFingerprint(sp) {
|
||||
invalid = "configuration_changed"
|
||||
}
|
||||
bgp := snapshotObservation(row.BGP.TelemetryObservation, now, staleAfter, invalid)
|
||||
view.ObservationOverride = &bgp
|
||||
view.Polled = !row.BGP.ReceivedAt.IsZero()
|
||||
view.ObservedAt, view.ReceivedAt, view.LastSuccessAt, view.Source = row.BGP.ObservedAt, row.BGP.ReceivedAt, row.BGP.LastSuccessAt, row.BGP.Source
|
||||
if bgp.Quality == "ok" {
|
||||
view.Sessions = row.BGP.Sessions
|
||||
} else {
|
||||
view.Error = bgp.ErrorCode
|
||||
}
|
||||
if row.Kind == "remote_agent" {
|
||||
h := snapshotObservation(row.Health.TelemetryObservation, now, staleAfter, invalid)
|
||||
result := nodedispatch.AgentHealthResult{OK: row.Health.OK && h.Quality == "ok", Error: row.Health.Error, LastSyncAt: row.Health.LastSyncAt, LastAppliedRevisionID: row.Health.LastAppliedRevisionID}
|
||||
if h.Quality != "ok" {
|
||||
result.Error = h.ErrorCode
|
||||
}
|
||||
if h.Quality == "ok" && !result.OK {
|
||||
h.Quality, h.ErrorCode = "unreachable", "reported_unhealthy"
|
||||
}
|
||||
health = &agentHealthPoll{result: result, observationOverride: &h}
|
||||
}
|
||||
}
|
||||
views = append(views, view)
|
||||
if sp != nil {
|
||||
statuses[id] = speakerLiveStatusWithPoll(sp, view, health)
|
||||
}
|
||||
}
|
||||
for _, sp := range speakers {
|
||||
if sp != nil {
|
||||
appendView(sp.ID, sp)
|
||||
}
|
||||
}
|
||||
if len(grouped[""]) > 0 {
|
||||
appendView("", nil)
|
||||
}
|
||||
return views, statuses
|
||||
}
|
||||
|
||||
func (s *Server) handleTelemetryRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok {
|
||||
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
||||
return
|
||||
}
|
||||
if !s.requirePerm(w, a, "bgp:network:read") {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
requested, err := s.store.RequestTelemetryRefresh(ctx, a.TenantID, 10*time.Second)
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry refresh unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"requested": requested})
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type telemetryHTTPStore struct {
|
||||
store.Backend
|
||||
discoveryWrites int
|
||||
snapshotErr error
|
||||
refreshCalls int
|
||||
}
|
||||
|
||||
func (s *telemetryHTTPStore) UpsertPeerDiscovery(tenant string, in *store.PeerDiscoveryUpsert) (*store.BGPPeerDiscovery, error) {
|
||||
s.discoveryWrites++
|
||||
return s.Backend.UpsertPeerDiscovery(tenant, in)
|
||||
}
|
||||
func (s *telemetryHTTPStore) ListTelemetry(ctx context.Context, tenant string) ([]store.TelemetrySnapshot, error) {
|
||||
if s.snapshotErr != nil {
|
||||
return nil, s.snapshotErr
|
||||
}
|
||||
return s.Backend.ListTelemetry(ctx, tenant)
|
||||
}
|
||||
func (s *telemetryHTTPStore) RequestTelemetryRefresh(ctx context.Context, tenant string, cooldown time.Duration) (bool, error) {
|
||||
s.refreshCalls++
|
||||
return s.Backend.RequestTelemetryRefresh(ctx, tenant, cooldown)
|
||||
}
|
||||
func telemetryRequest(method, path, tenant string) *http.Request {
|
||||
r := httptest.NewRequest(method, path, nil)
|
||||
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "viewer"}))
|
||||
}
|
||||
|
||||
func TestTelemetryGETsAndRefreshNeverPollAgents(t *testing.T) {
|
||||
var calls atomic.Int64
|
||||
agent := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
_, _ = w.Write([]byte(`{"ok":true,"sessions":[]}`))
|
||||
}))
|
||||
defer agent.Close()
|
||||
t.Setenv("EVOBGP_NODE_DISPATCH_INSECURE_TLS", "1")
|
||||
mem := store.NewMemory()
|
||||
mem.SeedDemo()
|
||||
tenant, _, _, _, _ := mem.DemoIDs()
|
||||
sp, err := mem.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: strings.TrimPrefix(agent.URL, "https://"), AgentSecret: "test"})})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Positive control: the legacy adapter reaches the fake configured agent.
|
||||
if result := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: agent.Client()}); !result.OK {
|
||||
t.Fatal(result)
|
||||
}
|
||||
calls.Store(0)
|
||||
st := &telemetryHTTPStore{Backend: mem}
|
||||
server := &Server{store: st}
|
||||
claims, err := mem.ClaimTelemetry(context.Background(), "collector", 1, time.Minute, "")
|
||||
if err != nil || len(claims) != 1 {
|
||||
t.Fatalf("claim: %v %v", claims, err)
|
||||
}
|
||||
for i := 0; i < 20; i++ {
|
||||
for _, tc := range []struct {
|
||||
path string
|
||||
handler http.HandlerFunc
|
||||
}{
|
||||
{"/v1/peers", server.handleListPeers}, {"/v1/peers?live=1", server.handleListPeers},
|
||||
{"/v1/speakers?live=1", server.handleListSpeakers}, {"/v1/peers/discovered?status=all", server.handleListPeerDiscoveries},
|
||||
} {
|
||||
w := httptest.NewRecorder()
|
||||
tc.handler(w, telemetryRequest(http.MethodGet, tc.path, tenant))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
server.handleTelemetryRefresh(w, telemetryRequest(http.MethodPost, "/v1/telemetry/refresh", tenant))
|
||||
if w.Code != http.StatusAccepted {
|
||||
t.Fatalf("refresh: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
want := `"requested":false`
|
||||
if i == 0 {
|
||||
want = `"requested":true`
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), want) {
|
||||
t.Fatalf("refresh coalescing: %s", w.Body.String())
|
||||
}
|
||||
}
|
||||
if calls.Load() != 0 || st.discoveryWrites != 0 {
|
||||
t.Fatalf("GET/refresh caused IO: agent=%d discovery=%d", calls.Load(), st.discoveryWrites)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryReadFailureAndRefreshPermission(t *testing.T) {
|
||||
mem := store.NewMemory()
|
||||
mem.SeedDemo()
|
||||
tenant, _, _, _, _ := mem.DemoIDs()
|
||||
st := &telemetryHTTPStore{Backend: mem, snapshotErr: errors.New("private database failure")}
|
||||
server := &Server{store: st}
|
||||
for _, handler := range []http.HandlerFunc{server.handleListPeers, server.handleListSpeakers} {
|
||||
w := httptest.NewRecorder()
|
||||
handler(w, telemetryRequest(http.MethodGet, "/v1/resource?live=1", tenant))
|
||||
if w.Code != http.StatusServiceUnavailable || strings.Contains(w.Body.String(), "private") {
|
||||
t.Fatalf("snapshot failure: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
r := httptest.NewRequest(http.MethodPost, "/v1/telemetry/refresh", nil)
|
||||
w := httptest.NewRecorder()
|
||||
server.handleTelemetryRefresh(w, r)
|
||||
if w.Code != http.StatusUnauthorized || st.refreshCalls != 0 {
|
||||
t.Fatal("unauthenticated refresh admitted")
|
||||
}
|
||||
r = r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Kind: AuthKindJWT}))
|
||||
w = httptest.NewRecorder()
|
||||
server.handleTelemetryRefresh(w, r)
|
||||
if w.Code != http.StatusForbidden || st.refreshCalls != 0 {
|
||||
t.Fatal("unprivileged refresh admitted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTelemetryBackgroundStartsOnceAndStopsBeforeClose(t *testing.T) {
|
||||
s, err := New(Options{SeedDemo: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
s.StartBackground(ctx)
|
||||
first := s.telemetryCollector
|
||||
if first == nil {
|
||||
t.Fatal("telemetry collector not started")
|
||||
}
|
||||
s.StartBackground(ctx)
|
||||
if s.telemetryCollector != first {
|
||||
t.Fatal("duplicate telemetry collector")
|
||||
}
|
||||
s.Close()
|
||||
s.StartBackground(ctx)
|
||||
if s.telemetryCollector != first {
|
||||
t.Fatal("collector started after close")
|
||||
}
|
||||
s.Close()
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/store"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTelemetryHydrationFreshnessAndConfiguration(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
sp := &store.Speaker{ID: "replica", Role: "replica", Endpoint: "old"}
|
||||
observation := store.TelemetryObservation{ObservedAt: now.Add(-time.Hour), ReceivedAt: now, LastSuccessAt: now, Source: "agent"}
|
||||
base := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: observation, Sessions: []birdfmt.BGPSession{{Name: "peer", State: "Established"}}}, Health: store.TelemetryHealth{TelemetryObservation: observation, OK: true}}
|
||||
cases := []struct {
|
||||
name string
|
||||
mutate func(*store.TelemetrySnapshot)
|
||||
bgp, health bool
|
||||
quality string
|
||||
}{
|
||||
{"fresh receipt despite remote clock", func(*store.TelemetrySnapshot) {}, true, true, "ok"},
|
||||
{"expired BGP independent of health", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(-2 * time.Minute) }, false, true, "stale"},
|
||||
{"expired health independent of BGP", func(s *store.TelemetrySnapshot) { s.Health.ReceivedAt = now.Add(-2 * time.Minute) }, true, false, "ok"},
|
||||
{"future receipt is not fresh", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(time.Hour) }, false, true, "stale"},
|
||||
{"configuration changed", func(s *store.TelemetrySnapshot) { s.ConfigFingerprint = "old" }, false, false, "unknown"},
|
||||
{"failed BGP", func(s *store.TelemetrySnapshot) { s.BGP.Error = "timeout" }, false, true, "stale"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
snapshot := base
|
||||
tc.mutate(&snapshot)
|
||||
views, statuses := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{snapshot}, now, time.Minute)
|
||||
status := statuses[sp.ID]
|
||||
if status["bgp_poll_ok"] != tc.bgp || status["agent_ok"] != tc.health || views[0].observation().Quality != tc.quality {
|
||||
t.Fatalf("status=%+v observation=%+v", status, views[0].observation())
|
||||
}
|
||||
if !tc.bgp && (status["bgp_established"] != nil || len(views[0].Sessions) != 0) {
|
||||
t.Fatal("stale BGP exposed live sessions")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryHydrationMissingAndAmbiguousLocal(t *testing.T) {
|
||||
now := time.Now()
|
||||
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||
for _, rows := range [][]store.TelemetrySnapshot{nil, {{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "a"}, {SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "b"}}} {
|
||||
views, statuses := hydrateTelemetry([]*store.Speaker{sp}, rows, now, time.Minute)
|
||||
if views[0].observation().Quality != "unknown" || statuses[sp.ID]["agent_ok"] == true || statuses[sp.ID]["bgp_poll_ok"] == true {
|
||||
t.Fatalf("unexpected healthy master: %+v", statuses)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryHydrationLocalAuthorityRestart(t *testing.T) {
|
||||
now := time.Now()
|
||||
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||
fresh := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "new", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now}}}
|
||||
old := fresh
|
||||
old.LocalAuthority = "old"
|
||||
old.BGP.ReceivedAt = now.Add(-2 * time.Minute)
|
||||
_, status := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute)
|
||||
if status[sp.ID]["bgp_poll_ok"] != true {
|
||||
t.Fatal("expired authority hides current local snapshot")
|
||||
}
|
||||
old.BGP.ReceivedAt = now
|
||||
_, status = hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute)
|
||||
if status[sp.ID]["bgp_poll_ok"] != false {
|
||||
t.Fatal("two current local authorities are ambiguous")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryPeerObservationPreservesUnknownConfiguration(t *testing.T) {
|
||||
now := time.Now()
|
||||
sp := &store.Speaker{ID: "replica", Role: "replica"}
|
||||
row := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: "previous", BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now, LastSuccessAt: now}}}
|
||||
views, _ := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{row}, now, time.Minute)
|
||||
observation := peerLiveObservation(views)
|
||||
if observation.Quality != "unknown" || observation.ErrorCode != "configuration_changed" {
|
||||
t.Fatalf("changed config is not a failed current poll: %+v", observation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryExpiryUsesConfiguredIntervalAndEarliestSource(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
old := now.Add(-30 * time.Second)
|
||||
a := snapshotObservation(store.TelemetryObservation{ReceivedAt: old}, now, 5*time.Minute, "")
|
||||
b := snapshotObservation(store.TelemetryObservation{ReceivedAt: now}, now, 5*time.Minute, "")
|
||||
want := old.Add(5 * time.Minute).Format(time.RFC3339Nano)
|
||||
if a.ExpiresAt == nil || *a.ExpiresAt != want {
|
||||
t.Fatalf("expiry ignores configured interval: %+v", a)
|
||||
}
|
||||
views := []speakerBGPLive{{Polled: true, ObservationOverride: &b}, {Polled: true, ObservationOverride: &a}}
|
||||
combined := peerLiveObservation(views)
|
||||
if combined.ExpiresAt == nil || *combined.ExpiresAt != want {
|
||||
t.Fatalf("aggregate extends oldest source: %+v", combined)
|
||||
}
|
||||
missing := snapshotObservation(store.TelemetryObservation{}, now, time.Minute, "")
|
||||
if missing.ExpiresAt != nil {
|
||||
t.Fatal("unobserved source has expiry")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// trustedProxyPrefixes returns no trusted peers when any configured CIDR is invalid.
|
||||
// An empty configuration deliberately treats every connection as direct.
|
||||
func trustedProxyPrefixes() []netip.Prefix {
|
||||
raw := strings.TrimSpace(os.Getenv("EVOBGP_TRUSTED_PROXY_CIDRS"))
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(raw, ",")
|
||||
prefixes := make([]netip.Prefix, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
prefix, err := netip.ParsePrefix(strings.TrimSpace(part))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if prefix.Addr().Is4In6() {
|
||||
if prefix.Bits() < 96 {
|
||||
return nil
|
||||
}
|
||||
prefix = netip.PrefixFrom(prefix.Addr().Unmap(), prefix.Bits()-96)
|
||||
}
|
||||
prefixes = append(prefixes, prefix.Masked())
|
||||
}
|
||||
return prefixes
|
||||
}
|
||||
|
||||
func peerAddress(r *http.Request) (netip.Addr, bool) {
|
||||
if r == nil {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
|
||||
if err != nil {
|
||||
host = strings.TrimSpace(r.RemoteAddr)
|
||||
}
|
||||
ip, err := netip.ParseAddr(host)
|
||||
if err != nil {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
return ip.WithZone("").Unmap(), true
|
||||
}
|
||||
|
||||
func isTrustedProxy(ip netip.Addr, prefixes []netip.Prefix) bool {
|
||||
for _, prefix := range prefixes {
|
||||
if prefix.Contains(ip) || prefix.Contains(ip.Unmap()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func clientIPWithTrustedProxies(r *http.Request, prefixes []netip.Prefix) string {
|
||||
peer, ok := peerAddress(r)
|
||||
if !ok {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(r.RemoteAddr)
|
||||
}
|
||||
if !isTrustedProxy(peer, prefixes) {
|
||||
return peer.String()
|
||||
}
|
||||
values := r.Header.Values("X-Forwarded-For")
|
||||
if len(values) != 1 || strings.TrimSpace(values[0]) == "" {
|
||||
return peer.String()
|
||||
}
|
||||
hops := strings.Split(values[0], ",")
|
||||
current := peer
|
||||
for i := len(hops) - 1; i >= 0; i-- {
|
||||
if !isTrustedProxy(current, prefixes) {
|
||||
return current.String()
|
||||
}
|
||||
next, err := netip.ParseAddr(strings.TrimSpace(hops[i]))
|
||||
if err != nil {
|
||||
return peer.String()
|
||||
}
|
||||
if next.Zone() != "" {
|
||||
return peer.String()
|
||||
}
|
||||
current = next.Unmap()
|
||||
}
|
||||
return current.String()
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestClientIPTrustedProxyChain(t *testing.T) {
|
||||
prefixes := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}
|
||||
tests := []struct {
|
||||
name string
|
||||
remote string
|
||||
xff string
|
||||
want string
|
||||
}{
|
||||
{"untrusted peer ignores spoof", "198.51.100.10:1234", "203.0.113.9", "198.51.100.10"},
|
||||
{"rightmost untrusted hop", "10.0.0.2:1234", "203.0.113.9, 198.51.100.8", "198.51.100.8"},
|
||||
{"trusted chain", "10.0.0.2:1234", "203.0.113.9, 10.0.0.3", "203.0.113.9"},
|
||||
{"mapped peer", "[::ffff:10.0.0.2]:1234", "203.0.113.9", "203.0.113.9"},
|
||||
{"mapped forwarded client", "10.0.0.2:1234", "::ffff:203.0.113.9", "203.0.113.9"},
|
||||
{"malformed nearest hop", "10.0.0.2:1234", "203.0.113.9, bogus", "10.0.0.2"},
|
||||
{"malformed farther hop ignored", "10.0.0.2:1234", "bogus, 198.51.100.8", "198.51.100.8"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest("GET", "http://example.test/", nil)
|
||||
r.RemoteAddr = tt.remote
|
||||
r.Header.Set("X-Forwarded-For", tt.xff)
|
||||
if got := clientIPWithTrustedProxies(r, prefixes); got != tt.want {
|
||||
t.Fatalf("client IP = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientIPConfigurationFailsClosed(t *testing.T) {
|
||||
r := httptest.NewRequest("GET", "http://example.test/", nil)
|
||||
r.RemoteAddr = "10.0.0.2:1234"
|
||||
r.Header.Set("X-Forwarded-For", "203.0.113.9")
|
||||
r.Header.Set("X-Real-IP", "203.0.113.8")
|
||||
if got := clientIPWithTrustedProxies(r, nil); got != "10.0.0.2" {
|
||||
t.Fatalf("direct client IP = %q", got)
|
||||
}
|
||||
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8, invalid")
|
||||
if got := clientIP(r); got != "10.0.0.2" {
|
||||
t.Fatalf("invalid CIDR client IP = %q", got)
|
||||
}
|
||||
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "::ffff:10.0.0.2/128")
|
||||
if got := clientIP(r); got != "203.0.113.9" {
|
||||
t.Fatalf("mapped CIDR client IP = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicControlPlaneURLTrust(t *testing.T) {
|
||||
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8")
|
||||
r := httptest.NewRequest("POST", "http://internal.local:8080/v1/speakers", nil)
|
||||
r.Header.Set("Origin", "https://evil.example")
|
||||
r.Header.Set("X-Forwarded-Proto", "https")
|
||||
r.Header.Set("X-Forwarded-Host", "bgp.example.test")
|
||||
r.RemoteAddr = "198.51.100.8:1234"
|
||||
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
|
||||
t.Fatalf("direct URL = %q", got)
|
||||
}
|
||||
r.RemoteAddr = "10.0.0.2:1234"
|
||||
if got := publicControlPlaneURL(r, ""); got != "https://bgp.example.test" {
|
||||
t.Fatalf("proxied URL = %q", got)
|
||||
}
|
||||
if got := publicControlPlaneURL(r, "https://explicit.example/"); got != "https://explicit.example" {
|
||||
t.Fatalf("override URL = %q", got)
|
||||
}
|
||||
for _, bad := range []struct{ proto, host string }{
|
||||
{"javascript", "bgp.example.test"},
|
||||
{"https,http", "bgp.example.test"},
|
||||
{"https", "evil.example/path"},
|
||||
{"https", "evil.example@bgp.example.test"},
|
||||
{"https", "evil.example,bgp.example.test"},
|
||||
{"https", "bgp.example.test:99999"},
|
||||
{"https", "bgp.example.test:"},
|
||||
} {
|
||||
r.Header.Set("X-Forwarded-Proto", bad.proto)
|
||||
r.Header.Set("X-Forwarded-Host", bad.host)
|
||||
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
|
||||
t.Errorf("bad forwarded headers (%q, %q): %q", bad.proto, bad.host, got)
|
||||
}
|
||||
}
|
||||
r.Header.Set("X-Forwarded-Proto", "https")
|
||||
r.Header.Set("X-Forwarded-Host", "bgp.example.test")
|
||||
r.Header.Add("X-Forwarded-Host", "evil.example")
|
||||
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
|
||||
t.Fatalf("duplicate forwarded host = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthRateLimiterManyIdentities(t *testing.T) {
|
||||
l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 1, window: time.Minute, maxClients: 32}
|
||||
now := time.Unix(1000, 0)
|
||||
for i := 0; i < 1000; i++ {
|
||||
allowed := l.allowAt(fmt.Sprintf("ip-%d", i), now)
|
||||
if allowed != (i < l.maxClients) {
|
||||
t.Fatalf("identity %d allowed = %t", i, allowed)
|
||||
}
|
||||
}
|
||||
if len(l.hits) != l.maxClients || l.allowAt("ip-0", now) {
|
||||
t.Fatalf("active counters changed under cardinality pressure: %d", len(l.hits))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthRateLimiterCapacityAndExpiry(t *testing.T) {
|
||||
l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 2, window: time.Minute, maxClients: 2}
|
||||
now := time.Unix(1000, 0)
|
||||
if !l.allowAt("a", now) || !l.allowAt("b", now) || !l.allowAt("a", now) {
|
||||
t.Fatal("initial requests rejected")
|
||||
}
|
||||
if l.allowAt("a", now) || l.allowAt("c", now) {
|
||||
t.Fatal("active limit or capacity bypassed")
|
||||
}
|
||||
if len(l.hits) != 2 {
|
||||
t.Fatalf("map size = %d", len(l.hits))
|
||||
}
|
||||
if !l.allowAt("c", now.Add(time.Minute)) {
|
||||
t.Fatal("expired entries did not release capacity")
|
||||
}
|
||||
if len(l.hits) != 1 {
|
||||
t.Fatalf("expired entries remained: %d", len(l.hits))
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user