feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed 2026-11-02 17:23:59 +07:00
1 parent 924f111e48
commit f8e0520817
197 files changed
+18655 -1416

No files matched your search

@@ -0,0 +1,39 @@
package httpapi
import (
"testing"
"evobgp/internal/store"
)
// These regressions ensure unobserved live state cannot inherit persisted health.
func TestStage0ReproPeerWithoutObservationIsNotEstablished(t *testing.T) {
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"}
cases := []struct {
name string
views []speakerBGPLive
}{
{name: "no_collector"},
{name: "all_timeout", views: []speakerBGPLive{{SpeakerID: "replica", Error: "timeout"}}},
{name: "session_absent", views: []speakerBGPLive{{SpeakerID: "replica", Label: "replica"}}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
row := map[string]any{"session_state": "Established"}
applyPeerLiveFields(row, peer, tc.views)
if row["session_state"] == "Established" {
t.Fatal("S01: persisted Established is exposed without a current session observation")
}
})
}
}
func TestStage0ReproUnpolledMasterIsNotHealthy(t *testing.T) {
sp := &store.Speaker{ID: "master", Role: "master"}
// collectSpeakerLiveStatus constructs this view when no poll result exists.
view := speakerBGPLive{SpeakerID: sp.ID, Label: "master"}
row := speakerLiveStatusJSON(sp, view, nil)
if row["bgp_poll_ok"] == true || row["agent_ok"] == true {
t.Fatal("S01: absent poll is exposed as successful BGP/agent health")
}
}
+8 -39
View File
@@ -7,9 +7,9 @@ import (
"time"
"evobgp/internal/db"
"evobgp/internal/httpclient"
"evobgp/internal/jobs"
"evobgp/internal/observability"
"evobgp/internal/pipeline"
"evobgp/internal/repository"
"evobgp/internal/store"
@@ -18,7 +18,7 @@ import (
// NewCDNHTTPClient returns the shared HTTP client for CDN and preview fetches (PERF-02 / ERR-03).
func NewCDNHTTPClient() *http.Client {
return httpclient.New(httpclient.DefaultTimeout)
return pipeline.NewCDNHTTPClient()
}
// BootstrapWorkers opens the same store.Backend and jobs.Registry as New (without HTTP or bundle keys).
@@ -50,46 +50,15 @@ func BootstrapWorkers(ctx context.Context, opts Options) (store.Backend, *jobs.R
backend = mem
}
cdnHTTP := NewCDNHTTPClient()
wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: cdnHTTP}
// Configured DoH profiles and RIPE service overrides retain the previous
// general HTTP/HTTPS transport, including explicitly configured private
// endpoints. CDN calls independently secure this client in UpstreamHTTPDo.
workerHTTP := pipeline.NewPipelineHTTPClient()
wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: workerHTTP}
reg := jobs.NewRegistry(wk.Process)
wk.Registry = reg
if pool != nil {
audit := repository.NewJobAuditWriter(pool)
jobMeta := func(j *jobs.Job) map[string]any {
if j == nil {
return nil
}
st := j.Snapshot()
meta, _ := st["meta"].(map[string]any)
return meta
}
reg.SetPersistHooks(
func(j *jobs.Job) {
if j == nil {
return
}
audit.UpsertQueued(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, j.ModuleID, jobMeta(j))
},
func(j *jobs.Job) {
if j == nil {
return
}
audit.UpsertRunning(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, jobMeta(j))
},
func(j *jobs.Job) {
if j == nil {
return
}
st := j.Snapshot()
status, _ := st["status"].(string)
var errMsg *string
if e, ok := st["error"].(string); ok && e != "" {
errMsg = &e
}
audit.MarkTerminal(context.Background(), j.TenantID, j.ID, status, errMsg, time.Now().UTC())
},
)
reg.UseDurableQueue(backend)
}
observability.RegisterStoreBackend(backend)
return backend, reg, pool, nil
+110
View File
@@ -0,0 +1,110 @@
package httpapi
import (
"context"
"encoding/base64"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"evobgp/internal/httpclient"
"evobgp/internal/jobs"
"evobgp/internal/pipeline"
"evobgp/internal/store"
"github.com/miekg/dns"
)
func TestBootstrapWorkerPreservesExplicitPrivateDOH(t *testing.T) {
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
t.Setenv("EVOBGP_BIRDC_SOCKET", "")
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
var calls atomic.Int32
resolver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
raw, err := base64.RawURLEncoding.DecodeString(r.URL.Query().Get("dns"))
if err != nil {
w.WriteHeader(400)
return
}
query := new(dns.Msg)
if err = query.Unpack(raw); err != nil || len(query.Question) == 0 {
w.WriteHeader(400)
return
}
answer := new(dns.Msg)
answer.SetReply(query)
if query.Question[0].Qtype == dns.TypeA {
answer.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: query.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.IPv4(192, 0, 2, 20)}}
}
body, err := answer.Pack()
if err != nil {
t.Error(err)
return
}
w.Header().Set("Content-Type", "application/dns-message")
_, _ = w.Write(body)
}))
defer resolver.Close()
st, reg, _, err := BootstrapWorkers(context.Background(), Options{SeedDemo: true})
if err != nil {
t.Fatal(err)
}
defer func() {
reg.RequestCancelAll()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if e := reg.Drain(ctx); e != nil {
t.Error(e)
}
}()
tenant, _, _, _, _ := st.DemoIDs()
profile, err := st.CreateDohProfile(tenant, &store.DohProfile{Name: "explicit private resolver", URL: resolver.URL})
if err != nil {
t.Fatal(err)
}
mod, err := st.CreateModule(tenant, &store.Module{Name: "private-doh", Type: "DOMAINS", Enabled: true, DohProfileIDs: []string{profile.ID}})
if err != nil {
t.Fatal(err)
}
if _, err = st.CreateDomainEntry(tenant, mod.ID, &store.DomainEntry{FQDN: "private-resolver.example.test"}); err != nil {
t.Fatal(err)
}
j, _, err := reg.Enqueue(tenant, jobs.KindModuleRefresh, nil, &mod.ID, map[string]any{"module_id": mod.ID})
if err != nil {
t.Fatal(err)
}
deadline := time.After(5 * time.Second)
ticker := time.NewTicker(5 * time.Millisecond)
defer ticker.Stop()
for {
snap := j.Snapshot()
status := snap["status"]
if status == jobs.StatusSucceeded {
break
}
if status == jobs.StatusFailed || status == jobs.StatusCancelled {
t.Fatalf("worker refresh failed: %+v", snap)
}
select {
case <-deadline:
t.Fatal("worker did not finish")
case <-ticker.C:
}
}
if calls.Load() < 2 {
t.Fatalf("bootstrap worker blocked private DoH: %d requests", calls.Load())
}
// The same general-client choice must not remove CDN's independent ACL.
cdn := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("private CDN reached") }))
defer cdn.Close()
req, err := http.NewRequest(http.MethodGet, cdn.URL, nil)
if err != nil {
t.Fatal(err)
}
if _, err = pipeline.UpstreamHTTPDo(context.Background(), httpclient.New(httpclient.DefaultTimeout), req); err == nil {
t.Fatal("private CDN accepted")
}
}
+81
View File
@@ -0,0 +1,81 @@
package httpapi
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"evobgp/internal/store"
)
func TestPreviewCDNSourceRejectsOversizedBody(t *testing.T) {
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = io.Copy(w, strings.NewReader(strings.Repeat("x", (8<<20)+1)))
}))
defer srv.Close()
st := store.NewMemory()
st.SeedDemo()
tenant, _, _, _, _ := st.DemoIDs()
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "oversized-preview", Enabled: true})
if err != nil {
t.Fatal(err)
}
s := &Server{store: st, cdnHTTP: srv.Client()}
body, err := json.Marshal(map[string]string{"url": srv.URL, "source_kind": "plain"})
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+mod.ID+"/cdn-sources/preview", strings.NewReader(string(body)))
req.SetPathValue("module_id", mod.ID)
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"}))
recorder := httptest.NewRecorder()
s.handlePreviewCDNSource(recorder, req)
if recorder.Code != http.StatusBadGateway {
t.Fatalf("oversized preview status=%d body=%s", recorder.Code, recorder.Body.String())
}
}
func TestPreviewCDNErrorBodyBoundAndQuotaBeforeFetch(t *testing.T) {
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
t.Setenv("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", "1")
m := store.NewMemory()
m.SeedDemo()
tenant, module, _, _, _ := m.DemoIDs()
var calls atomic.Int32
upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
w.WriteHeader(http.StatusNotFound)
_, _ = io.WriteString(w, strings.Repeat("x", 64<<10))
w.(http.Flusher).Flush()
// No EOF: an unbounded drain would wait until the caller deadline.
<-r.Context().Done()
}))
defer upstream.Close()
s := &Server{store: m, cdnHTTP: upstream.Client()}
request := func() *http.Request {
r := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/cdn-sources/preview", strings.NewReader(`{"url":"`+upstream.URL+`","source_kind":"plain"}`))
r.SetPathValue("module_id", module)
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"}))
}
first := httptest.NewRecorder()
firstRequest := request()
ctx, cancel := context.WithTimeout(firstRequest.Context(), 5*time.Second)
defer cancel()
s.handlePreviewCDNSource(first, firstRequest.WithContext(ctx))
if first.Code != http.StatusBadGateway || calls.Load() != 1 || ctx.Err() != nil {
t.Fatalf("unbounded drain status=%d calls=%d ctx=%v", first.Code, calls.Load(), ctx.Err())
}
second := httptest.NewRecorder()
s.handlePreviewCDNSource(second, request())
if second.Code != http.StatusTooManyRequests || calls.Load() != 1 || second.Header().Get("Retry-After") == "" {
t.Fatalf("quota fetch bypass status=%d calls=%d", second.Code, calls.Load())
}
}
+1
View File
@@ -39,6 +39,7 @@ func (s *Server) withCORS(h http.Handler) http.Handler {
}
w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key, Accept, X-Tenant-Id")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PATCH, DELETE, OPTIONS")
w.Header().Set("Access-Control-Expose-Headers", "Location, X-EvoBGP-Job-Status, Retry-After")
if r.Method == http.MethodOptions {
w.WriteHeader(http.StatusNoContent)
return
+56
View File
@@ -0,0 +1,56 @@
package httpapi
import (
"errors"
"net/http"
"evobgp/internal/jobs"
"evobgp/internal/store"
)
func (s *Server) replayJobResponse(w http.ResponseWriter, r *http.Request, in store.JobEnqueue) bool {
j, found, err := s.jobs.ReplayContext(r.Context(), in)
if err != nil {
writeJobEnqueueError(w, err)
return true
}
if !found {
return false
}
w.Header().Set("Location", "/v1/jobs/"+j.ID)
snap := j.Snapshot()
body := map[string]any{"job_id": j.ID, "status": snap["status"]}
if j.Kind == jobs.KindTenantRefresh {
ids := snap["meta"].(map[string]any)["module_ids"]
n := 0
switch v := ids.(type) {
case []any:
n = len(v)
case []string:
n = len(v)
}
body["module_ids"] = ids
body["modules_due"] = n
}
writeJSON(w, http.StatusAccepted, body)
return true
}
func writeJobEnqueueError(w http.ResponseWriter, err error) {
switch {
case errors.Is(err, store.ErrJobConflict):
writeProblem(w, http.StatusConflict, "Conflict", "idempotency key belongs to a different job request")
case errors.Is(err, store.ErrJobQueueFull):
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "job queue is full")
default:
writeInternalError(w, "job_enqueue", err)
}
}
func writeJobReadError(w http.ResponseWriter, err error) {
if errors.Is(err, jobs.ErrNotFound) || errors.Is(err, store.ErrJobNotFound) {
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
return
}
writeInternalError(w, "job_read", err)
}
+99
View File
@@ -0,0 +1,99 @@
package httpapi
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"evobgp/internal/jobs"
"evobgp/internal/store"
)
type unavailableJobQueue struct{ store.JobQueue }
func (q unavailableJobQueue) EnqueueJob(context.Context, store.JobEnqueue) (store.JobRecord, bool, error) {
return store.JobRecord{}, false, errors.New("database unavailable")
}
func TestJobEnqueueFailureNeverAccepts(t *testing.T) {
r := jobs.NewRegistry(nil)
r.UseDurableQueue(unavailableJobQueue{})
s := &Server{jobs: r}
req := httptest.NewRequest(http.MethodPost, "/v1/bird/reload", nil)
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "operator"}))
w := httptest.NewRecorder()
s.handleBirdReload(w, req)
if w.Code == http.StatusAccepted || w.Code < 500 {
t.Fatalf("DB failure returned %d: %s", w.Code, w.Body.String())
}
}
func TestAutomaticEnqueueFailureIsVisible(t *testing.T) {
r := jobs.NewRegistry(nil)
r.UseDurableQueue(unavailableJobQueue{})
s := &Server{jobs: r}
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/v1/peers", nil)
s.enqueuePeerReconcile(w, req, "tenant", "peer_create")
if got := w.Header().Get("X-EvoBGP-Job-Status"); got != "enqueue-failed" {
t.Fatalf("scheduling failure hidden: %q", got)
}
}
type selectorBackend struct {
store.Backend
modules []*store.Module
reads int
}
func (b *selectorBackend) ListModules(string) []*store.Module { b.reads++; return b.modules }
func TestTenantRefreshReplayBeforeDynamicSelection(t *testing.T) {
m := store.NewMemory()
b := &selectorBackend{Backend: m, modules: []*store.Module{{ID: "module-a", Enabled: true, RefreshIntervalSec: 1}}}
r := jobs.NewRegistry(nil)
r.UseDurableQueue(m)
s := &Server{store: b, jobs: r}
request := func() *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodPost, "/v1/tenant/refresh", strings.NewReader(`{}`))
req.Header.Set("Idempotency-Key", "same-request")
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "editor"}))
w := httptest.NewRecorder()
s.handleTenantRefresh(w, req)
return w
}
first := request()
if first.Code != http.StatusAccepted {
t.Fatalf("first=%d %s", first.Code, first.Body.String())
}
var accepted map[string]any
if err := json.Unmarshal(first.Body.Bytes(), &accepted); err != nil {
t.Fatal(err)
}
claimed, err := m.ClaimJobs(context.Background(), "worker", 1, time.Minute)
if err != nil || len(claimed) != 1 {
t.Fatalf("claim %+v %v", claimed, err)
}
claimed[0].Status = "succeeded"
if err = m.FinishJob(context.Background(), claimed[0]); err != nil {
t.Fatal(err)
}
b.modules = nil
reads := b.reads
replay := request()
if replay.Code != http.StatusAccepted {
t.Fatalf("replay=%d %s", replay.Code, replay.Body.String())
}
var got map[string]any
if err = json.Unmarshal(replay.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if got["job_id"] != accepted["job_id"] || got["status"] != "succeeded" || b.reads != reads {
t.Fatalf("replay re-resolved selector: %+v reads %d->%d", got, reads, b.reads)
}
}
+114
View File
@@ -0,0 +1,114 @@
package httpapi
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"evobgp/internal/jobs"
"evobgp/internal/store"
)
func TestJobReportDurableProgressAndTenantScope(t *testing.T) {
ctx := context.Background()
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, revision, speaker := m.DemoIDs()
j, _, err := m.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: jobs.KindDeployApply})
if err != nil {
t.Fatal(err)
}
claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
if err != nil || len(claimed) != 1 {
t.Fatalf("claim: %v %v", claimed, err)
}
f := store.JobFence{JobID: j.ID, TenantID: tenant, OwnerID: "worker", Token: claimed[0].FencingToken}
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
if err != nil {
t.Fatal(err)
}
if err := m.RecordDeploymentSpeaker(ctx, f, d, "unknown", "unconfirmed", "response lost"); err != nil {
t.Fatal(err)
}
// A fresh registry has no in-process job or progress metadata.
r := jobs.NewRegistry(nil)
r.UseDurableQueue(m)
s := &Server{store: m, jobs: r}
request := func(tenantID, jobID string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/v1/jobs/"+jobID+"/report", nil)
req.SetPathValue("job_id", jobID)
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenantID, Role: "operator"}))
w := httptest.NewRecorder()
s.handleGetJobReport(w, req)
return w
}
check := func(id string) {
t.Helper()
w := request(tenant, id)
if w.Code != http.StatusOK {
t.Fatalf("report: %d %s", w.Code, w.Body.String())
}
var body struct {
Progress []struct {
State, Confirmation string
JobID string `json:"job_id"`
} `json:"deployment_progress"`
}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatal(err)
}
if len(body.Progress) != 1 || body.Progress[0].JobID != j.ID || body.Progress[0].State != "unknown" || body.Progress[0].Confirmation != "unconfirmed" {
t.Fatalf("lost persisted progress: %s", w.Body.String())
}
}
check(j.ID)
recovery, _, err := r.EnqueueContext(ctx, tenant, jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID})
if err != nil {
t.Fatal(err)
}
check(recovery.ID)
// A later explicit completion releases the delivery reservation. Once recovery
// has its own persisted row, its report must not show the parent's old state.
if err := m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil {
t.Fatal(err)
}
recoveryRows, err := m.ClaimJobs(ctx, "recovery-worker", 1, time.Minute)
if err != nil || len(recoveryRows) != 1 || recoveryRows[0].ID != recovery.ID {
t.Fatalf("recovery claim: %v %v", recoveryRows, err)
}
rf := store.JobFence{JobID: recovery.ID, TenantID: tenant, OwnerID: "recovery-worker", Token: recoveryRows[0].FencingToken}
rd, err := m.BeginDeploymentSpeaker(ctx, rf, speaker, revision)
if err != nil {
t.Fatal(err)
}
if err := m.RecordDeploymentSpeaker(ctx, rf, rd, "applied", "agent_reported", ""); err != nil {
t.Fatal(err)
}
w := request(tenant, recovery.ID)
var report struct {
Progress []struct {
State string
JobID string `json:"job_id"`
} `json:"deployment_progress"`
}
if err := json.Unmarshal(w.Body.Bytes(), &report); err != nil {
t.Fatal(err)
}
if w.Code != http.StatusOK || len(report.Progress) != 1 || report.Progress[0].JobID != recovery.ID || report.Progress[0].State != "applied" {
t.Fatalf("recovery own progress hidden: %d %s", w.Code, w.Body.String())
}
if w := request("other-tenant", j.ID); w.Code != http.StatusNotFound {
t.Fatalf("cross-tenant report: %d", w.Code)
}
foreign, _, err := r.EnqueueContext(ctx, "other-tenant", jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID})
if err != nil {
t.Fatal(err)
}
if w := request("other-tenant", foreign.ID); w.Code != http.StatusNotFound {
t.Fatalf("cross-tenant source: %d %s", w.Code, w.Body.String())
}
}
@@ -0,0 +1,89 @@
package httpapi
import (
"context"
"crypto/ed25519"
"encoding/base64"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"net/http/httptest"
"evobgp/internal/bundle"
)
// TestLegacyNodeBinaryCurrentControlPlane exercises an independently built old
// CLI against today's HTTP handlers. It never calls apply or touches host BIRD.
func TestLegacyNodeBinaryCurrentControlPlane(t *testing.T) {
binary := os.Getenv("EVOBGP_TEST_LEGACY_NODE_BIN")
if binary == "" {
t.Skip("EVOBGP_TEST_LEGACY_NODE_BIN not set")
}
binary, err := filepath.Abs(binary)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(binary); err != nil {
t.Fatal(err)
}
srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed})
if err != nil {
t.Fatal(err)
}
defer srv.Close()
tenant, _, _, revision, speaker := srv.Store().DemoIDs()
mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,other-node|other-tenant|node")
if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil {
t.Fatal(err)
}
server := httptest.NewServer(srv.Handler())
defer server.Close()
run := func(wantSuccess bool, args ...string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
output, err := exec.CommandContext(ctx, binary, args...).CombinedOutput()
if ctx.Err() != nil {
t.Fatalf("legacy CLI timed out: %v", ctx.Err())
}
if wantSuccess && err != nil {
t.Fatalf("legacy CLI failed: %v: %s", err, output)
}
if !wantSuccess && err == nil {
t.Fatalf("legacy CLI unexpectedly accepted invalid request: %s", output)
}
}
path := filepath.Join(t.TempDir(), "current-bundle.tar.gz")
run(true, "pull-bundle", "-base-url", server.URL, "-token", "legacy-node", "-speaker-id", speaker, "-o", path)
run(true, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64())
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64())
if err != nil {
t.Fatal(err)
}
verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub))
if err != nil || verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker {
t.Fatalf("legacy CLI fetched the wrong revision/speaker or an invalid bundle: %v", err)
}
// The archive is valid, but verification with an unrelated key must fail.
run(false, "verify-bundle", "-f", path, "-pubkey-hex", strings.Repeat("00", ed25519.PublicKeySize))
for _, token := range []string{"invalid-node", "other-node"} {
denied := filepath.Join(t.TempDir(), "denied.tar.gz")
run(false, "pull-bundle", "-base-url", server.URL, "-token", token, "-speaker-id", speaker, "-o", denied)
if _, err := os.Stat(denied); !os.IsNotExist(err) {
t.Fatalf("unauthorized CLI wrote a bundle: %v", err)
}
}
// A damaged payload must not pass the old verifier with the correct key.
if err := os.WriteFile(path, []byte("not a signed bundle"), 0o600); err != nil {
t.Fatal(err)
}
run(false, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64())
}
+310
View File
@@ -0,0 +1,310 @@
package httpapi
import (
"context"
"crypto/ed25519"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"sort"
"strings"
"sync"
"evobgp/internal/bundle"
"evobgp/internal/pipeline"
"evobgp/internal/store"
)
const (
// nodeBundleCacheMaxEntries is the maximum number of finished node bundles retained.
nodeBundleCacheMaxEntries = 32
// nodeBundleCacheMaxBytes is the maximum total size of cached bundle bodies (64 MiB).
nodeBundleCacheMaxBytes = 64 << 20
)
// errNodeBundleRevisionMissing is returned when the revision cannot be loaded.
// It is not cached: the next pull tries again.
var errNodeBundleRevisionMissing = errors.New("httpapi: revision not found")
// nodeBundleOpError carries the writeInternalError operation name for a failed build.
type nodeBundleOpError struct {
op string
err error
}
func (e *nodeBundleOpError) Error() string {
if e == nil || e.err == nil {
return "httpapi: node bundle"
}
return fmt.Sprintf("httpapi: %s: %v", e.op, e.err)
}
func (e *nodeBundleOpError) Unwrap() error { return e.err }
// nodeBundleBuilder produces the gzip body served to evobgp-node.
// The default implementation is bundle.BuildGzippedTar, whose archive contains
// manifest.json and bundle.sig. Tests replace it to count rebuilds.
type nodeBundleBuilder func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error)
// nodeBundleCache stores finished gzip bodies. A hit returns the bytes of the
// build that filled the entry, signature included. Errors are not stored.
// Concurrent loads of one key share a single build.
type nodeBundleCache struct {
mu sync.Mutex
maxN int
maxBytes int
total int
items map[string]*nodeBundleEntry
lru []string // index 0 is the oldest idle entry
flights map[string]*nodeBundleFlight
}
type nodeBundleEntry struct {
body []byte
}
type nodeBundleFlight struct {
done chan struct{}
body []byte
err error
joined int
}
func newNodeBundleCache(maxEntries, maxBytes int) *nodeBundleCache {
return &nodeBundleCache{
maxN: maxEntries,
maxBytes: maxBytes,
items: make(map[string]*nodeBundleEntry),
flights: make(map[string]*nodeBundleFlight),
}
}
// nodeBundleCacheKey identifies one servable bundle.
// overlayFP is a digest of the speaker overlay output (router id / source addresses,
// node IPv4 fallback, peer and discovery render). Revision content itself is immutable,
// so the revision id stands in for the shared prefix files.
func nodeBundleCacheKey(tenantID, revisionID, speakerID, overlayFP string) string {
var b strings.Builder
b.Grow(len(tenantID) + len(revisionID) + len(speakerID) + len(overlayFP) + 3)
b.WriteString(tenantID)
b.WriteByte(0)
b.WriteString(revisionID)
b.WriteByte(0)
b.WriteString(speakerID)
b.WriteByte(0)
b.WriteString(overlayFP)
return b.String()
}
// nodeBundleOverlayFingerprint hashes the speaker overlay that handleNodeBundle
// would apply on top of an immutable revision. Empty base fragments are enough:
// bird.conf and the peers fragment are fully re-rendered from tenant settings,
// speaker source addresses, and the live peer set. Revision file bytes are not
// part of this digest.
func nodeBundleOverlayFingerprint(st store.Backend, tenantID, speakerID, revisionID string) (string, error) {
frags, err := pipeline.OverlayFragmentsForSpeaker(st, tenantID, speakerID, revisionID, nil)
if err != nil {
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
}
keys := make([]string, 0, len(frags))
for k := range frags {
keys = append(keys, k)
}
sort.Strings(keys)
h := sha256.New()
for _, k := range keys {
if err := hashWriteString(h, k); err != nil {
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
}
if err := hashWriteString(h, frags[k]); err != nil {
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
}
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func hashWriteString(h interface{ Write([]byte) (int, error) }, s string) error {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], uint64(len(s)))
if _, err := h.Write(buf[:]); err != nil {
return err
}
_, err := h.Write([]byte(s))
return err
}
// do returns the cached body or runs build once for key.
// build's bool is false when the result must not be stored (failure is also not stored).
// Callers must not mutate the returned slice.
func (c *nodeBundleCache) do(ctx context.Context, key string, build func(context.Context) ([]byte, bool, error)) (body []byte, err error) {
if c == nil {
return nil, errors.New("httpapi: nil node bundle cache")
}
if ctx == nil {
ctx = context.Background()
}
c.mu.Lock()
if e, ok := c.items[key]; ok {
c.touchLocked(key)
body = e.body
c.mu.Unlock()
return body, nil
}
if f, ok := c.flights[key]; ok {
f.joined++
c.mu.Unlock()
return waitNodeBundleFlight(ctx, f)
}
if err = ctx.Err(); err != nil {
c.mu.Unlock()
return nil, err
}
f := &nodeBundleFlight{done: make(chan struct{})}
c.flights[key] = f
c.mu.Unlock()
var cacheOK bool
var finished bool
finish := func() {
if finished {
return
}
finished = true
out := body
c.mu.Lock()
delete(c.flights, key)
if err == nil && cacheOK {
if stored, ok := c.storeLocked(key, out); ok {
out = stored
}
}
f.body = out
f.err = err
close(f.done)
c.mu.Unlock()
body = out
}
defer func() {
if rec := recover(); rec != nil {
body = nil
cacheOK = false
err = fmt.Errorf("httpapi: node bundle build: %v", rec)
}
finish()
}()
// Detach from the leader's cancel so one disconnected agent does not fail the shared build.
body, cacheOK, err = build(context.WithoutCancel(ctx))
return
}
func waitNodeBundleFlight(ctx context.Context, f *nodeBundleFlight) ([]byte, error) {
select {
case <-f.done:
return f.body, f.err
case <-ctx.Done():
select {
case <-f.done:
return f.body, f.err
default:
return nil, ctx.Err()
}
}
}
// storeLocked retains body when it fits under both caps, evicting the oldest idle
// entries first. body is copied. The stored slice is returned.
func (c *nodeBundleCache) storeLocked(key string, body []byte) ([]byte, bool) {
n := len(body)
if n == 0 || c.maxN <= 0 || c.maxBytes <= 0 || n > c.maxBytes {
return nil, false
}
if e, ok := c.items[key]; ok {
c.total -= len(e.body)
delete(c.items, key)
c.removeLRULocked(key)
}
for len(c.items) > 0 && (len(c.items) >= c.maxN || c.total+n > c.maxBytes) {
if len(c.lru) == 0 {
return nil, false
}
c.deleteOldestLocked()
}
if len(c.items) >= c.maxN || c.total+n > c.maxBytes {
return nil, false
}
stored := make([]byte, n)
copy(stored, body)
c.items[key] = &nodeBundleEntry{body: stored}
c.total += n
c.lru = append(c.lru, key)
return stored, true
}
func (c *nodeBundleCache) touchLocked(key string) {
c.removeLRULocked(key)
c.lru = append(c.lru, key)
}
func (c *nodeBundleCache) deleteOldestLocked() {
if len(c.lru) == 0 {
return
}
key := c.lru[0]
c.lru = c.lru[1:]
if e, ok := c.items[key]; ok {
c.total -= len(e.body)
delete(c.items, key)
}
}
func (c *nodeBundleCache) removeLRULocked(key string) {
for i, k := range c.lru {
if k != key {
continue
}
copy(c.lru[i:], c.lru[i+1:])
c.lru = c.lru[:len(c.lru)-1]
return
}
}
func (s *Server) buildNodeBundle(revisionID, speakerID string, fragments map[string]string) ([]byte, error) {
if s != nil && s.nodeBundleBuild != nil {
return s.nodeBundleBuild(revisionID, speakerID, fragments, s.bundlePriv)
}
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, s.bundlePriv)
}
// loadNodeBundle reads the revision, applies the speaker overlay, and signs the gzip.
// The bool is false when overlay inputs moved during the load, so the caller must not
// cache the body under the key captured before the load. Failed loads are not cacheable.
func (s *Server) loadNodeBundle(ctx context.Context, tenantID, speakerID, revisionID, overlayFP string) ([]byte, bool, error) {
if err := ctx.Err(); err != nil {
return nil, false, err
}
fpNow, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID)
if err != nil {
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
}
rev, err := s.store.GetRevision(tenantID, revisionID)
if err != nil {
return nil, false, errNodeBundleRevisionMissing
}
overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, tenantID, speakerID, revisionID, rev.PreviewFragments)
if err != nil {
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
}
fpAfter, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID)
if err != nil {
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
}
body, err := s.buildNodeBundle(revisionID, speakerID, overlaid)
if err != nil {
return nil, false, &nodeBundleOpError{op: "internal", err: err}
}
cacheable := fpNow == overlayFP && fpAfter == fpNow && len(body) > 0
return body, cacheable, nil
}
+791
View File
@@ -0,0 +1,791 @@
package httpapi
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/ed25519"
"encoding/base64"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"evobgp/internal/bundle"
"evobgp/internal/signing"
"evobgp/internal/store"
)
func TestNodeBundleCacheKeySeparatesTenantsRevisionsSpeakers(t *testing.T) {
a := nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay")
if a == nodeBundleCacheKey("tenant-b", "rev", "spk", "overlay") {
t.Fatal("tenants collided")
}
if a == nodeBundleCacheKey("tenant-a", "rev-2", "spk", "overlay") {
t.Fatal("revisions collided")
}
if a == nodeBundleCacheKey("tenant-a", "rev", "spk-2", "overlay") {
t.Fatal("speakers collided")
}
if a == nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay-2") {
t.Fatal("overlay inputs collided")
}
}
func TestNodeBundleCacheHitSingleflightAndErrors(t *testing.T) {
c := newNodeBundleCache(8, 1<<20)
var builds atomic.Int32
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
return []byte("signed-body"), true, nil
})
if err != nil {
t.Fatal(err)
}
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
return []byte("other"), true, nil
})
if err != nil {
t.Fatal(err)
}
if builds.Load() != 1 {
t.Fatalf("second pull rebuilt: builds=%d", builds.Load())
}
if !bytes.Equal(got, []byte("signed-body")) {
t.Fatalf("hit body = %q", got)
}
t.Run("singleflight", func(t *testing.T) {
c := newNodeBundleCache(8, 1<<20)
var builds atomic.Int32
release := make(chan struct{})
const n = 8
var wg sync.WaitGroup
bodies := make([][]byte, n)
errs := make([]error, n)
wg.Add(n)
for i := 0; i < n; i++ {
go func(i int) {
defer wg.Done()
bodies[i], errs[i] = c.do(context.Background(), "same", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
<-release
return []byte("signed-bundle"), true, nil
})
}(i)
}
deadline := time.Now().Add(2 * time.Second)
for {
c.mu.Lock()
joined := 0
if f := c.flights["same"]; f != nil {
joined = f.joined
}
c.mu.Unlock()
if builds.Load() > 1 {
close(release)
wg.Wait()
t.Fatalf("builds = %d before followers joined", builds.Load())
}
if builds.Load() == 1 && joined == n-1 {
break
}
if time.Now().After(deadline) {
close(release)
wg.Wait()
t.Fatalf("builds=%d joined=%d", builds.Load(), joined)
}
time.Sleep(time.Millisecond)
}
close(release)
wg.Wait()
if builds.Load() != 1 {
t.Fatalf("builds = %d, want 1", builds.Load())
}
for i := 0; i < n; i++ {
if errs[i] != nil {
t.Fatalf("waiter %d: %v", i, errs[i])
}
if !bytes.Equal(bodies[i], []byte("signed-bundle")) {
t.Fatalf("waiter %d body = %q", i, bodies[i])
}
}
})
t.Run("canceled waiter does not drop the build", func(t *testing.T) {
c := newNodeBundleCache(4, 1024)
release := make(chan struct{})
var builds atomic.Int32
leaderErr := make(chan error, 1)
go func() {
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
<-release
return []byte("body"), true, nil
})
leaderErr <- err
}()
deadline := time.Now().Add(2 * time.Second)
for {
c.mu.Lock()
_, ok := c.flights["k"]
c.mu.Unlock()
if ok || time.Now().After(deadline) {
break
}
time.Sleep(time.Millisecond)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := c.do(ctx, "k", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
return []byte("nope"), true, nil
})
if !errors.Is(err, context.Canceled) {
t.Fatalf("follower err = %v", err)
}
close(release)
if err := <-leaderErr; err != nil {
t.Fatal(err)
}
if builds.Load() != 1 {
t.Fatalf("builds = %d", builds.Load())
}
_, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
return []byte("other"), true, nil
})
if err != nil {
t.Fatal(err)
}
if builds.Load() != 1 {
t.Fatal("successful build was not cached after a canceled waiter")
}
})
t.Run("errors are not cached", func(t *testing.T) {
c := newNodeBundleCache(4, 1024)
calls := 0
boom := errors.New("httpapi: bundle build failed")
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
calls++
return nil, true, boom
})
if !errors.Is(err, boom) {
t.Fatalf("err = %v", err)
}
_, err = c.do(context.Background(), "miss", func(context.Context) ([]byte, bool, error) {
calls++
return nil, false, errNodeBundleRevisionMissing
})
if !errors.Is(err, errNodeBundleRevisionMissing) {
t.Fatalf("err = %v", err)
}
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
calls++
return []byte("ok"), true, nil
})
if err != nil {
t.Fatal(err)
}
if string(got) != "ok" || calls != 3 {
t.Fatalf("calls=%d body=%q", calls, got)
}
_, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
calls++
return []byte("other"), true, nil
})
if err != nil || calls != 3 {
t.Fatalf("cached error path rebuilt: calls=%d err=%v", calls, err)
}
})
t.Run("unstorable success is not cached", func(t *testing.T) {
c := newNodeBundleCache(4, 1024)
calls := 0
for i := 0; i < 2; i++ {
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
calls++
return []byte("fresh"), false, nil
})
if err != nil || string(got) != "fresh" {
t.Fatalf("i=%d err=%v body=%q", i, err, got)
}
}
if calls != 2 {
t.Fatalf("calls = %d, want 2", calls)
}
})
}
func TestNodeBundleCacheEvictsIdleAndRespectsCaps(t *testing.T) {
t.Run("entry cap evicts oldest idle", func(t *testing.T) {
c := newNodeBundleCache(2, 100)
put := func(key, body string) {
t.Helper()
got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) {
return []byte(body), true, nil
})
if err != nil || string(got) != body {
t.Fatalf("put %s: %v %q", key, err, got)
}
}
put("a", "A")
put("b", "B")
if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) {
t.Fatal("recent entry was rebuilt")
return nil, false, nil
}); err != nil {
t.Fatal(err)
}
put("c", "C")
assertBundleCacheAccounting(t, c)
if _, ok := c.items["b"]; ok {
t.Fatal("idle entry b was kept")
}
if _, ok := c.items["a"]; !ok || c.items["c"] == nil {
t.Fatalf("kept %v", cacheKeys(c))
}
if len(c.items) > 2 {
t.Fatalf("entry cap exceeded: %d", len(c.items))
}
})
t.Run("byte cap", func(t *testing.T) {
c := newNodeBundleCache(8, 10)
put := func(key string, n int) {
t.Helper()
body := bytes.Repeat([]byte("x"), n)
got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) {
return body, true, nil
})
if err != nil || !bytes.Equal(got, body) {
t.Fatalf("put %s: %v len=%d", key, err, len(got))
}
assertBundleCacheAccounting(t, c)
}
put("a", 4)
put("b", 4)
if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) {
t.Fatal("rebuilt a")
return nil, false, nil
}); err != nil {
t.Fatal(err)
}
put("c", 4)
assertBundleCacheAccounting(t, c)
if _, ok := c.items["b"]; ok {
t.Fatal("byte cap kept idle entry b")
}
if len(c.items) != 2 || c.total != 8 {
t.Fatalf("n=%d total=%d", len(c.items), c.total)
}
put("d", 6)
put("e", 6)
assertBundleCacheAccounting(t, c)
if len(c.items) != 1 || c.total != 6 {
t.Fatalf("after oversized pair n=%d total=%d keys=%v", len(c.items), c.total, cacheKeys(c))
}
})
t.Run("entry larger than the byte cap is not stored", func(t *testing.T) {
c := newNodeBundleCache(4, 8)
calls := 0
big := bytes.Repeat([]byte("y"), 9)
for i := 0; i < 2; i++ {
got, err := c.do(context.Background(), "big", func(context.Context) ([]byte, bool, error) {
calls++
return big, true, nil
})
if err != nil || !bytes.Equal(got, big) {
t.Fatalf("i=%d err=%v", i, err)
}
}
if calls != 2 {
t.Fatalf("oversize body was cached, calls=%d", calls)
}
if len(c.items) != 0 || c.total != 0 {
t.Fatalf("stored oversize n=%d total=%d", len(c.items), c.total)
}
})
}
func TestNodeBundleOverlayFingerprintInputs(t *testing.T) {
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, rev, speaker := m.DemoIDs()
base, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
again, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
if base != again {
t.Fatal("fingerprint changed without an overlay input change")
}
sp, err := m.GetSpeaker(tenant, speaker)
if err != nil {
t.Fatal(err)
}
dispatch := store.MergeSpeakerMetaJSON(sp.MetaJSON, store.SpeakerMeta{
LastDispatchAt: "2026-01-01T00:00:00Z",
LastDispatchStatus: "ok",
SyncStatus: "synced",
})
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &dispatch}); err != nil {
t.Fatal(err)
}
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base {
t.Fatalf("dispatch meta changed fingerprint: %v %s", err, got)
}
peers := m.ListPeers(tenant)
if len(peers) == 0 {
t.Fatal("expected demo peers")
}
state := "Active"
if _, err := m.UpdatePeer(tenant, peers[0].ID, &store.PeerPatch{SessionState: &state}); err != nil {
t.Fatal(err)
}
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base {
t.Fatalf("session state changed fingerprint: %v %s", err, got)
}
if err := m.PatchGlobalSettings(tenant, map[string]any{"bird_router_id": "203.0.113.7"}); err != nil {
t.Fatal(err)
}
routed, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
if routed == base {
t.Fatal("router id setting did not change the overlay fingerprint")
}
nodeMeta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"})
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeMeta}); err != nil {
t.Fatal(err)
}
nodeFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
if nodeFP == routed {
t.Fatal("node ipv4 did not change the overlay fingerprint")
}
sourceMeta := store.SpeakerMetaJSON(store.SpeakerMeta{
NodeIPv4: "198.51.100.9",
BirdBgpSourceIPv4: "203.0.113.40",
})
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &sourceMeta}); err != nil {
t.Fatal(err)
}
sourceFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
if sourceFP == nodeFP {
t.Fatal("source ipv4 did not change the overlay fingerprint")
}
nodeOnly := store.SpeakerMetaJSON(store.SpeakerMeta{
NodeIPv4: "198.51.100.8",
BirdBgpSourceIPv4: "203.0.113.40",
})
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeOnly}); err != nil {
t.Fatal(err)
}
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != sourceFP {
t.Fatalf("unused node ipv4 changed fingerprint: %v %s vs %s", err, got, sourceFP)
}
v6 := store.SpeakerMetaJSON(store.SpeakerMeta{
NodeIPv4: "198.51.100.8",
BirdBgpSourceIPv4: "203.0.113.40",
BirdBgpSourceIPv6: "2001:db8::40",
})
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &v6}); err != nil {
t.Fatal(err)
}
v6FP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
// Source v6 changes local v6. When no peer policy overrides it, the rendered
// peers fragment can stay the same. A difference means the overlay emitted it;
// equality means a rebuild would not change the bundle.
_ = v6FP
if _, err := m.CreatePeer(tenant, &store.BGPPeer{
Name: "cache-extra",
Neighbor: "203.0.113.50",
RemoteASN: 64512,
Enabled: true,
}); err != nil {
t.Fatal(err)
}
withPeer, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
if withPeer == v6FP {
t.Fatal("added peer did not change the overlay fingerprint")
}
sp2, err := m.CreateSpeaker(tenant, &store.Speaker{
Role: "replica",
Endpoint: "https://203.0.113.2",
MetaJSON: v6,
})
if err != nil {
t.Fatal(err)
}
other, err := nodeBundleOverlayFingerprint(m, tenant, sp2.ID, rev)
if err != nil {
t.Fatal(err)
}
if other == withPeer {
t.Fatal("speakers with the same addresses shared an overlay fingerprint")
}
otherRev, err := m.CreateRollbackRevision(tenant, rev)
if err != nil {
t.Fatal(err)
}
revFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, otherRev)
if err != nil {
t.Fatal(err)
}
if revFP == withPeer {
t.Fatal("revision id did not change the overlay fingerprint")
}
}
func TestHandleNodeBundleCache(t *testing.T) {
srv, ts, ids := startNodeBundleServer(t)
if srv.bundleCache.maxN != nodeBundleCacheMaxEntries || srv.bundleCache.maxBytes != nodeBundleCacheMaxBytes {
t.Fatalf("caps n=%d bytes=%d", srv.bundleCache.maxN, srv.bundleCache.maxBytes)
}
var builds atomic.Int32
srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
builds.Add(1)
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv)
}
client := ts.Client()
status, raw := pullNodeBundle(t, client, ts.URL, ids.speaker, "00000000-0000-0000-0000-000000000099")
if status != http.StatusNotFound {
t.Fatalf("missing revision status %d: %s", status, raw)
}
if builds.Load() != 0 {
t.Fatal("missing revision built a bundle")
}
body1 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
body2 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 1 {
t.Fatalf("second pull rebuilt: builds=%d", builds.Load())
}
if !bytes.Equal(body1, body2) {
t.Fatal("cache hit was not byte-for-byte identical")
}
verifyNodeBundle(t, srv, body2)
sp2, err := srv.Store().CreateSpeaker(ids.tenant, &store.Speaker{
Role: "replica",
Endpoint: "https://203.0.113.2",
MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "203.0.113.2"}),
})
if err != nil {
t.Fatal(err)
}
bodyB := mustPullNodeBundle(t, client, ts.URL, sp2.ID, ids.rev)
if builds.Load() != 2 {
t.Fatalf("second speaker builds=%d", builds.Load())
}
if bytes.Equal(body1, bodyB) {
t.Fatal("different speakers returned the same bundle")
}
if !strings.Contains(bundleFile(t, bodyB, "bird.conf"), "203.0.113.2") {
t.Fatal("speaker B bundle missing its router id")
}
body3 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 2 || !bytes.Equal(body1, body3) {
t.Fatalf("speaker A collided with B: builds=%d equal=%v", builds.Load(), bytes.Equal(body1, body3))
}
meta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"})
if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
t.Fatal(err)
}
body4 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 3 {
t.Fatalf("changed node ipv4 builds=%d, want 3", builds.Load())
}
if bytes.Equal(body1, body4) {
t.Fatal("changed overlay input reused the previous bundle")
}
conf := bundleFile(t, body4, "bird.conf")
if !strings.Contains(conf, "198.51.100.9") {
t.Fatalf("rebuilt bird.conf missing node ipv4: %s", conf)
}
verifyNodeBundle(t, srv, body4)
body5 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 3 || !bytes.Equal(body4, body5) {
t.Fatal("rebuilt bundle was not cached")
}
source := store.SpeakerMetaJSON(store.SpeakerMeta{
NodeIPv4: "198.51.100.9",
BirdBgpSourceIPv4: "203.0.113.40",
})
if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &source}); err != nil {
t.Fatal(err)
}
body6 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 4 {
t.Fatalf("changed source ipv4 builds=%d, want 4", builds.Load())
}
conf = bundleFile(t, body6, "bird.conf")
if !strings.Contains(conf, "203.0.113.40") {
t.Fatalf("source ipv4 not applied: %s", conf)
}
verifyNodeBundle(t, srv, body6)
otherRev, err := srv.Store().CreateRollbackRevision(ids.tenant, ids.rev)
if err != nil {
t.Fatal(err)
}
bodyRev := mustPullNodeBundle(t, client, ts.URL, ids.speaker, otherRev)
if builds.Load() != 5 {
t.Fatalf("other revision builds=%d", builds.Load())
}
if bytes.Equal(body6, bodyRev) {
t.Fatal("revisions shared a bundle body")
}
cached := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 5 || !bytes.Equal(body6, cached) {
t.Fatal("original revision was evicted or rebuilt")
}
}
func TestHandleNodeBundleSingleflight(t *testing.T) {
srv, ts, ids := startNodeBundleServer(t)
var builds atomic.Int32
release := make(chan struct{})
srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
builds.Add(1)
<-release
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv)
}
client := ts.Client()
const n = 2
bodies := make([][]byte, n)
errs := make([]error, n)
var wg sync.WaitGroup
wg.Add(n)
for i := 0; i < n; i++ {
go func(i int) {
defer wg.Done()
bodies[i], errs[i] = pullNodeBundleOK(client, ts.URL, ids.speaker, ids.rev)
}(i)
}
deadline := time.Now().Add(3 * time.Second)
joined := 0
for {
if builds.Load() > 1 {
close(release)
wg.Wait()
t.Fatalf("concurrent pulls built %d times", builds.Load())
}
srv.bundleCache.mu.Lock()
joined = 0
for _, f := range srv.bundleCache.flights {
joined += f.joined
}
srv.bundleCache.mu.Unlock()
if builds.Load() == 1 && joined >= 1 {
break
}
if time.Now().After(deadline) {
close(release)
wg.Wait()
t.Fatalf("timed out builds=%d joined=%d", builds.Load(), joined)
}
time.Sleep(time.Millisecond)
}
close(release)
wg.Wait()
if builds.Load() != 1 {
t.Fatalf("builds = %d, want 1", builds.Load())
}
for i := 0; i < n; i++ {
if errs[i] != nil {
t.Fatal(errs[i])
}
}
if !bytes.Equal(bodies[0], bodies[1]) {
t.Fatal("singleflight returned different bodies")
}
verifyNodeBundle(t, srv, bodies[0])
}
type nodeBundleIDs struct {
tenant, rev, speaker string
}
func startNodeBundleServer(t *testing.T) (*Server, *httptest.Server, nodeBundleIDs) {
t.Helper()
srv, err := New(Options{
InsecureDev: true,
SeedDemo: true,
BundleSeedHex: testBundleSeed,
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(srv.Close)
tenant, _, _, rev, speaker := srv.Store().DemoIDs()
mustSetTestAPIKeys(t, srv, "nodekey|"+tenant+"|node")
ts := httptest.NewServer(srv.Handler())
t.Cleanup(ts.Close)
return srv, ts, nodeBundleIDs{tenant: tenant, rev: rev, speaker: speaker}
}
func mustPullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) []byte {
t.Helper()
status, body := pullNodeBundle(t, client, base, speaker, rev)
if status != http.StatusOK {
t.Fatalf("status %d: %s", status, body)
}
return body
}
func pullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) (int, []byte) {
t.Helper()
req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Authorization", "Bearer nodekey")
resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
return resp.StatusCode, body
}
func pullNodeBundleOK(client *http.Client, base, speaker, rev string) ([]byte, error) {
req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer nodekey")
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, errors.New(resp.Status + ": " + string(body))
}
return body, nil
}
func verifyNodeBundle(t *testing.T, srv *Server, raw []byte) {
t.Helper()
pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64())
if err != nil {
t.Fatal(err)
}
if _, err := signing.VerifyGzippedTar(raw, ed25519.PublicKey(pub)); err != nil {
t.Fatal(err)
}
}
func bundleFile(t *testing.T, gz []byte, name string) string {
t.Helper()
gr, err := gzip.NewReader(bytes.NewReader(gz))
if err != nil {
t.Fatal(err)
}
defer func() { _ = gr.Close() }()
tr := tar.NewReader(gr)
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
t.Fatal(err)
}
if hdr.Name != name {
continue
}
var b strings.Builder
if _, err := io.Copy(&b, tr); err != nil {
t.Fatal(err)
}
return b.String()
}
t.Fatalf("archive missing %s", name)
return ""
}
func assertBundleCacheAccounting(t *testing.T, c *nodeBundleCache) {
t.Helper()
if len(c.items) != len(c.lru) {
t.Fatalf("items %d lru %d", len(c.items), len(c.lru))
}
sum := 0
seen := make(map[string]bool, len(c.lru))
for _, k := range c.lru {
e, ok := c.items[k]
if !ok {
t.Fatalf("lru key %q missing from items", k)
}
if seen[k] {
t.Fatalf("duplicate lru key %q", k)
}
seen[k] = true
sum += len(e.body)
}
if sum != c.total {
t.Fatalf("total=%d sum=%d", c.total, sum)
}
if c.maxN > 0 && len(c.items) > c.maxN {
t.Fatalf("entries %d over cap %d", len(c.items), c.maxN)
}
if c.maxBytes > 0 && c.total > c.maxBytes {
t.Fatalf("bytes %d over cap %d", c.total, c.maxBytes)
}
}
func cacheKeys(c *nodeBundleCache) []string {
keys := make([]string, 0, len(c.items))
for k := range c.items {
keys = append(keys, k)
}
return keys
}
@@ -0,0 +1,98 @@
package httpapi
import (
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"testing"
"evobgp/internal/bundle"
)
func TestLegacyCompatibilityNodePullAndAuth(t *testing.T) {
srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed})
if err != nil {
t.Fatal(err)
}
defer srv.Close()
tenant, _, _, revision, speaker := srv.Store().DemoIDs()
mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,operator|"+tenant+"|operator,other-node|other-tenant|node")
if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil {
t.Fatal(err)
}
handler := srv.Handler()
call := func(path, token string) (int, []byte, string) {
t.Helper()
req, err := http.NewRequest(http.MethodGet, path, nil)
if err != nil {
t.Fatal(err)
}
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
resp := w.Result()
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
return resp.StatusCode, body, resp.Header.Get("Content-Type")
}
latestPath := "/v1/speakers/" + speaker + "/revisions/latest"
bundlePath := "/v1/speakers/" + speaker + "/bundle/" + revision
for _, path := range []string{latestPath, bundlePath} {
for _, tc := range []struct {
token string
want int
}{{"", http.StatusUnauthorized}, {"operator", http.StatusForbidden}, {"other-node", http.StatusForbidden}} {
status, _, _ := call(path, tc.token)
if status != tc.want {
t.Fatalf("%s token %q: status %d, want %d", path, tc.token, status, tc.want)
}
}
}
status, raw, _ := call(latestPath, "legacy-node")
if status != http.StatusOK {
t.Fatalf("legacy latest: status %d: %s", status, raw)
}
var latest struct {
RevisionID string `json:"revision_id"`
PublishedAt string `json:"published_at"`
}
if err := json.Unmarshal(raw, &latest); err != nil {
t.Fatal(err)
}
if latest.RevisionID != revision || latest.PublishedAt == "" {
t.Fatalf("legacy latest response: %+v", latest)
}
status, raw, contentType := call("/v1/bundle/signing-public-key", "operator")
if status != http.StatusOK || contentType != "application/json" {
t.Fatalf("public key status %d, type %q: %s", status, contentType, raw)
}
var key struct {
PublicKeyBase64 string `json:"public_key_base64"`
}
if err := json.Unmarshal(raw, &key); err != nil {
t.Fatal(err)
}
pub, err := base64.StdEncoding.DecodeString(key.PublicKeyBase64)
if err != nil || len(pub) != ed25519.PublicKeySize {
t.Fatalf("legacy public key: len=%d err=%v", len(pub), err)
}
status, raw, contentType = call(bundlePath, "legacy-node")
if status != http.StatusOK || contentType != "application/gzip" {
t.Fatalf("legacy bundle: status %d, type %q: %s", status, contentType, raw)
}
verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub))
if err != nil {
t.Fatalf("legacy signed bundle verification: %v", err)
}
if verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker || len(verified.Files) == 0 {
t.Fatalf("legacy bundle manifest: %+v", verified.Manifest)
}
}
+105
View File
@@ -0,0 +1,105 @@
package httpapi
import (
"net/http"
"os"
"strconv"
"sync"
"time"
)
type expensiveOperation int
const (
lookupOperation expensiveOperation = iota
cdnPreviewOperation
revisionPreviewOperation
)
type tenantWork struct {
start time.Time
counts [3]int
active int
}
// operationLimiter is deliberately local to one API process. It never queues
// waiters or evicts active tenants to admit new identities.
type operationLimiter struct {
mu sync.Mutex
tenants map[string]*tenantWork
active, maxActive, maxTenantActive, maxTenants int
rates [3]int
nextSweep time.Time
}
func newOperationLimiter() *operationLimiter {
return &operationLimiter{
tenants: make(map[string]*tenantWork), maxTenants: 4096,
maxActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_MAX_CONCURRENT", 4, 64),
maxTenantActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", 2, 64),
rates: [3]int{operationEnvLimit("EVOBGP_API_LOOKUP_PER_MINUTE", 120, 100000), operationEnvLimit("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", 30, 100000), operationEnvLimit("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", 30, 100000)},
}
}
func operationEnvLimit(name string, fallback, maximum int) int {
n, err := strconv.Atoi(os.Getenv(name))
if err != nil || n < 1 || n > maximum {
return fallback
}
return n
}
func (l *operationLimiter) acquire(tenant string, op expensiveOperation, now time.Time) (func(), int) {
l.mu.Lock()
defer l.mu.Unlock()
if !now.Before(l.nextSweep) {
for key, entry := range l.tenants {
if entry.active == 0 && !now.Before(entry.start.Add(time.Minute)) {
delete(l.tenants, key)
}
}
l.nextSweep = now.Add(time.Second)
}
if l.active >= l.maxActive {
return nil, 1
}
entry := l.tenants[tenant]
if entry == nil {
if len(l.tenants) >= l.maxTenants {
return nil, 1
}
entry = &tenantWork{start: now}
l.tenants[tenant] = entry
}
if !now.Before(entry.start.Add(time.Minute)) {
entry.start = now
entry.counts = [3]int{}
}
if entry.active >= l.maxTenantActive {
return nil, 1
}
if entry.counts[op] >= l.rates[op] {
wait := entry.start.Add(time.Minute).Sub(now)
return nil, max(1, int((wait+time.Second-1)/time.Second))
}
entry.counts[op]++
entry.active++
l.active++
var once sync.Once
return func() { once.Do(func() { l.mu.Lock(); defer l.mu.Unlock(); entry.active--; l.active-- }) }, 0
}
func (s *Server) acquireOperation(w http.ResponseWriter, r *http.Request, tenant string, op expensiveOperation) (func(), bool) {
if r.Context().Err() != nil {
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request cancelled")
return nil, false
}
s.operationOnce.Do(func() { s.operationLimiter = newOperationLimiter() })
release, retry := s.operationLimiter.acquire(tenant, op, time.Now())
if release == nil {
w.Header().Set("Retry-After", strconv.Itoa(retry))
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "preview/lookup quota exceeded; retry later")
return nil, false
}
return release, true
}
+143
View File
@@ -0,0 +1,143 @@
package httpapi
import (
"context"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"evobgp/internal/store"
)
func TestOperationLimitsSharedCapacityAndWindows(t *testing.T) {
l := newOperationLimiter()
l.maxActive = 2
l.maxTenantActive = 1
l.rates = [3]int{1, 2, 2}
now := time.Now()
a, _ := l.acquire("a", lookupOperation, now)
if a == nil {
t.Fatal("first denied")
}
if r, _ := l.acquire("a", cdnPreviewOperation, now); r != nil {
t.Fatal("tenant concurrency bypass")
}
b, _ := l.acquire("b", revisionPreviewOperation, now)
if b == nil {
t.Fatal("other tenant denied")
}
if r, _ := l.acquire("c", lookupOperation, now); r != nil {
t.Fatal("global concurrency bypass")
}
a()
a()
b()
if l.active != 0 {
t.Fatal("release not idempotent")
}
if r, retry := l.acquire("a", lookupOperation, now.Add(time.Second)); r != nil || retry != 59 {
t.Fatalf("rate bypass/retry admitted=%t seconds=%d", r != nil, retry)
}
c, _ := l.acquire("a", cdnPreviewOperation, now.Add(time.Second))
if c == nil {
t.Fatal("independent operation denied")
}
c()
d, _ := l.acquire("a", lookupOperation, now.Add(time.Minute))
if d == nil {
t.Fatal("expired window not reset")
}
d()
}
func TestOperationTenantMapBoundWithoutActiveEviction(t *testing.T) {
l := newOperationLimiter()
l.maxTenants = 1
now := time.Now()
a, _ := l.acquire("a", lookupOperation, now)
if r, _ := l.acquire("b", lookupOperation, now.Add(2*time.Minute)); r != nil {
t.Fatal("active entry evicted")
}
a()
b, _ := l.acquire("b", lookupOperation, now.Add(3*time.Minute))
if b == nil || len(l.tenants) != 1 {
t.Fatal("idle expiry did not free bounded state")
}
b()
}
func TestOperationConcurrentAdmissionBound(t *testing.T) {
l := newOperationLimiter()
l.maxActive = 4
l.maxTenantActive = 4
l.rates = [3]int{100, 100, 100}
results := make(chan func(), 32)
var wg sync.WaitGroup
for range 32 {
wg.Add(1)
go func() { defer wg.Done(); r, _ := l.acquire("tenant", lookupOperation, time.Now()); results <- r }()
}
wg.Wait()
close(results)
accepted := 0
for r := range results {
if r != nil {
accepted++
r()
}
}
if accepted != 4 || l.active != 0 {
t.Fatalf("accepted=%d active=%d", accepted, l.active)
}
}
type waitingRevisionStore struct {
store.Backend
entered chan struct{}
unblock chan struct{}
}
func (s waitingRevisionStore) GetRevision(tenant, id string) (*store.Revision, error) {
s.entered <- struct{}{}
<-s.unblock
return s.Backend.GetRevision(tenant, id)
}
func TestRevisionQuotaHeldUntilWorkReturns(t *testing.T) {
t.Setenv("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", "1")
t.Setenv("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", "30")
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, revision, _ := m.DemoIDs()
entered := make(chan struct{}, 2)
unblock := make(chan struct{})
s := &Server{store: waitingRevisionStore{Backend: m, entered: entered, unblock: unblock}}
request := func() *http.Request {
r := httptest.NewRequest(http.MethodGet, "/v1/revisions/"+revision+"/preview", nil)
r.SetPathValue("revision_id", revision)
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"}))
}
ctx, cancel := context.WithCancel(request().Context())
defer cancel()
first := httptest.NewRecorder()
done := make(chan struct{})
go func() { defer close(done); s.handleRevisionPreview(first, request().WithContext(ctx)) }()
<-entered
cancel() // Backend deliberately ignores cancellation: capacity stays reserved.
second := httptest.NewRecorder()
s.handleRevisionPreview(second, request())
if second.Code != http.StatusTooManyRequests || second.Header().Get("Retry-After") == "" {
close(unblock)
<-done
t.Fatalf("quota: %d %s", second.Code, second.Body.String())
}
close(unblock)
<-done
third := httptest.NewRecorder()
s.handleRevisionPreview(third, request())
if third.Code != http.StatusOK {
t.Fatalf("release failed: %d %s", third.Code, third.Body.String())
}
}
+1 -47
View File
@@ -6,7 +6,6 @@ import (
"strings"
"time"
"evobgp/internal/birdfmt"
"evobgp/internal/store"
)
@@ -44,46 +43,6 @@ func peerDiscoveryJSON(d *store.BGPPeerDiscovery) map[string]any {
return m
}
// syncPeerDiscoveriesFromLive upserts pending discoveries from dynamic BGP sessions.
func (s *Server) syncPeerDiscoveriesFromLive(tenantID string, views []speakerBGPLive) {
if s.store == nil || tenantID == "" {
return
}
knownNeighbors := map[string]struct{}{}
for _, p := range s.store.ListPeers(tenantID) {
if p == nil {
continue
}
if n, ok := store.NormalizePeerNeighborString(p.Neighbor); ok {
knownNeighbors[n] = struct{}{}
}
}
now := time.Now().UTC()
for _, v := range views {
for _, sess := range v.Sessions {
if !birdfmt.IsDynamicDiscoverySession(sess.Name) {
continue
}
neighbor, ok := store.NormalizePeerNeighborString(sess.Neighbor)
if !ok {
continue
}
if _, known := knownNeighbors[neighbor]; known {
continue
}
_, _ = s.store.UpsertPeerDiscovery(tenantID, &store.PeerDiscoveryUpsert{
SpeakerID: v.SpeakerID,
NeighborID: strings.TrimSpace(sess.NeighborID),
Neighbor: neighbor,
RemoteASN: sess.NeighborAS,
ProtocolName: sess.Name,
SessionState: sess.State,
SeenAt: now,
})
}
}
}
func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Request) {
a, ok := authFromContext(r.Context())
if !ok {
@@ -97,11 +56,6 @@ func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Reques
if status == "" {
status = store.PeerDiscoveryPending
}
// Refresh live discovery when listing pending.
if status == store.PeerDiscoveryPending || status == "all" {
views := s.collectSpeakerBGPLive(r.Context(), a.TenantID, true)
s.syncPeerDiscoveriesFromLive(a.TenantID, views)
}
listStatus := status
if status == "all" {
listStatus = ""
@@ -138,7 +92,7 @@ func (s *Server) handleApprovePeerDiscovery(w http.ResponseWriter, r *http.Reque
writeStoreErr(w, err)
return
}
s.enqueuePeerReconcile(a.TenantID, "peer_discovery_approve")
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_approve")
s.recordCRUDAudit(r, a, "bgp.peer.discovery.approve", "Approved discovered peer "+peer.Neighbor, peer.ID, map[string]any{
"peer_id": peer.ID,
"discovery_id": disc.ID,
+185 -116
View File
@@ -1,47 +1,98 @@
package httpapi
import (
"context"
"net/netip"
"os"
"strings"
"sync"
"time"
"evobgp/internal/birdfmt"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
)
const peerLiveCacheTTL = 15 * time.Second
type speakerBGPLive struct {
SpeakerID string
Label string
Sessions []birdfmt.BGPSession
Error string
SpeakerID string
Label string
Sessions []birdfmt.BGPSession
Error string
Polled bool
ObservedAt time.Time
ReceivedAt time.Time
LastSuccessAt time.Time
Source string
ObservationOverride *liveObservation
}
type peerLiveCacheEntry struct {
at time.Time
views []speakerBGPLive
type liveObservation struct {
ObservedAt *string `json:"observed_at"`
ReceivedAt *string `json:"received_at"`
LastSuccessAt *string `json:"last_success_at"`
ExpiresAt *string `json:"expires_at,omitempty"`
Source string `json:"source"`
Quality string `json:"quality"`
ErrorCode string `json:"error_code,omitempty"`
}
var peerLiveCache sync.Map // tenantID -> peerLiveCacheEntry
func liveTime(t time.Time) *string {
if t.IsZero() {
return nil
}
v := t.UTC().Format(time.RFC3339Nano)
return &v
}
func (v speakerBGPLive) observed() bool {
return v.Polled || v.Error != "" || len(v.Sessions) > 0
}
func (v speakerBGPLive) successful() bool {
return v.observed() && v.Error == ""
}
func (v speakerBGPLive) observation() liveObservation {
if v.ObservationOverride != nil {
return *v.ObservationOverride
}
o := liveObservation{ObservedAt: liveTime(v.ObservedAt), ReceivedAt: liveTime(v.ReceivedAt), LastSuccessAt: liveTime(v.LastSuccessAt), Source: v.Source, Quality: "unknown"}
if o.Source == "" {
o.Source = "none"
}
switch {
case v.successful():
o.Quality = "ok"
case v.Error != "" && !v.LastSuccessAt.IsZero():
o.Quality = "stale"
o.ErrorCode = liveErrorCode(v.Error)
case v.Error != "":
o.Quality = "unreachable"
o.ErrorCode = liveErrorCode(v.Error)
default:
o.ErrorCode = "not_polled"
}
return o
}
func liveErrorCode(err string) string {
if strings.Contains(strings.ToLower(err), "not configured") || strings.Contains(strings.ToLower(err), "missing") {
return "not_configured"
}
return "poll_failed"
}
type peerSessionOnSpeaker struct {
SpeakerID string `json:"speaker_id"`
Label string `json:"label"`
State string `json:"state"`
PollError string `json:"poll_error,omitempty"`
SpeakerID string `json:"speaker_id"`
Label string `json:"label"`
State string `json:"state"`
PollError string `json:"poll_error,omitempty"`
Observation liveObservation `json:"observation"`
}
type liveSpeakerPoll struct {
SpeakerID string `json:"speaker_id"`
Label string `json:"label"`
OK bool `json:"ok"`
SessionCount int `json:"session_count"`
PollError string `json:"poll_error,omitempty"`
SpeakerID string `json:"speaker_id"`
Label string `json:"label"`
OK bool `json:"ok"`
SessionCount *int `json:"session_count"`
PollError string `json:"poll_error,omitempty"`
Observation liveObservation `json:"observation"`
}
func speakerDisplayLabel(sp *store.Speaker) string {
@@ -65,95 +116,21 @@ func speakerDisplayLabel(sp *store.Speaker) string {
return sp.ID
}
func masterSpeakerID(speakers []*store.Speaker) string {
for _, sp := range speakers {
if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
return sp.ID
}
}
return ""
}
func (s *Server) collectSpeakerBGPLive(ctx context.Context, tenantID string, fresh bool) []speakerBGPLive {
if !fresh {
if v, ok := peerLiveCache.Load(tenantID); ok {
ent := v.(peerLiveCacheEntry)
if time.Since(ent.at) < peerLiveCacheTTL {
return ent.views
}
}
}
speakers := s.store.ListSpeakersForTenant(tenantID)
views := make([]speakerBGPLive, 0, len(speakers)+1)
if sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")); sock != "" {
v := speakerBGPLive{Label: "CP (local BIRD)"}
if mid := masterSpeakerID(speakers); mid != "" {
v.SpeakerID = mid
for _, sp := range speakers {
if sp != nil && sp.ID == mid {
v.Label = speakerDisplayLabel(sp)
break
}
}
}
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")))
if err != nil {
v.Error = err.Error()
} else {
v.Sessions = birdfmt.ParseBGPSessions(out)
}
views = append(views, v)
}
opts := nodedispatch.Options{Timeout: 8 * time.Second}
type resWrap struct {
sp *store.Speaker
res nodedispatch.BirdProtocolsResult
}
ch := make(chan resWrap, len(speakers))
var wg sync.WaitGroup
for _, sp := range speakers {
if sp == nil {
continue
}
meta := store.ParseSpeakerMeta(sp.MetaJSON)
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
continue
}
wg.Add(1)
go func(speaker *store.Speaker) {
defer wg.Done()
ch <- resWrap{
sp: speaker,
res: nodedispatch.FetchBirdProtocols(ctx, speaker, opts),
}
}(sp)
}
wg.Wait()
close(ch)
for rw := range ch {
views = append(views, speakerBGPLive{
SpeakerID: rw.sp.ID,
Label: speakerDisplayLabel(rw.sp),
Sessions: rw.res.Sessions,
Error: rw.res.Error,
})
}
peerLiveCache.Store(tenantID, peerLiveCacheEntry{at: time.Now(), views: views})
return views
}
func liveSpeakerPollJSON(views []speakerBGPLive) []liveSpeakerPoll {
out := make([]liveSpeakerPoll, 0, len(views))
for _, v := range views {
var sessionCount *int
if v.successful() {
count := len(v.Sessions)
sessionCount = &count
}
out = append(out, liveSpeakerPoll{
SpeakerID: v.SpeakerID,
Label: v.Label,
OK: v.Error == "",
SessionCount: len(v.Sessions),
OK: v.successful(),
SessionCount: sessionCount,
PollError: v.Error,
Observation: v.observation(),
})
}
return out
@@ -183,27 +160,30 @@ func matchPeerOnSpeakers(peer *store.BGPPeer, views []speakerBGPLive) (
protoName := birdfmt.PeerProtocolName(peer.ID)
for _, v := range views {
if v.Error != "" && len(v.Sessions) == 0 {
if !v.successful() {
on = append(on, peerSessionOnSpeaker{
SpeakerID: v.SpeakerID,
Label: v.Label,
PollError: v.Error,
SpeakerID: v.SpeakerID,
Label: v.Label,
PollError: v.Error,
Observation: v.observation(),
})
continue
}
sess := findPeerSession(v.Sessions, protoName, neighbor, hasNeighbor)
if sess == nil {
on = append(on, peerSessionOnSpeaker{
SpeakerID: v.SpeakerID,
Label: v.Label,
State: "absent",
SpeakerID: v.SpeakerID,
Label: v.Label,
State: "absent",
Observation: v.observation(),
})
continue
}
hit := peerSessionOnSpeaker{
SpeakerID: v.SpeakerID,
Label: v.Label,
State: sess.State,
SpeakerID: v.SpeakerID,
Label: v.Label,
State: sess.State,
Observation: v.observation(),
}
on = append(on, hit)
if strings.EqualFold(strings.TrimSpace(sess.State), "Established") {
@@ -273,20 +253,109 @@ func sessionStateRank(state string) int {
}
func applyPeerLiveFields(row map[string]any, peer *store.BGPPeer, views []speakerBGPLive) {
state, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views)
expected := expectedPeerViews(peer, views)
state, _, _, _, _, _ := matchPeerOnSpeakers(peer, expected)
_, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views)
row["session_on_speakers"] = on
row["established_on_speakers"] = establishedOn
row["session_conflict"] = false
row["session_mismatch"] = mismatch
row["last_known_session_state"] = row["session_state"]
row["session_state"] = nil
if state != "" {
row["session_state"] = state
}
if state == "" && len(expected) > 0 && allSpeakerPollsSuccessful(expected) {
row["session_state"] = "absent"
}
row["live_observation"] = peerLiveObservation(expected)
if connLabel != "" {
row["connected_speaker_label"] = connLabel
}
row["connected_speaker_id"] = peerLiveSpeakerIDOrNull(connID)
}
func expectedPeerViews(peer *store.BGPPeer, views []speakerBGPLive) []speakerBGPLive {
if peer == nil || peer.SpeakerID == nil || strings.TrimSpace(*peer.SpeakerID) == "" {
return views
}
want := strings.TrimSpace(*peer.SpeakerID)
expected := make([]speakerBGPLive, 0, 1)
for _, v := range views {
if strings.EqualFold(v.SpeakerID, want) {
expected = append(expected, v)
}
}
return expected
}
func allSpeakerPollsSuccessful(views []speakerBGPLive) bool {
for _, v := range views {
if !v.successful() {
return false
}
}
return true
}
func peerLiveObservation(views []speakerBGPLive) liveObservation {
if len(views) == 0 {
return speakerBGPLive{}.observation()
}
result := liveObservation{Source: views[0].Source, Quality: "unknown"}
if result.Source == "" {
result.Source = "none"
}
var latestObserved, latestReceived, latestSuccess time.Time
allOK, allFailed, anyStale := true, true, false
for _, v := range views {
o := v.observation()
if v.ObservedAt.After(latestObserved) {
latestObserved = v.ObservedAt
}
if v.ReceivedAt.After(latestReceived) {
latestReceived = v.ReceivedAt
}
if v.LastSuccessAt.After(latestSuccess) {
latestSuccess = v.LastSuccessAt
}
allOK = allOK && v.successful()
allFailed = allFailed && o.Quality == "unreachable"
anyStale = anyStale || o.Quality == "stale"
if result.ErrorCode == "" && o.ErrorCode != "" {
result.ErrorCode = o.ErrorCode
}
if o.ExpiresAt != nil {
expires, err := time.Parse(time.RFC3339Nano, *o.ExpiresAt)
if err == nil {
previous := time.Time{}
if result.ExpiresAt != nil {
previous, _ = time.Parse(time.RFC3339Nano, *result.ExpiresAt)
}
if previous.IsZero() || expires.Before(previous) {
result.ExpiresAt = o.ExpiresAt
}
}
}
}
result.ObservedAt = liveTime(latestObserved)
result.ReceivedAt = liveTime(latestReceived)
result.LastSuccessAt = liveTime(latestSuccess)
if len(views) > 1 {
result.Source = "mixed"
}
switch {
case allOK:
result.Quality = "ok"
result.ErrorCode = ""
case anyStale:
result.Quality = "stale"
case allFailed:
result.Quality = "unreachable"
}
return result
}
func peerLiveSpeakerIDOrNull(id string) any {
if strings.TrimSpace(id) == "" {
return nil
+73
View File
@@ -2,11 +2,84 @@ package httpapi
import (
"testing"
"time"
"evobgp/internal/birdfmt"
"evobgp/internal/store"
)
func TestApplyPeerLiveFieldsObservationStates(t *testing.T) {
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"}
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
cases := []struct {
name string
views []speakerBGPLive
wantState any
wantQuality string
}{
{"no collector", nil, nil, "unknown"},
{"successful empty poll", []speakerBGPLive{{Polled: true, Source: "agent_bird_protocols", ObservedAt: now, ReceivedAt: now, LastSuccessAt: now}}, "absent", "ok"},
{"first failure", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now}}, nil, "unreachable"},
{"failure after success", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now, LastSuccessAt: now.Add(-time.Minute)}}, nil, "stale"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
row := map[string]any{"session_state": "Established"}
applyPeerLiveFields(row, peer, tc.views)
if row["session_state"] != tc.wantState || row["last_known_session_state"] != "Established" {
t.Fatalf("session_state=%v last_known=%v", row["session_state"], row["last_known_session_state"])
}
observation := row["live_observation"].(liveObservation)
if observation.Quality != tc.wantQuality {
t.Fatalf("observation=%+v", observation)
}
})
}
}
func TestApplyPeerLiveFieldsExpectedSpeakerCoverage(t *testing.T) {
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
assigned := "speaker-a"
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2", SpeakerID: &assigned}
viewAError := speakerBGPLive{SpeakerID: assigned, Polled: true, Error: "timeout", ObservedAt: now, Source: "agent_bird_protocols"}
viewAEmpty := speakerBGPLive{SpeakerID: assigned, Polled: true, ObservedAt: now, LastSuccessAt: now, Source: "agent_bird_protocols"}
viewBEmpty := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, ObservedAt: now.Add(time.Minute), LastSuccessAt: now.Add(time.Minute), Source: "agent_bird_protocols"}
viewBError := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, Error: "timeout", ObservedAt: now.Add(time.Minute), Source: "agent_bird_protocols"}
cases := []struct {
name string
assigned bool
views []speakerBGPLive
wantState any
wantQuality string
wantObserved string
}{
{"assigned timeout unrelated success", true, []speakerBGPLive{viewAError, viewBEmpty}, nil, "unreachable", now.Format(time.RFC3339Nano)},
{"assigned success unrelated failure", true, []speakerBGPLive{viewAEmpty, viewBError}, "absent", "ok", now.Format(time.RFC3339Nano)},
{"global mixed outcome", false, []speakerBGPLive{viewAEmpty, viewBError}, nil, "unknown", now.Add(time.Minute).Format(time.RFC3339Nano)},
{"assigned speaker not observed", true, []speakerBGPLive{viewBEmpty}, nil, "unknown", ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
candidate := *peer
if !tc.assigned {
candidate.SpeakerID = nil
}
row := map[string]any{"session_state": "Established"}
applyPeerLiveFields(row, &candidate, tc.views)
obs := row["live_observation"].(liveObservation)
if row["session_state"] != tc.wantState || obs.Quality != tc.wantQuality {
t.Fatalf("state=%v observation=%+v", row["session_state"], obs)
}
if tc.wantObserved == "" && obs.ObservedAt != nil {
t.Fatalf("unexpected timestamp %v", *obs.ObservedAt)
}
if tc.wantObserved != "" && (obs.ObservedAt == nil || *obs.ObservedAt != tc.wantObserved) {
t.Fatalf("observed_at=%v want=%q", obs.ObservedAt, tc.wantObserved)
}
})
}
}
func TestMatchPeerOnSpeakers_establishedOnReplica(t *testing.T) {
peer := &store.BGPPeer{
ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
+47 -20
View File
@@ -9,23 +9,33 @@ import (
"time"
)
// authRateLimiter is a simple per-IP token bucket for public auth-ish endpoints.
const maxAuthRateLimitClients = 4096
type authRateHit struct {
start time.Time
count int
}
// authRateLimiter keeps a bounded fixed-window counter per client IP.
type authRateLimiter struct {
mu sync.Mutex
hits map[string][]time.Time
limit int
window time.Duration
mu sync.Mutex
hits map[string]authRateHit
limit int
window time.Duration
maxClients int
nextSweep time.Time
}
func newAuthRateLimiter() *authRateLimiter {
limit := 60
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 {
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 && n <= 1000 {
limit = n
}
return &authRateLimiter{
hits: make(map[string][]time.Time),
limit: limit,
window: time.Minute,
hits: make(map[string]authRateHit),
limit: limit,
window: time.Minute,
maxClients: maxAuthRateLimitClients,
}
}
@@ -33,23 +43,40 @@ func (l *authRateLimiter) allow(ip string) bool {
if l == nil {
return true
}
now := time.Now()
return l.allowAt(ip, time.Now())
}
func (l *authRateLimiter) allowAt(ip string, now time.Time) bool {
l.mu.Lock()
defer l.mu.Unlock()
cut := now.Add(-l.window)
arr := l.hits[ip]
kept := arr[:0]
for _, t := range arr {
if t.After(cut) {
kept = append(kept, t)
if !l.nextSweep.IsZero() && !now.Before(l.nextSweep) {
l.nextSweep = time.Time{}
for key, hit := range l.hits {
expiry := hit.start.Add(l.window)
if !now.Before(expiry) {
delete(l.hits, key)
} else if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) {
l.nextSweep = expiry
}
}
}
if len(kept) >= l.limit {
l.hits[ip] = kept
hit, exists := l.hits[ip]
if !exists && len(l.hits) >= l.maxClients {
// Never evict an active counter: a new identity must wait for expiry.
return false
}
kept = append(kept, now)
l.hits[ip] = kept
if exists && hit.count >= l.limit {
return false
}
if !exists {
hit.start = now
expiry := now.Add(l.window)
if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) {
l.nextSweep = expiry
}
}
hit.count++
l.hits[ip] = hit
return true
}
+100
View File
@@ -0,0 +1,100 @@
package httpapi
import (
"bytes"
"context"
"errors"
"io"
"net"
"net/http"
"os"
"strconv"
"strings"
)
const requestBodyLimit int64 = 8 << 20
// withRequestBodyLimit must run inside authMiddleware. Reading the complete
// bounded body before dispatch prevents a valid JSON prefix from committing a
// mutation before a later size/read error is discovered. Content-Type is not a
// security boundary; CSV retains its additional row/field checks in its handler.
func withRequestBodyLimit(next http.Handler) http.Handler {
return withBodyLimitAndConcurrency(requestBodyLimit, requestBodyConcurrency(), next)
}
func withBodyLimit(limit int64, next http.Handler) http.Handler {
return withBodyLimitAndConcurrency(limit, 4, next)
}
func requestBodyConcurrency() int {
n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_API_BODY_MAX_CONCURRENT")))
if err == nil && n >= 1 && n <= 64 {
return n
}
return 4
}
func withBodyLimitAndConcurrency(limit int64, concurrency int, next http.Handler) http.Handler {
slots := make(chan struct{}, concurrency)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
original := r.Body
if original != nil {
defer func() { _ = original.Close() }()
}
if r.ContentLength > limit {
writeRequestBodyTooLarge(w)
return
}
if r.Context().Err() != nil {
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled")
return
}
if r.Body == nil || r.Body == http.NoBody {
next.ServeHTTP(w, r)
return
}
select {
case slots <- struct{}{}:
defer func() { <-slots }()
default:
w.Header().Set("Retry-After", "1")
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "request body processing capacity is full")
return
}
bounded := http.MaxBytesReader(w, r.Body, limit)
data, err := io.ReadAll(requestContextReader{ctx: r.Context(), reader: bounded})
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
writeRequestBodyTooLarge(w)
return
}
var timeout net.Error
if r.Context().Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timeout) && timeout.Timeout()) {
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled")
return
}
if err != nil {
writeProblem(w, http.StatusBadRequest, "Bad Request", "could not read request body")
return
}
r.Body = io.NopCloser(bytes.NewReader(data))
r.ContentLength = int64(len(data))
next.ServeHTTP(w, r)
})
}
func writeRequestBodyTooLarge(w http.ResponseWriter) {
writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "request body exceeds the 8 MiB limit")
}
type requestContextReader struct {
ctx context.Context
reader io.Reader
}
func (r requestContextReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.reader.Read(p)
}
+273
View File
@@ -0,0 +1,273 @@
package httpapi
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
)
func TestRequestBodyKnownAndUnknownBoundsBeforeMutation(t *testing.T) {
const limit = 32
for _, unknown := range []bool{false, true} {
for _, contentType := range []string{"", "application/json", "text/plain"} {
for _, extra := range []int{0, 1} {
label := "known"
if unknown {
label = "unknown"
}
t.Run(label+"/"+contentType+"/"+string(rune('0'+extra)), func(t *testing.T) {
mutations := 0
h := withBodyLimit(limit, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var input map[string]any
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
t.Error(err)
}
mutations++
w.WriteHeader(http.StatusCreated)
}))
body := `{"x":1}` + strings.Repeat(" ", limit-len(`{"x":1}`)+extra)
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
if unknown {
req.ContentLength = -1
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if extra == 0 {
if w.Code != http.StatusCreated || mutations != 1 {
t.Fatalf("exact limit rejected: %d mutations=%d", w.Code, mutations)
}
} else if w.Code != http.StatusRequestEntityTooLarge || mutations != 0 {
t.Fatalf("oversized tail mutated: %d mutations=%d", w.Code, mutations)
}
})
}
}
}
}
type checkedRequestBody struct {
reads int
closed bool
reader io.Reader
}
func (b *checkedRequestBody) Read(p []byte) (int, error) { b.reads++; return b.reader.Read(p) }
func (b *checkedRequestBody) Close() error { b.closed = true; return nil }
type failingRequestReader struct{}
func (failingRequestReader) Read([]byte) (int, error) {
return 0, errors.New("untrusted detailed read failure")
}
func TestRequestBodyErrorsDoNotMutate(t *testing.T) {
for _, cancelled := range []bool{false, true} {
calls := 0
b := &checkedRequestBody{reader: io.MultiReader(strings.NewReader(`{"x":1}`), failingRequestReader{})}
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
req.Body = b
req.ContentLength = -1
if cancelled {
ctx, cancel := context.WithCancel(req.Context())
cancel()
req = req.WithContext(ctx)
}
w := httptest.NewRecorder()
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { calls++ })).ServeHTTP(w, req)
want := http.StatusBadRequest
if cancelled {
want = http.StatusRequestTimeout
}
if w.Code != want || calls != 0 || !b.closed || strings.Contains(w.Body.String(), "untrusted detailed") {
t.Fatalf("error handling status=%d calls=%d body=%s", w.Code, calls, w.Body.String())
}
if cancelled && b.reads != 0 {
t.Fatal("read body after cancellation")
}
}
}
func TestRequestBodyAuthenticationRunsFirst(t *testing.T) {
b := &checkedRequestBody{reader: failingRequestReader{}}
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
req.Body = b
req.ContentLength = requestBodyLimit + 1
s := &Server{}
called := false
h := s.authMiddleware(withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusUnauthorized || b.reads != 0 || called {
t.Fatalf("body processed before auth: %d reads=%d", w.Code, b.reads)
}
}
func TestRequestBodyAppliesToGETAndIgnoresClaimedZeroLength(t *testing.T) {
for _, length := range []int64{-1, 0, requestBodyLimit + 1} {
req := httptest.NewRequest(http.MethodGet, "/v1/jobs", strings.NewReader(strings.Repeat("x", int(requestBodyLimit)+1)))
req.ContentLength = length
called := false
w := httptest.NewRecorder()
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })).ServeHTTP(w, req)
if w.Code != http.StatusRequestEntityTooLarge || called {
t.Fatalf("GET size bypass length=%d code=%d", length, w.Code)
}
}
}
func TestRequestBodyHandlerWiringAndNoMutation(t *testing.T) {
s, err := New(Options{SeedDemo: true, InsecureDev: true, BundleSeedHex: testBundleSeed})
if err != nil {
t.Fatal(err)
}
defer s.Close()
tenant, _, _, _, _ := s.store.DemoIDs()
before := len(s.store.ListModules(tenant))
h := s.Handler()
prefix := `{"type":"IP_RANGES","name":"bounded-create","enabled":false}`
for _, extra := range []int{1, 0} {
body := prefix + strings.Repeat(" ", int(requestBodyLimit)-len(prefix)+extra)
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
req.ContentLength = -1
req.Header.Set("Authorization", "Bearer dev")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if extra == 1 {
if w.Code != http.StatusRequestEntityTooLarge || len(s.store.ListModules(tenant)) != before {
t.Fatalf("oversized JSON committed: %d %s", w.Code, w.Body.String())
}
} else if w.Code != http.StatusCreated || len(s.store.ListModules(tenant)) != before+1 {
t.Fatalf("8MiB body rejected: %d %s", w.Code, w.Body.String())
}
}
}
func TestRequestBodyLeavesStreamingResponseUnbuffered(t *testing.T) {
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/v1/runtime-logs/stream", nil)
withRequestBodyLimit(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
f, ok := w.(http.Flusher)
if !ok {
t.Fatal("response lost Flusher")
}
w.Header().Set("Content-Type", "text/event-stream")
_, _ = io.WriteString(w, "data: ready\n\n")
f.Flush()
})).ServeHTTP(w, req)
if !w.Flushed || w.Body.String() != "data: ready\n\n" {
t.Fatal("stream response buffered")
}
}
type blockedAdmissionBody struct {
entered chan struct{}
resume chan struct{}
sent bool
}
func (b *blockedAdmissionBody) Read(p []byte) (int, error) {
if b.sent {
return 0, io.EOF
}
b.sent = true
close(b.entered)
<-b.resume
p[0] = 'x'
return 1, nil
}
func (*blockedAdmissionBody) Close() error { return nil }
func TestRequestBodyAdmissionRejectsWithoutWaitingAndHoldsThroughHandler(t *testing.T) {
readEntered, readResume := make(chan struct{}), make(chan struct{})
handlerEntered, handlerResume := make(chan struct{}), make(chan struct{})
var once sync.Once
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/empty" {
once.Do(func() { close(handlerEntered) })
<-handlerResume
}
w.WriteHeader(http.StatusNoContent)
}))
first := httptest.NewRequest(http.MethodPost, "/first", nil)
first.Body = &blockedAdmissionBody{entered: readEntered, resume: readResume}
first.ContentLength = 0 // A claimed zero must not bypass body admission.
done := make(chan struct{})
go func() { defer close(done); h.ServeHTTP(httptest.NewRecorder(), first) }()
select {
case <-readEntered:
case <-time.After(3 * time.Second):
t.Fatal("first body did not start")
}
reject := func() {
t.Helper()
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/second", strings.NewReader("x"))
h.ServeHTTP(w, req)
if w.Code != http.StatusTooManyRequests || w.Header().Get("Retry-After") != "1" {
t.Fatalf("admission did not reject: %d %s", w.Code, w.Body.String())
}
}
reject()
// An empty legacy GET remains available while all body slots are occupied.
empty := httptest.NewRecorder()
h.ServeHTTP(empty, httptest.NewRequest(http.MethodGet, "/empty", nil))
if empty.Code != http.StatusNoContent {
t.Fatal("empty GET consumed body slot")
}
close(readResume)
select {
case <-handlerEntered:
case <-time.After(3 * time.Second):
t.Fatal("handler did not start")
}
reject() // The fully read buffer is still retained by the running handler.
close(handlerResume)
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("first request did not finish")
}
w := httptest.NewRecorder()
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/third", strings.NewReader("x")))
if w.Code != http.StatusNoContent {
t.Fatalf("body slot not released: %d", w.Code)
}
}
func TestRequestBodyAdmissionReleasesOnReadError(t *testing.T) {
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("read error reached handler") }))
for range 2 {
req := httptest.NewRequest(http.MethodPost, "/bad", nil)
req.Body = io.NopCloser(failingRequestReader{})
req.ContentLength = -1
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusBadRequest {
t.Fatalf("read failure retained slot: %d", w.Code)
}
}
}
func TestRequestBodyAdmissionEnvironment(t *testing.T) {
for _, tc := range []struct {
raw string
want int
}{{"", 4}, {"1", 1}, {"64", 64}, {"65", 4}, {"0", 4}, {"-1", 4}, {"invalid", 4}} {
t.Run(tc.raw, func(t *testing.T) {
t.Setenv("EVOBGP_API_BODY_MAX_CONCURRENT", tc.raw)
if got := requestBodyConcurrency(); got != tc.want {
t.Fatalf("concurrency=%d want%d", got, tc.want)
}
})
}
}
+147 -79
View File
@@ -8,14 +8,12 @@ import (
"errors"
"io"
"net/http"
"os"
"sort"
"strconv"
"strings"
"time"
"evobgp/internal/birdfmt"
"evobgp/internal/bundle"
"evobgp/internal/jobs"
"evobgp/internal/observability"
"evobgp/internal/pipeline"
@@ -39,7 +37,7 @@ func (s *Server) Handler() http.Handler {
// Firewall subsystem moved to the standalone EvoFirewall service; see docs/firewall.md.
// Registered on the public mux so it wins over the "/v1/" subtree below regardless of auth.
s.mux.HandleFunc("/v1/firewall/", s.handleFirewallGone)
s.mux.Handle("/v1/", s.authMiddleware(wrappedV1))
s.mux.Handle("/v1/", s.authMiddleware(withRequestBodyLimit(wrappedV1)))
return s.withCORS(observability.HTTPMiddleware(s.mux))
}
@@ -60,6 +58,7 @@ func (s *Server) registerV1(m *http.ServeMux) {
m.HandleFunc("GET /modules/{module_id}", s.handleGetModule)
m.HandleFunc("GET /peers", s.handleListPeers)
m.HandleFunc("GET /speakers", s.handleListSpeakers)
m.HandleFunc("POST /telemetry/refresh", s.handleTelemetryRefresh)
m.HandleFunc("GET /bundle/signing-public-key", s.handleBundleSigningPublicKey)
m.HandleFunc("POST /modules/{module_id}/refresh", s.handleModuleRefresh)
m.HandleFunc("POST /tenant/refresh", s.handleTenantRefresh)
@@ -79,6 +78,7 @@ func (s *Server) registerV1(m *http.ServeMux) {
m.HandleFunc("GET /jobs/{job_id}", s.handleGetJob)
m.HandleFunc("GET /jobs/{job_id}/report", s.handleGetJobReport)
m.HandleFunc("POST /jobs/{job_id}/cancel", s.handleCancelJob)
m.HandleFunc("POST /jobs/{job_id}/retry", s.handleRetryJob)
m.HandleFunc("GET /speakers/{speaker_id}/revisions/latest", s.handleNodeLatestRevision)
m.HandleFunc("GET /speakers/{speaker_id}/bundle/{revision_id}", s.handleNodeBundle)
m.HandleFunc("POST /nodes/enroll", s.handleNodeEnroll)
@@ -295,12 +295,10 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) {
allPeers := s.store.ListPeers(a.TenantID)
allPeers = store.FilterOwned(allPeers, func(p *store.BGPPeer) string { return p.CreatedByUserID }, a.Kind, a.IsAdmin, a.UserID)
page, next, more := store.PaginateOffset(allPeers, r.URL.Query().Get("cursor"), parseListLimit(r))
fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1")
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second)
defer cancel()
liveViews := s.collectSpeakerBGPLive(ctx, a.TenantID, fresh)
if fresh {
s.syncPeerDiscoveriesFromLive(a.TenantID, liveViews)
liveViews, _, err := s.loadTelemetry(r.Context(), a.TenantID, s.store.ListSpeakersForTenant(a.TenantID))
if err != nil {
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable")
return
}
items := make([]map[string]any, 0, len(page))
for _, p := range page {
@@ -316,30 +314,6 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) {
})
}
func (s *Server) liveBGPProtocolStates(r *http.Request) map[string]string {
if r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1") {
return s.liveBGPProtocolStatesFresh(r.Context())
}
if cached, ok := observability.CachedBirdProtocolStates(90 * time.Second); ok {
return cached
}
return s.liveBGPProtocolStatesFresh(r.Context())
}
func (s *Server) liveBGPProtocolStatesFresh(ctx context.Context) map[string]string {
sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET"))
if sock == "" {
return map[string]string{}
}
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")))
if err != nil {
return map[string]string{}
}
states := birdfmt.ParseBGPProtocolStates(out)
observability.SetBirdProtocolStates(states)
return states
}
// parseBGPProtocolStates parses `birdc show protocols all` summary rows into protocol_name -> state.
func parseBGPProtocolStates(output string) map[string]string {
out := make(map[string]string)
@@ -403,9 +377,12 @@ func (s *Server) handleListSpeakers(w http.ResponseWriter, r *http.Request) {
fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1")
var liveByID map[string]map[string]any
if fresh {
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second)
defer cancel()
liveByID = s.collectSpeakerLiveStatus(ctx, a.TenantID, true, speakers)
var err error
_, liveByID, err = s.loadTelemetry(r.Context(), a.TenantID, speakers)
if err != nil {
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable")
return
}
}
items := make([]map[string]any, 0, len(speakers))
for _, sp := range speakers {
@@ -448,12 +425,12 @@ func (s *Server) handleModuleRefresh(w http.ResponseWriter, r *http.Request) {
idemPtr = &idem
}
mid := mod.ID
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{
"module_id": moduleID,
"trigger": "api",
})
if err != nil {
writeInternalError(w, "internal", err)
writeJobEnqueueError(w, err)
return
}
w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -480,6 +457,17 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
}
}
moduleIDs := body.ModuleIDs
sort.Strings(body.ModuleIDs)
requestIDs := append([]string{}, body.ModuleIDs...)
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
var idemPtr *string
if idem != "" {
idemPtr = &idem
}
request := map[string]any{"module_ids": requestIDs, "trigger": "api"}
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindTenantRefresh, IdempotencyKey: idemPtr, Request: request}) {
return
}
if len(moduleIDs) == 0 {
now := time.Now().UTC()
for _, mod := range s.store.ListModules(a.TenantID) {
@@ -488,21 +476,16 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
}
}
}
if len(moduleIDs) == 0 {
if len(moduleIDs) == 0 && idem == "" {
writeJSON(w, http.StatusNoContent, map[string]any{"message": "no modules due for refresh"})
return
}
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
var idemPtr *string
if idem != "" {
idemPtr = &idem
}
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{
j, _, err := s.jobs.EnqueueRequestContext(r.Context(), a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{
"module_ids": moduleIDs,
"trigger": "api",
})
}, request)
if err != nil {
writeInternalError(w, "internal", err)
writeJobEnqueueError(w, err)
return
}
w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -510,7 +493,7 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusAccepted, map[string]any{
"job_id": snap["job_id"],
"status": snap["status"],
"module_ids": moduleIDs,
"module_ids": j.Snapshot()["meta"].(map[string]any)["module_ids"],
"modules_due": len(moduleIDs),
})
}
@@ -560,8 +543,8 @@ func revisionJSON(rev *store.Revision) map[string]any {
return m
}
// enqueueModuleRefreshIfEnabled queues module_refresh when the module exists and is enabled (best-effort, no HTTP error).
func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger string) {
// enqueueModuleRefreshIfEnabled reports scheduling failure after a committed CRUD mutation.
func (s *Server) enqueueModuleRefreshIfEnabled(w http.ResponseWriter, r *http.Request, tenantID, moduleID, trigger string) {
if s.jobs == nil {
return
}
@@ -570,22 +553,35 @@ func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger strin
return
}
mid := moduleID
key := "module_refresh:" + moduleID
_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindModuleRefresh, &key, &mid, map[string]any{
// Each mutation is a new logical refresh. Reusing a module-wide key would
// incorrectly replay an older terminal job after later configuration changes.
_, _, err = s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindModuleRefresh, nil, &mid, map[string]any{
"module_id": moduleID,
"trigger": trigger,
})
s.reportAutomaticEnqueue(w, r, trigger, moduleID, err)
}
// enqueuePeerReconcile queues fast peer-only reconcile/render (best-effort, no HTTP error).
func (s *Server) enqueuePeerReconcile(tenantID, trigger string) {
func (s *Server) enqueuePeerReconcile(w http.ResponseWriter, r *http.Request, tenantID, trigger string) {
if s.jobs == nil {
return
}
_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{
_, _, err := s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{
"trigger": trigger,
"job_title": "Обновление BGP пиров",
})
s.reportAutomaticEnqueue(w, r, trigger, "", err)
}
func (s *Server) reportAutomaticEnqueue(w http.ResponseWriter, r *http.Request, trigger, target string, err error) {
if err == nil {
return
}
w.Header().Set("X-EvoBGP-Job-Status", "enqueue-failed")
if a, ok := authFromContext(r.Context()); ok {
s.recordCRUDAudit(r, a, "bgp.job.enqueue_failed", "Configuration saved; automatic job was not queued", target, map[string]any{"trigger": trigger, "error": err.Error()})
}
}
func (s *Server) handleGetRevision(w http.ResponseWriter, r *http.Request) {
@@ -614,6 +610,11 @@ func (s *Server) handleRevisionPreview(w http.ResponseWriter, r *http.Request) {
if !s.requirePerm(w, a, "bgp:operations:read") {
return
}
release, ok := s.acquireOperation(w, r, a.TenantID, revisionPreviewOperation)
if !ok {
return
}
defer release()
rev, err := s.store.GetRevision(a.TenantID, r.PathValue("revision_id"))
if err != nil {
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
@@ -706,11 +707,11 @@ func (s *Server) handleRevisionRollback(w http.ResponseWriter, r *http.Request)
if idem != "" {
idemPtr = &idem
}
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{
"source_revision_id": revID,
})
if err != nil {
writeInternalError(w, "internal", err)
writeJobEnqueueError(w, err)
return
}
w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -751,13 +752,13 @@ func (s *Server) handleApply(w http.ResponseWriter, r *http.Request) {
if idem != "" {
idemPtr = &idem
}
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
"revision_id": revID,
"strategy": body.Strategy,
"trigger": "api",
})
if err != nil {
writeInternalError(w, "internal", err)
writeJobEnqueueError(w, err)
return
}
w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -802,13 +803,13 @@ func (s *Server) handleSpeakerApply(w http.ResponseWriter, r *http.Request) {
if idem != "" {
idemPtr = &idem
}
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
"revision_id": revID,
"speaker_id": spkID,
"trigger": "api",
})
if err != nil {
writeInternalError(w, "internal", err)
writeJobEnqueueError(w, err)
return
}
w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -830,9 +831,9 @@ func (s *Server) handleBirdReload(w http.ResponseWriter, r *http.Request) {
if idem != "" {
idemPtr = &idem
}
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil)
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil)
if err != nil {
writeInternalError(w, "internal", err)
writeJobEnqueueError(w, err)
return
}
w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -867,7 +868,11 @@ func (s *Server) handleListJobs(w http.ResponseWriter, r *http.Request) {
status := r.URL.Query().Get("status")
kind := r.URL.Query().Get("kind")
cursor := r.URL.Query().Get("cursor")
list, next, more := s.jobs.List(a.TenantID, status, kind, cursor, limit)
list, next, more, err := s.jobs.ListContext(r.Context(), a.TenantID, status, kind, cursor, limit)
if err != nil {
writeInternalError(w, "jobs_list", err)
return
}
items := make([]map[string]any, 0, len(list))
for _, j := range list {
items = append(items, j.Snapshot())
@@ -886,9 +891,9 @@ func (s *Server) handleGetJob(w http.ResponseWriter, r *http.Request) {
if !s.requirePerm(w, a, "bgp:operations:read") {
return
}
j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id"))
j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id"))
if err != nil {
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
writeJobReadError(w, err)
return
}
writeJSON(w, http.StatusOK, j.Snapshot())
@@ -903,9 +908,9 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) {
if !s.requirePerm(w, a, "bgp:operations:read") {
return
}
j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id"))
j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id"))
if err != nil {
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
writeJobReadError(w, err)
return
}
snap := j.Snapshot()
@@ -929,6 +934,35 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) {
out["revision_id"] = v
}
}
sourceJobID := ""
if source, ok := meta["source_deployment_job_id"].(string); ok && source != "" {
// Resolve through the tenant-scoped registry before exposing linked progress.
if _, err := s.jobs.GetContext(r.Context(), a.TenantID, source); err != nil {
writeJobReadError(w, err)
return
}
sourceJobID = source
out["source_deployment_job_id"] = source
}
rows, err := s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, j.ID)
if err == nil && len(rows) == 0 && sourceJobID != "" {
rows, err = s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, sourceJobID)
}
if err != nil {
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "deployment progress unavailable")
return
}
progress := make([]map[string]any, 0, len(rows))
for _, d := range rows {
progress = append(progress, map[string]any{
"job_id": d.JobID, "speaker_id": d.SpeakerID, "revision_id": d.RevisionID,
"generation": d.Generation, "state": d.State, "confirmation": d.Confirmation,
"error": d.Error, "dispatch_attempts": d.DispatchAttempts, "retry_round": d.RetryRound,
"health_received_at": d.HealthReceivedAt, "health_revision_id": d.HealthRevisionID,
"health_source_at": d.HealthSourceAt, "health_error": d.HealthError,
})
}
out["deployment_progress"] = progress
writeJSON(w, http.StatusOK, out)
}
@@ -941,9 +975,34 @@ func (s *Server) handleCancelJob(w http.ResponseWriter, r *http.Request) {
if !s.requirePerm(w, a, "bgp:operations:admin") {
return
}
j, err := s.jobs.RequestCancel(a.TenantID, r.PathValue("job_id"))
j, err := s.jobs.RequestCancelContext(r.Context(), a.TenantID, r.PathValue("job_id"))
if err != nil {
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
writeJobReadError(w, err)
return
}
writeJSON(w, http.StatusAccepted, j.Snapshot())
}
func (s *Server) handleRetryJob(w http.ResponseWriter, r *http.Request) {
a, ok := authFromContext(r.Context())
if !ok {
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
return
}
if !s.requirePerm(w, a, "bgp:operations:admin") {
return
}
j, err := s.jobs.RetryContext(r.Context(), a.TenantID, r.PathValue("job_id"))
if err != nil {
if errors.Is(err, store.ErrJobQueueFull) {
writeJobEnqueueError(w, err)
return
}
if errors.Is(err, store.ErrJobConflict) {
writeProblem(w, http.StatusConflict, "Conflict", "job is not failed")
return
}
writeJobReadError(w, err)
return
}
writeJSON(w, http.StatusAccepted, j.Snapshot())
@@ -998,21 +1057,30 @@ func (s *Server) handleNodeBundle(w http.ResponseWriter, r *http.Request) {
writeProblem(w, http.StatusForbidden, "Forbidden", "speaker not in tenant scope")
return
}
rev, err := s.store.GetRevision(a.TenantID, rid)
if err != nil {
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
return
}
frags := rev.PreviewFragments
overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, a.TenantID, sid, rid, frags)
overlayFP, err := nodeBundleOverlayFingerprint(s.store, a.TenantID, sid, rid)
if err != nil {
writeInternalError(w, "bundle overlay", err)
return
}
frags = overlaid
tgz, err := bundle.BuildGzippedTar(rid, sid, frags, s.bundlePriv)
// Finished gzip, including manifest and bundle.sig. Revision bytes are immutable;
// the overlay fingerprint keeps speaker-specific renders from colliding.
tgz, err := s.bundleCache.do(r.Context(), nodeBundleCacheKey(a.TenantID, rid, sid, overlayFP), func(ctx context.Context) ([]byte, bool, error) {
return s.loadNodeBundle(ctx, a.TenantID, sid, rid, overlayFP)
})
if err != nil {
writeInternalError(w, "internal", err)
if errors.Is(err, errNodeBundleRevisionMissing) {
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
return
}
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return
}
op := "internal"
var opErr *nodeBundleOpError
if errors.As(err, &opErr) && opErr.op != "" {
op = opErr.op
}
writeInternalError(w, op, err)
return
}
w.Header().Set("Content-Type", "application/gzip")
+1 -18
View File
@@ -3,7 +3,6 @@ package httpapi
import (
"context"
"log"
"net"
"net/http"
"strings"
"time"
@@ -132,23 +131,7 @@ func (s *Server) pushAuditToPortal(entry *store.AuditEntry) {
}
func clientIP(r *http.Request) string {
if r == nil {
return ""
}
if xff := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); xff != "" {
parts := strings.Split(xff, ",")
if len(parts) > 0 {
return strings.TrimSpace(parts[0])
}
}
if xrip := strings.TrimSpace(r.Header.Get("X-Real-IP")); xrip != "" {
return xrip
}
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
if err != nil {
return strings.TrimSpace(r.RemoteAddr)
}
return host
return clientIPWithTrustedProxies(r, trustedProxyPrefixes())
}
// initAuditPusher wires portal push when URL and secret are configured.
+33 -24
View File
@@ -296,10 +296,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
return
}
if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
release, ok := s.acquireOperation(w, r, a.TenantID, cdnPreviewOperation)
if !ok {
return
}
defer release()
mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id"))
if err != nil {
writeStoreErr(w, err)
@@ -309,6 +310,10 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "module type must be CDN_CIDRS")
return
}
if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
return
}
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, u, nil)
if err != nil {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "invalid url")
@@ -321,11 +326,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, resp.Body)
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
writeBadGateway(w, "cdn preview fetch", fmt.Errorf("upstream status: %s", resp.Status))
return
}
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
raw, err := pipeline.ReadCDNBody(resp.Body)
if err != nil {
writeBadGateway(w, "cdn preview read body", err)
return
@@ -379,7 +384,7 @@ func (s *Server) handlePostCDNSource(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.cdn_source.create", "Created CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID, "url": x.URL})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_create")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_create")
writeJSON(w, http.StatusCreated, cdnSourceJSON(x))
}
@@ -410,7 +415,7 @@ func (s *Server) handlePatchCDNSource(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.cdn_source.update", "Updated CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_patch")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_patch")
writeJSON(w, http.StatusOK, cdnSourceJSON(x))
}
@@ -426,7 +431,7 @@ func (s *Server) handleDeleteCDNSource(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.cdn_source.delete", "Deleted CDN source", sourceID, map[string]any{"module_id": mid, "source_id": sourceID})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_delete")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_delete")
w.WriteHeader(http.StatusNoContent)
}
@@ -485,7 +490,7 @@ func (s *Server) handlePostAS(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.as_entry.create", "Created AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_create")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_create")
writeJSON(w, http.StatusCreated, asEntryJSON(x))
}
@@ -506,7 +511,7 @@ func (s *Server) handlePatchAS(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.as_entry.update", "Updated AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_patch")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_patch")
writeJSON(w, http.StatusOK, asEntryJSON(x))
}
@@ -522,7 +527,7 @@ func (s *Server) handleDeleteAS(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.as_entry.delete", "Deleted AS entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_delete")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_delete")
w.WriteHeader(http.StatusNoContent)
}
@@ -566,7 +571,7 @@ func (s *Server) handlePostDomain(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.domain_entry.create", "Created domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_create")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_create")
writeJSON(w, http.StatusCreated, domainEntryJSON(x))
}
@@ -587,7 +592,7 @@ func (s *Server) handlePatchDomain(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.domain_entry.update", "Updated domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_patch")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_patch")
writeJSON(w, http.StatusOK, domainEntryJSON(x))
}
@@ -603,7 +608,7 @@ func (s *Server) handleDeleteDomain(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.domain_entry.delete", "Deleted domain entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_delete")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_delete")
w.WriteHeader(http.StatusNoContent)
}
@@ -647,7 +652,7 @@ func (s *Server) handlePostIPRange(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.ip_range.create", "Created IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_create")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_create")
writeJSON(w, http.StatusCreated, ipRangeJSON(x))
}
@@ -668,7 +673,7 @@ func (s *Server) handlePatchIPRange(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.ip_range.update", "Updated IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_patch")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_patch")
writeJSON(w, http.StatusOK, ipRangeJSON(x))
}
@@ -684,7 +689,7 @@ func (s *Server) handleDeleteIPRange(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.ip_range.delete", "Deleted IP range entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_delete")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_delete")
w.WriteHeader(http.StatusNoContent)
}
@@ -777,8 +782,12 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req
return
}
moduleID := r.PathValue("module_id")
res, err := importer.ImportModuleEntriesCSV(s.store, a.TenantID, moduleID, r.Body)
res, err := importer.ImportModuleEntriesCSV(r.Context(), s.store, a.TenantID, moduleID, r.Body)
if err != nil {
if errors.Is(err, importer.ErrCSVTooLarge) {
writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "csv exceeds 8 MiB, 10000 data rows or 64 fields per record; split the import into smaller requests")
return
}
if errors.Is(err, store.ErrInvalidInput) {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv is empty")
return
@@ -802,11 +811,11 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req
if res.Imported > 0 {
switch res.ModuleType {
case "AS_PREFIXES":
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "as_entry_import_csv")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "as_entry_import_csv")
case "DOMAINS":
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "domain_entry_import_csv")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "domain_entry_import_csv")
case "IP_RANGES":
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "ip_range_import_csv")
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "ip_range_import_csv")
}
}
writeJSON(w, http.StatusOK, map[string]any{
@@ -1055,7 +1064,7 @@ func (s *Server) handlePostPeer(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.peer.create", "Created BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
s.enqueuePeerReconcile(a.TenantID, "peer_create")
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_create")
writeJSON(w, http.StatusCreated, peerJSON(x))
}
@@ -1099,7 +1108,7 @@ func (s *Server) handlePatchPeer(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.peer.update", "Updated BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
s.enqueuePeerReconcile(a.TenantID, "peer_patch")
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_patch")
writeJSON(w, http.StatusOK, peerJSON(x))
}
@@ -1120,7 +1129,7 @@ func (s *Server) handleDeletePeer(w http.ResponseWriter, r *http.Request) {
return
}
s.recordCRUDAudit(r, a, "bgp.peer.delete", "Deleted BGP peer", peerID, map[string]any{"peer_id": peerID})
s.enqueuePeerReconcile(a.TenantID, "peer_delete")
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_delete")
w.WriteHeader(http.StatusNoContent)
}
@@ -1295,7 +1304,7 @@ func (s *Server) handlePatchSettings(w http.ResponseWriter, r *http.Request) {
return
}
if peerDiscoverySettingsChanged(body) {
s.enqueuePeerReconcile(a.TenantID, "peer_discovery_settings")
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_settings")
}
s.recordCRUDAudit(r, a, "bgp.settings.update", "Updated tenant settings", a.TenantID, map[string]any{"keys": settingsAuditKeys(body)})
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+53
View File
@@ -1,6 +1,7 @@
package httpapi
import (
"context"
"encoding/csv"
"encoding/json"
"io"
@@ -8,8 +9,60 @@ import (
"net/http/httptest"
"strings"
"testing"
"evobgp/internal/importer"
"evobgp/internal/store"
)
func TestCSVImportLimitsBeforeMutations(t *testing.T) {
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, _, _ := m.DemoIDs()
mod, err := m.CreateModule(tenant, &store.Module{Type: "AS_PREFIXES", Name: "csv-limits"})
if err != nil {
t.Fatal(err)
}
module := mod.ID
s := &Server{store: m}
before, err := m.ListASEntries(tenant, module)
if err != nil {
t.Fatal(err)
}
header := "asn,community\n"
validPrefix := header + "64512,\n"
oversized := validPrefix + strings.Repeat("\n", importer.MaxCSVBytes-len(validPrefix)) + "64513,\n"
for _, tc := range []struct {
name, body string
unknownLength bool
status int
}{
{"exact", header + strings.Repeat("\n", importer.MaxCSVBytes-len(header)), false, http.StatusOK},
{"oversize", oversized, false, http.StatusRequestEntityTooLarge},
{"unknown_length", oversized, true, http.StatusRequestEntityTooLarge},
{"rows", header + strings.Repeat("64512,\n", importer.MaxCSVRows+1), true, http.StatusRequestEntityTooLarge},
{"fields", header + "64512," + strings.Repeat(",", importer.MaxCSVFields), false, http.StatusRequestEntityTooLarge},
} {
t.Run(tc.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/entries.csv", strings.NewReader(tc.body))
if tc.unknownLength {
req.ContentLength = -1
req.TransferEncoding = []string{"chunked"}
}
req.SetPathValue("module_id", module)
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"}))
w := httptest.NewRecorder()
s.handleImportModuleEntriesCSV(w, req)
if w.Code != tc.status {
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
}
after, e := m.ListASEntries(tenant, module)
if e != nil || len(after) != len(before) {
t.Fatalf("mutations on rejected/header-only request: count=%d err=%v", len(after), e)
}
})
}
}
func TestModuleEntriesCSVImportExportIPRanges(t *testing.T) {
srv, err := New(Options{
InsecureDev: true,
+5
View File
@@ -24,6 +24,11 @@ func (s *Server) handleLookup(w http.ResponseWriter, r *http.Request) {
writeProblem(w, http.StatusBadRequest, "Bad Request", "query parameter q is required")
return
}
release, ok := s.acquireOperation(w, r, a.TenantID, lookupOperation)
if !ok {
return
}
defer release()
res, err := lookup.Lookup(r.Context(), s.store, a.TenantID, q)
if err != nil {
if errors.Is(err, store.ErrInvalidInput) {
+2 -1
View File
@@ -1,6 +1,7 @@
package httpapi
import (
"context"
"encoding/json"
"io"
"net/http"
@@ -31,7 +32,7 @@ func TestLookupMembershipHTTP(t *testing.T) {
}); err != nil {
t.Fatal(err)
}
if err := srv.Store().SetModulePrefixSnapshot(tenant, modIP, "t", []store.PrefixRow{
if err := srv.Store().SetModulePrefixSnapshot(context.Background(), tenant, modIP, "t", []store.PrefixRow{
{Prefix: "198.51.100.0/24", CommunityID: &cid, Source: "ip_range"},
}); err != nil {
t.Fatal(err)
+9 -6
View File
@@ -242,10 +242,6 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request
return
}
kind := "maintenance_policy_run"
if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) {
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
return
}
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
var idemPtr *string
if idem != "" {
@@ -255,11 +251,18 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request
if dryRun {
title = "Maintenance policy dry-run"
}
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindMaintenancePolicyRun, IdempotencyKey: idemPtr, Meta: map[string]any{"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title}}) {
return
}
if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) {
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
return
}
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title,
})
if err != nil {
writeInternalError(w, "maintenance_policy_enqueue", err)
writeJobEnqueueError(w, err)
return
}
w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -11,6 +11,7 @@ import (
"evobgp/internal/jobs"
"evobgp/internal/pgmonitor"
"evobgp/internal/store"
)
var (
@@ -68,19 +69,22 @@ func (s *Server) enqueuePostgresMaint(w http.ResponseWriter, r *http.Request, a
if !s.requirePostgres(w) || !s.requireOperatorStrict(w, a) {
return
}
if !s.checkPgMaintRateLimit(a.TenantID, kind) {
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
return
}
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
var idemPtr *string
if idem != "" {
idemPtr = &idem
}
meta["actor_prefix"] = actorPrefix(a)
j, _, err := s.jobs.Enqueue(a.TenantID, kind, idemPtr, nil, meta)
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: kind, IdempotencyKey: idemPtr, Meta: meta}) {
return
}
if !s.checkPgMaintRateLimit(a.TenantID, kind) {
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
return
}
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, kind, idemPtr, nil, meta)
if err != nil {
writeInternalError(w, "postgres_maint_enqueue", err)
writeJobEnqueueError(w, err)
return
}
w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -184,12 +188,12 @@ func (s *Server) handlePostgresCleanup(w http.ResponseWriter, r *http.Request) {
if idem != "" {
idemPtr = &idem
}
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
"policy_id": policyID, "dry_run": body.DryRun, "actor_prefix": actorPrefix(a),
"job_title": "PostgreSQL cleanup (deprecated path)",
})
if err != nil {
writeInternalError(w, "postgres_maint_enqueue", err)
writeJobEnqueueError(w, err)
return
}
w.Header().Set("Location", "/v1/jobs/"+j.ID)
+61 -8
View File
@@ -8,15 +8,17 @@ import (
"errors"
"net/http"
"strings"
"sync"
"time"
"evobgp/internal/audit"
"evobgp/internal/jobs"
"evobgp/internal/logging"
"evobgp/internal/maintenance"
"evobgp/internal/pgmonitor"
"evobgp/internal/repository"
"evobgp/internal/runtimelogs"
"evobgp/internal/store"
"evobgp/internal/telemetry"
"github.com/jackc/pgx/v5/pgxpool"
)
@@ -30,7 +32,11 @@ type Server struct {
maintStats *maintenance.DBStatsProvider
jobs *jobs.Registry
authLimiter *authRateLimiter
operationOnce sync.Once
operationLimiter *operationLimiter
bundlePriv ed25519.PrivateKey
bundleCache *nodeBundleCache
nodeBundleBuild nodeBundleBuilder
keyResolver *apiKeyResolver
firewallResolver *firewallTokenResolver
bundleSeedHex string
@@ -39,6 +45,11 @@ type Server struct {
runtimeLogs *runtimelogs.Service
runtimeLogsPolicyTenant string
mux *http.ServeMux
backgroundMu sync.Mutex
backgroundStarted bool
backgroundClosed bool
backgroundCancel context.CancelFunc
telemetryCollector *telemetry.Collector
// Portal / dual-auth (JWT) configuration.
jwtSecret string
@@ -118,6 +129,7 @@ func New(opts Options) (*Server, error) {
maintStats: maintStats,
jobs: reg,
bundlePriv: priv,
bundleCache: newNodeBundleCache(nodeBundleCacheMaxEntries, nodeBundleCacheMaxBytes),
keyResolver: resolver,
firewallResolver: fwResolver,
bundleSeedHex: strings.TrimSpace(opts.BundleSeedHex),
@@ -143,6 +155,29 @@ func New(opts Options) (*Server, error) {
// Close releases database resources.
func (s *Server) Close() {
if s == nil {
return
}
s.backgroundMu.Lock()
if s.backgroundClosed {
s.backgroundMu.Unlock()
return
}
s.backgroundClosed = true
cancel := s.backgroundCancel
collector := s.telemetryCollector
s.backgroundMu.Unlock()
if cancel != nil {
cancel()
}
if collector != nil {
_ = collector.Close(context.Background())
}
// Do not close the pool while a worker is still persisting its final state.
if s.jobs != nil {
s.jobs.RequestCancelAll()
_ = s.jobs.Drain(context.Background())
}
if s.pgPool != nil {
s.pgPool.Close()
}
@@ -154,26 +189,44 @@ func (s *Server) Store() store.Backend { return s.store }
// Jobs exposes the in-process async job registry (for scheduler / evobgp-all).
func (s *Server) Jobs() *jobs.Registry { return s.jobs }
// StartBackground starts PostgreSQL monitoring and maintenance schedulers until ctx is cancelled.
// StartBackground starts shared schedulers and telemetry polling once until ctx is cancelled.
func (s *Server) StartBackground(ctx context.Context) {
if s != nil && s.pgPool != nil {
pgmonitor.StartScheduler(ctx, s.pgPool)
if s == nil {
return
}
s.backgroundMu.Lock()
defer s.backgroundMu.Unlock()
if s.backgroundClosed || s.backgroundStarted {
return
}
s.backgroundStarted = true
backgroundCtx, cancel := context.WithCancel(ctx)
s.backgroundCancel = cancel
if s.store != nil {
s.telemetryCollector = telemetry.Start(backgroundCtx, s.store, telemetry.ConfigFromEnv())
}
if s.pgPool != nil {
pgmonitor.StartScheduler(backgroundCtx, s.pgPool)
if s.jobs != nil {
jobs.StartDurableQueueLoop(ctx, s.jobs, repository.NewJobAuditWriter(s.pgPool))
jobs.StartDurableQueueLoop(backgroundCtx, s.jobs)
jobs.StartDeploymentReconciliationLoop(backgroundCtx, s.jobs, s.store)
}
}
if s != nil && s.maintConfig != nil && s.jobs != nil {
maintenance.StartScheduler(ctx, s.maintConfig, func(policyID string, dryRun bool, idem string) {
maintenance.StartScheduler(backgroundCtx, s.maintConfig, func(policyID string, dryRun bool, idem string) {
key := idem
_, _, _ = s.jobs.Enqueue("", jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{
_, _, err := s.jobs.EnqueueContext(backgroundCtx, store.SystemJobTenantID, jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{
"policy_id": policyID,
"dry_run": dryRun,
"trigger": "scheduler",
})
if err != nil {
logging.With("component", "maintenance.scheduler").Error("enqueue failed", "policy_id", policyID, "err", err)
}
}, 30*time.Second)
}
if s != nil && s.runtimeLogs != nil && s.store != nil {
runtimelogs.StartAutoCleanupScheduler(ctx, runtimelogs.SchedulerDeps{
runtimelogs.StartAutoCleanupScheduler(backgroundCtx, runtimelogs.SchedulerDeps{
Service: s.runtimeLogs,
Store: s.store,
PolicyTenant: s.runtimeLogsPolicyTenant,
+61 -16
View File
@@ -6,7 +6,11 @@ import (
"encoding/hex"
"encoding/json"
"errors"
"net"
"net/http"
"net/netip"
"net/url"
"strconv"
"strings"
"time"
@@ -165,24 +169,65 @@ func publicControlPlaneURL(r *http.Request, override string) string {
if s := strings.TrimSpace(override); s != "" {
return strings.TrimRight(s, "/")
}
if origin := strings.TrimSpace(r.Header.Get("Origin")); strings.HasPrefix(origin, "http://") || strings.HasPrefix(origin, "https://") {
return strings.TrimRight(origin, "/")
}
proto := strings.TrimSpace(r.Header.Get("X-Forwarded-Proto"))
if proto == "" {
proto = "https"
}
host := strings.TrimSpace(r.Header.Get("X-Forwarded-Host"))
if i := strings.Index(host, ","); i >= 0 {
host = strings.TrimSpace(host[:i])
}
if host == "" {
host = strings.TrimSpace(r.Host)
}
if host == "" {
if r == nil {
return ""
}
return proto + "://" + host
if peer, ok := peerAddress(r); ok && isTrustedProxy(peer, trustedProxyPrefixes()) {
protos := r.Header.Values("X-Forwarded-Proto")
hosts := r.Header.Values("X-Forwarded-Host")
if len(protos) == 1 && len(hosts) == 1 {
proto := strings.TrimSpace(protos[0])
host := strings.TrimSpace(hosts[0])
if (proto == "http" || proto == "https") && validControlPlaneHost(host) {
return proto + "://" + host
}
}
}
if !validControlPlaneHost(r.Host) {
return ""
}
proto := "http"
if r.TLS != nil {
proto = "https"
}
return proto + "://" + r.Host
}
func validControlPlaneHost(host string) bool {
if host == "" || strings.TrimSpace(host) != host || strings.HasSuffix(host, ":") || strings.ContainsAny(host, " ,/@?#\\\t\r\n") {
return false
}
u, err := url.Parse("http://" + host)
if err != nil || u.Host != host || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" {
return false
}
name := u.Hostname()
if name == "" {
return false
}
if strings.HasPrefix(host, "[") {
if _, err := netip.ParseAddr(name); err != nil {
return false
}
} else if net.ParseIP(name) == nil {
for _, label := range strings.Split(name, ".") {
if label == "" || label[0] == '-' || label[len(label)-1] == '-' {
return false
}
for _, c := range label {
if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') {
return false
}
}
}
}
if port := u.Port(); port != "" {
n, err := strconv.Atoi(port)
if err != nil || n < 1 || n > 65535 {
return false
}
}
return true
}
func (s *Server) attachReplicaInstall(resp map[string]any, sp *store.Speaker, nodeToken string, req speakerCreateRequest, r *http.Request) {
+59 -73
View File
@@ -1,9 +1,7 @@
package httpapi
import (
"context"
"strings"
"sync"
"time"
"evobgp/internal/birdfmt"
@@ -11,6 +9,34 @@ import (
"evobgp/internal/store"
)
type agentHealthPoll struct {
result nodedispatch.AgentHealthResult
at time.Time
lastSuccessAt time.Time
observationOverride *liveObservation
}
func agentHealthObservation(poll *agentHealthPoll) liveObservation {
if poll == nil {
return liveObservation{Source: "none", Quality: "unknown", ErrorCode: "not_polled"}
}
if poll.observationOverride != nil {
return *poll.observationOverride
}
o := liveObservation{ObservedAt: liveTime(poll.at), ReceivedAt: liveTime(poll.at), LastSuccessAt: liveTime(poll.lastSuccessAt), Source: "agent_health", Quality: "ok"}
if !poll.result.OK {
o.Quality = "unreachable"
if !poll.lastSuccessAt.IsZero() {
o.Quality = "stale"
}
o.ErrorCode = liveErrorCode(poll.result.Error)
if poll.result.Error == "agent reported ok=false" {
o.ErrorCode = "reported_unhealthy"
}
}
return o
}
func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) {
total = len(sessions)
for _, s := range sessions {
@@ -22,29 +48,48 @@ func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) {
}
func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *nodedispatch.AgentHealthResult) map[string]any {
var poll *agentHealthPoll
if health != nil {
poll = &agentHealthPoll{result: *health}
}
return speakerLiveStatusWithPoll(sp, view, poll)
}
func speakerLiveStatusWithPoll(sp *store.Speaker, view speakerBGPLive, health *agentHealthPoll) map[string]any {
total, established := countBGPSessions(view.Sessions)
bgpObservation := view.observation()
agentObservation := agentHealthObservation(health)
if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") && health == nil {
agentObservation = bgpObservation
}
m := map[string]any{
"label": view.Label,
"bgp_poll_ok": view.Error == "",
"bgp_sessions_total": total,
"bgp_established": established,
"bgp_poll_ok": view.successful(),
"bgp_sessions_total": nil,
"bgp_established": nil,
"bgp_observation": bgpObservation,
"agent_observation": agentObservation,
}
if view.successful() {
m["bgp_sessions_total"] = total
m["bgp_established"] = established
}
if view.Error != "" {
m["bgp_poll_error"] = view.Error
}
if health != nil {
m["agent_ok"] = health.OK
if health.Error != "" {
m["agent_error"] = health.Error
m["agent_ok"] = health.result.OK
if health.result.Error != "" {
m["agent_error"] = health.result.Error
}
if health.LastSyncAt != "" {
m["agent_last_sync_at"] = health.LastSyncAt
if health.result.LastSyncAt != "" {
m["agent_last_sync_at"] = health.result.LastSyncAt
}
if health.LastAppliedRevisionID != "" {
m["agent_last_applied_revision_id"] = health.LastAppliedRevisionID
if health.result.LastAppliedRevisionID != "" {
m["agent_last_applied_revision_id"] = health.result.LastAppliedRevisionID
}
} else if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
m["agent_ok"] = view.Error == ""
m["agent_ok"] = view.successful()
if view.Error != "" {
m["agent_error"] = view.Error
}
@@ -52,7 +97,7 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded
m["agent_ok"] = false
m["agent_error"] = "agent health not polled"
}
if len(view.Sessions) > 0 {
if view.successful() {
sess := make([]map[string]any, 0, len(view.Sessions))
for _, s := range view.Sessions {
row := map[string]any{
@@ -68,62 +113,3 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded
}
return m
}
func (s *Server) collectSpeakerLiveStatus(ctx context.Context, tenantID string, fresh bool, speakers []*store.Speaker) map[string]map[string]any {
views := s.collectSpeakerBGPLive(ctx, tenantID, fresh)
viewByID := make(map[string]speakerBGPLive, len(views))
for _, v := range views {
if v.SpeakerID != "" {
viewByID[v.SpeakerID] = v
}
}
opts := nodedispatch.Options{Timeout: 8 * time.Second}
type healthWrap struct {
id string
h nodedispatch.AgentHealthResult
}
healthCh := make(chan healthWrap, len(speakers))
var wg sync.WaitGroup
for _, sp := range speakers {
if sp == nil {
continue
}
meta := store.ParseSpeakerMeta(sp.MetaJSON)
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
continue
}
wg.Add(1)
go func(speaker *store.Speaker) {
defer wg.Done()
healthCh <- healthWrap{
id: speaker.ID,
h: nodedispatch.FetchAgentHealth(ctx, speaker, opts),
}
}(sp)
}
wg.Wait()
close(healthCh)
healthByID := make(map[string]nodedispatch.AgentHealthResult, len(speakers))
for hw := range healthCh {
healthByID[hw.id] = hw.h
}
out := make(map[string]map[string]any, len(speakers))
for _, sp := range speakers {
if sp == nil {
continue
}
view, ok := viewByID[sp.ID]
if !ok {
view = speakerBGPLive{SpeakerID: sp.ID, Label: speakerDisplayLabel(sp)}
}
var hp *nodedispatch.AgentHealthResult
if h, ok := healthByID[sp.ID]; ok {
hCopy := h
hp = &hCopy
}
out[sp.ID] = speakerLiveStatusJSON(sp, view, hp)
}
return out
}
+17
View File
@@ -2,12 +2,29 @@ package httpapi
import (
"testing"
"time"
"evobgp/internal/birdfmt"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
)
func TestSpeakerLiveStatusJSONEmptyAndUnpolled(t *testing.T) {
sp := &store.Speaker{ID: "master", Role: "master"}
unpolled := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID}, nil)
if unpolled["agent_ok"] != false || unpolled["bgp_poll_ok"] != false || unpolled["bgp_sessions_total"] != nil {
t.Fatalf("unpolled=%#v", unpolled)
}
if unpolled["bgp_observation"].(liveObservation).Quality != "unknown" {
t.Fatalf("unpolled observation=%#v", unpolled["bgp_observation"])
}
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
empty := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID, Polled: true, ObservedAt: now, ReceivedAt: now, LastSuccessAt: now, Source: "cp_birdc"}, nil)
if empty["agent_ok"] != true || empty["bgp_poll_ok"] != true || empty["bgp_sessions_total"] != 0 {
t.Fatalf("empty=%#v", empty)
}
}
func TestCountBGPSessions(t *testing.T) {
total, est := countBGPSessions([]birdfmt.BGPSession{
{Name: "p1", State: "Established"},
+154
View File
@@ -0,0 +1,154 @@
package httpapi
import (
"context"
"net/http"
"time"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
"evobgp/internal/telemetry"
)
func (s *Server) loadTelemetry(ctx context.Context, tenant string, speakers []*store.Speaker) ([]speakerBGPLive, map[string]map[string]any, error) {
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
snapshots, err := s.store.ListTelemetry(ctx, tenant)
if err != nil {
return nil, nil, err
}
views, statuses := hydrateTelemetry(speakers, snapshots, time.Now(), telemetry.ConfigFromEnv().StaleAfter)
return views, statuses, nil
}
// snapshotObservation uses control-plane receipt time, never the remote clock.
func snapshotObservation(o store.TelemetryObservation, now time.Time, staleAfter time.Duration, invalid string) liveObservation {
v := liveObservation{ObservedAt: liveTime(o.ObservedAt), ReceivedAt: liveTime(o.ReceivedAt), LastSuccessAt: liveTime(o.LastSuccessAt), Source: o.Source, Quality: "unknown"}
if !o.ReceivedAt.IsZero() {
v.ExpiresAt = liveTime(o.ReceivedAt.Add(staleAfter))
}
if v.Source == "" {
v.Source = "none"
}
switch {
case invalid != "":
v.ErrorCode = invalid
case o.ReceivedAt.IsZero():
v.ErrorCode = "not_polled"
case o.ReceivedAt.After(now) || now.Sub(o.ReceivedAt) > staleAfter:
v.Quality, v.ErrorCode = "stale", "snapshot_expired"
case o.Error != "":
v.Quality, v.ErrorCode = "unreachable", liveErrorCode(o.Error)
if !o.LastSuccessAt.IsZero() {
v.Quality = "stale"
}
default:
v.Quality = "ok"
}
return v
}
func hydrateTelemetry(speakers []*store.Speaker, snapshots []store.TelemetrySnapshot, now time.Time, staleAfter time.Duration) ([]speakerBGPLive, map[string]map[string]any) {
byID := make(map[string]*store.Speaker, len(speakers))
for _, sp := range speakers {
if sp != nil {
byID[sp.ID] = sp
}
}
grouped := make(map[string][]store.TelemetrySnapshot)
for _, snapshot := range snapshots {
if snapshot.SpeakerID != "" && byID[snapshot.SpeakerID] == nil {
continue
}
if snapshot.SpeakerID == "" && snapshot.Kind != "local_bird" {
continue
}
grouped[snapshot.SpeakerID] = append(grouped[snapshot.SpeakerID], snapshot)
}
views := make([]speakerBGPLive, 0, len(speakers)+1)
statuses := make(map[string]map[string]any, len(speakers))
appendView := func(id string, sp *store.Speaker) {
view := speakerBGPLive{SpeakerID: id, Label: speakerDisplayLabel(sp)}
if sp == nil {
view.Label = "CP (local)"
}
var health *agentHealthPoll
rows := grouped[id]
// A restarted local collector may leave an expired authority behind.
// Only simultaneously current authorities conflict; never prefer one
// of two current BIRD nodes arbitrarily.
if len(rows) > 1 {
current := make([]store.TelemetrySnapshot, 0, len(rows))
for _, row := range rows {
if row.Kind == "local_bird" && row.ConfigFingerprint == store.TelemetrySpeakerFingerprint(sp) && !row.BGP.ReceivedAt.IsZero() && !row.BGP.ReceivedAt.After(now) && now.Sub(row.BGP.ReceivedAt) <= staleAfter {
current = append(current, row)
}
}
if len(current) > 0 {
rows = current
}
}
if len(rows) > 1 {
view.Error = "ambiguous telemetry authorities"
view.ObservationOverride = &liveObservation{Source: "none", Quality: "unknown", ErrorCode: "ambiguous_authority"}
} else if len(rows) == 1 {
row := rows[0]
invalid := ""
if row.ConfigFingerprint != store.TelemetrySpeakerFingerprint(sp) {
invalid = "configuration_changed"
}
bgp := snapshotObservation(row.BGP.TelemetryObservation, now, staleAfter, invalid)
view.ObservationOverride = &bgp
view.Polled = !row.BGP.ReceivedAt.IsZero()
view.ObservedAt, view.ReceivedAt, view.LastSuccessAt, view.Source = row.BGP.ObservedAt, row.BGP.ReceivedAt, row.BGP.LastSuccessAt, row.BGP.Source
if bgp.Quality == "ok" {
view.Sessions = row.BGP.Sessions
} else {
view.Error = bgp.ErrorCode
}
if row.Kind == "remote_agent" {
h := snapshotObservation(row.Health.TelemetryObservation, now, staleAfter, invalid)
result := nodedispatch.AgentHealthResult{OK: row.Health.OK && h.Quality == "ok", Error: row.Health.Error, LastSyncAt: row.Health.LastSyncAt, LastAppliedRevisionID: row.Health.LastAppliedRevisionID}
if h.Quality != "ok" {
result.Error = h.ErrorCode
}
if h.Quality == "ok" && !result.OK {
h.Quality, h.ErrorCode = "unreachable", "reported_unhealthy"
}
health = &agentHealthPoll{result: result, observationOverride: &h}
}
}
views = append(views, view)
if sp != nil {
statuses[id] = speakerLiveStatusWithPoll(sp, view, health)
}
}
for _, sp := range speakers {
if sp != nil {
appendView(sp.ID, sp)
}
}
if len(grouped[""]) > 0 {
appendView("", nil)
}
return views, statuses
}
func (s *Server) handleTelemetryRefresh(w http.ResponseWriter, r *http.Request) {
a, ok := authFromContext(r.Context())
if !ok {
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
return
}
if !s.requirePerm(w, a, "bgp:network:read") {
return
}
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
defer cancel()
requested, err := s.store.RequestTelemetryRefresh(ctx, a.TenantID, 10*time.Second)
if err != nil {
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry refresh unavailable")
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"requested": requested})
}
+126
View File
@@ -0,0 +1,126 @@
package httpapi
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
)
type telemetryHTTPStore struct {
store.Backend
discoveryWrites int
snapshotErr error
refreshCalls int
}
func (s *telemetryHTTPStore) UpsertPeerDiscovery(tenant string, in *store.PeerDiscoveryUpsert) (*store.BGPPeerDiscovery, error) {
s.discoveryWrites++
return s.Backend.UpsertPeerDiscovery(tenant, in)
}
func (s *telemetryHTTPStore) ListTelemetry(ctx context.Context, tenant string) ([]store.TelemetrySnapshot, error) {
if s.snapshotErr != nil {
return nil, s.snapshotErr
}
return s.Backend.ListTelemetry(ctx, tenant)
}
func (s *telemetryHTTPStore) RequestTelemetryRefresh(ctx context.Context, tenant string, cooldown time.Duration) (bool, error) {
s.refreshCalls++
return s.Backend.RequestTelemetryRefresh(ctx, tenant, cooldown)
}
func telemetryRequest(method, path, tenant string) *http.Request {
r := httptest.NewRequest(method, path, nil)
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "viewer"}))
}
func TestTelemetryGETsAndRefreshNeverPollAgents(t *testing.T) {
var calls atomic.Int64
agent := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
_, _ = w.Write([]byte(`{"ok":true,"sessions":[]}`))
}))
defer agent.Close()
t.Setenv("EVOBGP_NODE_DISPATCH_INSECURE_TLS", "1")
mem := store.NewMemory()
mem.SeedDemo()
tenant, _, _, _, _ := mem.DemoIDs()
sp, err := mem.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: strings.TrimPrefix(agent.URL, "https://"), AgentSecret: "test"})})
if err != nil {
t.Fatal(err)
}
// Positive control: the legacy adapter reaches the fake configured agent.
if result := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: agent.Client()}); !result.OK {
t.Fatal(result)
}
calls.Store(0)
st := &telemetryHTTPStore{Backend: mem}
server := &Server{store: st}
claims, err := mem.ClaimTelemetry(context.Background(), "collector", 1, time.Minute, "")
if err != nil || len(claims) != 1 {
t.Fatalf("claim: %v %v", claims, err)
}
for i := 0; i < 20; i++ {
for _, tc := range []struct {
path string
handler http.HandlerFunc
}{
{"/v1/peers", server.handleListPeers}, {"/v1/peers?live=1", server.handleListPeers},
{"/v1/speakers?live=1", server.handleListSpeakers}, {"/v1/peers/discovered?status=all", server.handleListPeerDiscoveries},
} {
w := httptest.NewRecorder()
tc.handler(w, telemetryRequest(http.MethodGet, tc.path, tenant))
if w.Code != http.StatusOK {
t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String())
}
}
w := httptest.NewRecorder()
server.handleTelemetryRefresh(w, telemetryRequest(http.MethodPost, "/v1/telemetry/refresh", tenant))
if w.Code != http.StatusAccepted {
t.Fatalf("refresh: %d %s", w.Code, w.Body.String())
}
want := `"requested":false`
if i == 0 {
want = `"requested":true`
}
if !strings.Contains(w.Body.String(), want) {
t.Fatalf("refresh coalescing: %s", w.Body.String())
}
}
if calls.Load() != 0 || st.discoveryWrites != 0 {
t.Fatalf("GET/refresh caused IO: agent=%d discovery=%d", calls.Load(), st.discoveryWrites)
}
}
func TestTelemetryReadFailureAndRefreshPermission(t *testing.T) {
mem := store.NewMemory()
mem.SeedDemo()
tenant, _, _, _, _ := mem.DemoIDs()
st := &telemetryHTTPStore{Backend: mem, snapshotErr: errors.New("private database failure")}
server := &Server{store: st}
for _, handler := range []http.HandlerFunc{server.handleListPeers, server.handleListSpeakers} {
w := httptest.NewRecorder()
handler(w, telemetryRequest(http.MethodGet, "/v1/resource?live=1", tenant))
if w.Code != http.StatusServiceUnavailable || strings.Contains(w.Body.String(), "private") {
t.Fatalf("snapshot failure: %d %s", w.Code, w.Body.String())
}
}
r := httptest.NewRequest(http.MethodPost, "/v1/telemetry/refresh", nil)
w := httptest.NewRecorder()
server.handleTelemetryRefresh(w, r)
if w.Code != http.StatusUnauthorized || st.refreshCalls != 0 {
t.Fatal("unauthenticated refresh admitted")
}
r = r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Kind: AuthKindJWT}))
w = httptest.NewRecorder()
server.handleTelemetryRefresh(w, r)
if w.Code != http.StatusForbidden || st.refreshCalls != 0 {
t.Fatal("unprivileged refresh admitted")
}
}
@@ -0,0 +1,30 @@
package httpapi
import (
"context"
"testing"
)
func TestTelemetryBackgroundStartsOnceAndStopsBeforeClose(t *testing.T) {
s, err := New(Options{SeedDemo: true})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s.StartBackground(ctx)
first := s.telemetryCollector
if first == nil {
t.Fatal("telemetry collector not started")
}
s.StartBackground(ctx)
if s.telemetryCollector != first {
t.Fatal("duplicate telemetry collector")
}
s.Close()
s.StartBackground(ctx)
if s.telemetryCollector != first {
t.Fatal("collector started after close")
}
s.Close()
}
+102
View File
@@ -0,0 +1,102 @@
package httpapi
import (
"evobgp/internal/birdfmt"
"evobgp/internal/store"
"testing"
"time"
)
func TestTelemetryHydrationFreshnessAndConfiguration(t *testing.T) {
now := time.Now().UTC()
sp := &store.Speaker{ID: "replica", Role: "replica", Endpoint: "old"}
observation := store.TelemetryObservation{ObservedAt: now.Add(-time.Hour), ReceivedAt: now, LastSuccessAt: now, Source: "agent"}
base := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: observation, Sessions: []birdfmt.BGPSession{{Name: "peer", State: "Established"}}}, Health: store.TelemetryHealth{TelemetryObservation: observation, OK: true}}
cases := []struct {
name string
mutate func(*store.TelemetrySnapshot)
bgp, health bool
quality string
}{
{"fresh receipt despite remote clock", func(*store.TelemetrySnapshot) {}, true, true, "ok"},
{"expired BGP independent of health", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(-2 * time.Minute) }, false, true, "stale"},
{"expired health independent of BGP", func(s *store.TelemetrySnapshot) { s.Health.ReceivedAt = now.Add(-2 * time.Minute) }, true, false, "ok"},
{"future receipt is not fresh", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(time.Hour) }, false, true, "stale"},
{"configuration changed", func(s *store.TelemetrySnapshot) { s.ConfigFingerprint = "old" }, false, false, "unknown"},
{"failed BGP", func(s *store.TelemetrySnapshot) { s.BGP.Error = "timeout" }, false, true, "stale"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
snapshot := base
tc.mutate(&snapshot)
views, statuses := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{snapshot}, now, time.Minute)
status := statuses[sp.ID]
if status["bgp_poll_ok"] != tc.bgp || status["agent_ok"] != tc.health || views[0].observation().Quality != tc.quality {
t.Fatalf("status=%+v observation=%+v", status, views[0].observation())
}
if !tc.bgp && (status["bgp_established"] != nil || len(views[0].Sessions) != 0) {
t.Fatal("stale BGP exposed live sessions")
}
})
}
}
func TestTelemetryHydrationMissingAndAmbiguousLocal(t *testing.T) {
now := time.Now()
sp := &store.Speaker{ID: "master", Role: "master"}
for _, rows := range [][]store.TelemetrySnapshot{nil, {{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "a"}, {SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "b"}}} {
views, statuses := hydrateTelemetry([]*store.Speaker{sp}, rows, now, time.Minute)
if views[0].observation().Quality != "unknown" || statuses[sp.ID]["agent_ok"] == true || statuses[sp.ID]["bgp_poll_ok"] == true {
t.Fatalf("unexpected healthy master: %+v", statuses)
}
}
}
func TestTelemetryHydrationLocalAuthorityRestart(t *testing.T) {
now := time.Now()
sp := &store.Speaker{ID: "master", Role: "master"}
fresh := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "new", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now}}}
old := fresh
old.LocalAuthority = "old"
old.BGP.ReceivedAt = now.Add(-2 * time.Minute)
_, status := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute)
if status[sp.ID]["bgp_poll_ok"] != true {
t.Fatal("expired authority hides current local snapshot")
}
old.BGP.ReceivedAt = now
_, status = hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute)
if status[sp.ID]["bgp_poll_ok"] != false {
t.Fatal("two current local authorities are ambiguous")
}
}
func TestTelemetryPeerObservationPreservesUnknownConfiguration(t *testing.T) {
now := time.Now()
sp := &store.Speaker{ID: "replica", Role: "replica"}
row := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: "previous", BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now, LastSuccessAt: now}}}
views, _ := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{row}, now, time.Minute)
observation := peerLiveObservation(views)
if observation.Quality != "unknown" || observation.ErrorCode != "configuration_changed" {
t.Fatalf("changed config is not a failed current poll: %+v", observation)
}
}
func TestTelemetryExpiryUsesConfiguredIntervalAndEarliestSource(t *testing.T) {
now := time.Now().UTC()
old := now.Add(-30 * time.Second)
a := snapshotObservation(store.TelemetryObservation{ReceivedAt: old}, now, 5*time.Minute, "")
b := snapshotObservation(store.TelemetryObservation{ReceivedAt: now}, now, 5*time.Minute, "")
want := old.Add(5 * time.Minute).Format(time.RFC3339Nano)
if a.ExpiresAt == nil || *a.ExpiresAt != want {
t.Fatalf("expiry ignores configured interval: %+v", a)
}
views := []speakerBGPLive{{Polled: true, ObservationOverride: &b}, {Polled: true, ObservationOverride: &a}}
combined := peerLiveObservation(views)
if combined.ExpiresAt == nil || *combined.ExpiresAt != want {
t.Fatalf("aggregate extends oldest source: %+v", combined)
}
missing := snapshotObservation(store.TelemetryObservation{}, now, time.Minute, "")
if missing.ExpiresAt != nil {
t.Fatal("unobserved source has expiry")
}
}
+91
View File
@@ -0,0 +1,91 @@
package httpapi
import (
"net"
"net/http"
"net/netip"
"os"
"strings"
)
// trustedProxyPrefixes returns no trusted peers when any configured CIDR is invalid.
// An empty configuration deliberately treats every connection as direct.
func trustedProxyPrefixes() []netip.Prefix {
raw := strings.TrimSpace(os.Getenv("EVOBGP_TRUSTED_PROXY_CIDRS"))
if raw == "" {
return nil
}
parts := strings.Split(raw, ",")
prefixes := make([]netip.Prefix, 0, len(parts))
for _, part := range parts {
prefix, err := netip.ParsePrefix(strings.TrimSpace(part))
if err != nil {
return nil
}
if prefix.Addr().Is4In6() {
if prefix.Bits() < 96 {
return nil
}
prefix = netip.PrefixFrom(prefix.Addr().Unmap(), prefix.Bits()-96)
}
prefixes = append(prefixes, prefix.Masked())
}
return prefixes
}
func peerAddress(r *http.Request) (netip.Addr, bool) {
if r == nil {
return netip.Addr{}, false
}
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
if err != nil {
host = strings.TrimSpace(r.RemoteAddr)
}
ip, err := netip.ParseAddr(host)
if err != nil {
return netip.Addr{}, false
}
return ip.WithZone("").Unmap(), true
}
func isTrustedProxy(ip netip.Addr, prefixes []netip.Prefix) bool {
for _, prefix := range prefixes {
if prefix.Contains(ip) || prefix.Contains(ip.Unmap()) {
return true
}
}
return false
}
func clientIPWithTrustedProxies(r *http.Request, prefixes []netip.Prefix) string {
peer, ok := peerAddress(r)
if !ok {
if r == nil {
return ""
}
return strings.TrimSpace(r.RemoteAddr)
}
if !isTrustedProxy(peer, prefixes) {
return peer.String()
}
values := r.Header.Values("X-Forwarded-For")
if len(values) != 1 || strings.TrimSpace(values[0]) == "" {
return peer.String()
}
hops := strings.Split(values[0], ",")
current := peer
for i := len(hops) - 1; i >= 0; i-- {
if !isTrustedProxy(current, prefixes) {
return current.String()
}
next, err := netip.ParseAddr(strings.TrimSpace(hops[i]))
if err != nil {
return peer.String()
}
if next.Zone() != "" {
return peer.String()
}
current = next.Unmap()
}
return current.String()
}
+129
View File
@@ -0,0 +1,129 @@
package httpapi
import (
"fmt"
"net/http/httptest"
"net/netip"
"testing"
"time"
)
func TestClientIPTrustedProxyChain(t *testing.T) {
prefixes := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}
tests := []struct {
name string
remote string
xff string
want string
}{
{"untrusted peer ignores spoof", "198.51.100.10:1234", "203.0.113.9", "198.51.100.10"},
{"rightmost untrusted hop", "10.0.0.2:1234", "203.0.113.9, 198.51.100.8", "198.51.100.8"},
{"trusted chain", "10.0.0.2:1234", "203.0.113.9, 10.0.0.3", "203.0.113.9"},
{"mapped peer", "[::ffff:10.0.0.2]:1234", "203.0.113.9", "203.0.113.9"},
{"mapped forwarded client", "10.0.0.2:1234", "::ffff:203.0.113.9", "203.0.113.9"},
{"malformed nearest hop", "10.0.0.2:1234", "203.0.113.9, bogus", "10.0.0.2"},
{"malformed farther hop ignored", "10.0.0.2:1234", "bogus, 198.51.100.8", "198.51.100.8"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := httptest.NewRequest("GET", "http://example.test/", nil)
r.RemoteAddr = tt.remote
r.Header.Set("X-Forwarded-For", tt.xff)
if got := clientIPWithTrustedProxies(r, prefixes); got != tt.want {
t.Fatalf("client IP = %q, want %q", got, tt.want)
}
})
}
}
func TestClientIPConfigurationFailsClosed(t *testing.T) {
r := httptest.NewRequest("GET", "http://example.test/", nil)
r.RemoteAddr = "10.0.0.2:1234"
r.Header.Set("X-Forwarded-For", "203.0.113.9")
r.Header.Set("X-Real-IP", "203.0.113.8")
if got := clientIPWithTrustedProxies(r, nil); got != "10.0.0.2" {
t.Fatalf("direct client IP = %q", got)
}
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8, invalid")
if got := clientIP(r); got != "10.0.0.2" {
t.Fatalf("invalid CIDR client IP = %q", got)
}
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "::ffff:10.0.0.2/128")
if got := clientIP(r); got != "203.0.113.9" {
t.Fatalf("mapped CIDR client IP = %q", got)
}
}
func TestPublicControlPlaneURLTrust(t *testing.T) {
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8")
r := httptest.NewRequest("POST", "http://internal.local:8080/v1/speakers", nil)
r.Header.Set("Origin", "https://evil.example")
r.Header.Set("X-Forwarded-Proto", "https")
r.Header.Set("X-Forwarded-Host", "bgp.example.test")
r.RemoteAddr = "198.51.100.8:1234"
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
t.Fatalf("direct URL = %q", got)
}
r.RemoteAddr = "10.0.0.2:1234"
if got := publicControlPlaneURL(r, ""); got != "https://bgp.example.test" {
t.Fatalf("proxied URL = %q", got)
}
if got := publicControlPlaneURL(r, "https://explicit.example/"); got != "https://explicit.example" {
t.Fatalf("override URL = %q", got)
}
for _, bad := range []struct{ proto, host string }{
{"javascript", "bgp.example.test"},
{"https,http", "bgp.example.test"},
{"https", "evil.example/path"},
{"https", "evil.example@bgp.example.test"},
{"https", "evil.example,bgp.example.test"},
{"https", "bgp.example.test:99999"},
{"https", "bgp.example.test:"},
} {
r.Header.Set("X-Forwarded-Proto", bad.proto)
r.Header.Set("X-Forwarded-Host", bad.host)
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
t.Errorf("bad forwarded headers (%q, %q): %q", bad.proto, bad.host, got)
}
}
r.Header.Set("X-Forwarded-Proto", "https")
r.Header.Set("X-Forwarded-Host", "bgp.example.test")
r.Header.Add("X-Forwarded-Host", "evil.example")
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
t.Fatalf("duplicate forwarded host = %q", got)
}
}
func TestAuthRateLimiterManyIdentities(t *testing.T) {
l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 1, window: time.Minute, maxClients: 32}
now := time.Unix(1000, 0)
for i := 0; i < 1000; i++ {
allowed := l.allowAt(fmt.Sprintf("ip-%d", i), now)
if allowed != (i < l.maxClients) {
t.Fatalf("identity %d allowed = %t", i, allowed)
}
}
if len(l.hits) != l.maxClients || l.allowAt("ip-0", now) {
t.Fatalf("active counters changed under cardinality pressure: %d", len(l.hits))
}
}
func TestAuthRateLimiterCapacityAndExpiry(t *testing.T) {
l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 2, window: time.Minute, maxClients: 2}
now := time.Unix(1000, 0)
if !l.allowAt("a", now) || !l.allowAt("b", now) || !l.allowAt("a", now) {
t.Fatal("initial requests rejected")
}
if l.allowAt("a", now) || l.allowAt("c", now) {
t.Fatal("active limit or capacity bypassed")
}
if len(l.hits) != 2 {
t.Fatalf("map size = %d", len(l.hits))
}
if !l.allowAt("c", now.Add(time.Minute)) {
t.Fatal("expired entries did not release capacity")
}
if len(l.hits) != 1 {
t.Fatalf("expired entries remained: %d", len(l.hits))
}
}