feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -0,0 +1,273 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRequestBodyKnownAndUnknownBoundsBeforeMutation(t *testing.T) {
|
||||
const limit = 32
|
||||
for _, unknown := range []bool{false, true} {
|
||||
for _, contentType := range []string{"", "application/json", "text/plain"} {
|
||||
for _, extra := range []int{0, 1} {
|
||||
label := "known"
|
||||
if unknown {
|
||||
label = "unknown"
|
||||
}
|
||||
t.Run(label+"/"+contentType+"/"+string(rune('0'+extra)), func(t *testing.T) {
|
||||
mutations := 0
|
||||
h := withBodyLimit(limit, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var input map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
mutations++
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}))
|
||||
body := `{"x":1}` + strings.Repeat(" ", limit-len(`{"x":1}`)+extra)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
|
||||
if unknown {
|
||||
req.ContentLength = -1
|
||||
}
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if extra == 0 {
|
||||
if w.Code != http.StatusCreated || mutations != 1 {
|
||||
t.Fatalf("exact limit rejected: %d mutations=%d", w.Code, mutations)
|
||||
}
|
||||
} else if w.Code != http.StatusRequestEntityTooLarge || mutations != 0 {
|
||||
t.Fatalf("oversized tail mutated: %d mutations=%d", w.Code, mutations)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type checkedRequestBody struct {
|
||||
reads int
|
||||
closed bool
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (b *checkedRequestBody) Read(p []byte) (int, error) { b.reads++; return b.reader.Read(p) }
|
||||
func (b *checkedRequestBody) Close() error { b.closed = true; return nil }
|
||||
|
||||
type failingRequestReader struct{}
|
||||
|
||||
func (failingRequestReader) Read([]byte) (int, error) {
|
||||
return 0, errors.New("untrusted detailed read failure")
|
||||
}
|
||||
|
||||
func TestRequestBodyErrorsDoNotMutate(t *testing.T) {
|
||||
for _, cancelled := range []bool{false, true} {
|
||||
calls := 0
|
||||
b := &checkedRequestBody{reader: io.MultiReader(strings.NewReader(`{"x":1}`), failingRequestReader{})}
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
|
||||
req.Body = b
|
||||
req.ContentLength = -1
|
||||
if cancelled {
|
||||
ctx, cancel := context.WithCancel(req.Context())
|
||||
cancel()
|
||||
req = req.WithContext(ctx)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { calls++ })).ServeHTTP(w, req)
|
||||
want := http.StatusBadRequest
|
||||
if cancelled {
|
||||
want = http.StatusRequestTimeout
|
||||
}
|
||||
if w.Code != want || calls != 0 || !b.closed || strings.Contains(w.Body.String(), "untrusted detailed") {
|
||||
t.Fatalf("error handling status=%d calls=%d body=%s", w.Code, calls, w.Body.String())
|
||||
}
|
||||
if cancelled && b.reads != 0 {
|
||||
t.Fatal("read body after cancellation")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAuthenticationRunsFirst(t *testing.T) {
|
||||
b := &checkedRequestBody{reader: failingRequestReader{}}
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
|
||||
req.Body = b
|
||||
req.ContentLength = requestBodyLimit + 1
|
||||
s := &Server{}
|
||||
called := false
|
||||
h := s.authMiddleware(withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusUnauthorized || b.reads != 0 || called {
|
||||
t.Fatalf("body processed before auth: %d reads=%d", w.Code, b.reads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAppliesToGETAndIgnoresClaimedZeroLength(t *testing.T) {
|
||||
for _, length := range []int64{-1, 0, requestBodyLimit + 1} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/jobs", strings.NewReader(strings.Repeat("x", int(requestBodyLimit)+1)))
|
||||
req.ContentLength = length
|
||||
called := false
|
||||
w := httptest.NewRecorder()
|
||||
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })).ServeHTTP(w, req)
|
||||
if w.Code != http.StatusRequestEntityTooLarge || called {
|
||||
t.Fatalf("GET size bypass length=%d code=%d", length, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyHandlerWiringAndNoMutation(t *testing.T) {
|
||||
s, err := New(Options{SeedDemo: true, InsecureDev: true, BundleSeedHex: testBundleSeed})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
tenant, _, _, _, _ := s.store.DemoIDs()
|
||||
before := len(s.store.ListModules(tenant))
|
||||
h := s.Handler()
|
||||
prefix := `{"type":"IP_RANGES","name":"bounded-create","enabled":false}`
|
||||
for _, extra := range []int{1, 0} {
|
||||
body := prefix + strings.Repeat(" ", int(requestBodyLimit)-len(prefix)+extra)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
|
||||
req.ContentLength = -1
|
||||
req.Header.Set("Authorization", "Bearer dev")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if extra == 1 {
|
||||
if w.Code != http.StatusRequestEntityTooLarge || len(s.store.ListModules(tenant)) != before {
|
||||
t.Fatalf("oversized JSON committed: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
} else if w.Code != http.StatusCreated || len(s.store.ListModules(tenant)) != before+1 {
|
||||
t.Fatalf("8MiB body rejected: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyLeavesStreamingResponseUnbuffered(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/runtime-logs/stream", nil)
|
||||
withRequestBodyLimit(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
t.Fatal("response lost Flusher")
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = io.WriteString(w, "data: ready\n\n")
|
||||
f.Flush()
|
||||
})).ServeHTTP(w, req)
|
||||
if !w.Flushed || w.Body.String() != "data: ready\n\n" {
|
||||
t.Fatal("stream response buffered")
|
||||
}
|
||||
}
|
||||
|
||||
type blockedAdmissionBody struct {
|
||||
entered chan struct{}
|
||||
resume chan struct{}
|
||||
sent bool
|
||||
}
|
||||
|
||||
func (b *blockedAdmissionBody) Read(p []byte) (int, error) {
|
||||
if b.sent {
|
||||
return 0, io.EOF
|
||||
}
|
||||
b.sent = true
|
||||
close(b.entered)
|
||||
<-b.resume
|
||||
p[0] = 'x'
|
||||
return 1, nil
|
||||
}
|
||||
func (*blockedAdmissionBody) Close() error { return nil }
|
||||
|
||||
func TestRequestBodyAdmissionRejectsWithoutWaitingAndHoldsThroughHandler(t *testing.T) {
|
||||
readEntered, readResume := make(chan struct{}), make(chan struct{})
|
||||
handlerEntered, handlerResume := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/empty" {
|
||||
once.Do(func() { close(handlerEntered) })
|
||||
<-handlerResume
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
first := httptest.NewRequest(http.MethodPost, "/first", nil)
|
||||
first.Body = &blockedAdmissionBody{entered: readEntered, resume: readResume}
|
||||
first.ContentLength = 0 // A claimed zero must not bypass body admission.
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); h.ServeHTTP(httptest.NewRecorder(), first) }()
|
||||
select {
|
||||
case <-readEntered:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("first body did not start")
|
||||
}
|
||||
reject := func() {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/second", strings.NewReader("x"))
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusTooManyRequests || w.Header().Get("Retry-After") != "1" {
|
||||
t.Fatalf("admission did not reject: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
reject()
|
||||
// An empty legacy GET remains available while all body slots are occupied.
|
||||
empty := httptest.NewRecorder()
|
||||
h.ServeHTTP(empty, httptest.NewRequest(http.MethodGet, "/empty", nil))
|
||||
if empty.Code != http.StatusNoContent {
|
||||
t.Fatal("empty GET consumed body slot")
|
||||
}
|
||||
close(readResume)
|
||||
select {
|
||||
case <-handlerEntered:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("handler did not start")
|
||||
}
|
||||
reject() // The fully read buffer is still retained by the running handler.
|
||||
close(handlerResume)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("first request did not finish")
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/third", strings.NewReader("x")))
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Fatalf("body slot not released: %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAdmissionReleasesOnReadError(t *testing.T) {
|
||||
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("read error reached handler") }))
|
||||
for range 2 {
|
||||
req := httptest.NewRequest(http.MethodPost, "/bad", nil)
|
||||
req.Body = io.NopCloser(failingRequestReader{})
|
||||
req.ContentLength = -1
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("read failure retained slot: %d", w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAdmissionEnvironment(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
raw string
|
||||
want int
|
||||
}{{"", 4}, {"1", 1}, {"64", 64}, {"65", 4}, {"0", 4}, {"-1", 4}, {"invalid", 4}} {
|
||||
t.Run(tc.raw, func(t *testing.T) {
|
||||
t.Setenv("EVOBGP_API_BODY_MAX_CONCURRENT", tc.raw)
|
||||
if got := requestBodyConcurrency(); got != tc.want {
|
||||
t.Fatalf("concurrency=%d want%d", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user