feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
+109
-11
@@ -1,7 +1,10 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
@@ -10,6 +13,18 @@ import (
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
// MaxCSVBytes is the complete request limit, including the optional header.
|
||||
const MaxCSVBytes = 8 << 20
|
||||
|
||||
// MaxCSVRows bounds database work for a single import, excluding its header.
|
||||
const MaxCSVRows = 10000
|
||||
|
||||
// MaxCSVFields bounds CSV parser allocations for delimiter-heavy records.
|
||||
const MaxCSVFields = 64
|
||||
|
||||
// ErrCSVTooLarge means the entire import was rejected before creating entries.
|
||||
var ErrCSVTooLarge = errors.New("importer: csv exceeds byte, row or field limit")
|
||||
|
||||
// Result of importing module entries from CSV.
|
||||
type Result struct {
|
||||
Imported int
|
||||
@@ -17,18 +32,50 @@ type Result struct {
|
||||
}
|
||||
|
||||
// ImportModuleEntriesCSV parses and creates entries for AS_PREFIXES, DOMAINS, or IP_RANGES modules.
|
||||
func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io.Reader) (*Result, error) {
|
||||
func ImportModuleEntriesCSV(ctx context.Context, st store.Backend, tenantID, moduleID string, body io.Reader) (*Result, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mod, err := st.GetModule(tenantID, moduleID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cr := csv.NewReader(io.LimitReader(body, 8<<20))
|
||||
// Read one byte beyond the cap: a valid prefix must never be imported as if
|
||||
// it were the complete request. Check all resource limits before mutations.
|
||||
raw, err := io.ReadAll(io.LimitReader(body, MaxCSVBytes+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("importer: read csv: %w", err)
|
||||
}
|
||||
if len(raw) > MaxCSVBytes {
|
||||
return nil, ErrCSVTooLarge
|
||||
}
|
||||
if err := checkCSVFields(ctx, raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cr := csv.NewReader(bytes.NewReader(raw))
|
||||
cr.TrimLeadingSpace = true
|
||||
cr.FieldsPerRecord = -1
|
||||
rows, err := cr.ReadAll()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("importer: invalid csv: %w", err)
|
||||
rows := make([][]string, 0)
|
||||
headerRows := 0
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
record, err := cr.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("importer: invalid csv: %w", err)
|
||||
}
|
||||
if len(rows) == 0 && isCSVHeader(record) {
|
||||
headerRows = 1
|
||||
}
|
||||
if len(rows)-headerRows >= MaxCSVRows {
|
||||
return nil, ErrCSVTooLarge
|
||||
}
|
||||
rows = append(rows, record)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, fmt.Errorf("importer: %w", store.ErrInvalidInput)
|
||||
@@ -63,18 +110,17 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
||||
}
|
||||
|
||||
start := 0
|
||||
if len(rows[0]) >= 2 {
|
||||
key := strings.ToLower(strings.TrimSpace(rows[0][0]))
|
||||
switch key {
|
||||
case "asn", "domain", "iprange":
|
||||
start = 1
|
||||
}
|
||||
if isCSVHeader(rows[0]) {
|
||||
start = 1
|
||||
}
|
||||
|
||||
imported := 0
|
||||
switch mod.Type {
|
||||
case "AS_PREFIXES":
|
||||
for i := start; i < len(rows); i++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rec := rows[i]
|
||||
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
||||
continue
|
||||
@@ -97,6 +143,9 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
||||
}
|
||||
case "DOMAINS":
|
||||
for i := start; i < len(rows); i++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rec := rows[i]
|
||||
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
||||
continue
|
||||
@@ -119,6 +168,9 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
||||
}
|
||||
case "IP_RANGES":
|
||||
for i := start; i < len(rows); i++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rec := rows[i]
|
||||
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
||||
continue
|
||||
@@ -145,3 +197,49 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
||||
|
||||
return &Result{Imported: imported, ModuleType: mod.Type}, nil
|
||||
}
|
||||
|
||||
func isCSVHeader(record []string) bool {
|
||||
if len(record) < 2 {
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(record[0])) {
|
||||
case "asn", "domain", "iprange":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// checkCSVFields runs before encoding/csv allocates a string/index for each field.
|
||||
// Quoted commas, escaped quotes and multiline fields don't start new fields.
|
||||
// Syntax validation remains the responsibility of encoding/csv below.
|
||||
func checkCSVFields(ctx context.Context, raw []byte) error {
|
||||
quoted, fields := false, 1
|
||||
for i := 0; i < len(raw); i++ {
|
||||
if i%4096 == 0 {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch raw[i] {
|
||||
case '"':
|
||||
if quoted && i+1 < len(raw) && raw[i+1] == '"' {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
quoted = !quoted
|
||||
case ',':
|
||||
if !quoted {
|
||||
fields++
|
||||
if fields > MaxCSVFields {
|
||||
return ErrCSVTooLarge
|
||||
}
|
||||
}
|
||||
case '\n':
|
||||
if !quoted {
|
||||
fields = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type importBackend struct {
|
||||
store.Backend
|
||||
created int
|
||||
onCreate func()
|
||||
}
|
||||
|
||||
func (s *importBackend) GetModule(string, string) (*store.Module, error) {
|
||||
return &store.Module{Type: "AS_PREFIXES"}, nil
|
||||
}
|
||||
func (s *importBackend) ListCommunities(string) ([]*store.Community, error) { return nil, nil }
|
||||
func (s *importBackend) CreateASEntry(_, _ string, e *store.ASEntry) (*store.ASEntry, error) {
|
||||
s.created++
|
||||
if s.onCreate != nil {
|
||||
s.onCreate()
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func TestCSVByteLimitRejectsValidTruncatedPrefix(t *testing.T) {
|
||||
header := "asn,community\n"
|
||||
for _, extra := range []int{0, 1} {
|
||||
s := &importBackend{}
|
||||
body := header + strings.Repeat("\n", MaxCSVBytes-len(header)) + strings.Repeat("\n", extra)
|
||||
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(body))
|
||||
if extra == 0 && err != nil {
|
||||
t.Fatalf("exact limit: %v", err)
|
||||
}
|
||||
if extra == 1 && !errors.Is(err, ErrCSVTooLarge) {
|
||||
t.Fatalf("limit+1: %v", err)
|
||||
}
|
||||
if s.created != 0 {
|
||||
t.Fatal("oversized import created entries")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVRowLimitCheckedBeforeWrites(t *testing.T) {
|
||||
for _, header := range []string{"", "asn,community\n"} {
|
||||
for _, extra := range []int{0, 1} {
|
||||
s := &importBackend{}
|
||||
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(header+strings.Repeat("64512,\n", MaxCSVRows+extra)))
|
||||
if extra == 0 && (err != nil || s.created != MaxCSVRows) {
|
||||
t.Fatalf("exact rows: created=%d err=%v", s.created, err)
|
||||
}
|
||||
if extra == 1 && (!errors.Is(err, ErrCSVTooLarge) || s.created != 0) {
|
||||
t.Fatalf("rows+1: created=%d err=%v", s.created, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVCancellationStopsFurtherWrites(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
s := &importBackend{onCreate: cancel}
|
||||
_, err := ImportModuleEntriesCSV(ctx, s, "tenant", "module", strings.NewReader("asn,community\n64512,\n64513,\n"))
|
||||
if !errors.Is(err, context.Canceled) || s.created != 1 {
|
||||
t.Fatalf("created=%d err=%v", s.created, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVFieldLimitAndQuotedFields(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, input string
|
||||
oversized bool
|
||||
}{
|
||||
{"exact", "64512," + strings.Repeat(",", MaxCSVFields-2) + "\n", false},
|
||||
{"over", "64512," + strings.Repeat(",", MaxCSVFields-1) + "\n", true},
|
||||
{"many_fields", strings.Repeat(",", 1<<20) + "\n", true},
|
||||
{"quoted_extra", "64512,,\"" + strings.Repeat(",", 128) + "\nquoted \"\"text\"\"\"\n", false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := &importBackend{}
|
||||
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(tc.input))
|
||||
if tc.oversized {
|
||||
if !errors.Is(err, ErrCSVTooLarge) || s.created != 0 {
|
||||
t.Fatalf("created=%d err=%v", s.created, err)
|
||||
}
|
||||
} else if err != nil || s.created != 1 {
|
||||
t.Fatalf("created=%d err=%v", s.created, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user