feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -1,6 +1,9 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -185,3 +188,187 @@ func TestPeerReconcile_RendersFromLatestRevisionAndQueuesDeploy(t *testing.T) {
|
||||
t.Fatalf("expected deploy_apply_job_id in peer_reconcile meta, got %#v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
type renderHookStore struct {
|
||||
*store.Memory
|
||||
onCreate func(prefixes []store.PrefixRow)
|
||||
}
|
||||
|
||||
func (s *renderHookStore) CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parent *string, contentHash string, preview map[string]string, prefixes []store.PrefixRow) error {
|
||||
if err := s.Memory.CreateRenderRevision(ctx, revisionID, tenantID, moduleID, parent, contentHash, preview, prefixes); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.onCreate != nil {
|
||||
copied := append([]store.PrefixRow(nil), prefixes...)
|
||||
s.onCreate(copied)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newIPRenderFixture(t *testing.T) (tenant, module string, mem *store.Memory) {
|
||||
t.Helper()
|
||||
t.Setenv("EVOBGP_ASN_RESOLVE", "0")
|
||||
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
|
||||
mem = store.NewMemory()
|
||||
mem.SeedDemo()
|
||||
var modIP string
|
||||
tenant, _, modIP, _, _ = mem.DemoIDs()
|
||||
module = modIP
|
||||
for _, mod := range mem.ListModules(tenant) {
|
||||
if mod == nil || mod.ID == module {
|
||||
continue
|
||||
}
|
||||
disabled := false
|
||||
if _, err := mem.UpdateModule(tenant, mod.ID, &store.ModulePatch{Enabled: &disabled}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return tenant, module, mem
|
||||
}
|
||||
|
||||
func installModuleSnapshot(t *testing.T, mem *store.Memory, tenant, module, prefix string) {
|
||||
t.Helper()
|
||||
err := mem.SetModulePrefixSnapshot(context.Background(), tenant, module, "snap-"+prefix, []store.PrefixRow{{
|
||||
Prefix: prefix,
|
||||
Source: "ip_range",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot %s: %v", prefix, err)
|
||||
}
|
||||
}
|
||||
|
||||
func revisionHasPrefix(t *testing.T, mem *store.Memory, tenant, revID, prefix string) bool {
|
||||
t.Helper()
|
||||
rows, _, _ := mem.ListRevisionPrefixes(tenant, revID, "", 100)
|
||||
for _, row := range rows {
|
||||
if row.Prefix == prefix {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestRenderTenantUntilQuietOverlappingIngests(t *testing.T) {
|
||||
t.Run("latest snapshot", func(t *testing.T) {
|
||||
tenant, module, mem := newIPRenderFixture(t)
|
||||
prefixes := []string{"10.8.0.0/24", "10.9.0.0/24", "10.10.0.0/24", "10.11.0.0/24"}
|
||||
installModuleSnapshot(t, mem, tenant, module, prefixes[0])
|
||||
hook := &renderHookStore{Memory: mem}
|
||||
var wk *Worker
|
||||
step := 0
|
||||
hook.onCreate = func([]store.PrefixRow) {
|
||||
step++
|
||||
if step >= len(prefixes) {
|
||||
return
|
||||
}
|
||||
installModuleSnapshot(t, mem, tenant, module, prefixes[step])
|
||||
gate := wk.tenantRefreshGate(tenant)
|
||||
gate.mu.Lock()
|
||||
gate.dirty = true
|
||||
gate.mu.Unlock()
|
||||
}
|
||||
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
|
||||
j := &Job{ID: "refresh-overlap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
|
||||
wk.finishModuleRefreshSuccess(j, module)
|
||||
if j.Status != StatusSucceeded {
|
||||
errText := ""
|
||||
if j.Error != nil {
|
||||
errText = *j.Error
|
||||
}
|
||||
t.Fatalf("status=%s err=%s", j.Status, errText)
|
||||
}
|
||||
if step != len(prefixes) {
|
||||
t.Fatalf("render commits=%d, want %d so three overlapping ingests are included", step, len(prefixes))
|
||||
}
|
||||
rev := j.metaString("revision_id")
|
||||
if !revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-1]) {
|
||||
t.Fatalf("deployed revision %s missing latest prefix %s", rev, prefixes[len(prefixes)-1])
|
||||
}
|
||||
if revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-2]) {
|
||||
t.Fatalf("deployed revision %s still has stale prefix %s", rev, prefixes[len(prefixes)-2])
|
||||
}
|
||||
deployID := j.metaString("deploy_apply_job_id")
|
||||
deploy, err := wk.Registry.Get(tenant, deployID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if deploy.metaString("revision_id") != rev {
|
||||
t.Fatalf("deploy revision=%s, want %s", deploy.metaString("revision_id"), rev)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cancel does not deploy", func(t *testing.T) {
|
||||
tenant, module, mem := newIPRenderFixture(t)
|
||||
installModuleSnapshot(t, mem, tenant, module, "10.12.0.0/24")
|
||||
hook := &renderHookStore{Memory: mem}
|
||||
var wk *Worker
|
||||
var j *Job
|
||||
commits := 0
|
||||
hook.onCreate = func([]store.PrefixRow) {
|
||||
commits++
|
||||
installModuleSnapshot(t, mem, tenant, module, "10.13.0.0/24")
|
||||
gate := wk.tenantRefreshGate(tenant)
|
||||
gate.mu.Lock()
|
||||
gate.dirty = true
|
||||
gate.mu.Unlock()
|
||||
j.RequestCancel()
|
||||
}
|
||||
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
|
||||
j = &Job{ID: "refresh-cancel", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
|
||||
wk.finishModuleRefreshSuccess(j, module)
|
||||
if j.Status != StatusFailed {
|
||||
t.Fatalf("status=%s, want failed", j.Status)
|
||||
}
|
||||
if j.Error == nil || !strings.Contains(*j.Error, "newer snapshot") {
|
||||
t.Fatalf("error=%v, want superseded render", j.Error)
|
||||
}
|
||||
if commits != 1 {
|
||||
t.Fatalf("render commits=%d, want 1", commits)
|
||||
}
|
||||
if j.metaString("deploy_apply_job_id") != "" || j.metaString("revision_id") != "" {
|
||||
t.Fatalf("cancelled dirty render published meta %#v", j.metaCopy())
|
||||
}
|
||||
deploys, _, _ := wk.Registry.List(tenant, "", KindDeployApply, "", 10)
|
||||
if len(deploys) != 0 {
|
||||
t.Fatalf("deploy jobs=%d, want 0", len(deploys))
|
||||
}
|
||||
revs, _, _ := mem.ListRevisions(tenant, "", "", 50)
|
||||
for _, rev := range revs {
|
||||
if rev != nil && revisionHasPrefix(t, mem, tenant, rev.ID, "10.13.0.0/24") {
|
||||
t.Fatalf("revision %s contains the snapshot that landed during cancel", rev.ID)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cap does not publish", func(t *testing.T) {
|
||||
tenant, module, mem := newIPRenderFixture(t)
|
||||
installModuleSnapshot(t, mem, tenant, module, "10.20.0.0/24")
|
||||
hook := &renderHookStore{Memory: mem}
|
||||
var wk *Worker
|
||||
commits := 0
|
||||
hook.onCreate = func([]store.PrefixRow) {
|
||||
commits++
|
||||
installModuleSnapshot(t, mem, tenant, module, fmt.Sprintf("10.21.%d.0/24", commits))
|
||||
gate := wk.tenantRefreshGate(tenant)
|
||||
gate.mu.Lock()
|
||||
gate.dirty = true
|
||||
gate.mu.Unlock()
|
||||
}
|
||||
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
|
||||
j := &Job{ID: "refresh-cap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
|
||||
wk.finishModuleRefreshSuccess(j, module)
|
||||
if j.Status != StatusFailed || j.Error == nil || !strings.Contains(*j.Error, "did not quiesce") {
|
||||
errText := ""
|
||||
if j.Error != nil {
|
||||
errText = *j.Error
|
||||
}
|
||||
t.Fatalf("status=%s err=%s", j.Status, errText)
|
||||
}
|
||||
if commits != renderQuietPassLimit {
|
||||
t.Fatalf("render commits=%d, want cap %d", commits, renderQuietPassLimit)
|
||||
}
|
||||
if j.metaString("deploy_apply_job_id") != "" {
|
||||
t.Fatal("safety cap published a deploy")
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user