feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -0,0 +1,279 @@
|
||||
//go:build linux
|
||||
|
||||
package birddeploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func fakeCtl(t *testing.T, active string) *birdfmt.BirdCtl {
|
||||
t.Helper()
|
||||
bin := t.TempDir()
|
||||
parse := filepath.Join(bin, "bird")
|
||||
configure := filepath.Join(bin, "birdc")
|
||||
parseScript := `#!/bin/sh
|
||||
if grep -q FAIL_PARSE "$2"; then exit 1; fi
|
||||
exit 0
|
||||
`
|
||||
configureScript := fmt.Sprintf(`#!/bin/sh
|
||||
root=%q
|
||||
if [ -e "$root/.configure-wait" ]; then
|
||||
rm "$root/.configure-wait"
|
||||
touch "$root/.configure-started"
|
||||
exec sleep 5
|
||||
fi
|
||||
if [ -e "$root/.configure-fail-once" ]; then
|
||||
rm "$root/.configure-fail-once"
|
||||
exit 1
|
||||
fi
|
||||
mkdir "$root/.configuring" || exit 1
|
||||
sleep 0.05
|
||||
rmdir "$root/.configuring"
|
||||
`, active)
|
||||
for path, script := range map[string]string{parse: parseScript, configure: configureScript} {
|
||||
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return &birdfmt.BirdCtl{Bird: parse, Birdc: configure}
|
||||
}
|
||||
|
||||
func revision(label string) *store.Revision {
|
||||
return &store.Revision{PreviewFragments: map[string]string{
|
||||
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/*.conf\";\n# " + label + "\n",
|
||||
"bird.d/evobgp_new.conf": "# " + label + "\n",
|
||||
}}
|
||||
}
|
||||
|
||||
func setupActive(t *testing.T) (string, *birdfmt.BirdCtl, []byte) {
|
||||
t.Helper()
|
||||
active := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := []byte("router id 192.0.2.2;\ninclude \"bird.d/*.conf\";\n# old\n")
|
||||
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, content := range map[string]string{"evobgp_removed.conf": "# removed\n", "operator.conf": "# operator\n"} {
|
||||
if err := os.WriteFile(filepath.Join(active, "bird.d", name), []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return active, fakeCtl(t, active), old
|
||||
}
|
||||
|
||||
func TestApplyRevisionGenerationAndPrune(t *testing.T) {
|
||||
active, ctl, _ := setupActive(t)
|
||||
for _, label := range []string{"one", "two", "three"} {
|
||||
if err := ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(main), "# "+label) {
|
||||
t.Fatalf("wrong active generation: %s", main)
|
||||
}
|
||||
if strings.Contains(string(main), `include "bird.d/`) {
|
||||
t.Fatalf("include not pinned: %s", main)
|
||||
}
|
||||
}
|
||||
gens, err := filepath.Glob(filepath.Join(active, ".evobgp-generation-*"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(gens) != 2 {
|
||||
t.Fatalf("want active and previous generations, got %d", len(gens))
|
||||
}
|
||||
for _, gen := range gens {
|
||||
if _, err := os.Stat(filepath.Join(gen, "bird.d", "evobgp_removed.conf")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("removed fragment retained in %s: %v", gen, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(gen, "bird.d", "operator.conf")); err != nil {
|
||||
t.Fatalf("operator fragment missing: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRevisionFaults(t *testing.T) {
|
||||
for _, mode := range []string{"parse", "write", "disk-full", "configure", "cancel"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
active, ctl, old := setupActive(t)
|
||||
rev := revision(mode)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
cfg := Config{ActiveDir: active}
|
||||
switch mode {
|
||||
case "parse":
|
||||
rev.PreviewFragments["bird.conf"] += "FAIL_PARSE\n"
|
||||
case "write":
|
||||
rev.PreviewFragments["../escape.conf"] = "bad"
|
||||
case "disk-full":
|
||||
cfg.writeFile = func(path string, b []byte) error {
|
||||
if filepath.Base(path) == "previous-bird.conf" {
|
||||
return syscall.ENOSPC
|
||||
}
|
||||
return writeSynced(path, b)
|
||||
}
|
||||
case "configure":
|
||||
if err := os.WriteFile(filepath.Join(active, ".configure-fail-once"), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "cancel":
|
||||
if err := os.WriteFile(filepath.Join(active, ".configure-wait"), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
result := make(chan error, 1)
|
||||
go func() { result <- ApplyRevision(ctx, ctl, rev, cfg) }()
|
||||
if mode == "cancel" {
|
||||
deadline := time.After(3 * time.Second)
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(active, ".configure-started")); err == nil {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("configure did not start")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
err := <-result
|
||||
if err == nil {
|
||||
t.Fatal("expected apply failure")
|
||||
}
|
||||
main, readErr := os.ReadFile(filepath.Join(active, "bird.conf"))
|
||||
if readErr != nil {
|
||||
t.Fatal(readErr)
|
||||
}
|
||||
if string(main) != string(old) {
|
||||
t.Fatalf("old config not restored: %s; error: %v", main, err)
|
||||
}
|
||||
if mode == "configure" || mode == "cancel" {
|
||||
if !strings.Contains(err.Error(), "recovery configure succeeded") {
|
||||
t.Fatalf("unconfirmed recovery: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRevisionSerializes(t *testing.T) {
|
||||
active, ctl, _ := setupActive(t)
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 2)
|
||||
for _, label := range []string{"first", "second"} {
|
||||
wg.Add(1)
|
||||
go func(label string) {
|
||||
defer wg.Done()
|
||||
errs <- ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active})
|
||||
}(label)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLockHelperProcess(t *testing.T) {
|
||||
active := os.Getenv("EVOBGP_LOCK_HELPER_ACTIVE")
|
||||
if active == "" {
|
||||
return
|
||||
}
|
||||
lock, err := lockTarget(context.Background(), filepath.Join(active, ".evobgp-apply.lock"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = lock.Close() }()
|
||||
if err := os.WriteFile(filepath.Join(active, ".helper-ready"), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(active, ".helper-done")); err == nil {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRevisionCrossProcessLock(t *testing.T) {
|
||||
active, ctl, old := setupActive(t)
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestLockHelperProcess$")
|
||||
cmd.Env = append(os.Environ(), "EVOBGP_LOCK_HELPER_ACTIVE="+active)
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = os.WriteFile(filepath.Join(active, ".helper-done"), nil, 0o644)
|
||||
if err := cmd.Wait(); err != nil {
|
||||
t.Errorf("lock helper: %v", err)
|
||||
}
|
||||
}()
|
||||
deadline := time.After(3 * time.Second)
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(active, ".helper-ready")); err == nil {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("lock helper did not become ready")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := ApplyRevision(ctx, ctl, revision("blocked"), Config{ActiveDir: active}); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("want lock timeout, got %v", err)
|
||||
}
|
||||
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(main) != string(old) {
|
||||
t.Fatalf("apply bypassed cross-process lock: %s", main)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealBirdParseGeneration(t *testing.T) {
|
||||
bin := strings.TrimSpace(os.Getenv("EVOBGP_TEST_BIRD_BIN"))
|
||||
if bin == "" {
|
||||
t.Skip("EVOBGP_TEST_BIRD_BIN is not set")
|
||||
}
|
||||
active := t.TempDir()
|
||||
old := []byte("router id 192.0.2.2;\nprotocol device {}\n")
|
||||
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctl := fakeCtl(t, active)
|
||||
ctl.Bird = bin
|
||||
rev := &store.Revision{PreviewFragments: map[string]string{
|
||||
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/evobgp_new.conf\";\nprotocol device {}\n",
|
||||
"bird.d/evobgp_new.conf": "# complete generated fragment\n",
|
||||
}}
|
||||
if err := ApplyRevision(context.Background(), ctl, rev, Config{ActiveDir: active}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ctl.ParseCheck(context.Background(), filepath.Join(active, "bird.conf")); err != nil {
|
||||
t.Fatalf("activated config fails real bird -p: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user