feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <cursoragent@cursor.com>

internal/birddeploy/apply.go

@@ -1,152 +1,433 @@
-// Package birddeploy applies revision preview fragments to a shared BIRD volume with parse-check and LKG (plan §13).
 package birddeploy
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"regexp"
+	"sort"
 	"strings"
+	"time"
 
 	"evobgp/internal/birdfmt"
 	"evobgp/internal/store"
 )
 
-// Config drives two-phase deploy from a revision in the store.
+const recoveryTimeout = 30 * time.Second
+
+var includeLine = regexp.MustCompile(`(?m)^([ \t]*include[ \t]+")([^"]+)("[ \t]*;[ \t]*(?:#[^\n]*)?$)`)
+
+// Config identifies the BIRD instance. StagingDir remains for caller compatibility;
+// generations live inside ActiveDir so referenced files survive process restart.
 type Config struct {
-	ActiveDir  string // e.g. /etc/bird (live config tree)
-	StagingDir string // e.g. /tmp/evobgp-staging (same host/volume as ActiveDir)
+	ActiveDir  string
+	StagingDir string
 	BirdBin    string
 	BirdcBin   string
-	Socket     string // birdc -s
+	Socket     string
+	writeFile  func(string, []byte) error // test-only fault injection; nil uses synced writes
 }
 
-// ApplyRevision writes preview fragments to staging, runs bird -p, swaps into active, runs birdc configure; restores LKG on failure.
+// ApplyRevision installs a checked generation through one atomic bird.conf rename.
 func ApplyRevision(ctx context.Context, ctl *birdfmt.BirdCtl, rev *store.Revision, cfg Config) error {
 	if rev == nil || len(rev.PreviewFragments) == 0 {
 		return fmt.Errorf("birddeploy: no preview fragments")
 	}
-	active := strings.TrimSpace(cfg.ActiveDir)
-	staging := strings.TrimSpace(cfg.StagingDir)
-	if active == "" || staging == "" {
-		return fmt.Errorf("birddeploy: ActiveDir and StagingDir required")
+	if strings.TrimSpace(cfg.ActiveDir) == "" {
+		return fmt.Errorf("birddeploy: ActiveDir required")
 	}
-	if err := os.MkdirAll(staging, 0o755); err != nil {
-		return err
+	active, err := filepath.Abs(cfg.ActiveDir)
+	if err != nil {
+		return fmt.Errorf("birddeploy: resolve active directory: %w", err)
 	}
-	if err := os.MkdirAll(filepath.Join(active, "bird.d"), 0o755); err != nil {
-		return err
+	if err := os.MkdirAll(active, 0o755); err != nil {
+		return fmt.Errorf("birddeploy: create active directory: %w", err)
 	}
-	lkg := filepath.Join(active, ".evobgp_lkg")
-
-	// Phase 1: write staging tree
-	for rel, content := range rev.PreviewFragments {
-		rel = strings.TrimPrefix(rel, "/")
-		if !isDeployableBirdFragment(rel) {
-			continue
+	lock, err := lockTarget(ctx, filepath.Join(active, ".evobgp-apply.lock"))
+	if err != nil {
+		return fmt.Errorf("birddeploy: acquire apply lock: %w", err)
+	}
+	defer func() {
+		if err := lock.Close(); err != nil {
+			fmt.Fprintf(os.Stderr, "birddeploy: release apply lock: %v\n", err)
 		}
-		dst := filepath.Join(staging, rel)
-		if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
-			return err
-		}
-		if err := os.WriteFile(dst, []byte(content), 0o644); err != nil {
-			return err
-		}
-	}
-	mainConf := filepath.Join(staging, "bird.conf")
-	if _, err := os.Stat(mainConf); err != nil {
-		return fmt.Errorf("birddeploy: staging missing bird.conf: %w", err)
+	}()
+	if err := ctx.Err(); err != nil {
+		return fmt.Errorf("birddeploy: apply cancelled: %w", err)
 	}
 	bc := ctl
 	if bc == nil {
 		bc = &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
 	}
-	if err := bc.ParseCheck(ctx, mainConf); err != nil {
-		return fmt.Errorf("birddeploy: bird -p failed: %w", err)
+	writeFile := cfg.writeFile
+	if writeFile == nil {
+		writeFile = writeSynced
 	}
-
-	// Snapshot LKG (best-effort copy of current active bird.conf + bird.d)
-	_ = os.RemoveAll(lkg)
-	_ = snapshotBirdTree(active, lkg)
-
-	// Phase 2: atomic swap into active
-	for rel, content := range rev.PreviewFragments {
-		rel = strings.TrimPrefix(rel, "/")
-		if !isDeployableBirdFragment(rel) {
-			continue
+	mainPath := filepath.Join(active, "bird.conf")
+	mainInfo, err := os.Lstat(mainPath)
+	if err != nil {
+		return fmt.Errorf("birddeploy: inspect current bird.conf: %w", err)
+	}
+	if !mainInfo.Mode().IsRegular() {
+		return fmt.Errorf("birddeploy: current bird.conf must be a regular file; refusing to replace %s", mainInfo.Mode())
+	}
+	oldMain, err := os.ReadFile(mainPath)
+	if err != nil {
+		return fmt.Errorf("birddeploy: read current bird.conf: %w", err)
+	}
+	if err := bc.ParseCheck(ctx, mainPath); err != nil {
+		return fmt.Errorf("birddeploy: current configuration is not a usable LKG: %w", err)
+	}
+	gen, err := os.MkdirTemp(active, ".evobgp-generation-")
+	if err != nil {
+		return fmt.Errorf("birddeploy: create generation: %w", err)
+	}
+	activated := false
+	defer func() {
+		if !activated {
+			if err := os.RemoveAll(gen); err != nil {
+				fmt.Fprintf(os.Stderr, "birddeploy: remove unactivated generation %s: %v\n", gen, err)
+			}
 		}
-		dst := filepath.Join(active, rel)
-		if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
-			_ = restoreLKG(lkg, active)
-			return err
-		}
-		tmp := dst + ".tmp"
-		if err := os.WriteFile(tmp, []byte(content), 0o644); err != nil {
-			_ = restoreLKG(lkg, active)
-			return err
-		}
-		if err := os.Rename(tmp, dst); err != nil {
-			_ = restoreLKG(lkg, active)
-			return err
+	}()
+	if err := writeGenerationWith(active, gen, rev.PreviewFragments, writeFile); err != nil {
+		return err
+	}
+	newMain, err := os.ReadFile(filepath.Join(gen, "bird.conf"))
+	if err != nil {
+		return fmt.Errorf("birddeploy: read generated bird.conf: %w", err)
+	}
+	if err := bc.ParseCheck(ctx, filepath.Join(gen, "bird.conf")); err != nil {
+		return fmt.Errorf("birddeploy: staged bird -p failed: %w", err)
+	}
+	if err := writeFile(filepath.Join(gen, "previous-bird.conf"), oldMain); err != nil {
+		return fmt.Errorf("birddeploy: save LKG: %w", err)
+	}
+	if err := syncDir(gen); err != nil {
+		return fmt.Errorf("birddeploy: sync LKG generation: %w", err)
+	}
+	if err := ctx.Err(); err != nil {
+		return fmt.Errorf("birddeploy: apply cancelled: %w", err)
+	}
+	swapped, err := replaceMain(active, newMain)
+	activated = swapped
+	if err != nil {
+		if swapped {
+			return recoverPrevious(bc, active, gen, oldMain, fmt.Errorf("activate generation: %w", err))
 		}
+		return fmt.Errorf("birddeploy: activate generation: %w", err)
 	}
 	if err := bc.Configure(ctx); err != nil {
-		_ = restoreLKG(lkg, active)
-		_ = bc.Configure(ctx)
-		return fmt.Errorf("birddeploy: birdc configure failed, restored LKG: %w", err)
+		return recoverPrevious(bc, active, gen, oldMain, err)
+	}
+	if err := pruneGenerations(active, gen, oldMain); err != nil {
+		fmt.Fprintf(os.Stderr, "birddeploy: configuration applied; generation cleanup failed: %v\n", err)
 	}
 	return nil
 }
 
-// isDeployableBirdFragment skips UI-only preview keys (e.g. flattened full config) not referenced from bird.conf.
-func isDeployableBirdFragment(rel string) bool {
-	return rel != "" && !strings.HasPrefix(rel, "_")
+func recoverPrevious(bc *birdfmt.BirdCtl, active, gen string, oldMain []byte, cause error) error {
+	ctx, cancel := context.WithTimeout(context.Background(), recoveryTimeout)
+	defer cancel()
+	if _, err := replaceMain(active, oldMain); err != nil {
+		return fmt.Errorf("birddeploy: configure failed: %w; restore failed: %v", cause, err)
+	}
+	if err := bc.Configure(ctx); err != nil {
+		return fmt.Errorf("birddeploy: configure failed: %w; LKG restored on disk but recovery configure failed: %v", cause, err)
+	}
+	if err := os.RemoveAll(gen); err != nil {
+		fmt.Fprintf(os.Stderr, "birddeploy: LKG restored; remove failed generation %s: %v\n", gen, err)
+	}
+	return fmt.Errorf("birddeploy: configure failed: %w; LKG restored and recovery configure succeeded", cause)
 }
 
-func snapshotBirdTree(activeRoot, dstRoot string) error {
-	if err := os.MkdirAll(dstRoot, 0o755); err != nil {
-		return err
-	}
-	mc := filepath.Join(activeRoot, "bird.conf")
-	if b, err := os.ReadFile(mc); err == nil {
-		_ = os.WriteFile(filepath.Join(dstRoot, "bird.conf"), b, 0o644)
-	}
-	bd := filepath.Join(activeRoot, "bird.d")
-	if st, err := os.Stat(bd); err == nil && st.IsDir() {
-		_ = filepath.WalkDir(bd, func(path string, d fs.DirEntry, err error) error {
-			if err != nil || d.IsDir() {
-				return err
-			}
-			suffix, _ := filepath.Rel(activeRoot, path)
-			out := filepath.Join(dstRoot, suffix)
-			_ = os.MkdirAll(filepath.Dir(out), 0o755)
-			b, err := os.ReadFile(path)
-			if err != nil {
-				return nil
-			}
-			return os.WriteFile(out, b, 0o644)
-		})
-	}
-	return nil
+func writeGeneration(active, gen string, fragments map[string]string) error {
+	return writeGenerationWith(active, gen, fragments, writeSynced)
 }
 
-func restoreLKG(lkg, active string) error {
-	if _, err := os.Stat(filepath.Join(lkg, "bird.conf")); err != nil {
-		return err
+func writeGenerationWith(active, gen string, fragments map[string]string, writeFile func(string, []byte) error) error {
+	main, ok := fragments["bird.conf"]
+	if !ok {
+		return fmt.Errorf("birddeploy: preview missing bird.conf")
 	}
-	_ = filepath.WalkDir(lkg, func(path string, d fs.DirEntry, err error) error {
-		if err != nil || d.IsDir() {
-			return err
-		}
-		suffix, _ := filepath.Rel(lkg, path)
-		out := filepath.Join(active, suffix)
-		_ = os.MkdirAll(filepath.Dir(out), 0o755)
-		b, err := os.ReadFile(path)
-		if err != nil {
+	oldBirdD := filepath.Join(active, birdfmt.DirBirdD)
+	newBirdD := filepath.Join(gen, birdfmt.DirBirdD)
+	if err := os.MkdirAll(newBirdD, 0o755); err != nil {
+		return fmt.Errorf("birddeploy: create generation bird.d: %w", err)
+	}
+	if err := filepath.WalkDir(oldBirdD, func(path string, d fs.DirEntry, walkErr error) error {
+		if errors.Is(walkErr, fs.ErrNotExist) && path == oldBirdD {
 			return nil
 		}
-		return os.WriteFile(out, b, 0o644)
+		if walkErr != nil {
+			return walkErr
+		}
+		rel, err := filepath.Rel(oldBirdD, path)
+		if err != nil {
+			return err
+		}
+		if d.Type()&os.ModeSymlink != 0 {
+			return fmt.Errorf("operator snippet %q is a symlink", rel)
+		}
+		if rel == "." {
+			if !d.IsDir() {
+				return fmt.Errorf("birddeploy: bird.d is not a directory")
+			}
+			return nil
+		}
+		if d.IsDir() {
+			return os.MkdirAll(filepath.Join(newBirdD, rel), 0o755)
+		}
+		if !d.Type().IsRegular() {
+			return fmt.Errorf("operator snippet %q is not a regular file", rel)
+		}
+		_, replaced := fragments[filepath.ToSlash(filepath.Join(birdfmt.DirBirdD, rel))]
+		if strings.HasPrefix(filepath.Base(rel), "evobgp_") || replaced {
+			return nil
+		}
+		b, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		if countIncludeTokens(string(b)) > 0 {
+			return fmt.Errorf("operator snippet %q has a nested include; cannot pin the complete tree", rel)
+		}
+		return writeFile(filepath.Join(newBirdD, rel), b)
+	}); err != nil {
+		return fmt.Errorf("birddeploy: copy operator snippets: %w", err)
+	}
+	for rel, content := range fragments {
+		if rel == "bird.conf" || strings.HasPrefix(rel, "_") {
+			continue
+		}
+		if countIncludeTokens(content) > 0 {
+			return fmt.Errorf("birddeploy: nested include in preview fragment %q is unsupported", rel)
+		}
+		clean := filepath.Clean(filepath.FromSlash(rel))
+		if filepath.IsAbs(clean) || clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) || !strings.HasPrefix(clean, birdfmt.DirBirdD+string(os.PathSeparator)) || filepath.Ext(clean) != ".conf" {
+			return fmt.Errorf("birddeploy: unsafe preview path %q", rel)
+		}
+		dst := filepath.Join(gen, clean)
+		if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
+			return fmt.Errorf("birddeploy: create fragment directory: %w", err)
+		}
+		if err := writeFile(dst, []byte(content)); err != nil {
+			return fmt.Errorf("birddeploy: write %s: %w", rel, err)
+		}
+	}
+	var rewriteErr error
+	if !standaloneIncludesOnly(main) {
+		return fmt.Errorf("birddeploy: unsupported include syntax in bird.conf")
+	}
+	rewritten := includeLine.ReplaceAllStringFunc(main, func(line string) string {
+		m := includeLine.FindStringSubmatch(line)
+		inc := m[2]
+		if filepath.IsAbs(inc) {
+			rel, err := filepath.Rel(active, inc)
+			if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
+				rewriteErr = fmt.Errorf("birddeploy: external absolute include %q is unsupported", inc)
+				return line
+			}
+			inc = rel
+		}
+		clean := filepath.Clean(filepath.FromSlash(inc))
+		if clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) || !strings.HasPrefix(clean, birdfmt.DirBirdD+string(os.PathSeparator)) {
+			rewriteErr = fmt.Errorf("birddeploy: unsupported relative include %q", inc)
+			return line
+		}
+		return m[1] + filepath.ToSlash(filepath.Join(gen, clean)) + m[3]
 	})
+	if rewriteErr != nil {
+		return rewriteErr
+	}
+	if err := writeFile(filepath.Join(gen, ".evobgp-owned"), []byte("evobgp-generation-v1\n")); err != nil {
+		return fmt.Errorf("birddeploy: mark generation: %w", err)
+	}
+	if err := writeFile(filepath.Join(gen, "bird.conf"), []byte(rewritten)); err != nil {
+		return fmt.Errorf("birddeploy: write generated bird.conf: %w", err)
+	}
+	return syncTree(gen)
+}
+
+func standaloneIncludesOnly(config string) bool {
+	tokens := includeTokenOffsets(config)
+	matches := includeLine.FindAllStringIndex(config, -1)
+	if len(tokens) != len(matches) {
+		return false
+	}
+	for i, match := range matches {
+		word := strings.Index(config[match[0]:match[1]], "include")
+		if word < 0 || tokens[i] != match[0]+word {
+			return false
+		}
+	}
+	return true
+}
+
+func countIncludeTokens(config string) int { return len(includeTokenOffsets(config)) }
+
+// includeTokenOffsets skips BIRD comments and strings, retaining exact offsets
+// so a commented standalone include cannot mask a live inline include.
+func includeTokenOffsets(config string) []int {
+	var offsets []int
+	quote := byte(0)
+	lineComment, blockComment := false, false
+	for i := 0; i < len(config); {
+		c := config[i]
+		if lineComment {
+			if c == '\n' {
+				lineComment = false
+			}
+			i++
+			continue
+		}
+		if blockComment {
+			if c == '*' && i+1 < len(config) && config[i+1] == '/' {
+				blockComment = false
+				i += 2
+			} else {
+				i++
+			}
+			continue
+		}
+		if quote != 0 {
+			// BIRD quoted text does not use C-style backslash escapes.
+			// Skipping the following quote could hide a live include.
+			if c == quote {
+				quote = 0
+			}
+			i++
+			continue
+		}
+		if c == '#' {
+			lineComment = true
+			i++
+			continue
+		}
+		if c == '/' && i+1 < len(config) && config[i+1] == '*' {
+			blockComment = true
+			i += 2
+			continue
+		}
+		if c == '"' || c == '\'' {
+			quote = c
+			i++
+			continue
+		}
+		if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' {
+			start := i
+			for i < len(config) && ((config[i] >= 'a' && config[i] <= 'z') || (config[i] >= 'A' && config[i] <= 'Z') || (config[i] >= '0' && config[i] <= '9') || config[i] == '_') {
+				i++
+			}
+			if config[start:i] == "include" {
+				offsets = append(offsets, start)
+			}
+			continue
+		}
+		i++
+	}
+	return offsets
+}
+
+func syncTree(root string) error {
+	var dirs []string
+	if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			dirs = append(dirs, path)
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	sort.Slice(dirs, func(i, j int) bool { return len(dirs[i]) > len(dirs[j]) })
+	for _, dir := range dirs {
+		if err := syncDir(dir); err != nil {
+			return fmt.Errorf("sync %s: %w", dir, err)
+		}
+	}
 	return nil
 }
+
+func pruneGenerations(active, current string, previous []byte) error {
+	entries, err := os.ReadDir(active)
+	if err != nil {
+		return err
+	}
+	var stale []string
+	for _, entry := range entries {
+		if !entry.IsDir() || !strings.HasPrefix(entry.Name(), ".evobgp-generation-") {
+			continue
+		}
+		path := filepath.Join(active, entry.Name())
+		if path == current || strings.Contains(string(previous), filepath.ToSlash(path)+"/") {
+			continue
+		}
+		marker, err := os.ReadFile(filepath.Join(path, ".evobgp-owned"))
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return err
+		}
+		if string(marker) != "evobgp-generation-v1\n" {
+			continue
+		}
+		stale = append(stale, path)
+	}
+	sort.Strings(stale)
+	for _, path := range stale {
+		if err := os.RemoveAll(path); err != nil {
+			return fmt.Errorf("remove %s: %w", path, err)
+		}
+	}
+	return nil
+}
+
+func replaceMain(active string, content []byte) (bool, error) {
+	tmp, err := os.CreateTemp(active, ".evobgp-bird-conf-")
+	if err != nil {
+		return false, err
+	}
+	defer func() { _ = os.Remove(tmp.Name()) }()
+	if err := tmp.Chmod(0o644); err != nil {
+		_ = tmp.Close()
+		return false, err
+	}
+	if _, err := tmp.Write(content); err != nil {
+		_ = tmp.Close()
+		return false, err
+	}
+	if err := tmp.Sync(); err != nil {
+		_ = tmp.Close()
+		return false, err
+	}
+	if err := tmp.Close(); err != nil {
+		return false, err
+	}
+	if err := os.Rename(tmp.Name(), filepath.Join(active, "bird.conf")); err != nil {
+		return false, err
+	}
+	return true, syncDir(active)
+}
+
+func writeSynced(path string, b []byte) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
+	if err != nil {
+		return err
+	}
+	if _, err := f.Write(b); err != nil {
+		_ = f.Close()
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		_ = f.Close()
+		return err
+	}
+	return f.Close()
+}