feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -0,0 +1,312 @@
|
||||
package agentwatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
maxConfigFiles = 512
|
||||
maxConfigBytes = 64 << 20
|
||||
maxFileBytes = 16 << 20
|
||||
maxIncludeDepth = 16
|
||||
maxDirectoryEntries = 4096
|
||||
)
|
||||
|
||||
var errMainNotReady = errors.New("main bird.conf is not ready")
|
||||
|
||||
// Only standalone literal include statements are hashable. Other valid BIRD
|
||||
// grammar falls back to the legacy configure-on-every-tick path.
|
||||
var includeLine = regexp.MustCompile(`(?m)^[\t ]*include[\t ]+"([^"\r\n]+)"[\t ]*;[\t ]*(?:#[^\r\n]*)?\r?$`)
|
||||
|
||||
type configScanner struct {
|
||||
ctx context.Context
|
||||
root string
|
||||
h hash.Hash
|
||||
seen map[string]bool
|
||||
files int
|
||||
bytes int64
|
||||
}
|
||||
|
||||
func hashConfigTree(ctx context.Context, activeDir string) ([sha256.Size]byte, error) {
|
||||
var zero [sha256.Size]byte
|
||||
root, err := filepath.Abs(activeDir)
|
||||
if err != nil {
|
||||
return zero, fmt.Errorf("agentwatch: active directory: %w", err)
|
||||
}
|
||||
root, err = filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return zero, fmt.Errorf("agentwatch: %w: %v", errMainNotReady, err)
|
||||
}
|
||||
return zero, fmt.Errorf("agentwatch: resolve active directory: %w", err)
|
||||
}
|
||||
s := &configScanner{ctx: ctx, root: filepath.Clean(root), h: sha256.New(), seen: make(map[string]bool)}
|
||||
if err := s.visit(filepath.Join(s.root, "bird.conf"), 0); err != nil {
|
||||
return zero, err
|
||||
}
|
||||
var digest [sha256.Size]byte
|
||||
copy(digest[:], s.h.Sum(nil))
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
func (s *configScanner) visit(path string, depth int) error {
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if depth > maxIncludeDepth {
|
||||
return fmt.Errorf("agentwatch: include depth exceeds %d", maxIncludeDepth)
|
||||
}
|
||||
path = filepath.Clean(path)
|
||||
rel, err := filepath.Rel(s.root, path)
|
||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("agentwatch: include %q escapes active directory", path)
|
||||
}
|
||||
if s.seen[path] {
|
||||
return nil
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
if depth == 0 && errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("agentwatch: %w: %v", errMainNotReady, err)
|
||||
}
|
||||
return fmt.Errorf("agentwatch: inspect %q: %w", path, err)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("agentwatch: include %q is not a regular file", path)
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: resolve %q: %w", path, err)
|
||||
}
|
||||
if _, err := s.withinRoot(resolved); err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Size() > maxFileBytes {
|
||||
return fmt.Errorf("agentwatch: include %q exceeds %d bytes", path, maxFileBytes)
|
||||
}
|
||||
if s.files >= maxConfigFiles || s.bytes+info.Size() > maxConfigBytes {
|
||||
return fmt.Errorf("agentwatch: configuration tree exceeds %d files or %d bytes", maxConfigFiles, maxConfigBytes)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: open %q: %w", path, err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
limit := int64(maxFileBytes)
|
||||
if remaining := int64(maxConfigBytes) - s.bytes; remaining < limit {
|
||||
limit = remaining
|
||||
}
|
||||
content, err := readBounded(f, limit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: read %q: %w", path, err)
|
||||
}
|
||||
s.seen[path] = true
|
||||
s.files++
|
||||
s.bytes += int64(len(content))
|
||||
writeHash(s.h, filepath.ToSlash(rel), content)
|
||||
patterns, err := includePatterns(string(content))
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: %s: %w", rel, err)
|
||||
}
|
||||
for _, pattern := range patterns {
|
||||
if strings.ContainsAny(pattern, `[\`) {
|
||||
return fmt.Errorf("agentwatch: unsupported escaped include %q", pattern)
|
||||
}
|
||||
candidate := pattern
|
||||
if !filepath.IsAbs(candidate) {
|
||||
candidate = filepath.Join(filepath.Dir(path), filepath.FromSlash(candidate))
|
||||
}
|
||||
candidate = filepath.Clean(candidate)
|
||||
// The pattern itself changes the digest even when its glob currently matches nothing.
|
||||
writeHash(s.h, "include", []byte(filepath.ToSlash(candidate)))
|
||||
if _, err := s.withinRoot(candidate); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.ContainsAny(candidate, "*?") {
|
||||
if strings.ContainsAny(filepath.Dir(candidate), "*?") {
|
||||
return fmt.Errorf("agentwatch: unsupported include directory glob %q", pattern)
|
||||
}
|
||||
resolvedDir, err := filepath.EvalSymlinks(filepath.Dir(candidate))
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: resolve include directory %q: %w", pattern, err)
|
||||
}
|
||||
if _, err := s.withinRoot(resolvedDir); err != nil {
|
||||
return err
|
||||
}
|
||||
matches, err := boundedGlob(candidate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: include glob %q: %w", pattern, err)
|
||||
}
|
||||
for _, match := range matches {
|
||||
if err := s.visit(match, depth+1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := s.visit(candidate, depth+1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readBounded(r io.Reader, limit int64) ([]byte, error) {
|
||||
content, err := io.ReadAll(io.LimitReader(r, limit+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(content)) > limit {
|
||||
return nil, fmt.Errorf("configuration tree changed or exceeded size limit")
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
// boundedGlob only handles * and ? in a filename. Broader BIRD glob grammar
|
||||
// falls back to every-tick configure instead of risking a false unchanged hash.
|
||||
func boundedGlob(pattern string) ([]string, error) {
|
||||
dir, base := filepath.Dir(pattern), filepath.Base(pattern)
|
||||
f, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
names, err := f.Readdirnames(maxDirectoryEntries + 1)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, err
|
||||
}
|
||||
if len(names) > maxDirectoryEntries {
|
||||
return nil, fmt.Errorf("directory exceeds %d entries", maxDirectoryEntries)
|
||||
}
|
||||
matches := make([]string, 0)
|
||||
for _, name := range names {
|
||||
match, err := filepath.Match(base, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if match {
|
||||
matches = append(matches, filepath.Join(dir, name))
|
||||
if len(matches) > maxConfigFiles {
|
||||
return nil, fmt.Errorf("glob exceeds %d matches", maxConfigFiles)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(matches)
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
func (s *configScanner) withinRoot(path string) (string, error) {
|
||||
rel, err := filepath.Rel(s.root, path)
|
||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||
return "", fmt.Errorf("agentwatch: include %q escapes active directory", path)
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
func writeHash(h hash.Hash, name string, content []byte) {
|
||||
var size [8]byte
|
||||
binary.LittleEndian.PutUint64(size[:], uint64(len(name)))
|
||||
_, _ = h.Write(size[:])
|
||||
_, _ = h.Write([]byte(name))
|
||||
binary.LittleEndian.PutUint64(size[:], uint64(len(content)))
|
||||
_, _ = h.Write(size[:])
|
||||
_, _ = h.Write(content)
|
||||
}
|
||||
|
||||
func includePatterns(content string) ([]string, error) {
|
||||
tokens, unsupportedQuotedText := includeTokenOffsets(content)
|
||||
if unsupportedQuotedText {
|
||||
return nil, fmt.Errorf("unsupported quoted text or escape")
|
||||
}
|
||||
tokenSet := make(map[int]bool, len(tokens))
|
||||
for _, token := range tokens {
|
||||
tokenSet[token] = true
|
||||
}
|
||||
var patterns []string
|
||||
for _, match := range includeLine.FindAllStringSubmatchIndex(content, -1) {
|
||||
keyword := match[0] + strings.Index(content[match[0]:match[1]], "include")
|
||||
if !tokenSet[keyword] {
|
||||
continue
|
||||
} // text inside a block comment
|
||||
delete(tokenSet, keyword)
|
||||
patterns = append(patterns, content[match[2]:match[3]])
|
||||
}
|
||||
if len(tokenSet) != 0 {
|
||||
return nil, fmt.Errorf("unsupported include syntax")
|
||||
}
|
||||
return patterns, nil
|
||||
}
|
||||
|
||||
// Recognize include keywords outside BIRD comments and quoted strings.
|
||||
func includeTokenOffsets(config string) ([]int, bool) {
|
||||
var offsets []int
|
||||
quote := byte(0)
|
||||
lineComment, blockComment := false, false
|
||||
for i := 0; i < len(config); {
|
||||
c := config[i]
|
||||
if lineComment {
|
||||
if c == '\n' {
|
||||
lineComment = false
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if blockComment {
|
||||
if c == '*' && i+1 < len(config) && config[i+1] == '/' {
|
||||
blockComment = false
|
||||
i += 2
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if quote != 0 {
|
||||
if c == '\\' {
|
||||
return offsets, true
|
||||
}
|
||||
if c == quote {
|
||||
quote = 0
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if c == '#' || (c == '/' && i+1 < len(config) && config[i+1] == '/') {
|
||||
lineComment = true
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if c == '/' && i+1 < len(config) && config[i+1] == '*' {
|
||||
blockComment = true
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
if c == '"' || c == '\'' {
|
||||
quote = c
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' {
|
||||
start := i
|
||||
for i < len(config) && ((config[i] >= 'a' && config[i] <= 'z') || (config[i] >= 'A' && config[i] <= 'Z') || (config[i] >= '0' && config[i] <= '9') || config[i] == '_') {
|
||||
i++
|
||||
}
|
||||
if config[start:i] == "include" {
|
||||
offsets = append(offsets, start)
|
||||
}
|
||||
continue
|
||||
}
|
||||
i++
|
||||
}
|
||||
return offsets, quote != 0 || blockComment
|
||||
}
|
||||
Reference in new issue
Block a user