feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
+102
-5
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"evobgp/internal/observability"
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -38,6 +39,11 @@ type Job struct {
|
||||
Error *string
|
||||
ProgressPct *int16
|
||||
Meta map[string]any
|
||||
OwnerID string
|
||||
FencingToken int64
|
||||
RequestSequence int64
|
||||
Attempt int
|
||||
requestHash string
|
||||
cancelRequested bool
|
||||
mu sync.Mutex
|
||||
}
|
||||
@@ -225,6 +231,30 @@ type Registry struct {
|
||||
// have been enqueued but not yet finalized in finishModuleRefreshSuccess. Used for deterministic
|
||||
// deploy coalescing under tenantRefreshMu (instead of polling job statuses).
|
||||
inflightRefresh map[string]int
|
||||
queue store.JobQueue
|
||||
ownerID string
|
||||
durableMu sync.Mutex
|
||||
durableWG sync.WaitGroup
|
||||
durableLoopWG sync.WaitGroup
|
||||
durableStop bool
|
||||
}
|
||||
|
||||
// UseDurableQueue makes the shared store authoritative for new work and job reads.
|
||||
func (r *Registry) UseDurableQueue(q store.JobQueue) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.queue = q
|
||||
r.ownerID = uuid.NewString()
|
||||
}
|
||||
|
||||
func jobFromRecord(c store.JobRecord) *Job {
|
||||
return &Job{ID: c.ID, TenantID: c.TenantID, Kind: c.Kind, Status: c.Status, IdempotencyKey: c.IdempotencyKey, ModuleID: c.ModuleID, Meta: c.Meta, CreatedAt: c.CreatedAt, StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, Error: c.Error, OwnerID: c.OwnerID, FencingToken: c.FencingToken, RequestSequence: c.RequestSequence, Attempt: c.Attempt, cancelRequested: c.CancelRequested}
|
||||
}
|
||||
|
||||
func recordFromJob(j *Job) store.JobRecord {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return store.JobRecord{ID: j.ID, TenantID: j.TenantID, Kind: j.Kind, Status: j.Status, IdempotencyKey: j.IdempotencyKey, ModuleID: j.ModuleID, Meta: cloneMeta(j.Meta), CreatedAt: j.CreatedAt, StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, Error: j.Error, OwnerID: j.OwnerID, FencingToken: j.FencingToken, RequestSequence: j.RequestSequence, Attempt: j.Attempt, CancelRequested: j.cancelRequested}
|
||||
}
|
||||
|
||||
type idempoKey struct {
|
||||
@@ -310,6 +340,24 @@ func registryMaxConcurrentJobs() int {
|
||||
return 8
|
||||
}
|
||||
|
||||
func jobRegistryPendingLimit() int {
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_MAX_PENDING"))); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
return 10000
|
||||
}
|
||||
|
||||
func (r *Registry) ActiveCountUnsafe() int {
|
||||
n := 0
|
||||
for _, j := range r.byID {
|
||||
st := j.statusLocked()
|
||||
if st == StatusQueued || st == StatusRunning {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// pruneTerminalIfOver удаляет самые старые завершённые джобы (succeeded/failed/cancelled), пока len(byID) > maxJobs.
|
||||
func (r *Registry) pruneTerminalIfOver(maxJobs int) {
|
||||
if r == nil || maxJobs <= 0 || len(r.byID) <= maxJobs {
|
||||
@@ -352,6 +400,33 @@ func (r *Registry) pruneTerminalIfOver(maxJobs int) {
|
||||
|
||||
// Enqueue creates a job or returns an existing one for the same idempotency key.
|
||||
func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
|
||||
return r.EnqueueContext(context.Background(), tenantID, kind, idempotencyKey, moduleID, meta)
|
||||
}
|
||||
|
||||
// EnqueueContext commits a durable job before returning it to an HTTP caller.
|
||||
func (r *Registry) EnqueueContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
|
||||
return r.EnqueueRequestContext(ctx, tenantID, kind, idempotencyKey, moduleID, meta, nil)
|
||||
}
|
||||
|
||||
// EnqueueRequestContext preserves idempotency of dynamically resolved selectors.
|
||||
func (r *Registry) EnqueueRequestContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta, request map[string]any) (*Job, bool, error) {
|
||||
if r.queue != nil {
|
||||
r.durableMu.Lock()
|
||||
stopped := r.durableStop
|
||||
r.durableMu.Unlock()
|
||||
if stopped {
|
||||
return nil, false, fmt.Errorf("jobs: shutting down")
|
||||
}
|
||||
c, created, err := r.queue.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return jobFromRecord(c), created, nil
|
||||
}
|
||||
hash, err := store.JobRequestHash(store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
maxJobs := registryMaxJobsFromEnv()
|
||||
r.pruneTerminalIfOver(maxJobs)
|
||||
@@ -359,14 +434,17 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
|
||||
if idempotencyKey != nil && *idempotencyKey != "" {
|
||||
k := idempoKey{tenant: tenantID, key: *idempotencyKey}
|
||||
if existing, ok := r.byIdempo[k]; ok {
|
||||
st := existing.statusLocked()
|
||||
if st == StatusQueued || st == StatusRunning {
|
||||
r.mu.Unlock()
|
||||
return existing, false, nil
|
||||
r.mu.Unlock()
|
||||
if existing.requestHash != hash {
|
||||
return nil, false, store.ErrJobConflict
|
||||
}
|
||||
delete(r.byIdempo, k)
|
||||
return existing, false, nil
|
||||
}
|
||||
}
|
||||
if r.ActiveCountUnsafe() >= jobRegistryPendingLimit() {
|
||||
r.mu.Unlock()
|
||||
return nil, false, store.ErrJobQueueFull
|
||||
}
|
||||
|
||||
j := &Job{
|
||||
ID: uuid.NewString(),
|
||||
@@ -377,6 +455,7 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
|
||||
ModuleID: moduleID,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
Meta: cloneMeta(meta),
|
||||
requestHash: hash,
|
||||
}
|
||||
if idempotencyKey != nil && *idempotencyKey != "" {
|
||||
r.byIdempo[idempoKey{tenant: tenantID, key: *idempotencyKey}] = j
|
||||
@@ -571,6 +650,11 @@ func (r *Registry) RequestCancelAll() int {
|
||||
if r == nil {
|
||||
return 0
|
||||
}
|
||||
if r.queue != nil {
|
||||
r.durableMu.Lock()
|
||||
r.durableStop = true
|
||||
r.durableMu.Unlock()
|
||||
}
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
n := 0
|
||||
@@ -585,6 +669,9 @@ func (r *Registry) RequestCancelAll() int {
|
||||
j.RequestCancel()
|
||||
n++
|
||||
}
|
||||
if r.queue != nil && len(r.workerSem) > n {
|
||||
n = len(r.workerSem)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
@@ -614,6 +701,16 @@ func (r *Registry) Drain(ctx context.Context) error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
if r.queue != nil {
|
||||
done := make(chan struct{})
|
||||
go func() { r.durableLoopWG.Wait(); r.durableWG.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
ticker := time.NewTicker(50 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
|
||||
Reference in new issue
Block a user