feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-11-02 17:23:59 +07:00
1 parent 924f111e48
commit f8e0520817
@@ -2,136 +2,216 @@ package jobs

 import (
 	"context"
-	"os"
-	"strconv"
-	"strings"
-	"time"
-
+	"errors"
 	"evobgp/internal/logging"
 	"evobgp/internal/observability"
-	"evobgp/internal/repository"
+	"evobgp/internal/store"
+	"sync/atomic"
+	"time"
 )

-// Adopt registers a durable job_audit claim into the in-process registry and starts the worker.
-// Used after SKIP LOCKED claim so two API processes can share the queue without losing work on restart.
-func (r *Registry) Adopt(j *Job) bool {
-	if r == nil || j == nil || j.ID == "" {
-		return false
-	}
-	r.mu.Lock()
-	if _, exists := r.byID[j.ID]; exists {
-		r.mu.Unlock()
-		return false
-	}
-	if j.Status == "" {
-		j.Status = StatusQueued
-	}
-	if j.Meta == nil {
-		j.Meta = map[string]any{}
-	}
-	r.byID[j.ID] = j
-	if j.IdempotencyKey != nil && *j.IdempotencyKey != "" {
-		r.byIdempo[idempoKey{tenant: j.TenantID, key: *j.IdempotencyKey}] = j
-	}
-	if isRefreshKind(j.Kind) {
-		r.inflightRefresh[j.TenantID]++
-	}
-	workerStart := r.workerStart
-	r.mu.Unlock()
+const durableLease = time.Minute

-	if workerStart != nil {
-		go func() {
-			r.workerSem <- struct{}{}
-			active := len(r.workerSem)
-			capacity := cap(r.workerSem)
-			observability.RecordJobQueueDepth(active, capacity)
-			defer func() {
-				<-r.workerSem
-				observability.RecordJobQueueDepth(len(r.workerSem), capacity)
-			}()
-			workerStart(j)
-		}()
-	}
-	return true
-}
-
-// StartDurableQueueLoop periodically reclaims stale running rows and claims queued job_audit work (PG SKIP LOCKED).
-// No-op when audit is nil. Interval from EVOBGP_JOB_RECLAIM_INTERVAL (default 20s); stale from EVOBGP_JOB_STALE_AFTER (default 15m).
-func StartDurableQueueLoop(ctx context.Context, reg *Registry, audit *repository.JobAuditWriter) {
-	if ctx == nil || reg == nil || audit == nil {
+// StartDurableQueueLoop claims work only after reserving local capacity.
+func StartDurableQueueLoop(ctx context.Context, r *Registry) {
+	if r == nil || r.queue == nil {
 		return
 	}
-	interval := 20 * time.Second
-	if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_RECLAIM_INTERVAL")); s != "" {
-		if d, err := time.ParseDuration(s); err == nil && d > 0 {
-			interval = d
-		}
-	}
-	staleAfter := 15 * time.Minute
-	if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_STALE_AFTER")); s != "" {
-		if d, err := time.ParseDuration(s); err == nil && d > 0 {
-			staleAfter = d
-		}
-	}
-	limit := 8
-	if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_LIMIT"))); err == nil && n > 0 {
-		limit = n
-	}
-	grace := 30 * time.Second
-	if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_GRACE")); s != "" {
-		if d, err := time.ParseDuration(s); err == nil && d >= 0 {
-			grace = d
-		}
-	}
 	log := logging.With("component", "jobs.durable")
-	run := func() {
-		cctx, cancel := context.WithTimeout(ctx, 15*time.Second)
-		defer cancel()
-		n, err := audit.ReclaimStaleRunning(cctx, staleAfter)
-		if err != nil {
-			log.Warn("reclaim stale running failed", "err", err)
-		} else if n > 0 {
-			log.Info("reclaimed stale running jobs", "count", n)
-		}
-		claimed, err := audit.ClaimQueued(cctx, limit, grace)
-		if err != nil {
-			log.Warn("claim queued failed", "err", err)
-			return
-		}
-		for i := range claimed {
-			c := claimed[i]
-			j := &Job{
-				ID:             c.ID,
-				TenantID:       c.TenantID,
-				Kind:           c.Kind,
-				Status:         StatusQueued,
-				IdempotencyKey: c.IdempotencyKey,
-				ModuleID:       c.ModuleID,
-				CreatedAt:      c.CreatedAt,
-				Meta:           c.Meta,
-			}
-			if !reg.Adopt(j) {
-				// Already local — leave DB running; local worker owns it.
-				continue
-			}
-			log.Info("adopted durable job", "job_id", j.ID, "kind", j.Kind, "tenant_id", j.TenantID)
-		}
-		counts, err := audit.CountByStatus(cctx)
-		if err == nil {
-			observability.RecordJobAuditDepth(counts)
-		}
+	r.durableMu.Lock()
+	if r.durableStop {
+		r.durableMu.Unlock()
+		return
 	}
+	r.durableLoopWG.Add(2)
+	r.durableMu.Unlock()
+	go r.reapExpiredJobsLoop(ctx)
 	go func() {
-		run()
-		t := time.NewTicker(interval)
-		defer t.Stop()
+		defer r.durableLoopWG.Done()
+		ticker := time.NewTicker(time.Second)
+		defer ticker.Stop()
 		for {
+			if ctx.Err() != nil {
+				return
+			}
+			r.durableMu.Lock()
+			stopped := r.durableStop
+			r.durableMu.Unlock()
+			if stopped {
+				return
+			}
+			metricCtx, metricCancel := context.WithTimeout(ctx, 5*time.Second)
+			if counts, err := r.queue.CountJobsByStatus(metricCtx); err == nil {
+				observability.RecordJobAuditDepth(counts)
+			}
+			metricCancel()
+			free := 0
+		reserve:
+			for free < cap(r.workerSem) {
+				select {
+				case r.workerSem <- struct{}{}:
+					free++
+				default:
+					break reserve
+				}
+			}
+			if free > 0 {
+				qctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+				claimed, err := r.queue.ClaimJobs(qctx, r.ownerID, free, durableLease)
+				cancel()
+				if err != nil {
+					log.Warn("claim failed", "err", err)
+				}
+				for i := len(claimed); i < free; i++ {
+					<-r.workerSem
+				}
+				observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem))
+				for _, c := range claimed {
+					r.durableMu.Lock()
+					stopped = r.durableStop
+					if stopped || ctx.Err() != nil {
+						r.durableMu.Unlock()
+						releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 10*time.Second)
+						if err := r.queue.ReleaseJob(releaseCtx, c.ID, c.OwnerID, c.FencingToken); err != nil {
+							log.Warn("release during shutdown failed", "job_id", c.ID, "err", err)
+						}
+						releaseCancel()
+						<-r.workerSem
+						continue
+					}
+					j := jobFromRecord(c)
+					r.mu.Lock()
+					r.byID[j.ID] = j
+					if isRefreshKind(j.Kind) {
+						r.inflightRefresh[j.TenantID]++
+					}
+					r.mu.Unlock()
+					r.durableWG.Add(1)
+					go r.runClaimed(j)
+					r.durableMu.Unlock()
+				}
+			}
 			select {
 			case <-ctx.Done():
 				return
-			case <-t.C:
-				run()
+			case <-ticker.C:
 			}
 		}
 	}()
 }
+
+// reapExpiredJobsLoop runs the job_audit reaper every 15s, not on each claim.
+func (r *Registry) reapExpiredJobsLoop(ctx context.Context) {
+	defer r.durableLoopWG.Done()
+	r.reapExpiredJobs(ctx)
+	reap := time.NewTicker(15 * time.Second)
+	stop := time.NewTicker(time.Second)
+	defer reap.Stop()
+	defer stop.Stop()
+	for {
+		if r.durableStopped() || ctx.Err() != nil {
+			return
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-stop.C:
+			if r.durableStopped() {
+				return
+			}
+		case <-reap.C:
+			r.reapExpiredJobs(ctx)
+		}
+	}
+}
+
+func (r *Registry) durableStopped() bool {
+	if r == nil {
+		return true
+	}
+	r.durableMu.Lock()
+	defer r.durableMu.Unlock()
+	return r.durableStop
+}
+
+func (r *Registry) reapExpiredJobs(ctx context.Context) {
+	if r == nil || r.queue == nil {
+		return
+	}
+	cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	if err := r.queue.ReapExpiredJobs(cctx); err != nil && ctx.Err() == nil {
+		logging.With("component", "jobs.durable").Warn("reap expired jobs", "err", err)
+	}
+}
+
+func (r *Registry) runClaimed(j *Job) {
+	log := logging.With("component", "jobs.durable")
+	var leaseHealthy atomic.Bool
+	leaseHealthy.Store(true)
+	defer func() {
+		r.mu.Lock()
+		delete(r.byID, j.ID)
+		r.mu.Unlock()
+		<-r.workerSem
+		observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem))
+		r.durableWG.Done()
+	}()
+	stop := make(chan struct{})
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		t := time.NewTicker(durableLease / 4)
+		defer t.Stop()
+		for {
+			select {
+			case <-stop:
+				return
+			case <-t.C:
+				ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+				requested, err := r.queue.HeartbeatJob(ctx, j.ID, j.OwnerID, j.FencingToken, durableLease)
+				if err == nil {
+					err = r.queue.CheckpointJob(ctx, recordFromJob(j))
+				}
+				cancel()
+				if err != nil {
+					leaseHealthy.Store(false)
+					j.RequestCancel()
+					log.Warn("heartbeat failed", "job_id", j.ID, "err", err)
+					return
+				}
+				if requested {
+					j.RequestCancel()
+				}
+			}
+		}
+	}()
+	r.workerStart(j)
+	close(stop)
+	<-done
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	r.durableMu.Lock()
+	stopped := r.durableStop
+	r.durableMu.Unlock()
+	c := recordFromJob(j)
+	if stopped && c.Status != StatusSucceeded && c.Status != StatusFailed {
+		if err := r.queue.ReleaseJob(ctx, j.ID, j.OwnerID, j.FencingToken); err != nil && !errors.Is(err, store.ErrJobLeaseLost) {
+			log.Warn("release failed", "err", err)
+		}
+		return
+	}
+	if !leaseHealthy.Load() {
+		// Leave an uncertain lease for expiry/reclaim. This consumes an attempt,
+		// whereas a deliberate graceful shutdown releases without using the budget.
+		return
+	}
+	if c.Status != StatusSucceeded && c.Status != StatusFailed && c.Status != StatusCancelled {
+		c.Status = StatusFailed
+		msg := "worker returned without terminal status"
+		c.Error = &msg
+	}
+	if err := r.queue.FinishJob(ctx, c); err != nil {
+		log.Warn("finish failed; lease will retry", "job_id", j.ID, "err", err)
+	}
+}