feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <cursoragent@cursor.com>

internal/jobs/deployment_reconcile.go

@@ -0,0 +1,126 @@
+package jobs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"evobgp/internal/logging"
+	"evobgp/internal/nodedispatch"
+	"evobgp/internal/store"
+)
+
+func deploymentFenceForJob(j *Job) store.JobFence {
+	generation, _ := strconv.ParseInt(j.metaString("reconcile_generation"), 10, 64)
+	round, _ := strconv.Atoi(j.metaString("reconcile_round"))
+	return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken, ExpectedGeneration: generation, RetryRound: round}
+}
+
+func beginDeployment(ctx context.Context, st store.Deployments, f store.JobFence, speaker, revision string) (store.DeploymentSpeaker, error) {
+	for {
+		d, err := st.BeginDeploymentSpeaker(ctx, f, speaker, revision)
+		if !errors.Is(err, store.ErrDeploymentBusy) {
+			return d, err
+		}
+		timer := time.NewTimer(time.Second)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return store.DeploymentSpeaker{}, ctx.Err()
+		case <-timer.C:
+		}
+	}
+}
+
+// ReconcileDeployments schedules at most three recovery rounds for current desired
+// generations. Stable keys coalesce API replicas; the worker CAS checks generation
+// again before publishing, and preserves the original desired request sequence.
+func ReconcileDeployments(ctx context.Context, r *Registry, st store.Deployments) error {
+	rows, err := st.ListPendingDeployments(ctx, 100)
+	if err != nil {
+		return err
+	}
+	for _, d := range rows {
+		key := fmt.Sprintf("deployment_reconcile:%s:%s:%d", d.JobID, d.SpeakerID, d.RetryRound+1)
+		_, _, err = r.EnqueueContext(ctx, d.TenantID, KindDeployApply, &key, nil, map[string]any{"revision_id": d.RevisionID, "speaker_id": d.SpeakerID, "reconcile_generation": strconv.FormatInt(d.Generation, 10), "reconcile_round": strconv.Itoa(d.RetryRound + 1), "source_deployment_job_id": d.JobID})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// StartDeploymentReconciliationLoop belongs to the API server lifecycle. Static
+// ingest/deploy/scheduler processes must not start this job-producing loop.
+func StartDeploymentReconciliationLoop(ctx context.Context, r *Registry, st store.Backend) {
+	if r == nil || r.queue == nil || st == nil {
+		return
+	}
+	r.durableMu.Lock()
+	if r.durableStop {
+		r.durableMu.Unlock()
+		return
+	}
+	r.durableLoopWG.Add(1)
+	r.durableMu.Unlock()
+	go func() {
+		defer r.durableLoopWG.Done()
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+			}
+			r.durableMu.Lock()
+			stopped := r.durableStop
+			r.durableMu.Unlock()
+			if stopped {
+				return
+			}
+			qctx, cancel := context.WithTimeout(ctx, 40*time.Second)
+			err := ObserveDeployments(qctx, st, nil)
+			if err == nil {
+				err = ReconcileDeployments(qctx, r, st)
+			}
+			cancel()
+			if err != nil && ctx.Err() == nil {
+				logging.With("component", "jobs.deployment_reconcile").Warn("reconciliation failed", "err", err)
+			}
+		}
+	}()
+}
+
+// ObserveDeployments samples compatible health before reconciling drift. The CP
+// receipt time and optional agent source time are stored separately; missing
+// legacy revision fields never manufacture an applied/verified confirmation.
+func ObserveDeployments(ctx context.Context, st store.Backend, client *http.Client) error {
+	rows, err := st.ListDeploymentObservations(ctx, 10)
+	if err != nil {
+		return err
+	}
+	for _, d := range rows {
+		sp, e := st.GetSpeaker(d.TenantID, d.SpeakerID)
+		if e != nil {
+			return e
+		}
+		observation := store.DeploymentObservation{Error: "remote health unavailable for this speaker"}
+		if nodedispatch.Enabled() && store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON)) {
+			hctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+			h := nodedispatch.FetchAgentHealth(hctx, sp, nodedispatch.Options{HTTPClient: client, Timeout: 3 * time.Second})
+			cancel()
+			observation = store.DeploymentObservation{RevisionID: h.LastAppliedRevisionID, SourceAt: h.LastSyncAt, Error: h.Error}
+			if h.OK && observation.RevisionID == "" {
+				observation.Error = "legacy health did not report an applied revision"
+			}
+		}
+		if err = st.ObserveDeployment(ctx, d, observation); err != nil && !errors.Is(err, store.ErrJobLeaseLost) {
+			return err
+		}
+	}
+	return nil
+}