feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <cursoragent@cursor.com>

internal/jobs/deploy_confirmation_test.go

@@ -0,0 +1,231 @@
+package jobs
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"evobgp/internal/nodedispatch"
+	"evobgp/internal/store"
+
+	"github.com/google/uuid"
+)
+
+type deployRoundTrip func(*http.Request) (*http.Response, error)
+
+func (f deployRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func TestDeployApplyConfirmation(t *testing.T) {
+	for _, tc := range []struct {
+		name, response, confirmation, status string
+		dispatch                             bool
+		stale                                bool
+		wantApplied                          bool
+	}{
+		{name: "dispatch disabled", confirmation: "unconfirmed", status: StatusSucceeded},
+		{name: "bare success", dispatch: true, response: `{"ok":true}`, confirmation: "unconfirmed", status: StatusSucceeded},
+		{name: "matching ACK", dispatch: true, response: "matching", confirmation: "agent_reported", status: StatusSucceeded, wantApplied: true},
+		{name: "mismatched ACK", dispatch: true, response: `{"ok":true,"applied_revision_id":"other"}`, confirmation: "unconfirmed", status: StatusFailed},
+		{name: "stale desired", dispatch: true, response: "matching", confirmation: "unconfirmed", status: StatusFailed, stale: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
+			if tc.dispatch {
+				// A remote-only job must not touch the control plane BIRD tree.
+				t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", t.TempDir())
+				t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
+			} else {
+				t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
+			}
+			m := store.NewMemory()
+			m.SeedDemo()
+			tenant, _, module, previous, speakerID := m.DemoIDs()
+			if err := m.SetLastAppliedRevision(tenant, speakerID, previous); err != nil {
+				t.Fatal(err)
+			}
+			revision := uuid.NewString()
+			if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
+				t.Fatal(err)
+			}
+			newer := uuid.NewString()
+			if err := m.CreateRenderRevision(context.Background(), newer, tenant, module, &revision, "newer-hash", map[string]string{"bird.conf": "# newer\n"}, nil); err != nil {
+				t.Fatal(err)
+			}
+			meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
+			if _, err := m.UpdateSpeaker(tenant, speakerID, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
+				t.Fatal(err)
+			}
+			client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
+				if r.Method != http.MethodPost || r.URL.Path != "/v1/agent/sync" || r.Header.Get("Authorization") != "Bearer secret" {
+					t.Errorf("unexpected sync request: %s %s", r.Method, r.URL)
+				}
+				var body map[string]string
+				if err := json.NewDecoder(r.Body).Decode(&body); err != nil || len(body) != 1 || body["revision_id"] != revision {
+					t.Errorf("legacy sync body: %#v, %v", body, err)
+				}
+				if tc.stale {
+					if err := m.PublishRevisionForSpeaker(speakerID, newer); err != nil {
+						t.Error(err)
+					}
+				}
+				response := tc.response
+				if response == "matching" {
+					response = `{"ok":true,"applied_revision_id":"` + revision + `"}`
+				}
+				return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
+			})}
+			j := &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision, "speaker_id": speakerID}}
+			(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
+			if j.Status != tc.status {
+				t.Fatalf("job status = %s, want %s; error=%v", j.Status, tc.status, j.Error)
+			}
+			sp, err := m.GetSpeaker(tenant, speakerID)
+			if err != nil {
+				t.Fatal(err)
+			}
+			want := previous
+			if tc.wantApplied {
+				want = revision
+			}
+			if sp.LastAppliedRevisionID == nil || *sp.LastAppliedRevisionID != want {
+				t.Fatalf("applied = %v, want %s", sp.LastAppliedRevisionID, want)
+			}
+			published, _, err := m.LatestPublishedRevision(speakerID)
+			if err != nil {
+				t.Fatal(err)
+			}
+			wantPublished := revision
+			if tc.stale {
+				wantPublished = newer
+			}
+			if published != wantPublished {
+				t.Fatalf("published = %s, want %s", published, wantPublished)
+			}
+			results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
+			if len(results) != 1 || results[0].Confirmation != tc.confirmation {
+				t.Fatalf("results = %+v", results)
+			}
+		})
+	}
+}
+
+func TestDeployApplyPartialResult(t *testing.T) {
+	t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
+	t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, module, previous, first := m.DemoIDs()
+	revision := uuid.NewString()
+	if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
+		t.Fatal(err)
+	}
+	goodMeta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "good.example.test", AgentSecret: "secret"})
+	if _, err := m.UpdateSpeaker(tenant, first, &store.SpeakerPatch{MetaJSON: &goodMeta}); err != nil {
+		t.Fatal(err)
+	}
+	badMeta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "bad.example.test", AgentSecret: "secret"})
+	second, err := m.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: badMeta})
+	if err != nil {
+		t.Fatal(err)
+	}
+	client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
+		response := `{"ok":true,"applied_revision_id":"` + revision + `"}`
+		if strings.HasPrefix(r.URL.Host, "bad.") {
+			response = `{"ok":false,"error":"apply failed"}`
+		}
+		return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
+	})}
+	j := &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision}}
+	(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
+	if j.Status != StatusFailed {
+		t.Fatalf("job status = %s, want failed", j.Status)
+	}
+	summary := j.Meta["apply_summary"].(map[string]any)
+	if summary["applied_count"] != 1 || summary["failed_count"] != 1 || summary["published_count"] != 2 {
+		t.Fatalf("partial summary = %#v", summary)
+	}
+	firstSp, _ := m.GetSpeaker(tenant, first)
+	secondSp, _ := m.GetSpeaker(tenant, second.ID)
+	if firstSp.LastAppliedRevisionID == nil || *firstSp.LastAppliedRevisionID != revision || secondSp.LastAppliedRevisionID != nil {
+		t.Fatalf("partial applied state: first=%v second=%v", firstSp.LastAppliedRevisionID, secondSp.LastAppliedRevisionID)
+	}
+	results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
+	got := map[string]bool{}
+	for _, res := range results {
+		got[res.SpeakerID] = true
+	}
+	if len(results) != 2 || !got[first] || !got[second.ID] {
+		t.Fatalf("partial results=%+v", results)
+	}
+}
+
+func TestDeployApplyCancellationStopsFurtherPublication(t *testing.T) {
+	t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
+	t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, module, previous, seeded := m.DemoIDs()
+	if err := m.DeleteSpeaker(tenant, seeded); err != nil {
+		t.Fatal(err)
+	}
+	revision := uuid.NewString()
+	if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
+		t.Fatal(err)
+	}
+	meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
+	speakerCount := speakerWakeLimit + 2
+	for range speakerCount {
+		if _, err := m.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: meta}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	speakers := m.ListSpeakersForTenant(tenant)
+	if len(speakers) != speakerCount {
+		t.Fatalf("speakers=%d, want %d", len(speakers), speakerCount)
+	}
+	var j *Job
+	var calls atomic.Int32
+	client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
+		// Cancel only once the in-flight wake limit is busy, so later speakers stay unpublished.
+		if calls.Add(1) == int32(speakerWakeLimit) {
+			j.RequestCancel()
+		}
+		<-r.Context().Done()
+		return nil, r.Context().Err()
+	})}
+	j = &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision}}
+	(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
+	if j.Status != StatusCancelled || int(calls.Load()) != speakerWakeLimit {
+		t.Fatalf("cancelled job status=%s dispatch calls=%d", j.Status, calls.Load())
+	}
+	results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
+	gotResult := map[string]bool{}
+	for _, res := range results {
+		gotResult[res.SpeakerID] = res.Status == "cancelled"
+	}
+	if len(results) != speakerWakeLimit {
+		t.Fatalf("cancelled results=%+v", results)
+	}
+	for i, sp := range speakers {
+		got, _, err := m.LatestPublishedRevision(sp.ID)
+		if i < speakerWakeLimit {
+			if err != nil || got != revision || !gotResult[sp.ID] {
+				t.Fatalf("speaker %s published=%s err=%v result=%v", sp.ID, got, err, gotResult[sp.ID])
+			}
+			if sp.LastAppliedRevisionID != nil {
+				t.Fatalf("cancelled speaker %s marked applied", sp.ID)
+			}
+			continue
+		}
+		if err == nil {
+			t.Fatalf("speaker %s published %s after cancellation", sp.ID, got)
+		}
+	}
+	if j.Meta["apply_summary"].(map[string]any)["published_count"] != speakerWakeLimit {
+		t.Fatalf("cancelled summary = %#v", j.Meta["apply_summary"])
+	}
+}

internal/jobs/deployment_reconcile.go

@@ -0,0 +1,126 @@
+package jobs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"evobgp/internal/logging"
+	"evobgp/internal/nodedispatch"
+	"evobgp/internal/store"
+)
+
+func deploymentFenceForJob(j *Job) store.JobFence {
+	generation, _ := strconv.ParseInt(j.metaString("reconcile_generation"), 10, 64)
+	round, _ := strconv.Atoi(j.metaString("reconcile_round"))
+	return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken, ExpectedGeneration: generation, RetryRound: round}
+}
+
+func beginDeployment(ctx context.Context, st store.Deployments, f store.JobFence, speaker, revision string) (store.DeploymentSpeaker, error) {
+	for {
+		d, err := st.BeginDeploymentSpeaker(ctx, f, speaker, revision)
+		if !errors.Is(err, store.ErrDeploymentBusy) {
+			return d, err
+		}
+		timer := time.NewTimer(time.Second)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return store.DeploymentSpeaker{}, ctx.Err()
+		case <-timer.C:
+		}
+	}
+}
+
+// ReconcileDeployments schedules at most three recovery rounds for current desired
+// generations. Stable keys coalesce API replicas; the worker CAS checks generation
+// again before publishing, and preserves the original desired request sequence.
+func ReconcileDeployments(ctx context.Context, r *Registry, st store.Deployments) error {
+	rows, err := st.ListPendingDeployments(ctx, 100)
+	if err != nil {
+		return err
+	}
+	for _, d := range rows {
+		key := fmt.Sprintf("deployment_reconcile:%s:%s:%d", d.JobID, d.SpeakerID, d.RetryRound+1)
+		_, _, err = r.EnqueueContext(ctx, d.TenantID, KindDeployApply, &key, nil, map[string]any{"revision_id": d.RevisionID, "speaker_id": d.SpeakerID, "reconcile_generation": strconv.FormatInt(d.Generation, 10), "reconcile_round": strconv.Itoa(d.RetryRound + 1), "source_deployment_job_id": d.JobID})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// StartDeploymentReconciliationLoop belongs to the API server lifecycle. Static
+// ingest/deploy/scheduler processes must not start this job-producing loop.
+func StartDeploymentReconciliationLoop(ctx context.Context, r *Registry, st store.Backend) {
+	if r == nil || r.queue == nil || st == nil {
+		return
+	}
+	r.durableMu.Lock()
+	if r.durableStop {
+		r.durableMu.Unlock()
+		return
+	}
+	r.durableLoopWG.Add(1)
+	r.durableMu.Unlock()
+	go func() {
+		defer r.durableLoopWG.Done()
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+			}
+			r.durableMu.Lock()
+			stopped := r.durableStop
+			r.durableMu.Unlock()
+			if stopped {
+				return
+			}
+			qctx, cancel := context.WithTimeout(ctx, 40*time.Second)
+			err := ObserveDeployments(qctx, st, nil)
+			if err == nil {
+				err = ReconcileDeployments(qctx, r, st)
+			}
+			cancel()
+			if err != nil && ctx.Err() == nil {
+				logging.With("component", "jobs.deployment_reconcile").Warn("reconciliation failed", "err", err)
+			}
+		}
+	}()
+}
+
+// ObserveDeployments samples compatible health before reconciling drift. The CP
+// receipt time and optional agent source time are stored separately; missing
+// legacy revision fields never manufacture an applied/verified confirmation.
+func ObserveDeployments(ctx context.Context, st store.Backend, client *http.Client) error {
+	rows, err := st.ListDeploymentObservations(ctx, 10)
+	if err != nil {
+		return err
+	}
+	for _, d := range rows {
+		sp, e := st.GetSpeaker(d.TenantID, d.SpeakerID)
+		if e != nil {
+			return e
+		}
+		observation := store.DeploymentObservation{Error: "remote health unavailable for this speaker"}
+		if nodedispatch.Enabled() && store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON)) {
+			hctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+			h := nodedispatch.FetchAgentHealth(hctx, sp, nodedispatch.Options{HTTPClient: client, Timeout: 3 * time.Second})
+			cancel()
+			observation = store.DeploymentObservation{RevisionID: h.LastAppliedRevisionID, SourceAt: h.LastSyncAt, Error: h.Error}
+			if h.OK && observation.RevisionID == "" {
+				observation.Error = "legacy health did not report an applied revision"
+			}
+		}
+		if err = st.ObserveDeployment(ctx, d, observation); err != nil && !errors.Is(err, store.ErrJobLeaseLost) {
+			return err
+		}
+	}
+	return nil
+}

internal/jobs/deployment_reconcile_test.go

@@ -0,0 +1,140 @@
+package jobs
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"evobgp/internal/store"
+)
+
+func TestDeploymentReconcileDedupAndDesiredOrdering(t *testing.T) {
+	t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, revision, speaker := m.DemoIDs()
+	ctx := context.Background()
+	r := NewRegistry(nil)
+	r.UseDurableQueue(m)
+	initial, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
+	if err != nil {
+		t.Fatal(err)
+	}
+	claim := func(owner string) *Job {
+		t.Helper()
+		rows, e := m.ClaimJobs(ctx, owner, 1, time.Minute)
+		if e != nil || len(rows) != 1 {
+			t.Fatalf("claim: %+v %v", rows, e)
+		}
+		return jobFromRecord(rows[0])
+	}
+	first := claim("initial")
+	if first.ID != initial.ID {
+		t.Fatal("wrong initial job")
+	}
+	(&Worker{Store: m}).Process(first)
+	if err = m.FinishJob(ctx, recordFromJob(first)); err != nil {
+		t.Fatal(err)
+	}
+	// Enqueue an operator request before the synthetic recovery request. The latter
+	// must retain the earlier desired order despite its own larger queue sequence.
+	operator, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for range 2 {
+		if err = ReconcileDeployments(ctx, r, m); err != nil {
+			t.Fatal(err)
+		}
+	}
+	all, _, _, err := m.ListJobs(ctx, tenant, "", "", "", 100)
+	if err != nil || len(all) != 3 {
+		t.Fatalf("reconcile dedupe: %d %v", len(all), err)
+	}
+	newer := claim("operator")
+	if newer.ID != operator.ID {
+		t.Fatal("wrong operator job")
+	}
+	recovery := claim("recovery")
+	f := deploymentFenceForJob(recovery)
+	d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if d.RequestSequence != first.RequestSequence {
+		t.Fatal("recovery lost original desired order")
+	}
+	if err = m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil {
+		t.Fatal(err)
+	}
+	newD, err := m.BeginDeploymentSpeaker(ctx, deploymentFenceForJob(newer), speaker, revision)
+	if err != nil {
+		t.Fatalf("newer operator should supersede recovery despite smaller queue sequence: %v", err)
+	}
+	if newD.RequestSequence != newer.RequestSequence {
+		t.Fatal("operator desired sequence changed")
+	}
+	if _, err = m.BeginDeploymentSpeaker(ctx, f, speaker, revision); err == nil {
+		t.Fatal("stale recovery generation reused")
+	}
+}
+
+func TestDeploymentHealthDetectsDriftAndKeepsLegacyQuality(t *testing.T) {
+	t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, revision, speaker := m.DemoIDs()
+	ctx := context.Background()
+	r := NewRegistry(nil)
+	r.UseDurableQueue(m)
+	_, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
+	if err != nil {
+		t.Fatal(err)
+	}
+	rows, err := m.ClaimJobs(ctx, "owner", 1, time.Minute)
+	if err != nil || len(rows) != 1 {
+		t.Fatal(err)
+	}
+	j := jobFromRecord(rows[0])
+	(&Worker{Store: m}).Process(j)
+	if err = m.FinishJob(ctx, recordFromJob(j)); err != nil {
+		t.Fatal(err)
+	}
+	meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
+	if _, err = m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
+	response := `{"ok":true,"last_applied_revision_id":"` + revision + `"}`
+	client := &http.Client{Transport: deployRoundTrip(func(req *http.Request) (*http.Response, error) {
+		if req.Header.Get("Authorization") != "Bearer secret" || req.Method != "GET" {
+			t.Error("health must be authenticated GET")
+		}
+		return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
+	})}
+	if err = ObserveDeployments(ctx, m, client); err != nil {
+		t.Fatal(err)
+	}
+	progress, err := m.ListDeploymentSpeakers(ctx, tenant, j.ID)
+	if err != nil || len(progress) != 1 || progress[0].State != "applied" || progress[0].Confirmation != "agent_reported" || progress[0].HealthReceivedAt == nil || progress[0].HealthSourceAt != "" {
+		t.Fatalf("legacy health confirmation: %+v %v", progress, err)
+	}
+	response = `{"ok":true,"last_applied_revision_id":"older-late-operation"}`
+	if err = ObserveDeployments(ctx, m, client); err != nil {
+		t.Fatal(err)
+	}
+	progress, _ = m.ListDeploymentSpeakers(ctx, tenant, j.ID)
+	if progress[0].State != "unknown" || progress[0].Confirmation != "unconfirmed" {
+		t.Fatalf("late drift hidden: %+v", progress)
+	}
+	if err = ReconcileDeployments(ctx, r, m); err != nil {
+		t.Fatal(err)
+	}
+	jobs, _, _, _ := m.ListJobs(ctx, tenant, "queued", "", "", 100)
+	if len(jobs) != 1 {
+		t.Fatalf("drift recovery jobs: %d", len(jobs))
+	}
+}

internal/jobs/durable.go

@@ -2,136 +2,216 @@ package jobs
 
 import (
 	"context"
-	"os"
-	"strconv"
-	"strings"
-	"time"
-
+	"errors"
 	"evobgp/internal/logging"
 	"evobgp/internal/observability"
-	"evobgp/internal/repository"
+	"evobgp/internal/store"
+	"sync/atomic"
+	"time"
 )
 
-// Adopt registers a durable job_audit claim into the in-process registry and starts the worker.
-// Used after SKIP LOCKED claim so two API processes can share the queue without losing work on restart.
-func (r *Registry) Adopt(j *Job) bool {
-	if r == nil || j == nil || j.ID == "" {
-		return false
-	}
-	r.mu.Lock()
-	if _, exists := r.byID[j.ID]; exists {
-		r.mu.Unlock()
-		return false
-	}
-	if j.Status == "" {
-		j.Status = StatusQueued
-	}
-	if j.Meta == nil {
-		j.Meta = map[string]any{}
-	}
-	r.byID[j.ID] = j
-	if j.IdempotencyKey != nil && *j.IdempotencyKey != "" {
-		r.byIdempo[idempoKey{tenant: j.TenantID, key: *j.IdempotencyKey}] = j
-	}
-	if isRefreshKind(j.Kind) {
-		r.inflightRefresh[j.TenantID]++
-	}
-	workerStart := r.workerStart
-	r.mu.Unlock()
+const durableLease = time.Minute
 
-	if workerStart != nil {
-		go func() {
-			r.workerSem <- struct{}{}
-			active := len(r.workerSem)
-			capacity := cap(r.workerSem)
-			observability.RecordJobQueueDepth(active, capacity)
-			defer func() {
-				<-r.workerSem
-				observability.RecordJobQueueDepth(len(r.workerSem), capacity)
-			}()
-			workerStart(j)
-		}()
-	}
-	return true
-}
-
-// StartDurableQueueLoop periodically reclaims stale running rows and claims queued job_audit work (PG SKIP LOCKED).
-// No-op when audit is nil. Interval from EVOBGP_JOB_RECLAIM_INTERVAL (default 20s); stale from EVOBGP_JOB_STALE_AFTER (default 15m).
-func StartDurableQueueLoop(ctx context.Context, reg *Registry, audit *repository.JobAuditWriter) {
-	if ctx == nil || reg == nil || audit == nil {
+// StartDurableQueueLoop claims work only after reserving local capacity.
+func StartDurableQueueLoop(ctx context.Context, r *Registry) {
+	if r == nil || r.queue == nil {
 		return
 	}
-	interval := 20 * time.Second
-	if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_RECLAIM_INTERVAL")); s != "" {
-		if d, err := time.ParseDuration(s); err == nil && d > 0 {
-			interval = d
-		}
-	}
-	staleAfter := 15 * time.Minute
-	if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_STALE_AFTER")); s != "" {
-		if d, err := time.ParseDuration(s); err == nil && d > 0 {
-			staleAfter = d
-		}
-	}
-	limit := 8
-	if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_LIMIT"))); err == nil && n > 0 {
-		limit = n
-	}
-	grace := 30 * time.Second
-	if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_GRACE")); s != "" {
-		if d, err := time.ParseDuration(s); err == nil && d >= 0 {
-			grace = d
-		}
-	}
 	log := logging.With("component", "jobs.durable")
-	run := func() {
-		cctx, cancel := context.WithTimeout(ctx, 15*time.Second)
-		defer cancel()
-		n, err := audit.ReclaimStaleRunning(cctx, staleAfter)
-		if err != nil {
-			log.Warn("reclaim stale running failed", "err", err)
-		} else if n > 0 {
-			log.Info("reclaimed stale running jobs", "count", n)
-		}
-		claimed, err := audit.ClaimQueued(cctx, limit, grace)
-		if err != nil {
-			log.Warn("claim queued failed", "err", err)
-			return
-		}
-		for i := range claimed {
-			c := claimed[i]
-			j := &Job{
-				ID:             c.ID,
-				TenantID:       c.TenantID,
-				Kind:           c.Kind,
-				Status:         StatusQueued,
-				IdempotencyKey: c.IdempotencyKey,
-				ModuleID:       c.ModuleID,
-				CreatedAt:      c.CreatedAt,
-				Meta:           c.Meta,
-			}
-			if !reg.Adopt(j) {
-				// Already local — leave DB running; local worker owns it.
-				continue
-			}
-			log.Info("adopted durable job", "job_id", j.ID, "kind", j.Kind, "tenant_id", j.TenantID)
-		}
-		counts, err := audit.CountByStatus(cctx)
-		if err == nil {
-			observability.RecordJobAuditDepth(counts)
-		}
+	r.durableMu.Lock()
+	if r.durableStop {
+		r.durableMu.Unlock()
+		return
 	}
+	r.durableLoopWG.Add(2)
+	r.durableMu.Unlock()
+	go r.reapExpiredJobsLoop(ctx)
 	go func() {
-		run()
-		t := time.NewTicker(interval)
-		defer t.Stop()
+		defer r.durableLoopWG.Done()
+		ticker := time.NewTicker(time.Second)
+		defer ticker.Stop()
 		for {
+			if ctx.Err() != nil {
+				return
+			}
+			r.durableMu.Lock()
+			stopped := r.durableStop
+			r.durableMu.Unlock()
+			if stopped {
+				return
+			}
+			metricCtx, metricCancel := context.WithTimeout(ctx, 5*time.Second)
+			if counts, err := r.queue.CountJobsByStatus(metricCtx); err == nil {
+				observability.RecordJobAuditDepth(counts)
+			}
+			metricCancel()
+			free := 0
+		reserve:
+			for free < cap(r.workerSem) {
+				select {
+				case r.workerSem <- struct{}{}:
+					free++
+				default:
+					break reserve
+				}
+			}
+			if free > 0 {
+				qctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+				claimed, err := r.queue.ClaimJobs(qctx, r.ownerID, free, durableLease)
+				cancel()
+				if err != nil {
+					log.Warn("claim failed", "err", err)
+				}
+				for i := len(claimed); i < free; i++ {
+					<-r.workerSem
+				}
+				observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem))
+				for _, c := range claimed {
+					r.durableMu.Lock()
+					stopped = r.durableStop
+					if stopped || ctx.Err() != nil {
+						r.durableMu.Unlock()
+						releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 10*time.Second)
+						if err := r.queue.ReleaseJob(releaseCtx, c.ID, c.OwnerID, c.FencingToken); err != nil {
+							log.Warn("release during shutdown failed", "job_id", c.ID, "err", err)
+						}
+						releaseCancel()
+						<-r.workerSem
+						continue
+					}
+					j := jobFromRecord(c)
+					r.mu.Lock()
+					r.byID[j.ID] = j
+					if isRefreshKind(j.Kind) {
+						r.inflightRefresh[j.TenantID]++
+					}
+					r.mu.Unlock()
+					r.durableWG.Add(1)
+					go r.runClaimed(j)
+					r.durableMu.Unlock()
+				}
+			}
 			select {
 			case <-ctx.Done():
 				return
-			case <-t.C:
-				run()
+			case <-ticker.C:
 			}
 		}
 	}()
 }
+
+// reapExpiredJobsLoop runs the job_audit reaper every 15s, not on each claim.
+func (r *Registry) reapExpiredJobsLoop(ctx context.Context) {
+	defer r.durableLoopWG.Done()
+	r.reapExpiredJobs(ctx)
+	reap := time.NewTicker(15 * time.Second)
+	stop := time.NewTicker(time.Second)
+	defer reap.Stop()
+	defer stop.Stop()
+	for {
+		if r.durableStopped() || ctx.Err() != nil {
+			return
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-stop.C:
+			if r.durableStopped() {
+				return
+			}
+		case <-reap.C:
+			r.reapExpiredJobs(ctx)
+		}
+	}
+}
+
+func (r *Registry) durableStopped() bool {
+	if r == nil {
+		return true
+	}
+	r.durableMu.Lock()
+	defer r.durableMu.Unlock()
+	return r.durableStop
+}
+
+func (r *Registry) reapExpiredJobs(ctx context.Context) {
+	if r == nil || r.queue == nil {
+		return
+	}
+	cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	if err := r.queue.ReapExpiredJobs(cctx); err != nil && ctx.Err() == nil {
+		logging.With("component", "jobs.durable").Warn("reap expired jobs", "err", err)
+	}
+}
+
+func (r *Registry) runClaimed(j *Job) {
+	log := logging.With("component", "jobs.durable")
+	var leaseHealthy atomic.Bool
+	leaseHealthy.Store(true)
+	defer func() {
+		r.mu.Lock()
+		delete(r.byID, j.ID)
+		r.mu.Unlock()
+		<-r.workerSem
+		observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem))
+		r.durableWG.Done()
+	}()
+	stop := make(chan struct{})
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		t := time.NewTicker(durableLease / 4)
+		defer t.Stop()
+		for {
+			select {
+			case <-stop:
+				return
+			case <-t.C:
+				ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+				requested, err := r.queue.HeartbeatJob(ctx, j.ID, j.OwnerID, j.FencingToken, durableLease)
+				if err == nil {
+					err = r.queue.CheckpointJob(ctx, recordFromJob(j))
+				}
+				cancel()
+				if err != nil {
+					leaseHealthy.Store(false)
+					j.RequestCancel()
+					log.Warn("heartbeat failed", "job_id", j.ID, "err", err)
+					return
+				}
+				if requested {
+					j.RequestCancel()
+				}
+			}
+		}
+	}()
+	r.workerStart(j)
+	close(stop)
+	<-done
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	r.durableMu.Lock()
+	stopped := r.durableStop
+	r.durableMu.Unlock()
+	c := recordFromJob(j)
+	if stopped && c.Status != StatusSucceeded && c.Status != StatusFailed {
+		if err := r.queue.ReleaseJob(ctx, j.ID, j.OwnerID, j.FencingToken); err != nil && !errors.Is(err, store.ErrJobLeaseLost) {
+			log.Warn("release failed", "err", err)
+		}
+		return
+	}
+	if !leaseHealthy.Load() {
+		// Leave an uncertain lease for expiry/reclaim. This consumes an attempt,
+		// whereas a deliberate graceful shutdown releases without using the budget.
+		return
+	}
+	if c.Status != StatusSucceeded && c.Status != StatusFailed && c.Status != StatusCancelled {
+		c.Status = StatusFailed
+		msg := "worker returned without terminal status"
+		c.Error = &msg
+	}
+	if err := r.queue.FinishJob(ctx, c); err != nil {
+		log.Warn("finish failed; lease will retry", "job_id", j.ID, "err", err)
+	}
+}

internal/jobs/durable_queue_test.go

@@ -0,0 +1,132 @@
+package jobs
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"evobgp/internal/store"
+)
+
+type blockingClaimQueue struct {
+	store.JobQueue
+	entered chan int
+	resume  chan struct{}
+}
+
+func (q *blockingClaimQueue) ClaimJobs(ctx context.Context, owner string, limit int, lease time.Duration) ([]store.JobRecord, error) {
+	q.entered <- limit
+	<-q.resume
+	return q.JobQueue.ClaimJobs(context.Background(), owner, limit, lease)
+}
+
+func TestDurableShutdownWaitsInflightClaimAndReleases(t *testing.T) {
+	t.Setenv("EVOBGP_JOB_MAX_CONCURRENT", "1")
+	m := store.NewMemory()
+	queued, _, err := m.EnqueueJob(context.Background(), store.JobEnqueue{TenantID: "tenant", Kind: "test"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	q := &blockingClaimQueue{JobQueue: m, entered: make(chan int, 1), resume: make(chan struct{})}
+	var runs atomic.Int32
+	r := NewRegistry(func(j *Job) { runs.Add(1); j.Succeed() })
+	r.UseDurableQueue(q)
+	StartDurableQueueLoop(context.Background(), r)
+	select {
+	case limit := <-q.entered:
+		if limit != cap(r.workerSem) {
+			t.Fatalf("claim capacity=%d reserved=%d", limit, cap(r.workerSem))
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("no claim")
+	}
+	r.RequestCancelAll()
+	drained := make(chan error, 1)
+	go func() { drained <- r.Drain(context.Background()) }()
+	select {
+	case <-drained:
+		t.Fatal("drained before in-flight claim returned")
+	case <-time.After(20 * time.Millisecond):
+	}
+	close(q.resume)
+	select {
+	case err := <-drained:
+		if err != nil {
+			t.Fatal(err)
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("shutdown did not drain")
+	}
+	if runs.Load() != 0 {
+		t.Fatal("started worker after stop")
+	}
+	got, err := m.GetJob(context.Background(), "tenant", queued.ID)
+	if err != nil || got.Status != "queued" || got.OwnerID != "" || got.Attempt != 0 {
+		t.Fatalf("not released: %+v %v", got, err)
+	}
+}
+
+func TestDurableSharedTerminalAndRequestReplay(t *testing.T) {
+	ctx := context.Background()
+	m := store.NewMemory()
+	key := "request"
+	a := NewRegistry(nil)
+	a.UseDurableQueue(m)
+	j, _, err := a.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
+	if err != nil || len(claimed) != 1 {
+		t.Fatalf("claim %+v %v", claimed, err)
+	}
+	claimed[0].Status = StatusSucceeded
+	claimed[0].Meta = map[string]any{"result": "ok"}
+	if err = m.FinishJob(ctx, claimed[0]); err != nil {
+		t.Fatal(err)
+	}
+	b := NewRegistry(nil)
+	b.UseDurableQueue(m)
+	got, err := b.GetContext(ctx, "tenant", j.ID)
+	if err != nil || got.Status != StatusSucceeded {
+		t.Fatalf("shared get %+v %v", got, err)
+	}
+	list, _, _, err := b.ListContext(ctx, "tenant", StatusSucceeded, "", "", 10)
+	if err != nil || len(list) != 1 {
+		t.Fatalf("shared list %+v %v", list, err)
+	}
+	replay, created, err := b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
+	if err != nil || created || replay.ID != j.ID || replay.Status != StatusSucceeded {
+		t.Fatalf("terminal replay %+v %v %v", replay, created, err)
+	}
+	if _, _, err = b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"input": "different"}); !errors.Is(err, store.ErrJobConflict) {
+		t.Fatalf("changed payload accepted: %v", err)
+	}
+}
+
+func TestParentRetryRecoversCommittedDeployChild(t *testing.T) {
+	ctx := context.Background()
+	m := store.NewMemory()
+	r := NewRegistry(nil)
+	r.UseDurableQueue(m)
+	parent, _, err := r.EnqueueContext(ctx, "tenant", KindRevisionRollback, nil, nil, map[string]any{"revision_id": "original"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	key := "deploy_from:" + parent.ID
+	child, _, err := r.EnqueueContext(ctx, "tenant", KindDeployApply, &key, nil, map[string]any{"revision_id": "already-rendered"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	worker := &Worker{Store: m, Registry: r}
+	worker.Process(parent)
+	if parent.statusLocked() != StatusSucceeded || parent.metaString("revision_id") != "already-rendered" || parent.metaString("deploy_apply_job_id") != child.ID {
+		t.Fatalf("did not recover child: %+v", parent.Snapshot())
+	}
+	items, _, _, err := m.ListJobs(ctx, "tenant", "", KindDeployApply, "", 10)
+	if err != nil || len(items) != 1 {
+		t.Fatalf("duplicate child: %+v %v", items, err)
+	}
+}

internal/jobs/job.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"evobgp/internal/observability"
+	"evobgp/internal/store"
 
 	"github.com/google/uuid"
 )
@@ -38,6 +39,11 @@ type Job struct {
 	Error           *string
 	ProgressPct     *int16
 	Meta            map[string]any
+	OwnerID         string
+	FencingToken    int64
+	RequestSequence int64
+	Attempt         int
+	requestHash     string
 	cancelRequested bool
 	mu              sync.Mutex
 }
@@ -225,6 +231,30 @@ type Registry struct {
 	// have been enqueued but not yet finalized in finishModuleRefreshSuccess. Used for deterministic
 	// deploy coalescing under tenantRefreshMu (instead of polling job statuses).
 	inflightRefresh map[string]int
+	queue           store.JobQueue
+	ownerID         string
+	durableMu       sync.Mutex
+	durableWG       sync.WaitGroup
+	durableLoopWG   sync.WaitGroup
+	durableStop     bool
+}
+
+// UseDurableQueue makes the shared store authoritative for new work and job reads.
+func (r *Registry) UseDurableQueue(q store.JobQueue) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.queue = q
+	r.ownerID = uuid.NewString()
+}
+
+func jobFromRecord(c store.JobRecord) *Job {
+	return &Job{ID: c.ID, TenantID: c.TenantID, Kind: c.Kind, Status: c.Status, IdempotencyKey: c.IdempotencyKey, ModuleID: c.ModuleID, Meta: c.Meta, CreatedAt: c.CreatedAt, StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, Error: c.Error, OwnerID: c.OwnerID, FencingToken: c.FencingToken, RequestSequence: c.RequestSequence, Attempt: c.Attempt, cancelRequested: c.CancelRequested}
+}
+
+func recordFromJob(j *Job) store.JobRecord {
+	j.mu.Lock()
+	defer j.mu.Unlock()
+	return store.JobRecord{ID: j.ID, TenantID: j.TenantID, Kind: j.Kind, Status: j.Status, IdempotencyKey: j.IdempotencyKey, ModuleID: j.ModuleID, Meta: cloneMeta(j.Meta), CreatedAt: j.CreatedAt, StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, Error: j.Error, OwnerID: j.OwnerID, FencingToken: j.FencingToken, RequestSequence: j.RequestSequence, Attempt: j.Attempt, CancelRequested: j.cancelRequested}
 }
 
 type idempoKey struct {
@@ -310,6 +340,24 @@ func registryMaxConcurrentJobs() int {
 	return 8
 }
 
+func jobRegistryPendingLimit() int {
+	if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_MAX_PENDING"))); err == nil && n > 0 {
+		return n
+	}
+	return 10000
+}
+
+func (r *Registry) ActiveCountUnsafe() int {
+	n := 0
+	for _, j := range r.byID {
+		st := j.statusLocked()
+		if st == StatusQueued || st == StatusRunning {
+			n++
+		}
+	}
+	return n
+}
+
 // pruneTerminalIfOver удаляет самые старые завершённые джобы (succeeded/failed/cancelled), пока len(byID) > maxJobs.
 func (r *Registry) pruneTerminalIfOver(maxJobs int) {
 	if r == nil || maxJobs <= 0 || len(r.byID) <= maxJobs {
@@ -352,6 +400,33 @@ func (r *Registry) pruneTerminalIfOver(maxJobs int) {
 
 // Enqueue creates a job or returns an existing one for the same idempotency key.
 func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
+	return r.EnqueueContext(context.Background(), tenantID, kind, idempotencyKey, moduleID, meta)
+}
+
+// EnqueueContext commits a durable job before returning it to an HTTP caller.
+func (r *Registry) EnqueueContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
+	return r.EnqueueRequestContext(ctx, tenantID, kind, idempotencyKey, moduleID, meta, nil)
+}
+
+// EnqueueRequestContext preserves idempotency of dynamically resolved selectors.
+func (r *Registry) EnqueueRequestContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta, request map[string]any) (*Job, bool, error) {
+	if r.queue != nil {
+		r.durableMu.Lock()
+		stopped := r.durableStop
+		r.durableMu.Unlock()
+		if stopped {
+			return nil, false, fmt.Errorf("jobs: shutting down")
+		}
+		c, created, err := r.queue.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request})
+		if err != nil {
+			return nil, false, err
+		}
+		return jobFromRecord(c), created, nil
+	}
+	hash, err := store.JobRequestHash(store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request})
+	if err != nil {
+		return nil, false, err
+	}
 	r.mu.Lock()
 	maxJobs := registryMaxJobsFromEnv()
 	r.pruneTerminalIfOver(maxJobs)
@@ -359,14 +434,17 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
 	if idempotencyKey != nil && *idempotencyKey != "" {
 		k := idempoKey{tenant: tenantID, key: *idempotencyKey}
 		if existing, ok := r.byIdempo[k]; ok {
-			st := existing.statusLocked()
-			if st == StatusQueued || st == StatusRunning {
-				r.mu.Unlock()
-				return existing, false, nil
+			r.mu.Unlock()
+			if existing.requestHash != hash {
+				return nil, false, store.ErrJobConflict
 			}
-			delete(r.byIdempo, k)
+			return existing, false, nil
 		}
 	}
+	if r.ActiveCountUnsafe() >= jobRegistryPendingLimit() {
+		r.mu.Unlock()
+		return nil, false, store.ErrJobQueueFull
+	}
 
 	j := &Job{
 		ID:             uuid.NewString(),
@@ -377,6 +455,7 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
 		ModuleID:       moduleID,
 		CreatedAt:      time.Now().UTC(),
 		Meta:           cloneMeta(meta),
+		requestHash:    hash,
 	}
 	if idempotencyKey != nil && *idempotencyKey != "" {
 		r.byIdempo[idempoKey{tenant: tenantID, key: *idempotencyKey}] = j
@@ -571,6 +650,11 @@ func (r *Registry) RequestCancelAll() int {
 	if r == nil {
 		return 0
 	}
+	if r.queue != nil {
+		r.durableMu.Lock()
+		r.durableStop = true
+		r.durableMu.Unlock()
+	}
 	r.mu.RLock()
 	defer r.mu.RUnlock()
 	n := 0
@@ -585,6 +669,9 @@ func (r *Registry) RequestCancelAll() int {
 		j.RequestCancel()
 		n++
 	}
+	if r.queue != nil && len(r.workerSem) > n {
+		n = len(r.workerSem)
+	}
 	return n
 }
 
@@ -614,6 +701,16 @@ func (r *Registry) Drain(ctx context.Context) error {
 	if r == nil {
 		return nil
 	}
+	if r.queue != nil {
+		done := make(chan struct{})
+		go func() { r.durableLoopWG.Wait(); r.durableWG.Wait(); close(done) }()
+		select {
+		case <-done:
+			return nil
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
 	ticker := time.NewTicker(50 * time.Millisecond)
 	defer ticker.Stop()
 	for {

internal/jobs/registry_shared.go

@@ -0,0 +1,115 @@
+package jobs
+
+import (
+	"context"
+	"errors"
+
+	"evobgp/internal/store"
+)
+
+// FindIdempotentContext looks up a retained logical operation without inserting.
+func (r *Registry) FindIdempotentContext(ctx context.Context, tenant, key string) (*Job, error) {
+	if r.queue != nil {
+		c, err := r.queue.FindJobByIdempotencyKey(ctx, tenant, key)
+		if err != nil {
+			return nil, err
+		}
+		j := jobFromRecord(c)
+		j.requestHash = c.RequestHash
+		return j, nil
+	}
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	j := r.byIdempo[idempoKey{tenant: tenant, key: key}]
+	if j == nil {
+		return nil, store.ErrJobNotFound
+	}
+	return j, nil
+}
+
+// ReplayContext validates an immutable request before expensive live resolution.
+func (r *Registry) ReplayContext(ctx context.Context, in store.JobEnqueue) (*Job, bool, error) {
+	if in.IdempotencyKey == nil || *in.IdempotencyKey == "" {
+		return nil, false, nil
+	}
+	j, err := r.FindIdempotentContext(ctx, in.TenantID, *in.IdempotencyKey)
+	if errors.Is(err, store.ErrJobNotFound) {
+		return nil, false, nil
+	}
+	if err != nil {
+		return nil, false, err
+	}
+	h, err := store.JobRequestHash(in)
+	if err != nil {
+		return nil, false, err
+	}
+	if j.requestHash != h {
+		return nil, false, store.ErrJobConflict
+	}
+	return j, true, nil
+}
+
+// GetContext reads a job from the shared store when durability is enabled.
+func (r *Registry) GetContext(ctx context.Context, tenant, id string) (*Job, error) {
+	if r.queue == nil {
+		return r.Get(tenant, id)
+	}
+	c, err := r.queue.GetJob(ctx, tenant, id)
+	if errors.Is(err, store.ErrJobNotFound) {
+		return nil, ErrNotFound
+	}
+	if err != nil {
+		return nil, err
+	}
+	return jobFromRecord(c), nil
+}
+
+// ListContext returns a bounded tenant page from the authoritative queue.
+func (r *Registry) ListContext(ctx context.Context, tenant, status, kind, cursor string, limit int) ([]*Job, string, bool, error) {
+	if r.queue == nil {
+		j, n, m := r.List(tenant, status, kind, cursor, limit)
+		return j, n, m, nil
+	}
+	c, next, more, err := r.queue.ListJobs(ctx, tenant, status, kind, cursor, limit)
+	if err != nil {
+		return nil, "", false, err
+	}
+	out := make([]*Job, 0, len(c))
+	for _, v := range c {
+		out = append(out, jobFromRecord(v))
+	}
+	return out, next, more, nil
+}
+
+// RequestCancelContext persists a cancel request visible to any worker replica.
+func (r *Registry) RequestCancelContext(ctx context.Context, tenant, id string) (*Job, error) {
+	if r.queue == nil {
+		return r.RequestCancel(tenant, id)
+	}
+	c, err := r.queue.CancelJob(ctx, tenant, id)
+	if errors.Is(err, store.ErrJobNotFound) {
+		return nil, ErrNotFound
+	}
+	if err != nil {
+		return nil, err
+	}
+	r.mu.RLock()
+	local := r.byID[id]
+	r.mu.RUnlock()
+	if local != nil {
+		local.RequestCancel()
+	}
+	return jobFromRecord(c), nil
+}
+
+// RetryContext schedules an explicit retry of a failed shared operation.
+func (r *Registry) RetryContext(ctx context.Context, tenant, id string) (*Job, error) {
+	if r.queue == nil {
+		return nil, store.ErrJobConflict
+	}
+	c, err := r.queue.RetryJob(ctx, tenant, id)
+	if err != nil {
+		return nil, err
+	}
+	return jobFromRecord(c), nil
+}

internal/jobs/worker.go

@@ -2,6 +2,7 @@ package jobs
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/netip"
@@ -66,13 +67,18 @@ const (
 
 // Worker executes queued jobs against store.Backend (memory or SQL).
 type Worker struct {
-	Store      store.Backend
-	PgPool     *pgxpool.Pool
-	HTTPClient *http.Client // optional; CDN refresh uses this (default 45s timeout).
+	Store          store.Backend
+	PgPool         *pgxpool.Pool
+	HTTPClient     *http.Client // optional; CDN refresh uses this (default 45s timeout).
+	NodeHTTPClient *http.Client // optional; agent dispatch only, never shared with CDN transport.
 	// Registry is set after BootstrapWorkers creates the job queue; used to chain deploy_apply after refresh/rollback.
 	Registry *Registry
-	// refreshGate serializes deploy_apply gating after module_refresh per tenant (see finishModuleRefreshSuccess).
-	refreshGate sync.Map // map[string]*sync.Mutex
+	// refreshGate decides which module refresh renders for a tenant.
+	// The mutex covers only that decision; the render itself runs outside the lock.
+	refreshGate sync.Map // map[string]*tenantRefreshGate
+	// dispatchGate serializes publish and deployment state for one speaker in this process.
+	// Agent wake runs outside the mutex.
+	dispatchGate sync.Map // map[string]*sync.Mutex
 }
 
 type revisionLogEntry struct {
@@ -116,6 +122,23 @@ func (w *Worker) Process(j *Job) {
 		j.MarkCancelled()
 		return
 	}
+	// A committed child is the durable checkpoint for a parent that crashed
+	// before its own terminal write. Reuse it before rendering another revision.
+	if w.Registry != nil && (isRefreshKind(j.Kind) || j.Kind == KindRevisionRollback || j.Kind == KindPeerReconcile) {
+		child, err := w.Registry.FindIdempotentContext(context.Background(), j.TenantID, "deploy_from:"+j.ID)
+		if err == nil {
+			j.mergeMeta(map[string]any{"revision_id": child.metaString("revision_id"), "deploy_apply_job_id": child.ID, "recovered_child": true})
+			if isRefreshKind(j.Kind) {
+				w.Registry.finalizeRefreshCoalesce(j.TenantID)
+			}
+			j.Succeed()
+			return
+		}
+		if !errors.Is(err, store.ErrJobNotFound) {
+			j.Fail(err.Error())
+			return
+		}
+	}
 
 	switch j.Kind {
 	case KindModuleRefresh:
@@ -185,50 +208,24 @@ func (w *Worker) runPeerReconcile(j *Job) {
 	const peerJobTitle = "Обновление BGP пиров"
 	j.mergeMeta(map[string]any{"job_title": peerJobTitle})
 
-	var revID string
 	latest, _, _ := w.Store.ListRevisions(j.TenantID, "", "", 1)
 	triggerModuleID, err := w.peerTriggerModuleID(j.TenantID, latest)
 	if err != nil {
 		j.Fail(err.Error())
 		return
 	}
-	if len(latest) == 0 {
-		ctx, cancel := j.workContext()
-		defer cancel()
-		rid, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
-		if err != nil {
-			if ctx.Err() != nil {
-				j.MarkCancelled()
-				return
-			}
-			j.Fail(err.Error())
+	// Peer edits bump render generation. Render from module snapshots so the
+	// new revision includes current prefixes without paging the previous one.
+	ctx, cancel := j.workContext()
+	defer cancel()
+	revID, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
+	if err != nil {
+		if ctx.Err() != nil {
+			j.MarkCancelled()
 			return
 		}
-		revID = rid
-	} else {
-		baseRevID := latest[0].ID
-		rows := make([]store.PrefixRow, 0, 1024)
-		cursor := ""
-		for {
-			page, next, more := w.Store.ListRevisionPrefixes(j.TenantID, baseRevID, cursor, 2000)
-			rows = append(rows, page...)
-			if !more || strings.TrimSpace(next) == "" {
-				break
-			}
-			cursor = next
-		}
-		ctx, cancel := j.workContext()
-		defer cancel()
-		rid, err := pipeline.RenderTenantRevisionFromPrefixes(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID, rows)
-		if err != nil {
-			if ctx.Err() != nil {
-				j.MarkCancelled()
-				return
-			}
-			j.Fail(err.Error())
-			return
-		}
-		revID = rid
+		j.Fail(err.Error())
+		return
 	}
 	j.mergeMeta(map[string]any{"revision_id": revID})
 	if entries, total, err := w.buildRevisionLogEntries(j.TenantID, revID); err == nil {
@@ -240,7 +237,10 @@ func (w *Worker) runPeerReconcile(j *Job) {
 	} else {
 		j.mergeMeta(map[string]any{"log_build_error": err.Error()})
 	}
-	w.enqueueDeployAllSpeakers(j, j.TenantID, revID)
+	if err := w.enqueueDeployAllSpeakers(j, j.TenantID, revID); err != nil {
+		j.Fail(err.Error())
+		return
+	}
 	j.Succeed()
 }
 
@@ -269,8 +269,23 @@ func (w *Worker) peerTriggerModuleID(tenantID string, latest []*store.Revision)
 	return "", fmt.Errorf("missing module_id for peer reconcile")
 }
 
+type tenantRefreshGate struct {
+	mu        sync.Mutex
+	rendering bool
+	dirty     bool
+}
+
+func (w *Worker) tenantRefreshGate(tenantID string) *tenantRefreshGate {
+	v, _ := w.refreshGate.LoadOrStore(tenantID, &tenantRefreshGate{})
+	return v.(*tenantRefreshGate)
+}
+
 func (w *Worker) tenantRefreshMu(tenantID string) *sync.Mutex {
-	v, _ := w.refreshGate.LoadOrStore(tenantID, &sync.Mutex{})
+	return &w.tenantRefreshGate(tenantID).mu
+}
+
+func (w *Worker) speakerDispatchMu(speakerID string) *sync.Mutex {
+	v, _ := w.dispatchGate.LoadOrStore(speakerID, &sync.Mutex{})
 	return v.(*sync.Mutex)
 }
 
@@ -327,7 +342,8 @@ func (w *Worker) runTenantRefresh(j *Job) {
 
 	moduleIDs := moduleIDsFromJobMeta(j.metaCopy())
 	if len(moduleIDs) == 0 {
-		j.Fail("missing module_ids in job meta")
+		j.mergeMeta(map[string]any{"modules_refreshed": 0})
+		j.Succeed()
 		return
 	}
 	ctx, cancel := j.workContext()
@@ -382,20 +398,19 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
 		j.Succeed()
 		return
 	}
-	mu := w.tenantRefreshMu(j.TenantID)
-	mu.Lock()
-	defer mu.Unlock()
-	ctx, cancel := j.workContext()
-	defer cancel()
-	// Determine whether this is the last outstanding refresh for the tenant. The counter is
-	// incremented in Enqueue (under r.mu) and decremented here, so the "last one" decision is
-	// deterministic regardless of ingest timing — unlike the previous status-polling approach
-	// (CountOtherActiveRefresh) which could race under -race.
+	gate := w.tenantRefreshGate(j.TenantID)
+	gate.mu.Lock()
+	// The counter is incremented in Enqueue and decremented here, so the "last one"
+	// decision does not depend on job status polling. Render runs after the unlock.
 	isLastRefresh := true
 	if w.Registry != nil {
 		isLastRefresh = w.Registry.finalizeRefreshCoalesce(j.TenantID)
 	}
-	if !isLastRefresh {
+	if !isLastRefresh || gate.rendering {
+		if gate.rendering {
+			gate.dirty = true
+		}
+		gate.mu.Unlock()
 		j.mergeMeta(map[string]any{
 			"deploy_apply_deferred":     true,
 			"deploy_apply_defer_reason": "parallel_module_refresh",
@@ -403,16 +418,33 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
 		j.Succeed()
 		return
 	}
+	gate.rendering = true
+	gate.mu.Unlock()
+	defer func() {
+		gate.mu.Lock()
+		gate.rendering = false
+		gate.mu.Unlock()
+	}()
 
-	rev, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
+	ctx, cancel := j.workContext()
+	defer cancel()
+	rev, err := w.renderTenantUntilQuiet(ctx, j, triggerModuleID)
 	if err != nil {
-		if ctx.Err() != nil {
+		if errors.Is(err, errRenderSuperseded) {
+			j.Fail(err.Error())
+			return
+		}
+		if ctx.Err() != nil || j.IsCancelRequested() {
 			j.MarkCancelled()
 			return
 		}
 		j.Fail(err.Error())
 		return
 	}
+	if ctx.Err() != nil || j.IsCancelRequested() {
+		j.MarkCancelled()
+		return
+	}
 	j.mergeMeta(map[string]any{"revision_id": rev})
 	if entries, total, err := w.buildRevisionLogEntries(j.TenantID, rev); err == nil {
 		j.mergeMeta(map[string]any{
@@ -423,50 +455,240 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
 	} else {
 		j.mergeMeta(map[string]any{"log_build_error": err.Error()})
 	}
-	w.enqueueDeployAllSpeakers(j, j.TenantID, rev)
+	if err := w.enqueueDeployAllSpeakers(j, j.TenantID, rev); err != nil {
+		j.Fail(err.Error())
+		return
+	}
 	j.Succeed()
 }
 
+const (
+	// renderQuietPassLimit stops a render loop that never observes a quiet input set.
+	// Hitting the cap fails the job; the last revision must not be published.
+	renderQuietPassLimit = 32
+	// speakerWakeLimit is the maximum number of concurrent agent wakes in one deploy.
+	speakerWakeLimit = 4
+)
+
+// errRenderSuperseded marks a render that must not be deployed. A newer snapshot
+// landed, the job was cancelled while inputs were still changing, or the quiet
+// pass cap was exhausted.
+var errRenderSuperseded = errors.New("newer snapshot still pending")
+
+// renderTenantUntilQuiet builds a revision outside the tenant gate.
+// It keeps rendering while another ingest marks the gate dirty or the snapshot
+// generation moves, and returns a revision only after a fresh check shows the
+// inputs are still the ones just rendered. Cancellation or the safety cap while
+// inputs are still changing fails the render instead of publishing a stale revision.
+func (w *Worker) renderTenantUntilQuiet(ctx context.Context, j *Job, triggerModuleID string) (string, error) {
+	gate := w.tenantRefreshGate(j.TenantID)
+	var rev string
+	for pass := 0; pass < renderQuietPassLimit; pass++ {
+		if abort := renderJobAborted(ctx, j); abort != nil {
+			if takeRefreshDirty(gate) {
+				return "", errRenderWhileDirty(abort)
+			}
+			return "", abort
+		}
+		before, err := w.captureRenderInputs(ctx, j.TenantID)
+		if err != nil {
+			if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
+				return "", errRenderWhileDirty(abort)
+			}
+			return "", err
+		}
+		rev, err = pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
+		if err != nil {
+			dirty := takeRefreshDirty(gate)
+			if abort := renderJobAborted(ctx, j); abort != nil && dirty {
+				return "", errRenderWhileDirty(abort)
+			}
+			if abort := renderJobAborted(ctx, j); abort != nil {
+				return "", abort
+			}
+			if dirty {
+				continue
+			}
+			return "", err
+		}
+		after, err := w.captureRenderInputs(ctx, j.TenantID)
+		if err != nil {
+			if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
+				return "", errRenderWhileDirty(abort)
+			}
+			return "", err
+		}
+		dirty := takeRefreshDirty(gate)
+		if dirty || renderInputsChanged(before, after) {
+			if abort := renderJobAborted(ctx, j); abort != nil {
+				return "", errRenderWhileDirty(abort)
+			}
+			continue
+		}
+		confirm, err := w.captureRenderInputs(ctx, j.TenantID)
+		if err != nil {
+			if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
+				return "", errRenderWhileDirty(abort)
+			}
+			return "", err
+		}
+		if takeRefreshDirty(gate) || renderInputsChanged(after, confirm) {
+			if abort := renderJobAborted(ctx, j); abort != nil {
+				return "", errRenderWhileDirty(abort)
+			}
+			continue
+		}
+		if abort := renderJobAborted(ctx, j); abort != nil {
+			return "", abort
+		}
+		return rev, nil
+	}
+	if abort := renderJobAborted(ctx, j); abort != nil {
+		return "", errRenderWhileDirty(abort)
+	}
+	return "", errRenderWhileDirty(nil)
+}
+
+func errRenderWhileDirty(cause error) error {
+	if cause == nil {
+		return fmt.Errorf("jobs: tenant render did not quiesce after %d passes: %w", renderQuietPassLimit, errRenderSuperseded)
+	}
+	return fmt.Errorf("jobs: tenant render cancelled while a newer snapshot was still pending: %w", errRenderSuperseded)
+}
+
+func renderJobAborted(ctx context.Context, j *Job) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if j != nil && j.IsCancelRequested() {
+		return context.Canceled
+	}
+	return nil
+}
+
+func takeRefreshDirty(gate *tenantRefreshGate) bool {
+	if gate == nil {
+		return false
+	}
+	gate.mu.Lock()
+	defer gate.mu.Unlock()
+	dirty := gate.dirty
+	gate.dirty = false
+	return dirty
+}
+
+type renderInputView struct {
+	generation int64
+	snapshots  []store.ModuleSnapshotUse
+}
+
+func (w *Worker) captureRenderInputs(ctx context.Context, tenantID string) (renderInputView, error) {
+	if w == nil || w.Store == nil {
+		return renderInputView{}, fmt.Errorf("jobs: render store unavailable")
+	}
+	gen, err := w.Store.GetRenderGeneration(ctx, tenantID)
+	if err != nil {
+		if !errors.Is(err, store.ErrNotFound) {
+			return renderInputView{}, fmt.Errorf("jobs: read render generation: %w", err)
+		}
+		gen = 0
+	}
+	headers, err := w.Store.ListModuleSnapshotHeaders(ctx, tenantID)
+	if err != nil {
+		return renderInputView{}, fmt.Errorf("jobs: read snapshot headers: %w", err)
+	}
+	return renderInputView{generation: gen, snapshots: headers}, nil
+}
+
+func renderInputsChanged(before, after renderInputView) bool {
+	if before.generation != after.generation {
+		return true
+	}
+	return !sameSnapshotHeaders(before.snapshots, after.snapshots)
+}
+
+func sameSnapshotHeaders(a, b []store.ModuleSnapshotUse) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	aa := append([]store.ModuleSnapshotUse(nil), a...)
+	bb := append([]store.ModuleSnapshotUse(nil), b...)
+	sort.Slice(aa, func(i, j int) bool { return aa[i].ModuleID < aa[j].ModuleID })
+	sort.Slice(bb, func(i, j int) bool { return bb[i].ModuleID < bb[j].ModuleID })
+	for i := range aa {
+		if aa[i].ModuleID != bb[i].ModuleID || aa[i].SnapshotVersion != bb[i].SnapshotVersion || aa[i].InputHash != bb[i].InputHash {
+			return false
+		}
+	}
+	return true
+}
+
 // enqueueDeployAllSpeakers queues the same work as POST /v1/apply (all speakers, no speaker_id).
-func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) {
+func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) error {
 	if w == nil || w.Registry == nil {
-		return
+		return fmt.Errorf("jobs: deploy queue unavailable")
 	}
 	revID = strings.TrimSpace(revID)
 	if revID == "" {
-		return
+		return fmt.Errorf("jobs: missing revision for deploy")
 	}
 	meta := map[string]any{"revision_id": revID}
 	if tr := strings.TrimSpace(j.metaString("trigger")); tr != "" {
 		meta["trigger"] = tr
 	}
-	applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, nil, nil, meta)
+	key := "deploy_from:" + j.ID
+	applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, &key, nil, meta)
 	if err != nil {
 		j.mergeMeta(map[string]any{"deploy_apply_enqueue_error": err.Error()})
-		return
+		return fmt.Errorf("jobs: enqueue deploy: %w", err)
 	}
 	if applyJob != nil {
 		j.mergeMeta(map[string]any{"deploy_apply_job_id": applyJob.ID})
 	}
+	return nil
 }
 
 func (w *Worker) runDeployApply(j *Job) {
 	revID := strings.TrimSpace(j.metaString("revision_id"))
 	spk := strings.TrimSpace(j.metaString("speaker_id"))
-	hasSpeaker := spk != ""
 	if revID == "" {
 		j.Fail("missing revision_id in job meta")
 		return
 	}
+	if j.OwnerID == "" {
+		if _, developmentMemory := w.Store.(*store.Memory); !developmentMemory {
+			j.Fail("deployment requires a current durable job lease")
+			return
+		}
+	}
+
 	ctx, cancel := j.workContext()
 	defer cancel()
-	activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR"))
-	if activeDir != "" {
-		revObj, err := w.Store.GetRevision(j.TenantID, revID)
+	cancelled := func() bool { return ctx.Err() != nil || j.IsCancelRequested() }
+	var speakers []*store.Speaker
+	if spk != "" {
+		sp, err := w.Store.GetSpeaker(j.TenantID, spk)
 		if err != nil {
 			j.Fail(err.Error())
 			return
 		}
+		speakers = []*store.Speaker{sp}
+	} else {
+		speakers = w.Store.ListSpeakersForTenant(j.TenantID)
+	}
+	activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR"))
+	localAttempted := false
+	var localErr error
+	applyLocal := func() error {
+		if localAttempted {
+			return localErr
+		}
+		localAttempted = true
+		revObj, err := w.Store.GetRevision(j.TenantID, revID)
+		if err != nil {
+			localErr = err
+			return err
+		}
 		staging := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_STAGING_DIR"))
 		if staging == "" {
 			staging = os.TempDir() + "/evobgp-bird-staging"
@@ -480,99 +702,322 @@ func (w *Worker) runDeployApply(j *Job) {
 		}
 		ctl := &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
 		if err := birddeploy.ApplyRevision(ctx, ctl, revObj, cfg); err != nil {
-			if ctx.Err() != nil {
-				j.MarkCancelled()
-				return
-			}
-			j.Fail(err.Error())
-			return
-		}
-	}
-	applied := make([]string, 0, 8)
-	var dispatchResults []nodedispatch.Result
-	applyOne := func(speakerID string) error {
-		if err := w.Store.SetLastAppliedRevision(j.TenantID, speakerID, revID); err != nil {
+			localErr = err
 			return err
 		}
-		// Replica / node pulls use LatestPublishedRevision; keep pointer in sync with successful deploy.
-		if err := w.Store.PublishRevisionForSpeaker(speakerID, revID); err != nil {
-			return err
-		}
-		applied = append(applied, speakerID)
 		return nil
 	}
-	dispatchSpeaker := func(sp *store.Speaker) {
-		if !nodedispatch.Enabled() || sp == nil {
-			return
+	tally := &deployTally{applied: make([]string, 0, len(speakers))}
+	slots := make(chan struct{}, speakerWakeLimit)
+	var wakeGroup sync.WaitGroup
+	acquireWake := func() bool {
+		select {
+		case slots <- struct{}{}:
+		case <-ctx.Done():
+			return false
 		}
-		meta := store.ParseSpeakerMeta(sp.MetaJSON)
-		if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
-			return
-		}
-		ctx2, cancel := context.WithTimeout(ctx, 35*time.Second)
-		defer cancel()
-		res := nodedispatch.WakeSpeaker(ctx2, sp, nodedispatch.Options{RevisionID: revID})
-		dispatchResults = append(dispatchResults, res)
-		patch := store.SpeakerMeta{
-			LastDispatchAt:     time.Now().UTC().Format(time.RFC3339Nano),
-			LastDispatchStatus: res.Status,
-		}
-		if res.Error != "" {
-			patch.LastDispatchError = res.Error
-			patch.SyncStatus = "error"
-		} else if res.Status == "ok" {
-			patch.LastDispatchError = ""
-			patch.SyncStatus = "synced"
-		}
-		merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch)
-		_, _ = w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged})
-	}
-	if hasSpeaker && spk != "" {
-		if err := applyOne(spk); err != nil {
-			j.Fail(err.Error())
-			return
-		}
-		if sp, err := w.Store.GetSpeaker(j.TenantID, spk); err == nil {
-			dispatchSpeaker(sp)
-		}
-		if len(dispatchResults) > 0 {
-			j.mergeMeta(map[string]any{"node_dispatch": map[string]any{
-				"revision_id": revID,
-				"results":     dispatchResults,
-			}})
-		}
-		mergeBirdPostApplyMeta(j)
-		j.Succeed()
-		return
-	}
-	speakers := w.Store.ListSpeakersForTenant(j.TenantID)
-	for _, sp := range speakers {
-		if err := applyOne(sp.ID); err != nil {
-			j.Fail(err.Error())
-			return
+		if cancelled() {
+			<-slots
+			return false
 		}
+		return true
 	}
 	for _, sp := range speakers {
-		dispatchSpeaker(sp)
+		if cancelled() {
+			break
+		}
+		if sp == nil {
+			continue
+		}
+		held := false
+		if speakerRemoteWake(sp) {
+			if !acquireWake() {
+				break
+			}
+			held = true
+		}
+		staged := w.stageSpeakerDeploy(ctx, j, sp, revID, activeDir, cancelled, applyLocal)
+		if staged.wake && !staged.stop {
+			wakeGroup.Add(1)
+			go func(sp *store.Speaker, deployment store.DeploymentSpeaker) {
+				defer wakeGroup.Done()
+				defer func() { <-slots }()
+				wakeCtx, cancelWake := context.WithTimeout(ctx, 35*time.Second)
+				res := nodedispatch.WakeSpeaker(wakeCtx, sp, nodedispatch.Options{RevisionID: revID, HTTPClient: w.NodeHTTPClient, MaxRetries: 1})
+				cancelWake()
+				out, include, failedN, unconfirmedN, appliedID := w.finishSpeakerWake(ctx, j, sp, revID, deployment, res, cancelled)
+				tally.add(out, include, 0, failedN, unconfirmedN, appliedID)
+			}(sp, staged.deployment)
+		} else if held {
+			<-slots
+		}
+		tally.add(staged.result, staged.hasResult, staged.published, staged.failed, staged.unconfirmed, staged.appliedID)
+		if staged.stop {
+			break
+		}
 	}
+	wakeGroup.Wait()
+	results, applied, published, failed, unconfirmed := tally.snapshot()
 	j.mergeMeta(map[string]any{
 		"apply_summary": map[string]any{
-			"revision_id":    revID,
-			"speakers_count": len(applied),
-			"speaker_ids":    applied,
-			"message":        fmt.Sprintf("Ревизия %s применена на %d спикерах", shortID(revID), len(applied)),
+			"revision_id":       revID,
+			"speakers_count":    len(speakers),
+			"published_count":   published,
+			"applied_count":     len(applied),
+			"unconfirmed_count": unconfirmed,
+			"failed_count":      failed,
+			"speaker_ids":       applied,
+			"message":           fmt.Sprintf("Ревизия %s: подтверждено %d, без подтверждения %d, ошибок %d", shortID(revID), len(applied), unconfirmed, failed),
 		},
+		"node_dispatch": map[string]any{"revision_id": revID, "results": results},
 	})
-	if len(dispatchResults) > 0 {
-		j.mergeMeta(map[string]any{"node_dispatch": map[string]any{
-			"revision_id": revID,
-			"results":     dispatchResults,
-		}})
+	if cancelled() {
+		j.MarkCancelled()
+		return
 	}
 	mergeBirdPostApplyMeta(j)
+	if cancelled() {
+		j.MarkCancelled()
+		return
+	}
+	if failed > 0 {
+		j.Fail(fmt.Sprintf("deploy delivery failed for %d of %d speakers", failed, len(speakers)))
+		return
+	}
 	j.Succeed()
 }
 
+type speakerStage struct {
+	stop        bool
+	wake        bool
+	deployment  store.DeploymentSpeaker
+	result      nodedispatch.Result
+	hasResult   bool
+	published   int
+	failed      int
+	unconfirmed int
+	appliedID   string
+}
+
+type deployTally struct {
+	mu          sync.Mutex
+	results     []nodedispatch.Result
+	applied     []string
+	published   int
+	failed      int
+	unconfirmed int
+}
+
+func (t *deployTally) add(res nodedispatch.Result, include bool, published, failed, unconfirmed int, appliedID string) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if include {
+		t.results = append(t.results, res)
+	}
+	t.published += published
+	t.failed += failed
+	t.unconfirmed += unconfirmed
+	if appliedID != "" {
+		t.applied = append(t.applied, appliedID)
+	}
+}
+
+func (t *deployTally) snapshot() (results []nodedispatch.Result, applied []string, published, failed, unconfirmed int) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	results = append([]nodedispatch.Result(nil), t.results...)
+	applied = append([]string(nil), t.applied...)
+	if applied == nil {
+		applied = []string{}
+	}
+	sort.Slice(results, func(i, j int) bool { return results[i].SpeakerID < results[j].SpeakerID })
+	sort.Strings(applied)
+	return results, applied, t.published, t.failed, t.unconfirmed
+}
+
+func speakerRemoteWake(sp *store.Speaker) bool {
+	if sp == nil || !nodedispatch.Enabled() {
+		return false
+	}
+	return store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON))
+}
+
+func (w *Worker) bindDeploymentWriters(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker) (func(string, string, string) error, func(string) error) {
+	fence := deploymentFenceForJob(j)
+	record := func(state, confirmation, message string) error {
+		if j.OwnerID == "" {
+			return nil
+		}
+		return w.Store.RecordDeploymentSpeaker(ctx, fence, deployment, state, confirmation, message)
+	}
+	markApplied := func(confirmation string) error {
+		if j.OwnerID == "" {
+			return w.Store.SetLastAppliedRevision(j.TenantID, sp.ID, revID)
+		}
+		return record("applied", confirmation, "")
+	}
+	return record, markApplied
+}
+
+// stageSpeakerDeploy publishes one speaker and records terminal states.
+// Remote wakes are left to the caller so the speaker lock is not held across the network.
+func (w *Worker) stageSpeakerDeploy(ctx context.Context, j *Job, sp *store.Speaker, revID, activeDir string, cancelled func() bool, applyLocal func() error) speakerStage {
+	var staged speakerStage
+	mu := w.speakerDispatchMu(sp.ID)
+	mu.Lock()
+	defer mu.Unlock()
+	if cancelled() {
+		staged.stop = true
+		return staged
+	}
+	var deployment store.DeploymentSpeaker
+	var publishErr error
+	if j.OwnerID != "" {
+		deployment, publishErr = beginDeployment(ctx, w.Store, deploymentFenceForJob(j), sp.ID, revID)
+	} else {
+		publishErr = w.Store.PublishRevisionForSpeaker(sp.ID, revID)
+	}
+	staged.deployment = deployment
+	record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
+	if publishErr != nil {
+		staged.failed = 1
+		staged.hasResult = true
+		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: publishErr.Error()}
+		return staged
+	}
+	staged.published = 1
+	if deployment.State == "applied" {
+		staged.appliedID = sp.ID
+		staged.hasResult = true
+		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: deployment.Confirmation, AppliedRevisionID: revID}
+		return staged
+	}
+	if cancelled() {
+		staged.stop = true
+		return staged
+	}
+	if !speakerRemoteWake(sp) {
+		if activeDir != "" && strings.EqualFold(sp.Role, "master") {
+			err := applyLocal()
+			if cancelled() {
+				staged.stop = true
+				return staged
+			}
+			if err != nil {
+				_ = record("failed", "unconfirmed", err.Error())
+				staged.failed = 1
+				staged.hasResult = true
+				staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
+				return staged
+			}
+			if err := markApplied("local_applied"); err != nil {
+				staged.failed = 1
+				staged.hasResult = true
+				staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
+				return staged
+			}
+			staged.appliedID = sp.ID
+			staged.hasResult = true
+			staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: "local_applied", AppliedRevisionID: revID}
+			return staged
+		}
+		if err := record("unconfirmed", "unconfirmed", "dispatch disabled or speaker has no remote agent"); err != nil {
+			staged.failed = 1
+		}
+		staged.unconfirmed = 1
+		staged.hasResult = true
+		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "skipped", Confirmation: "unconfirmed"}
+		return staged
+	}
+	if err := record("dispatched", "unconfirmed", "awaiting agent acknowledgement"); err != nil {
+		staged.failed = 1
+		staged.hasResult = true
+		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
+		return staged
+	}
+	if deployment.DispatchAttempts >= 3 {
+		_ = record("unknown", "unconfirmed", "delivery attempt budget exhausted")
+		staged.failed = 1
+		staged.hasResult = true
+		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Confirmation: "unconfirmed", Error: "delivery attempt budget exhausted"}
+		return staged
+	}
+	staged.wake = true
+	return staged
+}
+
+// finishSpeakerWake records the agent acknowledgement. The speaker lock covers the
+// state update and is not held while WakeSpeaker is in flight.
+func (w *Worker) finishSpeakerWake(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker, res nodedispatch.Result, cancelled func() bool) (nodedispatch.Result, bool, int, int, string) {
+	mu := w.speakerDispatchMu(sp.ID)
+	mu.Lock()
+	defer mu.Unlock()
+	if cancelled != nil && cancelled() {
+		return nodedispatch.Result{SpeakerID: sp.ID, Endpoint: res.Endpoint, Status: "cancelled", Confirmation: "unconfirmed"}, true, 0, 0, ""
+	}
+	record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
+	appliedID := ""
+	if res.Status == "ok" && res.AppliedRevisionID != "" {
+		if res.AppliedRevisionID != revID {
+			res.Status = "error"
+			res.Confirmation = "unconfirmed"
+			res.Error = "agent reported a different applied revision"
+		} else if latest, _, err := w.Store.LatestPublishedRevision(sp.ID); err != nil {
+			res.Status = "error"
+			res.Confirmation = "unconfirmed"
+			res.Error = err.Error()
+		} else if latest != revID {
+			res.Status = "error"
+			res.Confirmation = "unconfirmed"
+			res.Error = "stale agent acknowledgement: desired revision changed"
+		} else if err := markApplied("agent_reported"); err != nil {
+			res.Status = "error"
+			res.Confirmation = "unconfirmed"
+			res.Error = err.Error()
+		} else {
+			appliedID = sp.ID
+		}
+	}
+	if res.Status != "ok" || res.Confirmation != "agent_reported" {
+		state := "unconfirmed"
+		if res.Status != "ok" {
+			state = "unknown"
+		}
+		if err := record(state, "unconfirmed", res.Error); err != nil {
+			res.Status = "error"
+			res.Error = err.Error()
+		}
+	}
+	patch := store.SpeakerMeta{
+		LastDispatchAt:     time.Now().UTC().Format(time.RFC3339Nano),
+		LastDispatchStatus: res.Status,
+	}
+	if res.Error != "" {
+		patch.LastDispatchError = res.Error
+		patch.SyncStatus = "error"
+	} else if res.Status == "ok" && res.Confirmation == "agent_reported" {
+		patch.LastDispatchError = ""
+		patch.SyncStatus = "synced"
+	} else if res.Status == "ok" {
+		patch.SyncStatus = "unconfirmed"
+	}
+	merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch)
+	if j.OwnerID == "" {
+		if _, err := w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged}); err != nil {
+			res.Status = "error"
+			res.Error = "dispatch metadata update: " + err.Error()
+		}
+	}
+	failed := 0
+	unconfirmed := 0
+	if res.Status == "ok" && res.Confirmation == "unconfirmed" {
+		unconfirmed = 1
+	} else if res.Status != "ok" {
+		failed = 1
+	}
+	return res, true, failed, unconfirmed, appliedID
+}
+
 func (w *Worker) runRollback(j *Job) {
 	src := strings.TrimSpace(j.metaString("source_revision_id"))
 	if src == "" {
@@ -592,7 +1037,10 @@ func (w *Worker) runRollback(j *Job) {
 			"message":            fmt.Sprintf("Откат %s → %s", shortID(src), shortID(newID)),
 		},
 	})
-	w.enqueueDeployAllSpeakers(j, j.TenantID, newID)
+	if err := w.enqueueDeployAllSpeakers(j, j.TenantID, newID); err != nil {
+		j.Fail(err.Error())
+		return
+	}
 	j.Succeed()
 }
 

internal/jobs/worker_test.go

@@ -1,6 +1,9 @@
 package jobs
 
 import (
+	"context"
+	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -185,3 +188,187 @@ func TestPeerReconcile_RendersFromLatestRevisionAndQueuesDeploy(t *testing.T) {
 		t.Fatalf("expected deploy_apply_job_id in peer_reconcile meta, got %#v", meta)
 	}
 }
+
+type renderHookStore struct {
+	*store.Memory
+	onCreate func(prefixes []store.PrefixRow)
+}
+
+func (s *renderHookStore) CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parent *string, contentHash string, preview map[string]string, prefixes []store.PrefixRow) error {
+	if err := s.Memory.CreateRenderRevision(ctx, revisionID, tenantID, moduleID, parent, contentHash, preview, prefixes); err != nil {
+		return err
+	}
+	if s.onCreate != nil {
+		copied := append([]store.PrefixRow(nil), prefixes...)
+		s.onCreate(copied)
+	}
+	return nil
+}
+
+func newIPRenderFixture(t *testing.T) (tenant, module string, mem *store.Memory) {
+	t.Helper()
+	t.Setenv("EVOBGP_ASN_RESOLVE", "0")
+	t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
+	mem = store.NewMemory()
+	mem.SeedDemo()
+	var modIP string
+	tenant, _, modIP, _, _ = mem.DemoIDs()
+	module = modIP
+	for _, mod := range mem.ListModules(tenant) {
+		if mod == nil || mod.ID == module {
+			continue
+		}
+		disabled := false
+		if _, err := mem.UpdateModule(tenant, mod.ID, &store.ModulePatch{Enabled: &disabled}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return tenant, module, mem
+}
+
+func installModuleSnapshot(t *testing.T, mem *store.Memory, tenant, module, prefix string) {
+	t.Helper()
+	err := mem.SetModulePrefixSnapshot(context.Background(), tenant, module, "snap-"+prefix, []store.PrefixRow{{
+		Prefix: prefix,
+		Source: "ip_range",
+	}})
+	if err != nil {
+		t.Fatalf("snapshot %s: %v", prefix, err)
+	}
+}
+
+func revisionHasPrefix(t *testing.T, mem *store.Memory, tenant, revID, prefix string) bool {
+	t.Helper()
+	rows, _, _ := mem.ListRevisionPrefixes(tenant, revID, "", 100)
+	for _, row := range rows {
+		if row.Prefix == prefix {
+			return true
+		}
+	}
+	return false
+}
+
+func TestRenderTenantUntilQuietOverlappingIngests(t *testing.T) {
+	t.Run("latest snapshot", func(t *testing.T) {
+		tenant, module, mem := newIPRenderFixture(t)
+		prefixes := []string{"10.8.0.0/24", "10.9.0.0/24", "10.10.0.0/24", "10.11.0.0/24"}
+		installModuleSnapshot(t, mem, tenant, module, prefixes[0])
+		hook := &renderHookStore{Memory: mem}
+		var wk *Worker
+		step := 0
+		hook.onCreate = func([]store.PrefixRow) {
+			step++
+			if step >= len(prefixes) {
+				return
+			}
+			installModuleSnapshot(t, mem, tenant, module, prefixes[step])
+			gate := wk.tenantRefreshGate(tenant)
+			gate.mu.Lock()
+			gate.dirty = true
+			gate.mu.Unlock()
+		}
+		wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
+		j := &Job{ID: "refresh-overlap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
+		wk.finishModuleRefreshSuccess(j, module)
+		if j.Status != StatusSucceeded {
+			errText := ""
+			if j.Error != nil {
+				errText = *j.Error
+			}
+			t.Fatalf("status=%s err=%s", j.Status, errText)
+		}
+		if step != len(prefixes) {
+			t.Fatalf("render commits=%d, want %d so three overlapping ingests are included", step, len(prefixes))
+		}
+		rev := j.metaString("revision_id")
+		if !revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-1]) {
+			t.Fatalf("deployed revision %s missing latest prefix %s", rev, prefixes[len(prefixes)-1])
+		}
+		if revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-2]) {
+			t.Fatalf("deployed revision %s still has stale prefix %s", rev, prefixes[len(prefixes)-2])
+		}
+		deployID := j.metaString("deploy_apply_job_id")
+		deploy, err := wk.Registry.Get(tenant, deployID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if deploy.metaString("revision_id") != rev {
+			t.Fatalf("deploy revision=%s, want %s", deploy.metaString("revision_id"), rev)
+		}
+	})
+
+	t.Run("cancel does not deploy", func(t *testing.T) {
+		tenant, module, mem := newIPRenderFixture(t)
+		installModuleSnapshot(t, mem, tenant, module, "10.12.0.0/24")
+		hook := &renderHookStore{Memory: mem}
+		var wk *Worker
+		var j *Job
+		commits := 0
+		hook.onCreate = func([]store.PrefixRow) {
+			commits++
+			installModuleSnapshot(t, mem, tenant, module, "10.13.0.0/24")
+			gate := wk.tenantRefreshGate(tenant)
+			gate.mu.Lock()
+			gate.dirty = true
+			gate.mu.Unlock()
+			j.RequestCancel()
+		}
+		wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
+		j = &Job{ID: "refresh-cancel", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
+		wk.finishModuleRefreshSuccess(j, module)
+		if j.Status != StatusFailed {
+			t.Fatalf("status=%s, want failed", j.Status)
+		}
+		if j.Error == nil || !strings.Contains(*j.Error, "newer snapshot") {
+			t.Fatalf("error=%v, want superseded render", j.Error)
+		}
+		if commits != 1 {
+			t.Fatalf("render commits=%d, want 1", commits)
+		}
+		if j.metaString("deploy_apply_job_id") != "" || j.metaString("revision_id") != "" {
+			t.Fatalf("cancelled dirty render published meta %#v", j.metaCopy())
+		}
+		deploys, _, _ := wk.Registry.List(tenant, "", KindDeployApply, "", 10)
+		if len(deploys) != 0 {
+			t.Fatalf("deploy jobs=%d, want 0", len(deploys))
+		}
+		revs, _, _ := mem.ListRevisions(tenant, "", "", 50)
+		for _, rev := range revs {
+			if rev != nil && revisionHasPrefix(t, mem, tenant, rev.ID, "10.13.0.0/24") {
+				t.Fatalf("revision %s contains the snapshot that landed during cancel", rev.ID)
+			}
+		}
+	})
+
+	t.Run("cap does not publish", func(t *testing.T) {
+		tenant, module, mem := newIPRenderFixture(t)
+		installModuleSnapshot(t, mem, tenant, module, "10.20.0.0/24")
+		hook := &renderHookStore{Memory: mem}
+		var wk *Worker
+		commits := 0
+		hook.onCreate = func([]store.PrefixRow) {
+			commits++
+			installModuleSnapshot(t, mem, tenant, module, fmt.Sprintf("10.21.%d.0/24", commits))
+			gate := wk.tenantRefreshGate(tenant)
+			gate.mu.Lock()
+			gate.dirty = true
+			gate.mu.Unlock()
+		}
+		wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
+		j := &Job{ID: "refresh-cap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
+		wk.finishModuleRefreshSuccess(j, module)
+		if j.Status != StatusFailed || j.Error == nil || !strings.Contains(*j.Error, "did not quiesce") {
+			errText := ""
+			if j.Error != nil {
+				errText = *j.Error
+			}
+			t.Fatalf("status=%s err=%s", j.Status, errText)
+		}
+		if commits != renderQuietPassLimit {
+			t.Fatalf("render commits=%d, want cap %d", commits, renderQuietPassLimit)
+		}
+		if j.metaString("deploy_apply_job_id") != "" {
+			t.Fatal("safety cap published a deploy")
+		}
+	})
+}