feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <cursoragent@cursor.com>

internal/jobs/worker.go

@@ -2,6 +2,7 @@ package jobs
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/netip"
@@ -66,13 +67,18 @@ const (
 
 // Worker executes queued jobs against store.Backend (memory or SQL).
 type Worker struct {
-	Store      store.Backend
-	PgPool     *pgxpool.Pool
-	HTTPClient *http.Client // optional; CDN refresh uses this (default 45s timeout).
+	Store          store.Backend
+	PgPool         *pgxpool.Pool
+	HTTPClient     *http.Client // optional; CDN refresh uses this (default 45s timeout).
+	NodeHTTPClient *http.Client // optional; agent dispatch only, never shared with CDN transport.
 	// Registry is set after BootstrapWorkers creates the job queue; used to chain deploy_apply after refresh/rollback.
 	Registry *Registry
-	// refreshGate serializes deploy_apply gating after module_refresh per tenant (see finishModuleRefreshSuccess).
-	refreshGate sync.Map // map[string]*sync.Mutex
+	// refreshGate decides which module refresh renders for a tenant.
+	// The mutex covers only that decision; the render itself runs outside the lock.
+	refreshGate sync.Map // map[string]*tenantRefreshGate
+	// dispatchGate serializes publish and deployment state for one speaker in this process.
+	// Agent wake runs outside the mutex.
+	dispatchGate sync.Map // map[string]*sync.Mutex
 }
 
 type revisionLogEntry struct {
@@ -116,6 +122,23 @@ func (w *Worker) Process(j *Job) {
 		j.MarkCancelled()
 		return
 	}
+	// A committed child is the durable checkpoint for a parent that crashed
+	// before its own terminal write. Reuse it before rendering another revision.
+	if w.Registry != nil && (isRefreshKind(j.Kind) || j.Kind == KindRevisionRollback || j.Kind == KindPeerReconcile) {
+		child, err := w.Registry.FindIdempotentContext(context.Background(), j.TenantID, "deploy_from:"+j.ID)
+		if err == nil {
+			j.mergeMeta(map[string]any{"revision_id": child.metaString("revision_id"), "deploy_apply_job_id": child.ID, "recovered_child": true})
+			if isRefreshKind(j.Kind) {
+				w.Registry.finalizeRefreshCoalesce(j.TenantID)
+			}
+			j.Succeed()
+			return
+		}
+		if !errors.Is(err, store.ErrJobNotFound) {
+			j.Fail(err.Error())
+			return
+		}
+	}
 
 	switch j.Kind {
 	case KindModuleRefresh:
@@ -185,50 +208,24 @@ func (w *Worker) runPeerReconcile(j *Job) {
 	const peerJobTitle = "Обновление BGP пиров"
 	j.mergeMeta(map[string]any{"job_title": peerJobTitle})
 
-	var revID string
 	latest, _, _ := w.Store.ListRevisions(j.TenantID, "", "", 1)
 	triggerModuleID, err := w.peerTriggerModuleID(j.TenantID, latest)
 	if err != nil {
 		j.Fail(err.Error())
 		return
 	}
-	if len(latest) == 0 {
-		ctx, cancel := j.workContext()
-		defer cancel()
-		rid, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
-		if err != nil {
-			if ctx.Err() != nil {
-				j.MarkCancelled()
-				return
-			}
-			j.Fail(err.Error())
+	// Peer edits bump render generation. Render from module snapshots so the
+	// new revision includes current prefixes without paging the previous one.
+	ctx, cancel := j.workContext()
+	defer cancel()
+	revID, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
+	if err != nil {
+		if ctx.Err() != nil {
+			j.MarkCancelled()
 			return
 		}
-		revID = rid
-	} else {
-		baseRevID := latest[0].ID
-		rows := make([]store.PrefixRow, 0, 1024)
-		cursor := ""
-		for {
-			page, next, more := w.Store.ListRevisionPrefixes(j.TenantID, baseRevID, cursor, 2000)
-			rows = append(rows, page...)
-			if !more || strings.TrimSpace(next) == "" {
-				break
-			}
-			cursor = next
-		}
-		ctx, cancel := j.workContext()
-		defer cancel()
-		rid, err := pipeline.RenderTenantRevisionFromPrefixes(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID, rows)
-		if err != nil {
-			if ctx.Err() != nil {
-				j.MarkCancelled()
-				return
-			}
-			j.Fail(err.Error())
-			return
-		}
-		revID = rid
+		j.Fail(err.Error())
+		return
 	}
 	j.mergeMeta(map[string]any{"revision_id": revID})
 	if entries, total, err := w.buildRevisionLogEntries(j.TenantID, revID); err == nil {
@@ -240,7 +237,10 @@ func (w *Worker) runPeerReconcile(j *Job) {
 	} else {
 		j.mergeMeta(map[string]any{"log_build_error": err.Error()})
 	}
-	w.enqueueDeployAllSpeakers(j, j.TenantID, revID)
+	if err := w.enqueueDeployAllSpeakers(j, j.TenantID, revID); err != nil {
+		j.Fail(err.Error())
+		return
+	}
 	j.Succeed()
 }
 
@@ -269,8 +269,23 @@ func (w *Worker) peerTriggerModuleID(tenantID string, latest []*store.Revision)
 	return "", fmt.Errorf("missing module_id for peer reconcile")
 }
 
+type tenantRefreshGate struct {
+	mu        sync.Mutex
+	rendering bool
+	dirty     bool
+}
+
+func (w *Worker) tenantRefreshGate(tenantID string) *tenantRefreshGate {
+	v, _ := w.refreshGate.LoadOrStore(tenantID, &tenantRefreshGate{})
+	return v.(*tenantRefreshGate)
+}
+
 func (w *Worker) tenantRefreshMu(tenantID string) *sync.Mutex {
-	v, _ := w.refreshGate.LoadOrStore(tenantID, &sync.Mutex{})
+	return &w.tenantRefreshGate(tenantID).mu
+}
+
+func (w *Worker) speakerDispatchMu(speakerID string) *sync.Mutex {
+	v, _ := w.dispatchGate.LoadOrStore(speakerID, &sync.Mutex{})
 	return v.(*sync.Mutex)
 }
 
@@ -327,7 +342,8 @@ func (w *Worker) runTenantRefresh(j *Job) {
 
 	moduleIDs := moduleIDsFromJobMeta(j.metaCopy())
 	if len(moduleIDs) == 0 {
-		j.Fail("missing module_ids in job meta")
+		j.mergeMeta(map[string]any{"modules_refreshed": 0})
+		j.Succeed()
 		return
 	}
 	ctx, cancel := j.workContext()
@@ -382,20 +398,19 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
 		j.Succeed()
 		return
 	}
-	mu := w.tenantRefreshMu(j.TenantID)
-	mu.Lock()
-	defer mu.Unlock()
-	ctx, cancel := j.workContext()
-	defer cancel()
-	// Determine whether this is the last outstanding refresh for the tenant. The counter is
-	// incremented in Enqueue (under r.mu) and decremented here, so the "last one" decision is
-	// deterministic regardless of ingest timing — unlike the previous status-polling approach
-	// (CountOtherActiveRefresh) which could race under -race.
+	gate := w.tenantRefreshGate(j.TenantID)
+	gate.mu.Lock()
+	// The counter is incremented in Enqueue and decremented here, so the "last one"
+	// decision does not depend on job status polling. Render runs after the unlock.
 	isLastRefresh := true
 	if w.Registry != nil {
 		isLastRefresh = w.Registry.finalizeRefreshCoalesce(j.TenantID)
 	}
-	if !isLastRefresh {
+	if !isLastRefresh || gate.rendering {
+		if gate.rendering {
+			gate.dirty = true
+		}
+		gate.mu.Unlock()
 		j.mergeMeta(map[string]any{
 			"deploy_apply_deferred":     true,
 			"deploy_apply_defer_reason": "parallel_module_refresh",
@@ -403,16 +418,33 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
 		j.Succeed()
 		return
 	}
+	gate.rendering = true
+	gate.mu.Unlock()
+	defer func() {
+		gate.mu.Lock()
+		gate.rendering = false
+		gate.mu.Unlock()
+	}()
 
-	rev, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
+	ctx, cancel := j.workContext()
+	defer cancel()
+	rev, err := w.renderTenantUntilQuiet(ctx, j, triggerModuleID)
 	if err != nil {
-		if ctx.Err() != nil {
+		if errors.Is(err, errRenderSuperseded) {
+			j.Fail(err.Error())
+			return
+		}
+		if ctx.Err() != nil || j.IsCancelRequested() {
 			j.MarkCancelled()
 			return
 		}
 		j.Fail(err.Error())
 		return
 	}
+	if ctx.Err() != nil || j.IsCancelRequested() {
+		j.MarkCancelled()
+		return
+	}
 	j.mergeMeta(map[string]any{"revision_id": rev})
 	if entries, total, err := w.buildRevisionLogEntries(j.TenantID, rev); err == nil {
 		j.mergeMeta(map[string]any{
@@ -423,50 +455,240 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
 	} else {
 		j.mergeMeta(map[string]any{"log_build_error": err.Error()})
 	}
-	w.enqueueDeployAllSpeakers(j, j.TenantID, rev)
+	if err := w.enqueueDeployAllSpeakers(j, j.TenantID, rev); err != nil {
+		j.Fail(err.Error())
+		return
+	}
 	j.Succeed()
 }
 
+const (
+	// renderQuietPassLimit stops a render loop that never observes a quiet input set.
+	// Hitting the cap fails the job; the last revision must not be published.
+	renderQuietPassLimit = 32
+	// speakerWakeLimit is the maximum number of concurrent agent wakes in one deploy.
+	speakerWakeLimit = 4
+)
+
+// errRenderSuperseded marks a render that must not be deployed. A newer snapshot
+// landed, the job was cancelled while inputs were still changing, or the quiet
+// pass cap was exhausted.
+var errRenderSuperseded = errors.New("newer snapshot still pending")
+
+// renderTenantUntilQuiet builds a revision outside the tenant gate.
+// It keeps rendering while another ingest marks the gate dirty or the snapshot
+// generation moves, and returns a revision only after a fresh check shows the
+// inputs are still the ones just rendered. Cancellation or the safety cap while
+// inputs are still changing fails the render instead of publishing a stale revision.
+func (w *Worker) renderTenantUntilQuiet(ctx context.Context, j *Job, triggerModuleID string) (string, error) {
+	gate := w.tenantRefreshGate(j.TenantID)
+	var rev string
+	for pass := 0; pass < renderQuietPassLimit; pass++ {
+		if abort := renderJobAborted(ctx, j); abort != nil {
+			if takeRefreshDirty(gate) {
+				return "", errRenderWhileDirty(abort)
+			}
+			return "", abort
+		}
+		before, err := w.captureRenderInputs(ctx, j.TenantID)
+		if err != nil {
+			if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
+				return "", errRenderWhileDirty(abort)
+			}
+			return "", err
+		}
+		rev, err = pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
+		if err != nil {
+			dirty := takeRefreshDirty(gate)
+			if abort := renderJobAborted(ctx, j); abort != nil && dirty {
+				return "", errRenderWhileDirty(abort)
+			}
+			if abort := renderJobAborted(ctx, j); abort != nil {
+				return "", abort
+			}
+			if dirty {
+				continue
+			}
+			return "", err
+		}
+		after, err := w.captureRenderInputs(ctx, j.TenantID)
+		if err != nil {
+			if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
+				return "", errRenderWhileDirty(abort)
+			}
+			return "", err
+		}
+		dirty := takeRefreshDirty(gate)
+		if dirty || renderInputsChanged(before, after) {
+			if abort := renderJobAborted(ctx, j); abort != nil {
+				return "", errRenderWhileDirty(abort)
+			}
+			continue
+		}
+		confirm, err := w.captureRenderInputs(ctx, j.TenantID)
+		if err != nil {
+			if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
+				return "", errRenderWhileDirty(abort)
+			}
+			return "", err
+		}
+		if takeRefreshDirty(gate) || renderInputsChanged(after, confirm) {
+			if abort := renderJobAborted(ctx, j); abort != nil {
+				return "", errRenderWhileDirty(abort)
+			}
+			continue
+		}
+		if abort := renderJobAborted(ctx, j); abort != nil {
+			return "", abort
+		}
+		return rev, nil
+	}
+	if abort := renderJobAborted(ctx, j); abort != nil {
+		return "", errRenderWhileDirty(abort)
+	}
+	return "", errRenderWhileDirty(nil)
+}
+
+func errRenderWhileDirty(cause error) error {
+	if cause == nil {
+		return fmt.Errorf("jobs: tenant render did not quiesce after %d passes: %w", renderQuietPassLimit, errRenderSuperseded)
+	}
+	return fmt.Errorf("jobs: tenant render cancelled while a newer snapshot was still pending: %w", errRenderSuperseded)
+}
+
+func renderJobAborted(ctx context.Context, j *Job) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if j != nil && j.IsCancelRequested() {
+		return context.Canceled
+	}
+	return nil
+}
+
+func takeRefreshDirty(gate *tenantRefreshGate) bool {
+	if gate == nil {
+		return false
+	}
+	gate.mu.Lock()
+	defer gate.mu.Unlock()
+	dirty := gate.dirty
+	gate.dirty = false
+	return dirty
+}
+
+type renderInputView struct {
+	generation int64
+	snapshots  []store.ModuleSnapshotUse
+}
+
+func (w *Worker) captureRenderInputs(ctx context.Context, tenantID string) (renderInputView, error) {
+	if w == nil || w.Store == nil {
+		return renderInputView{}, fmt.Errorf("jobs: render store unavailable")
+	}
+	gen, err := w.Store.GetRenderGeneration(ctx, tenantID)
+	if err != nil {
+		if !errors.Is(err, store.ErrNotFound) {
+			return renderInputView{}, fmt.Errorf("jobs: read render generation: %w", err)
+		}
+		gen = 0
+	}
+	headers, err := w.Store.ListModuleSnapshotHeaders(ctx, tenantID)
+	if err != nil {
+		return renderInputView{}, fmt.Errorf("jobs: read snapshot headers: %w", err)
+	}
+	return renderInputView{generation: gen, snapshots: headers}, nil
+}
+
+func renderInputsChanged(before, after renderInputView) bool {
+	if before.generation != after.generation {
+		return true
+	}
+	return !sameSnapshotHeaders(before.snapshots, after.snapshots)
+}
+
+func sameSnapshotHeaders(a, b []store.ModuleSnapshotUse) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	aa := append([]store.ModuleSnapshotUse(nil), a...)
+	bb := append([]store.ModuleSnapshotUse(nil), b...)
+	sort.Slice(aa, func(i, j int) bool { return aa[i].ModuleID < aa[j].ModuleID })
+	sort.Slice(bb, func(i, j int) bool { return bb[i].ModuleID < bb[j].ModuleID })
+	for i := range aa {
+		if aa[i].ModuleID != bb[i].ModuleID || aa[i].SnapshotVersion != bb[i].SnapshotVersion || aa[i].InputHash != bb[i].InputHash {
+			return false
+		}
+	}
+	return true
+}
+
 // enqueueDeployAllSpeakers queues the same work as POST /v1/apply (all speakers, no speaker_id).
-func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) {
+func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) error {
 	if w == nil || w.Registry == nil {
-		return
+		return fmt.Errorf("jobs: deploy queue unavailable")
 	}
 	revID = strings.TrimSpace(revID)
 	if revID == "" {
-		return
+		return fmt.Errorf("jobs: missing revision for deploy")
 	}
 	meta := map[string]any{"revision_id": revID}
 	if tr := strings.TrimSpace(j.metaString("trigger")); tr != "" {
 		meta["trigger"] = tr
 	}
-	applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, nil, nil, meta)
+	key := "deploy_from:" + j.ID
+	applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, &key, nil, meta)
 	if err != nil {
 		j.mergeMeta(map[string]any{"deploy_apply_enqueue_error": err.Error()})
-		return
+		return fmt.Errorf("jobs: enqueue deploy: %w", err)
 	}
 	if applyJob != nil {
 		j.mergeMeta(map[string]any{"deploy_apply_job_id": applyJob.ID})
 	}
+	return nil
 }
 
 func (w *Worker) runDeployApply(j *Job) {
 	revID := strings.TrimSpace(j.metaString("revision_id"))
 	spk := strings.TrimSpace(j.metaString("speaker_id"))
-	hasSpeaker := spk != ""
 	if revID == "" {
 		j.Fail("missing revision_id in job meta")
 		return
 	}
+	if j.OwnerID == "" {
+		if _, developmentMemory := w.Store.(*store.Memory); !developmentMemory {
+			j.Fail("deployment requires a current durable job lease")
+			return
+		}
+	}
+
 	ctx, cancel := j.workContext()
 	defer cancel()
-	activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR"))
-	if activeDir != "" {
-		revObj, err := w.Store.GetRevision(j.TenantID, revID)
+	cancelled := func() bool { return ctx.Err() != nil || j.IsCancelRequested() }
+	var speakers []*store.Speaker
+	if spk != "" {
+		sp, err := w.Store.GetSpeaker(j.TenantID, spk)
 		if err != nil {
 			j.Fail(err.Error())
 			return
 		}
+		speakers = []*store.Speaker{sp}
+	} else {
+		speakers = w.Store.ListSpeakersForTenant(j.TenantID)
+	}
+	activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR"))
+	localAttempted := false
+	var localErr error
+	applyLocal := func() error {
+		if localAttempted {
+			return localErr
+		}
+		localAttempted = true
+		revObj, err := w.Store.GetRevision(j.TenantID, revID)
+		if err != nil {
+			localErr = err
+			return err
+		}
 		staging := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_STAGING_DIR"))
 		if staging == "" {
 			staging = os.TempDir() + "/evobgp-bird-staging"
@@ -480,99 +702,322 @@ func (w *Worker) runDeployApply(j *Job) {
 		}
 		ctl := &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
 		if err := birddeploy.ApplyRevision(ctx, ctl, revObj, cfg); err != nil {
-			if ctx.Err() != nil {
-				j.MarkCancelled()
-				return
-			}
-			j.Fail(err.Error())
-			return
-		}
-	}
-	applied := make([]string, 0, 8)
-	var dispatchResults []nodedispatch.Result
-	applyOne := func(speakerID string) error {
-		if err := w.Store.SetLastAppliedRevision(j.TenantID, speakerID, revID); err != nil {
+			localErr = err
 			return err
 		}
-		// Replica / node pulls use LatestPublishedRevision; keep pointer in sync with successful deploy.
-		if err := w.Store.PublishRevisionForSpeaker(speakerID, revID); err != nil {
-			return err
-		}
-		applied = append(applied, speakerID)
 		return nil
 	}
-	dispatchSpeaker := func(sp *store.Speaker) {
-		if !nodedispatch.Enabled() || sp == nil {
-			return
+	tally := &deployTally{applied: make([]string, 0, len(speakers))}
+	slots := make(chan struct{}, speakerWakeLimit)
+	var wakeGroup sync.WaitGroup
+	acquireWake := func() bool {
+		select {
+		case slots <- struct{}{}:
+		case <-ctx.Done():
+			return false
 		}
-		meta := store.ParseSpeakerMeta(sp.MetaJSON)
-		if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
-			return
-		}
-		ctx2, cancel := context.WithTimeout(ctx, 35*time.Second)
-		defer cancel()
-		res := nodedispatch.WakeSpeaker(ctx2, sp, nodedispatch.Options{RevisionID: revID})
-		dispatchResults = append(dispatchResults, res)
-		patch := store.SpeakerMeta{
-			LastDispatchAt:     time.Now().UTC().Format(time.RFC3339Nano),
-			LastDispatchStatus: res.Status,
-		}
-		if res.Error != "" {
-			patch.LastDispatchError = res.Error
-			patch.SyncStatus = "error"
-		} else if res.Status == "ok" {
-			patch.LastDispatchError = ""
-			patch.SyncStatus = "synced"
-		}
-		merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch)
-		_, _ = w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged})
-	}
-	if hasSpeaker && spk != "" {
-		if err := applyOne(spk); err != nil {
-			j.Fail(err.Error())
-			return
-		}
-		if sp, err := w.Store.GetSpeaker(j.TenantID, spk); err == nil {
-			dispatchSpeaker(sp)
-		}
-		if len(dispatchResults) > 0 {
-			j.mergeMeta(map[string]any{"node_dispatch": map[string]any{
-				"revision_id": revID,
-				"results":     dispatchResults,
-			}})
-		}
-		mergeBirdPostApplyMeta(j)
-		j.Succeed()
-		return
-	}
-	speakers := w.Store.ListSpeakersForTenant(j.TenantID)
-	for _, sp := range speakers {
-		if err := applyOne(sp.ID); err != nil {
-			j.Fail(err.Error())
-			return
+		if cancelled() {
+			<-slots
+			return false
 		}
+		return true
 	}
 	for _, sp := range speakers {
-		dispatchSpeaker(sp)
+		if cancelled() {
+			break
+		}
+		if sp == nil {
+			continue
+		}
+		held := false
+		if speakerRemoteWake(sp) {
+			if !acquireWake() {
+				break
+			}
+			held = true
+		}
+		staged := w.stageSpeakerDeploy(ctx, j, sp, revID, activeDir, cancelled, applyLocal)
+		if staged.wake && !staged.stop {
+			wakeGroup.Add(1)
+			go func(sp *store.Speaker, deployment store.DeploymentSpeaker) {
+				defer wakeGroup.Done()
+				defer func() { <-slots }()
+				wakeCtx, cancelWake := context.WithTimeout(ctx, 35*time.Second)
+				res := nodedispatch.WakeSpeaker(wakeCtx, sp, nodedispatch.Options{RevisionID: revID, HTTPClient: w.NodeHTTPClient, MaxRetries: 1})
+				cancelWake()
+				out, include, failedN, unconfirmedN, appliedID := w.finishSpeakerWake(ctx, j, sp, revID, deployment, res, cancelled)
+				tally.add(out, include, 0, failedN, unconfirmedN, appliedID)
+			}(sp, staged.deployment)
+		} else if held {
+			<-slots
+		}
+		tally.add(staged.result, staged.hasResult, staged.published, staged.failed, staged.unconfirmed, staged.appliedID)
+		if staged.stop {
+			break
+		}
 	}
+	wakeGroup.Wait()
+	results, applied, published, failed, unconfirmed := tally.snapshot()
 	j.mergeMeta(map[string]any{
 		"apply_summary": map[string]any{
-			"revision_id":    revID,
-			"speakers_count": len(applied),
-			"speaker_ids":    applied,
-			"message":        fmt.Sprintf("Ревизия %s применена на %d спикерах", shortID(revID), len(applied)),
+			"revision_id":       revID,
+			"speakers_count":    len(speakers),
+			"published_count":   published,
+			"applied_count":     len(applied),
+			"unconfirmed_count": unconfirmed,
+			"failed_count":      failed,
+			"speaker_ids":       applied,
+			"message":           fmt.Sprintf("Ревизия %s: подтверждено %d, без подтверждения %d, ошибок %d", shortID(revID), len(applied), unconfirmed, failed),
 		},
+		"node_dispatch": map[string]any{"revision_id": revID, "results": results},
 	})
-	if len(dispatchResults) > 0 {
-		j.mergeMeta(map[string]any{"node_dispatch": map[string]any{
-			"revision_id": revID,
-			"results":     dispatchResults,
-		}})
+	if cancelled() {
+		j.MarkCancelled()
+		return
 	}
 	mergeBirdPostApplyMeta(j)
+	if cancelled() {
+		j.MarkCancelled()
+		return
+	}
+	if failed > 0 {
+		j.Fail(fmt.Sprintf("deploy delivery failed for %d of %d speakers", failed, len(speakers)))
+		return
+	}
 	j.Succeed()
 }
 
+type speakerStage struct {
+	stop        bool
+	wake        bool
+	deployment  store.DeploymentSpeaker
+	result      nodedispatch.Result
+	hasResult   bool
+	published   int
+	failed      int
+	unconfirmed int
+	appliedID   string
+}
+
+type deployTally struct {
+	mu          sync.Mutex
+	results     []nodedispatch.Result
+	applied     []string
+	published   int
+	failed      int
+	unconfirmed int
+}
+
+func (t *deployTally) add(res nodedispatch.Result, include bool, published, failed, unconfirmed int, appliedID string) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if include {
+		t.results = append(t.results, res)
+	}
+	t.published += published
+	t.failed += failed
+	t.unconfirmed += unconfirmed
+	if appliedID != "" {
+		t.applied = append(t.applied, appliedID)
+	}
+}
+
+func (t *deployTally) snapshot() (results []nodedispatch.Result, applied []string, published, failed, unconfirmed int) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	results = append([]nodedispatch.Result(nil), t.results...)
+	applied = append([]string(nil), t.applied...)
+	if applied == nil {
+		applied = []string{}
+	}
+	sort.Slice(results, func(i, j int) bool { return results[i].SpeakerID < results[j].SpeakerID })
+	sort.Strings(applied)
+	return results, applied, t.published, t.failed, t.unconfirmed
+}
+
+func speakerRemoteWake(sp *store.Speaker) bool {
+	if sp == nil || !nodedispatch.Enabled() {
+		return false
+	}
+	return store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON))
+}
+
+func (w *Worker) bindDeploymentWriters(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker) (func(string, string, string) error, func(string) error) {
+	fence := deploymentFenceForJob(j)
+	record := func(state, confirmation, message string) error {
+		if j.OwnerID == "" {
+			return nil
+		}
+		return w.Store.RecordDeploymentSpeaker(ctx, fence, deployment, state, confirmation, message)
+	}
+	markApplied := func(confirmation string) error {
+		if j.OwnerID == "" {
+			return w.Store.SetLastAppliedRevision(j.TenantID, sp.ID, revID)
+		}
+		return record("applied", confirmation, "")
+	}
+	return record, markApplied
+}
+
+// stageSpeakerDeploy publishes one speaker and records terminal states.
+// Remote wakes are left to the caller so the speaker lock is not held across the network.
+func (w *Worker) stageSpeakerDeploy(ctx context.Context, j *Job, sp *store.Speaker, revID, activeDir string, cancelled func() bool, applyLocal func() error) speakerStage {
+	var staged speakerStage
+	mu := w.speakerDispatchMu(sp.ID)
+	mu.Lock()
+	defer mu.Unlock()
+	if cancelled() {
+		staged.stop = true
+		return staged
+	}
+	var deployment store.DeploymentSpeaker
+	var publishErr error
+	if j.OwnerID != "" {
+		deployment, publishErr = beginDeployment(ctx, w.Store, deploymentFenceForJob(j), sp.ID, revID)
+	} else {
+		publishErr = w.Store.PublishRevisionForSpeaker(sp.ID, revID)
+	}
+	staged.deployment = deployment
+	record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
+	if publishErr != nil {
+		staged.failed = 1
+		staged.hasResult = true
+		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: publishErr.Error()}
+		return staged
+	}
+	staged.published = 1
+	if deployment.State == "applied" {
+		staged.appliedID = sp.ID
+		staged.hasResult = true
+		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: deployment.Confirmation, AppliedRevisionID: revID}
+		return staged
+	}
+	if cancelled() {
+		staged.stop = true
+		return staged
+	}
+	if !speakerRemoteWake(sp) {
+		if activeDir != "" && strings.EqualFold(sp.Role, "master") {
+			err := applyLocal()
+			if cancelled() {
+				staged.stop = true
+				return staged
+			}
+			if err != nil {
+				_ = record("failed", "unconfirmed", err.Error())
+				staged.failed = 1
+				staged.hasResult = true
+				staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
+				return staged
+			}
+			if err := markApplied("local_applied"); err != nil {
+				staged.failed = 1
+				staged.hasResult = true
+				staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
+				return staged
+			}
+			staged.appliedID = sp.ID
+			staged.hasResult = true
+			staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: "local_applied", AppliedRevisionID: revID}
+			return staged
+		}
+		if err := record("unconfirmed", "unconfirmed", "dispatch disabled or speaker has no remote agent"); err != nil {
+			staged.failed = 1
+		}
+		staged.unconfirmed = 1
+		staged.hasResult = true
+		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "skipped", Confirmation: "unconfirmed"}
+		return staged
+	}
+	if err := record("dispatched", "unconfirmed", "awaiting agent acknowledgement"); err != nil {
+		staged.failed = 1
+		staged.hasResult = true
+		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
+		return staged
+	}
+	if deployment.DispatchAttempts >= 3 {
+		_ = record("unknown", "unconfirmed", "delivery attempt budget exhausted")
+		staged.failed = 1
+		staged.hasResult = true
+		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Confirmation: "unconfirmed", Error: "delivery attempt budget exhausted"}
+		return staged
+	}
+	staged.wake = true
+	return staged
+}
+
+// finishSpeakerWake records the agent acknowledgement. The speaker lock covers the
+// state update and is not held while WakeSpeaker is in flight.
+func (w *Worker) finishSpeakerWake(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker, res nodedispatch.Result, cancelled func() bool) (nodedispatch.Result, bool, int, int, string) {
+	mu := w.speakerDispatchMu(sp.ID)
+	mu.Lock()
+	defer mu.Unlock()
+	if cancelled != nil && cancelled() {
+		return nodedispatch.Result{SpeakerID: sp.ID, Endpoint: res.Endpoint, Status: "cancelled", Confirmation: "unconfirmed"}, true, 0, 0, ""
+	}
+	record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
+	appliedID := ""
+	if res.Status == "ok" && res.AppliedRevisionID != "" {
+		if res.AppliedRevisionID != revID {
+			res.Status = "error"
+			res.Confirmation = "unconfirmed"
+			res.Error = "agent reported a different applied revision"
+		} else if latest, _, err := w.Store.LatestPublishedRevision(sp.ID); err != nil {
+			res.Status = "error"
+			res.Confirmation = "unconfirmed"
+			res.Error = err.Error()
+		} else if latest != revID {
+			res.Status = "error"
+			res.Confirmation = "unconfirmed"
+			res.Error = "stale agent acknowledgement: desired revision changed"
+		} else if err := markApplied("agent_reported"); err != nil {
+			res.Status = "error"
+			res.Confirmation = "unconfirmed"
+			res.Error = err.Error()
+		} else {
+			appliedID = sp.ID
+		}
+	}
+	if res.Status != "ok" || res.Confirmation != "agent_reported" {
+		state := "unconfirmed"
+		if res.Status != "ok" {
+			state = "unknown"
+		}
+		if err := record(state, "unconfirmed", res.Error); err != nil {
+			res.Status = "error"
+			res.Error = err.Error()
+		}
+	}
+	patch := store.SpeakerMeta{
+		LastDispatchAt:     time.Now().UTC().Format(time.RFC3339Nano),
+		LastDispatchStatus: res.Status,
+	}
+	if res.Error != "" {
+		patch.LastDispatchError = res.Error
+		patch.SyncStatus = "error"
+	} else if res.Status == "ok" && res.Confirmation == "agent_reported" {
+		patch.LastDispatchError = ""
+		patch.SyncStatus = "synced"
+	} else if res.Status == "ok" {
+		patch.SyncStatus = "unconfirmed"
+	}
+	merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch)
+	if j.OwnerID == "" {
+		if _, err := w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged}); err != nil {
+			res.Status = "error"
+			res.Error = "dispatch metadata update: " + err.Error()
+		}
+	}
+	failed := 0
+	unconfirmed := 0
+	if res.Status == "ok" && res.Confirmation == "unconfirmed" {
+		unconfirmed = 1
+	} else if res.Status != "ok" {
+		failed = 1
+	}
+	return res, true, failed, unconfirmed, appliedID
+}
+
 func (w *Worker) runRollback(j *Job) {
 	src := strings.TrimSpace(j.metaString("source_revision_id"))
 	if src == "" {
@@ -592,7 +1037,10 @@ func (w *Worker) runRollback(j *Job) {
 			"message":            fmt.Sprintf("Откат %s → %s", shortID(src), shortID(newID)),
 		},
 	})
-	w.enqueueDeployAllSpeakers(j, j.TenantID, newID)
+	if err := w.enqueueDeployAllSpeakers(j, j.TenantID, newID); err != nil {
+		j.Fail(err.Error())
+		return
+	}
 	j.Succeed()
 }