EvoBGP/internal/jobs/worker.go

package jobs

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/netip"
	"os"
	"sort"
	"strings"
	"sync"
	"time"

	"evobgp/internal/birddeploy"
	"evobgp/internal/birdfmt"
	"evobgp/internal/httpclient"
	"evobgp/internal/nodedispatch"
	"evobgp/internal/observability"
	"evobgp/internal/pipeline"
	"evobgp/internal/store"

	"github.com/jackc/pgx/v5/pgxpool"
)

// mergeBirdPostApplyMeta attaches a birdc snapshot after deploy/reload (best-effort).
func mergeBirdPostApplyMeta(j *Job) {
	if strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")) == "" {
		j.mergeMeta(map[string]any{"bird_post_apply_check": "skipped_no_birdc_socket"})
		return
	}
	ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
	defer cancel()
	st := birdfmt.InspectLocalBird(ctx)
	inner := map[string]any{
		"bgp_established":    st.BGPEstablished,
		"bgp_sessions_total": st.BGPSessionsTotal,
	}
	if st.Error != "" {
		inner["ok"] = false
		inner["error"] = st.Error
	} else {
		inner["ok"] = true
	}
	j.mergeMeta(map[string]any{"bird_post_apply": inner})
}

const (
	KindModuleRefresh          = "module_refresh"
	KindTenantRefresh          = "tenant_refresh"
	KindPeerReconcile          = "peer_reconcile"
	KindDeployApply            = "deploy_apply"
	KindRevisionRollback       = "revision_rollback"
	KindBirdReload             = "bird_reload"
	KindPostgresMetricsRefresh = "postgres_metrics_refresh"
	KindPostgresSlowQueryAgg   = "postgres_slow_query_aggregate"
	KindPostgresTableBloat     = "postgres_table_bloat_estimate"
	KindPostgresIndexUsage     = "postgres_index_usage_analyze"
	KindPostgresAutovacuumLag  = "postgres_autovacuum_lag_detect"
	KindPostgresVacuum         = "postgres_vacuum"
	KindPostgresVacuumAnalyze  = "postgres_vacuum_analyze"
	KindPostgresAnalyze        = "postgres_analyze"
	KindPostgresReindex        = "postgres_reindex"
	KindPostgresCleanup        = "postgres_cleanup"
	KindMaintenancePolicyRun   = "maintenance_policy_run"
)

// Worker executes queued jobs against store.Backend (memory or SQL).
type Worker struct {
	Store          store.Backend
	PgPool         *pgxpool.Pool
	HTTPClient     *http.Client // optional; CDN refresh uses this (default 45s timeout).
	NodeHTTPClient *http.Client // optional; agent dispatch only, never shared with CDN transport.
	// Registry is set after BootstrapWorkers creates the job queue; used to chain deploy_apply after refresh/rollback.
	Registry *Registry
	// refreshGate decides which module refresh renders for a tenant.
	// The mutex covers only that decision; the render itself runs outside the lock.
	refreshGate sync.Map // map[string]*tenantRefreshGate
	// dispatchGate serializes publish and deployment state for one speaker in this process.
	// Agent wake runs outside the mutex.
	dispatchGate sync.Map // map[string]*sync.Mutex
}

type revisionLogEntry struct {
	Kind           string   `json:"kind"`
	Source         string   `json:"source"`
	Community      string   `json:"community"`
	CommunityLabel string   `json:"community_label"`
	PrefixCount    int      `json:"prefix_count"`
	Sample         []string `json:"sample,omitempty"`
	Message        string   `json:"message"`
}

var defaultWorkerHTTP = httpclient.New(httpclient.DefaultTimeout)

func (w *Worker) httpClient() *http.Client {
	if w != nil && w.HTTPClient != nil {
		return w.HTTPClient
	}
	return defaultWorkerHTTP
}

// Process is registered as Registry.workerStart.
func (w *Worker) Process(j *Job) {
	defer func() {
		observability.RecordJobTerminal(j.Kind, j.statusLocked())
		if w != nil && w.Registry != nil {
			w.Registry.fireTerminal(j)
		}
	}()

	if w == nil || w.Store == nil {
		j.MarkRunning()
		j.Fail("worker not configured")
		return
	}
	j.MarkRunning()
	if w != nil && w.Registry != nil {
		w.Registry.fireRunning(j)
	}
	if j.IsCancelRequested() {
		j.MarkCancelled()
		return
	}
	// A committed child is the durable checkpoint for a parent that crashed
	// before its own terminal write. Reuse it before rendering another revision.
	if w.Registry != nil && (isRefreshKind(j.Kind) || j.Kind == KindRevisionRollback || j.Kind == KindPeerReconcile) {
		child, err := w.Registry.FindIdempotentContext(context.Background(), j.TenantID, "deploy_from:"+j.ID)
		if err == nil {
			j.mergeMeta(map[string]any{"revision_id": child.metaString("revision_id"), "deploy_apply_job_id": child.ID, "recovered_child": true})
			if isRefreshKind(j.Kind) {
				w.Registry.finalizeRefreshCoalesce(j.TenantID)
			}
			j.Succeed()
			return
		}
		if !errors.Is(err, store.ErrJobNotFound) {
			j.Fail(err.Error())
			return
		}
	}

	switch j.Kind {
	case KindModuleRefresh:
		w.runModuleRefresh(j)
	case KindTenantRefresh:
		w.runTenantRefresh(j)
	case KindPeerReconcile:
		w.runPeerReconcile(j)
	case KindDeployApply:
		w.runDeployApply(j)
	case KindRevisionRollback:
		w.runRollback(j)
	case KindBirdReload:
		sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET"))
		if sock == "" {
			j.Succeed()
			return
		}
		ctx, cancel := j.workContext()
		defer cancel()
		ctl := &birdfmt.BirdCtl{
			Socket: sock,
			Birdc:  strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")),
		}
		if err := ctl.Configure(ctx); err != nil {
			if ctx.Err() != nil {
				j.MarkCancelled()
				return
			}
			j.Fail(err.Error())
			return
		}
		mergeBirdPostApplyMeta(j)
		j.Succeed()
	case KindPostgresMetricsRefresh:
		w.runPostgresMetricsRefresh(j)
	case KindPostgresSlowQueryAgg:
		w.runPostgresSlowQueryAgg(j)
	case KindPostgresTableBloat:
		w.runPostgresTableBloat(j)
	case KindPostgresIndexUsage:
		w.runPostgresIndexUsage(j)
	case KindPostgresAutovacuumLag:
		w.runPostgresAutovacuumLag(j)
	case KindPostgresVacuum:
		w.runPostgresMaint(j, "vacuum")
	case KindPostgresVacuumAnalyze:
		w.runPostgresMaint(j, "vacuum_analyze")
	case KindPostgresAnalyze:
		w.runPostgresMaint(j, "analyze")
	case KindPostgresReindex:
		w.runPostgresMaint(j, "reindex")
	case KindPostgresCleanup:
		w.runPostgresCleanup(j)
	case KindMaintenancePolicyRun:
		w.runMaintenancePolicy(j)
	default:
		j.Fail("unknown job kind")
	}
}

func (w *Worker) runPeerReconcile(j *Job) {
	if w == nil || w.Store == nil {
		j.Fail("worker not configured")
		return
	}
	const peerJobTitle = "Обновление BGP пиров"
	j.mergeMeta(map[string]any{"job_title": peerJobTitle})

	latest, _, _ := w.Store.ListRevisions(j.TenantID, "", "", 1)
	triggerModuleID, err := w.peerTriggerModuleID(j.TenantID, latest)
	if err != nil {
		j.Fail(err.Error())
		return
	}
	// Peer edits bump render generation. Render from module snapshots so the
	// new revision includes current prefixes without paging the previous one.
	ctx, cancel := j.workContext()
	defer cancel()
	revID, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
	if err != nil {
		if ctx.Err() != nil {
			j.MarkCancelled()
			return
		}
		j.Fail(err.Error())
		return
	}
	j.mergeMeta(map[string]any{"revision_id": revID})
	if entries, total, err := w.buildRevisionLogEntries(j.TenantID, revID); err == nil {
		j.mergeMeta(map[string]any{
			"log_entries":   entries,
			"log_total":     total,
			"log_generated": time.Now().UTC().Format(time.RFC3339Nano),
		})
	} else {
		j.mergeMeta(map[string]any{"log_build_error": err.Error()})
	}
	if err := w.enqueueDeployAllSpeakers(j, j.TenantID, revID); err != nil {
		j.Fail(err.Error())
		return
	}
	j.Succeed()
}

func (w *Worker) peerTriggerModuleID(tenantID string, latest []*store.Revision) (string, error) {
	if len(latest) > 0 {
		if mid := strings.TrimSpace(latest[0].ModuleID); mid != "" {
			return mid, nil
		}
	}
	for _, mod := range w.Store.ListModules(tenantID) {
		if mod == nil || !mod.Enabled {
			continue
		}
		if strings.TrimSpace(mod.ID) != "" {
			return mod.ID, nil
		}
	}
	for _, mod := range w.Store.ListModules(tenantID) {
		if mod == nil {
			continue
		}
		if strings.TrimSpace(mod.ID) != "" {
			return mod.ID, nil
		}
	}
	return "", fmt.Errorf("missing module_id for peer reconcile")
}

type tenantRefreshGate struct {
	mu        sync.Mutex
	rendering bool
	dirty     bool
}

func (w *Worker) tenantRefreshGate(tenantID string) *tenantRefreshGate {
	v, _ := w.refreshGate.LoadOrStore(tenantID, &tenantRefreshGate{})
	return v.(*tenantRefreshGate)
}

func (w *Worker) tenantRefreshMu(tenantID string) *sync.Mutex {
	return &w.tenantRefreshGate(tenantID).mu
}

func (w *Worker) speakerDispatchMu(speakerID string) *sync.Mutex {
	v, _ := w.dispatchGate.LoadOrStore(speakerID, &sync.Mutex{})
	return v.(*sync.Mutex)
}

// finishModuleRefreshSuccess marks the refresh job and, for the last active refresh in tenant,
// creates one aggregate revision and enqueues a single deploy_apply.
// runModuleRefresh handles a single module_refresh job and guarantees the per-tenant inflight
// slot is released exactly once — even on failure/cancellation before finishModuleRefreshSuccess.
func (w *Worker) runModuleRefresh(j *Job) {
	coalesceFinalized := false
	defer func() {
		if !coalesceFinalized && w != nil && w.Registry != nil {
			// Refresh failed/was cancelled before reaching finishModuleRefreshSuccess.
			// Decrement the counter under the tenant mutex so the "last one" logic stays sound.
			mu := w.tenantRefreshMu(j.TenantID)
			mu.Lock()
			w.Registry.finalizeRefreshCoalesce(j.TenantID)
			mu.Unlock()
		}
	}()

	mid := strings.TrimSpace(j.metaString("module_id"))
	if mid == "" {
		j.Fail("missing module_id in job meta")
		return
	}
	ctx, cancel := j.workContext()
	defer cancel()
	if ctx.Err() != nil {
		j.MarkCancelled()
		return
	}
	if err := pipeline.RefreshModuleIngest(ctx, w.Store, w.httpClient(), j.TenantID, mid); err != nil {
		if ctx.Err() != nil {
			j.MarkCancelled()
			return
		}
		j.Fail(err.Error())
		return
	}
	w.finishModuleRefreshSuccess(j, mid)
	coalesceFinalized = true
}

func (w *Worker) runTenantRefresh(j *Job) {
	coalesceFinalized := false
	defer func() {
		if !coalesceFinalized && w != nil && w.Registry != nil {
			mu := w.tenantRefreshMu(j.TenantID)
			mu.Lock()
			w.Registry.finalizeRefreshCoalesce(j.TenantID)
			mu.Unlock()
		}
	}()

	moduleIDs := moduleIDsFromJobMeta(j.metaCopy())
	if len(moduleIDs) == 0 {
		j.mergeMeta(map[string]any{"modules_refreshed": 0})
		j.Succeed()
		return
	}
	ctx, cancel := j.workContext()
	defer cancel()
	if err := pipeline.RefreshTenantModules(ctx, w.Store, w.httpClient(), j.TenantID, moduleIDs); err != nil {
		if ctx.Err() != nil {
			j.MarkCancelled()
			return
		}
		j.Fail(err.Error())
		return
	}
	trigger, err := pipeline.PickTenantRefreshTriggerModule(w.Store, j.TenantID, moduleIDs)
	if err != nil {
		j.Fail(err.Error())
		return
	}
	j.mergeMeta(map[string]any{"module_ids": moduleIDs, "modules_refreshed": len(moduleIDs)})
	w.finishModuleRefreshSuccess(j, trigger)
	coalesceFinalized = true
}

func moduleIDsFromJobMeta(meta map[string]any) []string {
	if meta == nil {
		return nil
	}
	raw, ok := meta["module_ids"]
	if !ok {
		return nil
	}
	switch v := raw.(type) {
	case []string:
		return v
	case []any:
		var out []string
		for _, x := range v {
			if s, ok := x.(string); ok && strings.TrimSpace(s) != "" {
				out = append(out, strings.TrimSpace(s))
			}
		}
		return out
	default:
		return nil
	}
}

func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
	if w == nil || w.Store == nil {
		if w != nil && w.Registry != nil {
			w.Registry.finalizeRefreshCoalesce(j.TenantID)
		}
		j.Succeed()
		return
	}
	gate := w.tenantRefreshGate(j.TenantID)
	gate.mu.Lock()
	// The counter is incremented in Enqueue and decremented here, so the "last one"
	// decision does not depend on job status polling. Render runs after the unlock.
	isLastRefresh := true
	if w.Registry != nil {
		isLastRefresh = w.Registry.finalizeRefreshCoalesce(j.TenantID)
	}
	if !isLastRefresh || gate.rendering {
		if gate.rendering {
			gate.dirty = true
		}
		gate.mu.Unlock()
		j.mergeMeta(map[string]any{
			"deploy_apply_deferred":     true,
			"deploy_apply_defer_reason": "parallel_module_refresh",
		})
		j.Succeed()
		return
	}
	gate.rendering = true
	gate.mu.Unlock()
	defer func() {
		gate.mu.Lock()
		gate.rendering = false
		gate.mu.Unlock()
	}()

	ctx, cancel := j.workContext()
	defer cancel()
	rev, err := w.renderTenantUntilQuiet(ctx, j, triggerModuleID)
	if err != nil {
		if errors.Is(err, errRenderSuperseded) {
			j.Fail(err.Error())
			return
		}
		if ctx.Err() != nil || j.IsCancelRequested() {
			j.MarkCancelled()
			return
		}
		j.Fail(err.Error())
		return
	}
	if ctx.Err() != nil || j.IsCancelRequested() {
		j.MarkCancelled()
		return
	}
	j.mergeMeta(map[string]any{"revision_id": rev})
	if entries, total, err := w.buildRevisionLogEntries(j.TenantID, rev); err == nil {
		j.mergeMeta(map[string]any{
			"log_entries":   entries,
			"log_total":     total,
			"log_generated": time.Now().UTC().Format(time.RFC3339Nano),
		})
	} else {
		j.mergeMeta(map[string]any{"log_build_error": err.Error()})
	}
	if err := w.enqueueDeployAllSpeakers(j, j.TenantID, rev); err != nil {
		j.Fail(err.Error())
		return
	}
	j.Succeed()
}

const (
	// renderQuietPassLimit stops a render loop that never observes a quiet input set.
	// Hitting the cap fails the job; the last revision must not be published.
	renderQuietPassLimit = 32
	// speakerWakeLimit is the maximum number of concurrent agent wakes in one deploy.
	speakerWakeLimit = 4
)

// errRenderSuperseded marks a render that must not be deployed. A newer snapshot
// landed, the job was cancelled while inputs were still changing, or the quiet
// pass cap was exhausted.
var errRenderSuperseded = errors.New("newer snapshot still pending")

// renderTenantUntilQuiet builds a revision outside the tenant gate.
// It keeps rendering while another ingest marks the gate dirty or the snapshot
// generation moves, and returns a revision only after a fresh check shows the
// inputs are still the ones just rendered. Cancellation or the safety cap while
// inputs are still changing fails the render instead of publishing a stale revision.
func (w *Worker) renderTenantUntilQuiet(ctx context.Context, j *Job, triggerModuleID string) (string, error) {
	gate := w.tenantRefreshGate(j.TenantID)
	var rev string
	for pass := 0; pass < renderQuietPassLimit; pass++ {
		if abort := renderJobAborted(ctx, j); abort != nil {
			if takeRefreshDirty(gate) {
				return "", errRenderWhileDirty(abort)
			}
			return "", abort
		}
		before, err := w.captureRenderInputs(ctx, j.TenantID)
		if err != nil {
			if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
				return "", errRenderWhileDirty(abort)
			}
			return "", err
		}
		rev, err = pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
		if err != nil {
			dirty := takeRefreshDirty(gate)
			if abort := renderJobAborted(ctx, j); abort != nil && dirty {
				return "", errRenderWhileDirty(abort)
			}
			if abort := renderJobAborted(ctx, j); abort != nil {
				return "", abort
			}
			if dirty {
				continue
			}
			return "", err
		}
		after, err := w.captureRenderInputs(ctx, j.TenantID)
		if err != nil {
			if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
				return "", errRenderWhileDirty(abort)
			}
			return "", err
		}
		dirty := takeRefreshDirty(gate)
		if dirty || renderInputsChanged(before, after) {
			if abort := renderJobAborted(ctx, j); abort != nil {
				return "", errRenderWhileDirty(abort)
			}
			continue
		}
		confirm, err := w.captureRenderInputs(ctx, j.TenantID)
		if err != nil {
			if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
				return "", errRenderWhileDirty(abort)
			}
			return "", err
		}
		if takeRefreshDirty(gate) || renderInputsChanged(after, confirm) {
			if abort := renderJobAborted(ctx, j); abort != nil {
				return "", errRenderWhileDirty(abort)
			}
			continue
		}
		if abort := renderJobAborted(ctx, j); abort != nil {
			return "", abort
		}
		return rev, nil
	}
	if abort := renderJobAborted(ctx, j); abort != nil {
		return "", errRenderWhileDirty(abort)
	}
	return "", errRenderWhileDirty(nil)
}

func errRenderWhileDirty(cause error) error {
	if cause == nil {
		return fmt.Errorf("jobs: tenant render did not quiesce after %d passes: %w", renderQuietPassLimit, errRenderSuperseded)
	}
	return fmt.Errorf("jobs: tenant render cancelled while a newer snapshot was still pending: %w", errRenderSuperseded)
}

func renderJobAborted(ctx context.Context, j *Job) error {
	if err := ctx.Err(); err != nil {
		return err
	}
	if j != nil && j.IsCancelRequested() {
		return context.Canceled
	}
	return nil
}

func takeRefreshDirty(gate *tenantRefreshGate) bool {
	if gate == nil {
		return false
	}
	gate.mu.Lock()
	defer gate.mu.Unlock()
	dirty := gate.dirty
	gate.dirty = false
	return dirty
}

type renderInputView struct {
	generation int64
	snapshots  []store.ModuleSnapshotUse
}

func (w *Worker) captureRenderInputs(ctx context.Context, tenantID string) (renderInputView, error) {
	if w == nil || w.Store == nil {
		return renderInputView{}, fmt.Errorf("jobs: render store unavailable")
	}
	gen, err := w.Store.GetRenderGeneration(ctx, tenantID)
	if err != nil {
		if !errors.Is(err, store.ErrNotFound) {
			return renderInputView{}, fmt.Errorf("jobs: read render generation: %w", err)
		}
		gen = 0
	}
	headers, err := w.Store.ListModuleSnapshotHeaders(ctx, tenantID)
	if err != nil {
		return renderInputView{}, fmt.Errorf("jobs: read snapshot headers: %w", err)
	}
	return renderInputView{generation: gen, snapshots: headers}, nil
}

func renderInputsChanged(before, after renderInputView) bool {
	if before.generation != after.generation {
		return true
	}
	return !sameSnapshotHeaders(before.snapshots, after.snapshots)
}

func sameSnapshotHeaders(a, b []store.ModuleSnapshotUse) bool {
	if len(a) != len(b) {
		return false
	}
	aa := append([]store.ModuleSnapshotUse(nil), a...)
	bb := append([]store.ModuleSnapshotUse(nil), b...)
	sort.Slice(aa, func(i, j int) bool { return aa[i].ModuleID < aa[j].ModuleID })
	sort.Slice(bb, func(i, j int) bool { return bb[i].ModuleID < bb[j].ModuleID })
	for i := range aa {
		if aa[i].ModuleID != bb[i].ModuleID || aa[i].SnapshotVersion != bb[i].SnapshotVersion || aa[i].InputHash != bb[i].InputHash {
			return false
		}
	}
	return true
}

// enqueueDeployAllSpeakers queues the same work as POST /v1/apply (all speakers, no speaker_id).
func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) error {
	if w == nil || w.Registry == nil {
		return fmt.Errorf("jobs: deploy queue unavailable")
	}
	revID = strings.TrimSpace(revID)
	if revID == "" {
		return fmt.Errorf("jobs: missing revision for deploy")
	}
	meta := map[string]any{"revision_id": revID}
	if tr := strings.TrimSpace(j.metaString("trigger")); tr != "" {
		meta["trigger"] = tr
	}
	key := "deploy_from:" + j.ID
	applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, &key, nil, meta)
	if err != nil {
		j.mergeMeta(map[string]any{"deploy_apply_enqueue_error": err.Error()})
		return fmt.Errorf("jobs: enqueue deploy: %w", err)
	}
	if applyJob != nil {
		j.mergeMeta(map[string]any{"deploy_apply_job_id": applyJob.ID})
	}
	return nil
}

func (w *Worker) runDeployApply(j *Job) {
	revID := strings.TrimSpace(j.metaString("revision_id"))
	spk := strings.TrimSpace(j.metaString("speaker_id"))
	if revID == "" {
		j.Fail("missing revision_id in job meta")
		return
	}
	if j.OwnerID == "" {
		if _, developmentMemory := w.Store.(*store.Memory); !developmentMemory {
			j.Fail("deployment requires a current durable job lease")
			return
		}
	}

	ctx, cancel := j.workContext()
	defer cancel()
	cancelled := func() bool { return ctx.Err() != nil || j.IsCancelRequested() }
	var speakers []*store.Speaker
	if spk != "" {
		sp, err := w.Store.GetSpeaker(j.TenantID, spk)
		if err != nil {
			j.Fail(err.Error())
			return
		}
		speakers = []*store.Speaker{sp}
	} else {
		speakers = w.Store.ListSpeakersForTenant(j.TenantID)
	}
	activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR"))
	localAttempted := false
	var localErr error
	applyLocal := func() error {
		if localAttempted {
			return localErr
		}
		localAttempted = true
		revObj, err := w.Store.GetRevision(j.TenantID, revID)
		if err != nil {
			localErr = err
			return err
		}
		staging := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_STAGING_DIR"))
		if staging == "" {
			staging = os.TempDir() + "/evobgp-bird-staging"
		}
		cfg := birddeploy.Config{
			ActiveDir:  activeDir,
			StagingDir: staging,
			BirdBin:    strings.TrimSpace(os.Getenv("EVOBGP_BIRD_BIN")),
			BirdcBin:   strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")),
			Socket:     strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")),
		}
		ctl := &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
		if err := birddeploy.ApplyRevision(ctx, ctl, revObj, cfg); err != nil {
			localErr = err
			return err
		}
		return nil
	}
	tally := &deployTally{applied: make([]string, 0, len(speakers))}
	slots := make(chan struct{}, speakerWakeLimit)
	var wakeGroup sync.WaitGroup
	acquireWake := func() bool {
		select {
		case slots <- struct{}{}:
		case <-ctx.Done():
			return false
		}
		if cancelled() {
			<-slots
			return false
		}
		return true
	}
	for _, sp := range speakers {
		if cancelled() {
			break
		}
		if sp == nil {
			continue
		}
		held := false
		if speakerRemoteWake(sp) {
			if !acquireWake() {
				break
			}
			held = true
		}
		staged := w.stageSpeakerDeploy(ctx, j, sp, revID, activeDir, cancelled, applyLocal)
		if staged.wake && !staged.stop {
			wakeGroup.Add(1)
			go func(sp *store.Speaker, deployment store.DeploymentSpeaker) {
				defer wakeGroup.Done()
				defer func() { <-slots }()
				wakeCtx, cancelWake := context.WithTimeout(ctx, 35*time.Second)
				res := nodedispatch.WakeSpeaker(wakeCtx, sp, nodedispatch.Options{RevisionID: revID, HTTPClient: w.NodeHTTPClient, MaxRetries: 1})
				cancelWake()
				out, include, failedN, unconfirmedN, appliedID := w.finishSpeakerWake(ctx, j, sp, revID, deployment, res, cancelled)
				tally.add(out, include, 0, failedN, unconfirmedN, appliedID)
			}(sp, staged.deployment)
		} else if held {
			<-slots
		}
		tally.add(staged.result, staged.hasResult, staged.published, staged.failed, staged.unconfirmed, staged.appliedID)
		if staged.stop {
			break
		}
	}
	wakeGroup.Wait()
	results, applied, published, failed, unconfirmed := tally.snapshot()
	j.mergeMeta(map[string]any{
		"apply_summary": map[string]any{
			"revision_id":       revID,
			"speakers_count":    len(speakers),
			"published_count":   published,
			"applied_count":     len(applied),
			"unconfirmed_count": unconfirmed,
			"failed_count":      failed,
			"speaker_ids":       applied,
			"message":           fmt.Sprintf("Ревизия %s: подтверждено %d, без подтверждения %d, ошибок %d", shortID(revID), len(applied), unconfirmed, failed),
		},
		"node_dispatch": map[string]any{"revision_id": revID, "results": results},
	})
	if cancelled() {
		j.MarkCancelled()
		return
	}
	mergeBirdPostApplyMeta(j)
	if cancelled() {
		j.MarkCancelled()
		return
	}
	if failed > 0 {
		j.Fail(fmt.Sprintf("deploy delivery failed for %d of %d speakers", failed, len(speakers)))
		return
	}
	j.Succeed()
}

type speakerStage struct {
	stop        bool
	wake        bool
	deployment  store.DeploymentSpeaker
	result      nodedispatch.Result
	hasResult   bool
	published   int
	failed      int
	unconfirmed int
	appliedID   string
}

type deployTally struct {
	mu          sync.Mutex
	results     []nodedispatch.Result
	applied     []string
	published   int
	failed      int
	unconfirmed int
}

func (t *deployTally) add(res nodedispatch.Result, include bool, published, failed, unconfirmed int, appliedID string) {
	t.mu.Lock()
	defer t.mu.Unlock()
	if include {
		t.results = append(t.results, res)
	}
	t.published += published
	t.failed += failed
	t.unconfirmed += unconfirmed
	if appliedID != "" {
		t.applied = append(t.applied, appliedID)
	}
}

func (t *deployTally) snapshot() (results []nodedispatch.Result, applied []string, published, failed, unconfirmed int) {
	t.mu.Lock()
	defer t.mu.Unlock()
	results = append([]nodedispatch.Result(nil), t.results...)
	applied = append([]string(nil), t.applied...)
	if applied == nil {
		applied = []string{}
	}
	sort.Slice(results, func(i, j int) bool { return results[i].SpeakerID < results[j].SpeakerID })
	sort.Strings(applied)
	return results, applied, t.published, t.failed, t.unconfirmed
}

func speakerRemoteWake(sp *store.Speaker) bool {
	if sp == nil || !nodedispatch.Enabled() {
		return false
	}
	return store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON))
}

func (w *Worker) bindDeploymentWriters(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker) (func(string, string, string) error, func(string) error) {
	fence := deploymentFenceForJob(j)
	record := func(state, confirmation, message string) error {
		if j.OwnerID == "" {
			return nil
		}
		return w.Store.RecordDeploymentSpeaker(ctx, fence, deployment, state, confirmation, message)
	}
	markApplied := func(confirmation string) error {
		if j.OwnerID == "" {
			return w.Store.SetLastAppliedRevision(j.TenantID, sp.ID, revID)
		}
		return record("applied", confirmation, "")
	}
	return record, markApplied
}

// stageSpeakerDeploy publishes one speaker and records terminal states.
// Remote wakes are left to the caller so the speaker lock is not held across the network.
func (w *Worker) stageSpeakerDeploy(ctx context.Context, j *Job, sp *store.Speaker, revID, activeDir string, cancelled func() bool, applyLocal func() error) speakerStage {
	var staged speakerStage
	mu := w.speakerDispatchMu(sp.ID)
	mu.Lock()
	defer mu.Unlock()
	if cancelled() {
		staged.stop = true
		return staged
	}
	var deployment store.DeploymentSpeaker
	var publishErr error
	if j.OwnerID != "" {
		deployment, publishErr = beginDeployment(ctx, w.Store, deploymentFenceForJob(j), sp.ID, revID)
	} else {
		publishErr = w.Store.PublishRevisionForSpeaker(sp.ID, revID)
	}
	staged.deployment = deployment
	record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
	if publishErr != nil {
		staged.failed = 1
		staged.hasResult = true
		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: publishErr.Error()}
		return staged
	}
	staged.published = 1
	if deployment.State == "applied" {
		staged.appliedID = sp.ID
		staged.hasResult = true
		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: deployment.Confirmation, AppliedRevisionID: revID}
		return staged
	}
	if cancelled() {
		staged.stop = true
		return staged
	}
	if !speakerRemoteWake(sp) {
		if activeDir != "" && strings.EqualFold(sp.Role, "master") {
			err := applyLocal()
			if cancelled() {
				staged.stop = true
				return staged
			}
			if err != nil {
				_ = record("failed", "unconfirmed", err.Error())
				staged.failed = 1
				staged.hasResult = true
				staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
				return staged
			}
			if err := markApplied("local_applied"); err != nil {
				staged.failed = 1
				staged.hasResult = true
				staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
				return staged
			}
			staged.appliedID = sp.ID
			staged.hasResult = true
			staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: "local_applied", AppliedRevisionID: revID}
			return staged
		}
		if err := record("unconfirmed", "unconfirmed", "dispatch disabled or speaker has no remote agent"); err != nil {
			staged.failed = 1
		}
		staged.unconfirmed = 1
		staged.hasResult = true
		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "skipped", Confirmation: "unconfirmed"}
		return staged
	}
	if err := record("dispatched", "unconfirmed", "awaiting agent acknowledgement"); err != nil {
		staged.failed = 1
		staged.hasResult = true
		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
		return staged
	}
	if deployment.DispatchAttempts >= 3 {
		_ = record("unknown", "unconfirmed", "delivery attempt budget exhausted")
		staged.failed = 1
		staged.hasResult = true
		staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Confirmation: "unconfirmed", Error: "delivery attempt budget exhausted"}
		return staged
	}
	staged.wake = true
	return staged
}

// finishSpeakerWake records the agent acknowledgement. The speaker lock covers the
// state update and is not held while WakeSpeaker is in flight.
func (w *Worker) finishSpeakerWake(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker, res nodedispatch.Result, cancelled func() bool) (nodedispatch.Result, bool, int, int, string) {
	mu := w.speakerDispatchMu(sp.ID)
	mu.Lock()
	defer mu.Unlock()
	if cancelled != nil && cancelled() {
		return nodedispatch.Result{SpeakerID: sp.ID, Endpoint: res.Endpoint, Status: "cancelled", Confirmation: "unconfirmed"}, true, 0, 0, ""
	}
	record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
	appliedID := ""
	if res.Status == "ok" && res.AppliedRevisionID != "" {
		if res.AppliedRevisionID != revID {
			res.Status = "error"
			res.Confirmation = "unconfirmed"
			res.Error = "agent reported a different applied revision"
		} else if latest, _, err := w.Store.LatestPublishedRevision(sp.ID); err != nil {
			res.Status = "error"
			res.Confirmation = "unconfirmed"
			res.Error = err.Error()
		} else if latest != revID {
			res.Status = "error"
			res.Confirmation = "unconfirmed"
			res.Error = "stale agent acknowledgement: desired revision changed"
		} else if err := markApplied("agent_reported"); err != nil {
			res.Status = "error"
			res.Confirmation = "unconfirmed"
			res.Error = err.Error()
		} else {
			appliedID = sp.ID
		}
	}
	if res.Status != "ok" || res.Confirmation != "agent_reported" {
		state := "unconfirmed"
		if res.Status != "ok" {
			state = "unknown"
		}
		if err := record(state, "unconfirmed", res.Error); err != nil {
			res.Status = "error"
			res.Error = err.Error()
		}
	}
	patch := store.SpeakerMeta{
		LastDispatchAt:     time.Now().UTC().Format(time.RFC3339Nano),
		LastDispatchStatus: res.Status,
	}
	if res.Error != "" {
		patch.LastDispatchError = res.Error
		patch.SyncStatus = "error"
	} else if res.Status == "ok" && res.Confirmation == "agent_reported" {
		patch.LastDispatchError = ""
		patch.SyncStatus = "synced"
	} else if res.Status == "ok" {
		patch.SyncStatus = "unconfirmed"
	}
	merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch)
	if j.OwnerID == "" {
		if _, err := w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged}); err != nil {
			res.Status = "error"
			res.Error = "dispatch metadata update: " + err.Error()
		}
	}
	failed := 0
	unconfirmed := 0
	if res.Status == "ok" && res.Confirmation == "unconfirmed" {
		unconfirmed = 1
	} else if res.Status != "ok" {
		failed = 1
	}
	return res, true, failed, unconfirmed, appliedID
}

func (w *Worker) runRollback(j *Job) {
	src := strings.TrimSpace(j.metaString("source_revision_id"))
	if src == "" {
		j.Fail("missing source_revision_id in job meta")
		return
	}
	newID, err := w.Store.CreateRollbackRevision(j.TenantID, src)
	if err != nil {
		j.Fail(err.Error())
		return
	}
	j.mergeMeta(map[string]any{"new_revision_id": newID})
	j.mergeMeta(map[string]any{
		"rollback_summary": map[string]any{
			"source_revision_id": src,
			"new_revision_id":    newID,
			"message":            fmt.Sprintf("Откат %s → %s", shortID(src), shortID(newID)),
		},
	})
	if err := w.enqueueDeployAllSpeakers(j, j.TenantID, newID); err != nil {
		j.Fail(err.Error())
		return
	}
	j.Succeed()
}

// buildCommunityLabelMap maps community UUID -> human-readable title (or BGP community string).
func buildCommunityLabelMap(st store.Backend, tenantID string) map[string]string {
	out := make(map[string]string)
	if st == nil {
		return out
	}
	list, err := st.ListCommunities(tenantID)
	if err != nil || list == nil {
		return out
	}
	for _, c := range list {
		if c == nil {
			continue
		}
		label := strings.TrimSpace(c.Title)
		if label == "" {
			label = strings.TrimSpace(c.Community)
		}
		if label == "" {
			label = c.ID
		}
		out[c.ID] = label
	}
	return out
}

func resolveCommunityLabel(commID string, byID map[string]string) string {
	if commID == "" || commID == "none" {
		return "без community"
	}
	if lbl, ok := byID[commID]; ok && strings.TrimSpace(lbl) != "" {
		return strings.TrimSpace(lbl)
	}
	return commID
}

func (w *Worker) buildRevisionLogEntries(tenantID, revID string) ([]map[string]any, int, error) {
	if w == nil || w.Store == nil {
		return nil, 0, fmt.Errorf("store not configured")
	}
	commLabels := buildCommunityLabelMap(w.Store, tenantID)
	type agg struct {
		kind      string
		source    string
		community string
		count     int
		sample    []string
	}
	groups := map[string]*agg{}
	total := 0
	cursor := ""
	for {
		rows, next, more := w.Store.ListRevisionPrefixes(tenantID, revID, cursor, 1000)
		for _, p := range rows {
			total++
			src := strings.TrimSpace(p.Source)
			comm := "none"
			if p.CommunityID != nil && strings.TrimSpace(*p.CommunityID) != "" {
				comm = strings.TrimSpace(*p.CommunityID)
			}
			kind, sourceName := classifySource(src)
			k := kind + "|" + sourceName + "|" + comm
			g, ok := groups[k]
			if !ok {
				g = &agg{kind: kind, source: sourceName, community: comm}
				groups[k] = g
			}
			g.count++
			if len(g.sample) < 5 {
				g.sample = append(g.sample, p.Prefix)
			}
		}
		if !more {
			break
		}
		cursor = next
		if strings.TrimSpace(cursor) == "" {
			break
		}
	}
	keys := make([]string, 0, len(groups))
	for k := range groups {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	out := make([]map[string]any, 0, len(keys))
	for _, k := range keys {
		g := groups[k]
		cl := resolveCommunityLabel(g.community, commLabels)
		msg := humanLogMessage(g.kind, g.source, g.count, cl, g.sample)
		out = append(out, map[string]any{
			"kind":            g.kind,
			"source":          g.source,
			"community":       g.community,
			"community_label": cl,
			"prefix_count":    g.count,
			"sample":          g.sample,
			"message":         msg,
		})
	}
	return out, total, nil
}

func classifySource(src string) (kind, name string) {
	switch {
	case strings.HasPrefix(src, "as:"):
		return "asn", strings.TrimPrefix(src, "as:")
	case strings.HasPrefix(src, "domain:"):
		return "domain", strings.TrimPrefix(src, "domain:")
	case strings.HasPrefix(src, "cdn:"):
		return "cdn", strings.TrimPrefix(src, "cdn:")
	case src == "ip_range":
		return "ip_range", "manual_ranges"
	default:
		if src == "" {
			return "unknown", "unknown"
		}
		return "source", src
	}
}

// humanLogMessage builds a Russian log line; communityLabel is already resolved (title or BGP value).
func humanLogMessage(kind, source string, count int, communityLabel string, sample []string) string {
	switch kind {
	case "asn":
		return fmt.Sprintf("AS%s: добавлено %d префиксов в сообщество «%s»", source, count, communityLabel)
	case "domain":
		ips := strings.Join(prettyDomainSample(sample), " ")
		if ips == "" {
			ips = "—"
		}
		return fmt.Sprintf("%s: IP (%s) → добавлено в сообщество «%s»", source, ips, communityLabel)
	case "cdn":
		return fmt.Sprintf("CDN «%s»: добавлено %d префиксов в сообщество «%s»", source, count, communityLabel)
	case "ip_range":
		return fmt.Sprintf("Статические диапазоны: добавлено %d префиксов в сообщество «%s»", count, communityLabel)
	default:
		return fmt.Sprintf("%s: добавлено %d префиксов в сообщество «%s»", source, count, communityLabel)
	}
}

func prettyDomainSample(sample []string) []string {
	out := make([]string, 0, len(sample))
	for _, s := range sample {
		p, err := netip.ParsePrefix(strings.TrimSpace(s))
		if err != nil {
			out = append(out, s)
			continue
		}
		if (p.Addr().Is4() && p.Bits() == 32) || (p.Addr().Is6() && p.Bits() == 128) {
			out = append(out, p.Addr().String())
			continue
		}
		out = append(out, s)
	}
	return out
}

func shortID(id string) string {
	s := strings.TrimSpace(id)
	if len(s) <= 8 {
		return s
	}
	return s[:8]
}