feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <cursoragent@cursor.com>

internal/jobs/durable_queue_test.go

@@ -0,0 +1,132 @@
+package jobs
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"evobgp/internal/store"
+)
+
+type blockingClaimQueue struct {
+	store.JobQueue
+	entered chan int
+	resume  chan struct{}
+}
+
+func (q *blockingClaimQueue) ClaimJobs(ctx context.Context, owner string, limit int, lease time.Duration) ([]store.JobRecord, error) {
+	q.entered <- limit
+	<-q.resume
+	return q.JobQueue.ClaimJobs(context.Background(), owner, limit, lease)
+}
+
+func TestDurableShutdownWaitsInflightClaimAndReleases(t *testing.T) {
+	t.Setenv("EVOBGP_JOB_MAX_CONCURRENT", "1")
+	m := store.NewMemory()
+	queued, _, err := m.EnqueueJob(context.Background(), store.JobEnqueue{TenantID: "tenant", Kind: "test"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	q := &blockingClaimQueue{JobQueue: m, entered: make(chan int, 1), resume: make(chan struct{})}
+	var runs atomic.Int32
+	r := NewRegistry(func(j *Job) { runs.Add(1); j.Succeed() })
+	r.UseDurableQueue(q)
+	StartDurableQueueLoop(context.Background(), r)
+	select {
+	case limit := <-q.entered:
+		if limit != cap(r.workerSem) {
+			t.Fatalf("claim capacity=%d reserved=%d", limit, cap(r.workerSem))
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("no claim")
+	}
+	r.RequestCancelAll()
+	drained := make(chan error, 1)
+	go func() { drained <- r.Drain(context.Background()) }()
+	select {
+	case <-drained:
+		t.Fatal("drained before in-flight claim returned")
+	case <-time.After(20 * time.Millisecond):
+	}
+	close(q.resume)
+	select {
+	case err := <-drained:
+		if err != nil {
+			t.Fatal(err)
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("shutdown did not drain")
+	}
+	if runs.Load() != 0 {
+		t.Fatal("started worker after stop")
+	}
+	got, err := m.GetJob(context.Background(), "tenant", queued.ID)
+	if err != nil || got.Status != "queued" || got.OwnerID != "" || got.Attempt != 0 {
+		t.Fatalf("not released: %+v %v", got, err)
+	}
+}
+
+func TestDurableSharedTerminalAndRequestReplay(t *testing.T) {
+	ctx := context.Background()
+	m := store.NewMemory()
+	key := "request"
+	a := NewRegistry(nil)
+	a.UseDurableQueue(m)
+	j, _, err := a.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
+	if err != nil || len(claimed) != 1 {
+		t.Fatalf("claim %+v %v", claimed, err)
+	}
+	claimed[0].Status = StatusSucceeded
+	claimed[0].Meta = map[string]any{"result": "ok"}
+	if err = m.FinishJob(ctx, claimed[0]); err != nil {
+		t.Fatal(err)
+	}
+	b := NewRegistry(nil)
+	b.UseDurableQueue(m)
+	got, err := b.GetContext(ctx, "tenant", j.ID)
+	if err != nil || got.Status != StatusSucceeded {
+		t.Fatalf("shared get %+v %v", got, err)
+	}
+	list, _, _, err := b.ListContext(ctx, "tenant", StatusSucceeded, "", "", 10)
+	if err != nil || len(list) != 1 {
+		t.Fatalf("shared list %+v %v", list, err)
+	}
+	replay, created, err := b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
+	if err != nil || created || replay.ID != j.ID || replay.Status != StatusSucceeded {
+		t.Fatalf("terminal replay %+v %v %v", replay, created, err)
+	}
+	if _, _, err = b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"input": "different"}); !errors.Is(err, store.ErrJobConflict) {
+		t.Fatalf("changed payload accepted: %v", err)
+	}
+}
+
+func TestParentRetryRecoversCommittedDeployChild(t *testing.T) {
+	ctx := context.Background()
+	m := store.NewMemory()
+	r := NewRegistry(nil)
+	r.UseDurableQueue(m)
+	parent, _, err := r.EnqueueContext(ctx, "tenant", KindRevisionRollback, nil, nil, map[string]any{"revision_id": "original"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	key := "deploy_from:" + parent.ID
+	child, _, err := r.EnqueueContext(ctx, "tenant", KindDeployApply, &key, nil, map[string]any{"revision_id": "already-rendered"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	worker := &Worker{Store: m, Registry: r}
+	worker.Process(parent)
+	if parent.statusLocked() != StatusSucceeded || parent.metaString("revision_id") != "already-rendered" || parent.metaString("deploy_apply_job_id") != child.ID {
+		t.Fatalf("did not recover child: %+v", parent.Snapshot())
+	}
+	items, _, _, err := m.ListJobs(ctx, "tenant", "", KindDeployApply, "", 10)
+	if err != nil || len(items) != 1 {
+		t.Fatalf("duplicate child: %+v %v", items, err)
+	}
+}