package jobs import ( "context" "errors" "sync/atomic" "testing" "time" "evobgp/internal/store" ) type blockingClaimQueue struct { store.JobQueue entered chan int resume chan struct{} } func (q *blockingClaimQueue) ClaimJobs(ctx context.Context, owner string, limit int, lease time.Duration) ([]store.JobRecord, error) { q.entered <- limit <-q.resume return q.JobQueue.ClaimJobs(context.Background(), owner, limit, lease) } func TestDurableShutdownWaitsInflightClaimAndReleases(t *testing.T) { t.Setenv("EVOBGP_JOB_MAX_CONCURRENT", "1") m := store.NewMemory() queued, _, err := m.EnqueueJob(context.Background(), store.JobEnqueue{TenantID: "tenant", Kind: "test"}) if err != nil { t.Fatal(err) } q := &blockingClaimQueue{JobQueue: m, entered: make(chan int, 1), resume: make(chan struct{})} var runs atomic.Int32 r := NewRegistry(func(j *Job) { runs.Add(1); j.Succeed() }) r.UseDurableQueue(q) StartDurableQueueLoop(context.Background(), r) select { case limit := <-q.entered: if limit != cap(r.workerSem) { t.Fatalf("claim capacity=%d reserved=%d", limit, cap(r.workerSem)) } case <-time.After(3 * time.Second): t.Fatal("no claim") } r.RequestCancelAll() drained := make(chan error, 1) go func() { drained <- r.Drain(context.Background()) }() select { case <-drained: t.Fatal("drained before in-flight claim returned") case <-time.After(20 * time.Millisecond): } close(q.resume) select { case err := <-drained: if err != nil { t.Fatal(err) } case <-time.After(3 * time.Second): t.Fatal("shutdown did not drain") } if runs.Load() != 0 { t.Fatal("started worker after stop") } got, err := m.GetJob(context.Background(), "tenant", queued.ID) if err != nil || got.Status != "queued" || got.OwnerID != "" || got.Attempt != 0 { t.Fatalf("not released: %+v %v", got, err) } } func TestDurableSharedTerminalAndRequestReplay(t *testing.T) { ctx := context.Background() m := store.NewMemory() key := "request" a := NewRegistry(nil) a.UseDurableQueue(m) j, _, err := a.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}}) if err != nil { t.Fatal(err) } claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute) if err != nil || len(claimed) != 1 { t.Fatalf("claim %+v %v", claimed, err) } claimed[0].Status = StatusSucceeded claimed[0].Meta = map[string]any{"result": "ok"} if err = m.FinishJob(ctx, claimed[0]); err != nil { t.Fatal(err) } b := NewRegistry(nil) b.UseDurableQueue(m) got, err := b.GetContext(ctx, "tenant", j.ID) if err != nil || got.Status != StatusSucceeded { t.Fatalf("shared get %+v %v", got, err) } list, _, _, err := b.ListContext(ctx, "tenant", StatusSucceeded, "", "", 10) if err != nil || len(list) != 1 { t.Fatalf("shared list %+v %v", list, err) } replay, created, err := b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}}) if err != nil || created || replay.ID != j.ID || replay.Status != StatusSucceeded { t.Fatalf("terminal replay %+v %v %v", replay, created, err) } if _, _, err = b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"input": "different"}); !errors.Is(err, store.ErrJobConflict) { t.Fatalf("changed payload accepted: %v", err) } } func TestParentRetryRecoversCommittedDeployChild(t *testing.T) { ctx := context.Background() m := store.NewMemory() r := NewRegistry(nil) r.UseDurableQueue(m) parent, _, err := r.EnqueueContext(ctx, "tenant", KindRevisionRollback, nil, nil, map[string]any{"revision_id": "original"}) if err != nil { t.Fatal(err) } key := "deploy_from:" + parent.ID child, _, err := r.EnqueueContext(ctx, "tenant", KindDeployApply, &key, nil, map[string]any{"revision_id": "already-rendered"}) if err != nil { t.Fatal(err) } worker := &Worker{Store: m, Registry: r} worker.Process(parent) if parent.statusLocked() != StatusSucceeded || parent.metaString("revision_id") != "already-rendered" || parent.metaString("deploy_apply_job_id") != child.ID { t.Fatalf("did not recover child: %+v", parent.Snapshot()) } items, _, _, err := m.ListJobs(ctx, "tenant", "", KindDeployApply, "", 10) if err != nil || len(items) != 1 { t.Fatalf("duplicate child: %+v %v", items, err) } }