EvoBGP/internal/jobs/durable_queue_test.go

package jobs

import (
	"context"
	"errors"
	"sync/atomic"
	"testing"
	"time"

	"evobgp/internal/store"
)

type blockingClaimQueue struct {
	store.JobQueue
	entered chan int
	resume  chan struct{}
}

func (q *blockingClaimQueue) ClaimJobs(ctx context.Context, owner string, limit int, lease time.Duration) ([]store.JobRecord, error) {
	q.entered <- limit
	<-q.resume
	return q.JobQueue.ClaimJobs(context.Background(), owner, limit, lease)
}

func TestDurableShutdownWaitsInflightClaimAndReleases(t *testing.T) {
	t.Setenv("EVOBGP_JOB_MAX_CONCURRENT", "1")
	m := store.NewMemory()
	queued, _, err := m.EnqueueJob(context.Background(), store.JobEnqueue{TenantID: "tenant", Kind: "test"})
	if err != nil {
		t.Fatal(err)
	}
	q := &blockingClaimQueue{JobQueue: m, entered: make(chan int, 1), resume: make(chan struct{})}
	var runs atomic.Int32
	r := NewRegistry(func(j *Job) { runs.Add(1); j.Succeed() })
	r.UseDurableQueue(q)
	StartDurableQueueLoop(context.Background(), r)
	select {
	case limit := <-q.entered:
		if limit != cap(r.workerSem) {
			t.Fatalf("claim capacity=%d reserved=%d", limit, cap(r.workerSem))
		}
	case <-time.After(3 * time.Second):
		t.Fatal("no claim")
	}
	r.RequestCancelAll()
	drained := make(chan error, 1)
	go func() { drained <- r.Drain(context.Background()) }()
	select {
	case <-drained:
		t.Fatal("drained before in-flight claim returned")
	case <-time.After(20 * time.Millisecond):
	}
	close(q.resume)
	select {
	case err := <-drained:
		if err != nil {
			t.Fatal(err)
		}
	case <-time.After(3 * time.Second):
		t.Fatal("shutdown did not drain")
	}
	if runs.Load() != 0 {
		t.Fatal("started worker after stop")
	}
	got, err := m.GetJob(context.Background(), "tenant", queued.ID)
	if err != nil || got.Status != "queued" || got.OwnerID != "" || got.Attempt != 0 {
		t.Fatalf("not released: %+v %v", got, err)
	}
}

func TestDurableSharedTerminalAndRequestReplay(t *testing.T) {
	ctx := context.Background()
	m := store.NewMemory()
	key := "request"
	a := NewRegistry(nil)
	a.UseDurableQueue(m)
	j, _, err := a.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
	if err != nil {
		t.Fatal(err)
	}
	claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
	if err != nil || len(claimed) != 1 {
		t.Fatalf("claim %+v %v", claimed, err)
	}
	claimed[0].Status = StatusSucceeded
	claimed[0].Meta = map[string]any{"result": "ok"}
	if err = m.FinishJob(ctx, claimed[0]); err != nil {
		t.Fatal(err)
	}
	b := NewRegistry(nil)
	b.UseDurableQueue(m)
	got, err := b.GetContext(ctx, "tenant", j.ID)
	if err != nil || got.Status != StatusSucceeded {
		t.Fatalf("shared get %+v %v", got, err)
	}
	list, _, _, err := b.ListContext(ctx, "tenant", StatusSucceeded, "", "", 10)
	if err != nil || len(list) != 1 {
		t.Fatalf("shared list %+v %v", list, err)
	}
	replay, created, err := b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
	if err != nil || created || replay.ID != j.ID || replay.Status != StatusSucceeded {
		t.Fatalf("terminal replay %+v %v %v", replay, created, err)
	}
	if _, _, err = b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"input": "different"}); !errors.Is(err, store.ErrJobConflict) {
		t.Fatalf("changed payload accepted: %v", err)
	}
}

func TestParentRetryRecoversCommittedDeployChild(t *testing.T) {
	ctx := context.Background()
	m := store.NewMemory()
	r := NewRegistry(nil)
	r.UseDurableQueue(m)
	parent, _, err := r.EnqueueContext(ctx, "tenant", KindRevisionRollback, nil, nil, map[string]any{"revision_id": "original"})
	if err != nil {
		t.Fatal(err)
	}
	key := "deploy_from:" + parent.ID
	child, _, err := r.EnqueueContext(ctx, "tenant", KindDeployApply, &key, nil, map[string]any{"revision_id": "already-rendered"})
	if err != nil {
		t.Fatal(err)
	}
	worker := &Worker{Store: m, Registry: r}
	worker.Process(parent)
	if parent.statusLocked() != StatusSucceeded || parent.metaString("revision_id") != "already-rendered" || parent.metaString("deploy_apply_job_id") != child.ID {
		t.Fatalf("did not recover child: %+v", parent.Snapshot())
	}
	items, _, _, err := m.ListJobs(ctx, "tenant", "", KindDeployApply, "", 10)
	if err != nil || len(items) != 1 {
		t.Fatalf("duplicate child: %+v %v", items, err)
	}
}