package repository import ( "context" "errors" "testing" "time" "evobgp/internal/store" "github.com/google/uuid" ) func TestDeploymentLeaseGenerationAndRestartIntegration(t *testing.T) { p, tenant := queueIntegration(t) ctx := context.Background() speaker, revision, newer := uuid.NewString(), uuid.NewString(), uuid.NewString() for _, id := range []string{revision, newer} { if _, err := p.pool.Exec(ctx, `INSERT INTO config_revision(id,tenant_id,content_hash) VALUES($1,$2,'deployment-test')`, id, tenant); err != nil { t.Fatal(err) } } if _, err := p.pool.Exec(ctx, `INSERT INTO bgp_speaker(id,tenant_id,role) VALUES($1,$2,'replica')`, speaker, tenant); err != nil { t.Fatal(err) } enqueue := func() store.JobRecord { t.Helper() j, _, err := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "deploy_apply"}) if err != nil { t.Fatal(err) } return j } claim := func(owner string) store.JobRecord { t.Helper() rows, err := p.ClaimJobs(ctx, owner, 1, time.Minute) if err != nil || len(rows) != 1 { t.Fatalf("claim: %+v %v", rows, err) } return rows[0] } fence := func(j store.JobRecord) store.JobFence { return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken} } first := enqueue() first = claim("deployment-one") d, err := p.BeginDeploymentSpeaker(ctx, fence(first), speaker, revision) if err != nil { t.Fatal(err) } second := enqueue() second = claim("deployment-two") if _, err = p.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer); !errors.Is(err, store.ErrDeploymentBusy) { t.Fatalf("overlapping network ownership: %v", err) } if err = p.RecordDeploymentSpeaker(ctx, fence(first), d, "unknown", "unconfirmed", "response lost"); err != nil { t.Fatal(err) } if _, err = p.pool.Exec(ctx, `UPDATE job_audit SET lease_until=clock_timestamp()-interval '1 second' WHERE id=$1`, first.ID); err != nil { t.Fatal(err) } if err = p.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "agent_reported", ""); !errors.Is(err, store.ErrJobLeaseLost) { t.Fatalf("expired ACK accepted: %v", err) } reclaimed := claim("deployment-reclaimed") if reclaimed.ID != first.ID || reclaimed.FencingToken <= first.FencingToken { t.Fatal("job not reclaimed with fresh token") } if _, err = p.BeginDeploymentSpeaker(ctx, fence(reclaimed), speaker, revision); !errors.Is(err, store.ErrDeploymentBusy) { t.Fatalf("reclaim overlaps uncertain request: %v", err) } if _, err = p.pool.Exec(ctx, `UPDATE bgp_speaker SET delivery_until=clock_timestamp()-interval '1 second' WHERE id=$1`, speaker); err != nil { t.Fatal(err) } // A fresh repository object sees the committed partial progress after restart. restarted := &Postgres{pool: p.pool} rows, err := restarted.ListDeploymentSpeakers(ctx, tenant, first.ID) if err != nil || len(rows) != 1 || rows[0].State != "unknown" { t.Fatalf("restart progress: %+v %v", rows, err) } d2, err := restarted.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer) if err != nil { t.Fatal(err) } if d2.Generation <= d.Generation { t.Fatal("speaker generation did not advance") } if err = p.RecordDeploymentSpeaker(ctx, fence(reclaimed), d, "applied", "agent_reported", ""); !errors.Is(err, store.ErrJobLeaseLost) { t.Fatalf("late old-generation ACK accepted: %v", err) } if err = p.RecordDeploymentSpeaker(ctx, fence(second), d2, "applied", "agent_reported", ""); err != nil { t.Fatal(err) } var published, applied string if err = p.pool.QueryRow(ctx, `SELECT published_revision_id::text,last_applied_revision_id::text FROM bgp_speaker WHERE id=$1`, speaker).Scan(&published, &applied); err != nil || published != newer || applied != newer { t.Fatalf("fenced pointers: %s %s %v", published, applied, err) } d2Replay, err := p.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer) if err != nil || d2Replay.State != "applied" || d2Replay.Generation != d2.Generation { t.Fatalf("applied replay: %+v %v", d2Replay, err) } second.Status = "succeeded" if err = p.FinishJob(ctx, second); err != nil { t.Fatal(err) } oldPoll, err := p.ListDeploymentObservations(ctx, 10) if err != nil || len(oldPoll) != 1 { t.Fatalf("old poll %+v %v", oldPoll, err) } newPoll, err := p.ListDeploymentObservations(ctx, 10) if err != nil || len(newPoll) != 1 { t.Fatalf("new poll %+v %v", newPoll, err) } // A stale snapshot must not downgrade the current acknowledged state when // the latest poll times out or the old agent omits revision fields. newPoll[0].State = "unknown" newPoll[0].Confirmation = "unconfirmed" if err = p.ObserveDeployment(ctx, newPoll[0], store.DeploymentObservation{Error: "timeout"}); err != nil { t.Fatal(err) } if err = p.ObserveDeployment(ctx, oldPoll[0], store.DeploymentObservation{RevisionID: revision}); !errors.Is(err, store.ErrJobLeaseLost) { t.Fatalf("late health accepted: %v", err) } progress, err := p.ListDeploymentSpeakers(ctx, tenant, second.ID) if err != nil || len(progress) != 1 || progress[0].State != "applied" || progress[0].Confirmation != "agent_reported" || progress[0].HealthError != "timeout" { t.Fatalf("health lost ACK %+v %v", progress, err) } // Historical deployment records must survive ordinary revision retention. if _, err = p.pool.Exec(ctx, `DELETE FROM config_revision WHERE id=$1::uuid`, revision); err != nil { t.Fatalf("history blocks revision pruning: %v", err) } history, err := p.ListDeploymentSpeakers(ctx, tenant, first.ID) if err != nil || len(history) != 1 || history[0].RevisionID != revision { t.Fatalf("pruning removed history %+v %v", history, err) } }