feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -0,0 +1,135 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestDeploymentLeaseGenerationAndRestartIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
ctx := context.Background()
|
||||
speaker, revision, newer := uuid.NewString(), uuid.NewString(), uuid.NewString()
|
||||
for _, id := range []string{revision, newer} {
|
||||
if _, err := p.pool.Exec(ctx, `INSERT INTO config_revision(id,tenant_id,content_hash) VALUES($1,$2,'deployment-test')`, id, tenant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := p.pool.Exec(ctx, `INSERT INTO bgp_speaker(id,tenant_id,role) VALUES($1,$2,'replica')`, speaker, tenant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
enqueue := func() store.JobRecord {
|
||||
t.Helper()
|
||||
j, _, err := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "deploy_apply"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return j
|
||||
}
|
||||
claim := func(owner string) store.JobRecord {
|
||||
t.Helper()
|
||||
rows, err := p.ClaimJobs(ctx, owner, 1, time.Minute)
|
||||
if err != nil || len(rows) != 1 {
|
||||
t.Fatalf("claim: %+v %v", rows, err)
|
||||
}
|
||||
return rows[0]
|
||||
}
|
||||
fence := func(j store.JobRecord) store.JobFence {
|
||||
return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken}
|
||||
}
|
||||
first := enqueue()
|
||||
first = claim("deployment-one")
|
||||
d, err := p.BeginDeploymentSpeaker(ctx, fence(first), speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := enqueue()
|
||||
second = claim("deployment-two")
|
||||
if _, err = p.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer); !errors.Is(err, store.ErrDeploymentBusy) {
|
||||
t.Fatalf("overlapping network ownership: %v", err)
|
||||
}
|
||||
if err = p.RecordDeploymentSpeaker(ctx, fence(first), d, "unknown", "unconfirmed", "response lost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = p.pool.Exec(ctx, `UPDATE job_audit SET lease_until=clock_timestamp()-interval '1 second' WHERE id=$1`, first.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = p.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "agent_reported", ""); !errors.Is(err, store.ErrJobLeaseLost) {
|
||||
t.Fatalf("expired ACK accepted: %v", err)
|
||||
}
|
||||
reclaimed := claim("deployment-reclaimed")
|
||||
if reclaimed.ID != first.ID || reclaimed.FencingToken <= first.FencingToken {
|
||||
t.Fatal("job not reclaimed with fresh token")
|
||||
}
|
||||
if _, err = p.BeginDeploymentSpeaker(ctx, fence(reclaimed), speaker, revision); !errors.Is(err, store.ErrDeploymentBusy) {
|
||||
t.Fatalf("reclaim overlaps uncertain request: %v", err)
|
||||
}
|
||||
if _, err = p.pool.Exec(ctx, `UPDATE bgp_speaker SET delivery_until=clock_timestamp()-interval '1 second' WHERE id=$1`, speaker); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A fresh repository object sees the committed partial progress after restart.
|
||||
restarted := &Postgres{pool: p.pool}
|
||||
rows, err := restarted.ListDeploymentSpeakers(ctx, tenant, first.ID)
|
||||
if err != nil || len(rows) != 1 || rows[0].State != "unknown" {
|
||||
t.Fatalf("restart progress: %+v %v", rows, err)
|
||||
}
|
||||
d2, err := restarted.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d2.Generation <= d.Generation {
|
||||
t.Fatal("speaker generation did not advance")
|
||||
}
|
||||
if err = p.RecordDeploymentSpeaker(ctx, fence(reclaimed), d, "applied", "agent_reported", ""); !errors.Is(err, store.ErrJobLeaseLost) {
|
||||
t.Fatalf("late old-generation ACK accepted: %v", err)
|
||||
}
|
||||
if err = p.RecordDeploymentSpeaker(ctx, fence(second), d2, "applied", "agent_reported", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var published, applied string
|
||||
if err = p.pool.QueryRow(ctx, `SELECT published_revision_id::text,last_applied_revision_id::text FROM bgp_speaker WHERE id=$1`, speaker).Scan(&published, &applied); err != nil || published != newer || applied != newer {
|
||||
t.Fatalf("fenced pointers: %s %s %v", published, applied, err)
|
||||
}
|
||||
d2Replay, err := p.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer)
|
||||
if err != nil || d2Replay.State != "applied" || d2Replay.Generation != d2.Generation {
|
||||
t.Fatalf("applied replay: %+v %v", d2Replay, err)
|
||||
}
|
||||
second.Status = "succeeded"
|
||||
if err = p.FinishJob(ctx, second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldPoll, err := p.ListDeploymentObservations(ctx, 10)
|
||||
if err != nil || len(oldPoll) != 1 {
|
||||
t.Fatalf("old poll %+v %v", oldPoll, err)
|
||||
}
|
||||
newPoll, err := p.ListDeploymentObservations(ctx, 10)
|
||||
if err != nil || len(newPoll) != 1 {
|
||||
t.Fatalf("new poll %+v %v", newPoll, err)
|
||||
}
|
||||
// A stale snapshot must not downgrade the current acknowledged state when
|
||||
// the latest poll times out or the old agent omits revision fields.
|
||||
newPoll[0].State = "unknown"
|
||||
newPoll[0].Confirmation = "unconfirmed"
|
||||
if err = p.ObserveDeployment(ctx, newPoll[0], store.DeploymentObservation{Error: "timeout"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = p.ObserveDeployment(ctx, oldPoll[0], store.DeploymentObservation{RevisionID: revision}); !errors.Is(err, store.ErrJobLeaseLost) {
|
||||
t.Fatalf("late health accepted: %v", err)
|
||||
}
|
||||
progress, err := p.ListDeploymentSpeakers(ctx, tenant, second.ID)
|
||||
if err != nil || len(progress) != 1 || progress[0].State != "applied" || progress[0].Confirmation != "agent_reported" || progress[0].HealthError != "timeout" {
|
||||
t.Fatalf("health lost ACK %+v %v", progress, err)
|
||||
}
|
||||
// Historical deployment records must survive ordinary revision retention.
|
||||
if _, err = p.pool.Exec(ctx, `DELETE FROM config_revision WHERE id=$1::uuid`, revision); err != nil {
|
||||
t.Fatalf("history blocks revision pruning: %v", err)
|
||||
}
|
||||
history, err := p.ListDeploymentSpeakers(ctx, tenant, first.ID)
|
||||
if err != nil || len(history) != 1 || history[0].RevisionID != revision {
|
||||
t.Fatalf("pruning removed history %+v %v", history, err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user