feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -0,0 +1,260 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"evobgp/internal/db"
|
||||
"evobgp/internal/store"
|
||||
"evobgp/migrations"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestJobQueueFreshSeedIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("EVOBGP_TEST_DATABASE_URL not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
admin, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer admin.Close()
|
||||
schema := "queue_seed_" + uuid.NewString()
|
||||
quoted := `"` + schema + `"`
|
||||
if _, err = admin.Exec(ctx, `CREATE SCHEMA `+quoted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if _, e := admin.Exec(ctx, `DROP SCHEMA `+quoted+` CASCADE`); e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
}()
|
||||
cfg, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.ConnConfig.RuntimeParams["search_path"] = schema
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err = db.ApplyPostgresMigrations(ctx, pool, migrations.Postgres, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := NewPostgres(ctx, pool, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tenant, cdn, ip, revision, speaker := p.DemoIDs()
|
||||
if tenant == "" || tenant == store.SystemJobTenantID || cdn == "" || ip == "" || revision == "" || speaker == "" {
|
||||
t.Fatalf("system tenant blocked demo seed: %q %q %q %q %q", tenant, cdn, ip, revision, speaker)
|
||||
}
|
||||
}
|
||||
|
||||
func queueIntegration(t *testing.T) (*Postgres, string) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("EVOBGP_TEST_DATABASE_URL not set")
|
||||
}
|
||||
pool, e := db.OpenPostgresPool(context.Background(), dsn)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
tenant := uuid.NewString()
|
||||
if _, e = pool.Exec(context.Background(), `INSERT INTO tenant(id,name,slug) VALUES($1,'queue integration',$2)`, tenant, "queue-"+tenant); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = pool.Exec(context.Background(), `DELETE FROM tenant WHERE id=$1`, tenant) })
|
||||
return &Postgres{pool: pool}, tenant
|
||||
}
|
||||
|
||||
func TestJobAuditClaimAndRestartIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
ctx := context.Background()
|
||||
j, _, e := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan []store.JobRecord, 2)
|
||||
errs := make(chan error, 2)
|
||||
for _, owner := range []string{"replica-a", "replica-b"} {
|
||||
wg.Add(1)
|
||||
go func(owner string) {
|
||||
defer wg.Done()
|
||||
items, e := p.ClaimJobs(ctx, owner, 1, time.Minute)
|
||||
results <- items
|
||||
errs <- e
|
||||
}(owner)
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
close(errs)
|
||||
for e := range errs {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
var first store.JobRecord
|
||||
count := 0
|
||||
for items := range results {
|
||||
for _, item := range items {
|
||||
if item.ID != j.ID {
|
||||
t.Fatalf("unexpected job %+v", item)
|
||||
}
|
||||
first = item
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("claims=%d", count)
|
||||
}
|
||||
if _, e = p.HeartbeatJob(ctx, j.ID, first.OwnerID, first.FencingToken, time.Hour); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if items, e := p.ClaimJobs(ctx, "other", 1, time.Minute); e != nil || len(items) != 0 {
|
||||
t.Fatalf("stole live lease: %+v %v", items, e)
|
||||
}
|
||||
if _, e = p.pool.Exec(ctx, `UPDATE job_audit SET lease_until=now()-interval '1 second' WHERE id=$1`, j.ID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
pool, e := db.OpenPostgresPool(ctx, os.Getenv("EVOBGP_TEST_DATABASE_URL"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer pool.Close()
|
||||
restarted := &Postgres{pool: pool}
|
||||
items, e := restarted.ClaimJobs(ctx, "restarted", 1, time.Minute)
|
||||
if e != nil || len(items) != 1 {
|
||||
t.Fatalf("restart claim: %+v %v", items, e)
|
||||
}
|
||||
if items[0].FencingToken <= first.FencingToken {
|
||||
t.Fatal("token did not advance")
|
||||
}
|
||||
first.Status = "succeeded"
|
||||
if e = p.FinishJob(ctx, first); !errors.Is(e, store.ErrJobLeaseLost) {
|
||||
t.Fatalf("stale finish: %v", e)
|
||||
}
|
||||
if _, e = p.CancelJob(ctx, tenant, j.ID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items[0].Status = "succeeded"
|
||||
if e = restarted.FinishJob(ctx, items[0]); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
got, e := p.GetJob(ctx, tenant, j.ID)
|
||||
if e != nil || got.Status != "cancelled" {
|
||||
t.Fatalf("cancel/finish race: %+v %v", got, e)
|
||||
}
|
||||
listed, _, _, e := restarted.ListJobs(ctx, tenant, "cancelled", "", "", 10)
|
||||
if e != nil || len(listed) != 1 {
|
||||
t.Fatalf("shared terminal list %+v %v", listed, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobAuditIdempotencyRowIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
ctx := context.Background()
|
||||
key := uuid.NewString()
|
||||
in := store.JobEnqueue{TenantID: tenant, Kind: "test", IdempotencyKey: &key, Meta: map[string]any{"input": "same"}}
|
||||
var wg sync.WaitGroup
|
||||
ids := make(chan string, 8)
|
||||
errs := make(chan error, 8)
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); j, _, e := p.EnqueueJob(ctx, in); ids <- j.ID; errs <- e }()
|
||||
}
|
||||
wg.Wait()
|
||||
close(ids)
|
||||
close(errs)
|
||||
for e := range errs {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
id := ""
|
||||
for got := range ids {
|
||||
if id == "" {
|
||||
id = got
|
||||
}
|
||||
if got != id {
|
||||
t.Fatalf("different IDs %s / %s", id, got)
|
||||
}
|
||||
}
|
||||
items, e := p.ClaimJobs(ctx, "replica", 1, time.Minute)
|
||||
if e != nil || len(items) != 1 {
|
||||
t.Fatalf("claim %+v %v", items, e)
|
||||
}
|
||||
items[0].Status = "succeeded"
|
||||
items[0].Meta = map[string]any{"result": "done"}
|
||||
if e = p.FinishJob(ctx, items[0]); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
got, created, e := p.EnqueueJob(ctx, in)
|
||||
if e != nil || created || got.ID != id || got.Status != "succeeded" {
|
||||
t.Fatalf("terminal replay %+v %v %v", got, created, e)
|
||||
}
|
||||
in.Meta = map[string]any{"input": "changed"}
|
||||
if _, _, e = p.EnqueueJob(ctx, in); !errors.Is(e, store.ErrJobConflict) {
|
||||
t.Fatalf("changed request accepted: %v", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobQueueRetryBoundsAndFailureIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
ctx := context.Background()
|
||||
t.Setenv("EVOBGP_JOB_MAX_TENANT_PENDING", "1")
|
||||
j, _, e := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, _, e = p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"}); !errors.Is(e, store.ErrJobQueueFull) {
|
||||
t.Fatalf("unbounded queue: %v", e)
|
||||
}
|
||||
if _, e = p.pool.Exec(ctx, `UPDATE job_audit SET attempt=3 WHERE id=$1`, j.ID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = p.ReapExpiredJobs(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items, e := p.ClaimJobs(ctx, "worker", 1, time.Minute)
|
||||
if e != nil || len(items) != 0 {
|
||||
t.Fatalf("exhausted claim %+v %v", items, e)
|
||||
}
|
||||
got, e := p.GetJob(ctx, tenant, j.ID)
|
||||
if e != nil || got.Status != "failed" {
|
||||
t.Fatalf("exhausted status %+v %v", got, e)
|
||||
}
|
||||
if _, e = p.RetryJob(ctx, tenant, j.ID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items, e = p.ClaimJobs(ctx, "worker", 1, time.Minute)
|
||||
if e != nil || len(items) != 1 || items[0].Attempt != 1 {
|
||||
t.Fatalf("operator retry %+v %v", items, e)
|
||||
}
|
||||
if e = p.ReleaseJob(ctx, j.ID, items[0].OwnerID, items[0].FencingToken); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items, e = p.ClaimJobs(ctx, "worker-2", 1, time.Minute)
|
||||
if e != nil || len(items) != 1 || items[0].Attempt != 1 {
|
||||
t.Fatalf("shutdown consumed retry %+v %v", items, e)
|
||||
}
|
||||
failedPool, e := db.OpenPostgresPool(ctx, os.Getenv("EVOBGP_TEST_DATABASE_URL"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
failedPool.Close()
|
||||
broken := &Postgres{pool: failedPool}
|
||||
if accepted, created, e := broken.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"}); e == nil || created || accepted.ID != "" {
|
||||
t.Fatalf("accepted on failed DB: %+v %v %v", accepted, created, e)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user