feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -8,6 +8,11 @@ import (
|
||||
|
||||
// Backend is the persistence abstraction for the control plane (memory, PostgreSQL, SQLite).
|
||||
type Backend interface {
|
||||
JobQueue
|
||||
Deployments
|
||||
TelemetryStore
|
||||
// ObserveStoreMetrics returns historical revision and peer inventory aggregates or an error.
|
||||
ObserveStoreMetrics(ctx context.Context) (max, sum, peers int, states map[string]int, err error)
|
||||
MaterializedPrefixStats() (max int, sum int)
|
||||
PeerCount() int
|
||||
PeerSessionCountsByState() map[string]int
|
||||
@@ -16,6 +21,14 @@ type Backend interface {
|
||||
|
||||
// ListTenantIDs returns distinct tenant identifiers (for background workers).
|
||||
ListTenantIDs() ([]string, error)
|
||||
// GetRenderGeneration returns the tenant's monotonic render-input barrier.
|
||||
GetRenderGeneration(ctx context.Context, tenantID string) (int64, error)
|
||||
// ListModuleSnapshotHeaders returns snapshot versions for the tenant, ordered by module id.
|
||||
// It does not load prefix rows.
|
||||
ListModuleSnapshotHeaders(ctx context.Context, tenantID string) ([]ModuleSnapshotUse, error)
|
||||
// LatestRenderManifest returns the manifest stored with the newest config revision.
|
||||
// ok is false when that revision has no manifest yet.
|
||||
LatestRenderManifest(ctx context.Context, tenantID string) (*RenderManifestRecord, bool, error)
|
||||
|
||||
// ListModules returns all modules for a tenant (control plane may paginate in httpapi).
|
||||
ListModules(tenantID string) []*Module
|
||||
@@ -91,11 +104,15 @@ type Backend interface {
|
||||
ListRevisionPrefixes(tenantID, revisionID string, cursor string, limit int) (prefixes []PrefixRow, next string, more bool)
|
||||
CreateRollbackRevision(tenantID, sourceRevisionID string) (newID string, err error)
|
||||
// CreateRenderRevision inserts a new config_revision (revID must be unique) with materialized prefixes and preview fragments.
|
||||
CreateRenderRevision(revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error
|
||||
// The same transaction records config_generation and snapshot versions in render_revision_manifest.
|
||||
// ctx cancellation aborts the write. prefix_snapshot is keyed by a hash of prefixes only.
|
||||
CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error
|
||||
RevisionDiff(tenantID, aID, bID string) (map[string]any, error)
|
||||
EstimateRevisionPrune(tenantID string, cutoff time.Time, retentionMinutes int) (RevisionPruneEstimate, error)
|
||||
PruneRevisionsWithStats(tenantID string, cutoff time.Time) (RevisionPruneResult, error)
|
||||
PruneRevisionsBefore(tenantID string, cutoff time.Time) (deleted int, err error)
|
||||
// PruneRevisionsBefore deletes revisions older than cutoff, keeping the newest
|
||||
// revision and any revision a speaker still references. ctx bounds the delete.
|
||||
PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (deleted int, err error)
|
||||
SetLastAppliedRevision(tenantID, speakerID, revisionID string) error
|
||||
PublishRevisionForSpeaker(speakerID, revisionID string) error
|
||||
LatestPublishedRevision(speakerID string) (revisionID string, publishedAt time.Time, err error)
|
||||
@@ -113,13 +130,16 @@ type Backend interface {
|
||||
TouchAPIKeyLastUsed(id string) error
|
||||
|
||||
// Module prefix snapshots cache last successful collect per module (pipeline ingest/render).
|
||||
GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error)
|
||||
SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error
|
||||
DeleteModulePrefixSnapshot(tenantID, moduleID string) error
|
||||
GetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error)
|
||||
// ListModuleSnapshotSourceRows returns persisted rows for one source key.
|
||||
// It does not load the rest of the module snapshot.
|
||||
ListModuleSnapshotSourceRows(ctx context.Context, tenantID, moduleID, source string) ([]PrefixRow, error)
|
||||
SetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID, inputHash string, prefixes []PrefixRow) error
|
||||
DeleteModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) error
|
||||
// UpdateModulePrefixSnapshot runs a pure mutation against the latest snapshot atomically.
|
||||
UpdateModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string, update ModuleSnapshotUpdate) error
|
||||
// SetModuleInputHash stores the ingest fingerprint on the module row (O(1) snapshot check).
|
||||
SetModuleInputHash(tenantID, moduleID, hash string) error
|
||||
// LockModuleSnapshot serializes read-modify-write of one module snapshot; unlock must be called.
|
||||
LockModuleSnapshot(tenantID, moduleID string) (unlock func())
|
||||
SetModuleInputHash(ctx context.Context, tenantID, moduleID, hash string) error
|
||||
|
||||
// ASNPrefixCache stores RIPEstat announced-prefixes per ASN (global TTL cache).
|
||||
GetASNPrefixCache(asn int64) (*ASNPrefixCacheEntry, bool, error)
|
||||
@@ -197,6 +217,7 @@ type DomainResolveCacheEntry struct {
|
||||
type ModulePrefixSnapshot struct {
|
||||
InputHash string
|
||||
CollectedAt time.Time
|
||||
Version int64
|
||||
Prefixes []PrefixRow
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrDeploymentBusy means a previous legacy request may still be executing.
|
||||
var ErrDeploymentBusy = errors.New("store: speaker delivery reservation is active")
|
||||
|
||||
// DeploymentDeliveryWindow retains delivery ownership after an ambiguous response.
|
||||
const DeploymentDeliveryWindow = 2 * time.Minute
|
||||
|
||||
// JobFence identifies the currently leased deployment attempt.
|
||||
type JobFence struct {
|
||||
JobID, TenantID, OwnerID string
|
||||
Token int64
|
||||
ExpectedGeneration int64
|
||||
RetryRound int
|
||||
}
|
||||
|
||||
// DeploymentSpeaker preserves delivery progress independently from job result metadata.
|
||||
// Generation is monotonic per speaker, including across different jobs.
|
||||
type DeploymentSpeaker struct {
|
||||
ObservationToken int64
|
||||
JobID, TenantID, SpeakerID, RevisionID string
|
||||
Generation int64
|
||||
RequestSequence int64
|
||||
State, Confirmation, Error string
|
||||
DispatchAttempts int
|
||||
RetryRound int
|
||||
HealthReceivedAt *time.Time
|
||||
HealthRevisionID, HealthSourceAt, HealthError string
|
||||
}
|
||||
|
||||
// DeploymentObservation is a compatible authenticated health report. SourceAt may
|
||||
// be absent on legacy agents; ReceivedAt is generated by the control-plane store.
|
||||
type DeploymentObservation struct{ RevisionID, SourceAt, Error string }
|
||||
|
||||
// Deployments atomically fences publication and acknowledgement against the job
|
||||
// lease and the speaker's current deployment generation.
|
||||
type Deployments interface {
|
||||
BeginDeploymentSpeaker(context.Context, JobFence, string, string) (DeploymentSpeaker, error)
|
||||
RecordDeploymentSpeaker(context.Context, JobFence, DeploymentSpeaker, string, string, string) error
|
||||
ListDeploymentSpeakers(context.Context, string, string) ([]DeploymentSpeaker, error)
|
||||
ListPendingDeployments(context.Context, int) ([]DeploymentSpeaker, error)
|
||||
ListDeploymentObservations(context.Context, int) ([]DeploymentSpeaker, error)
|
||||
ObserveDeployment(context.Context, DeploymentSpeaker, DeploymentObservation) error
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDeploymentLeaseGenerationAndReplay(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, revision, speaker := m.DemoIDs()
|
||||
ctx := context.Background()
|
||||
enqueue := func() JobRecord {
|
||||
t.Helper()
|
||||
j, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: tenant, Kind: "deploy_apply"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return j
|
||||
}
|
||||
claim := func(owner string) JobRecord {
|
||||
t.Helper()
|
||||
rows, err := m.ClaimJobs(ctx, owner, 1, time.Minute)
|
||||
if err != nil || len(rows) != 1 {
|
||||
t.Fatalf("claim: %v %v", rows, err)
|
||||
}
|
||||
return rows[0]
|
||||
}
|
||||
fence := func(j JobRecord) JobFence {
|
||||
return JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken}
|
||||
}
|
||||
first := enqueue()
|
||||
first = claim("one")
|
||||
d, err := m.BeginDeploymentSpeaker(ctx, fence(first), speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = m.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "verified", ""); !errors.Is(err, ErrInvalidInput) {
|
||||
t.Fatalf("unproved verified: %v", err)
|
||||
}
|
||||
if err = m.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "agent_reported", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replayed, err := m.BeginDeploymentSpeaker(ctx, fence(first), speaker, revision)
|
||||
if err != nil || replayed.Generation != d.Generation || replayed.State != "applied" {
|
||||
t.Fatalf("replay: %+v %v", replayed, err)
|
||||
}
|
||||
if err = m.ReleaseJob(ctx, first.ID, first.OwnerID, first.FencingToken); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reclaimed := claim("two")
|
||||
if err = m.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "agent_reported", ""); !errors.Is(err, ErrJobLeaseLost) {
|
||||
t.Fatalf("old owner ACK: %v", err)
|
||||
}
|
||||
if _, err = m.BeginDeploymentSpeaker(ctx, fence(reclaimed), speaker, revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := enqueue()
|
||||
second = claim("three")
|
||||
d2, err := m.BeginDeploymentSpeaker(ctx, fence(second), speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d2.Generation <= d.Generation {
|
||||
t.Fatal("generation failed to advance across independent job fencing tokens")
|
||||
}
|
||||
if err = m.RecordDeploymentSpeaker(ctx, fence(reclaimed), d, "applied", "agent_reported", ""); !errors.Is(err, ErrJobLeaseLost) {
|
||||
t.Fatalf("old generation ACK: %v", err)
|
||||
}
|
||||
if _, err = m.BeginDeploymentSpeaker(ctx, fence(reclaimed), speaker, revision); !errors.Is(err, ErrJobLeaseLost) {
|
||||
t.Fatalf("old generation republish: %v", err)
|
||||
}
|
||||
rows, err := m.ListDeploymentSpeakers(ctx, tenant, first.ID)
|
||||
if err != nil || len(rows) != 1 {
|
||||
t.Fatalf("durable progress: %v %v", rows, err)
|
||||
}
|
||||
if err = m.RecordDeploymentSpeaker(ctx, fence(second), d2, "unknown", "unconfirmed", "response lost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentObservationTokenRejectsLateHealth(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, revision, speaker := m.DemoIDs()
|
||||
ctx := context.Background()
|
||||
_, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: tenant, Kind: "deploy_apply"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs, err := m.ClaimJobs(ctx, "owner", 1, time.Minute)
|
||||
if err != nil || len(jobs) != 1 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := jobs[0]
|
||||
f := JobFence{JobID: j.ID, TenantID: tenant, OwnerID: j.OwnerID, Token: j.FencingToken}
|
||||
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = m.RecordDeploymentSpeaker(ctx, f, d, "applied", "agent_reported", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j.Status = "succeeded"
|
||||
if err = m.FinishJob(ctx, j); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old, err := m.ListDeploymentObservations(ctx, 1)
|
||||
if err != nil || len(old) != 1 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
latest, err := m.ListDeploymentObservations(ctx, 1)
|
||||
if err != nil || len(latest) != 1 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = m.ObserveDeployment(ctx, latest[0], DeploymentObservation{Error: "timeout"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = m.ObserveDeployment(ctx, old[0], DeploymentObservation{RevisionID: "older"}); !errors.Is(err, ErrJobLeaseLost) {
|
||||
t.Fatalf("late report accepted: %v", err)
|
||||
}
|
||||
rows, err := m.ListDeploymentSpeakers(ctx, tenant, j.ID)
|
||||
if err != nil || rows[0].State != "applied" || rows[0].Confirmation != "agent_reported" || rows[0].HealthError != "timeout" {
|
||||
t.Fatalf("error downgraded confirmation: %+v %v", rows, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentReconciliationRetryBudget(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, revision, speaker := m.DemoIDs()
|
||||
ctx := context.Background()
|
||||
var previous DeploymentSpeaker
|
||||
for round := 0; round <= 3; round++ {
|
||||
_, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: tenant, Kind: "deploy_apply"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
|
||||
if err != nil || len(jobs) != 1 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := jobs[0]
|
||||
f := JobFence{JobID: j.ID, TenantID: tenant, OwnerID: j.OwnerID, Token: j.FencingToken, RetryRound: round}
|
||||
if round > 0 {
|
||||
f.ExpectedGeneration = previous.Generation
|
||||
}
|
||||
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = m.RecordDeploymentSpeaker(ctx, f, d, "failed", "unconfirmed", "unreachable"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j.Status = "failed"
|
||||
if err = m.FinishJob(ctx, j); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err := m.ListPendingDeployments(ctx, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if (round < 3 && len(pending) != 1) || (round == 3 && len(pending) != 0) {
|
||||
t.Fatalf("round %d pending=%d", round, len(pending))
|
||||
}
|
||||
previous = d
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SystemJobTenantID owns global maintenance jobs outside user tenant scope.
|
||||
const SystemJobTenantID = "00000000-0000-0000-0000-000000000000"
|
||||
|
||||
// JobRequestHash identifies the immutable requested operation, excluding result metadata.
|
||||
func JobRequestHash(in JobEnqueue) (string, error) {
|
||||
meta := in.Meta
|
||||
if in.Request != nil {
|
||||
meta = in.Request
|
||||
}
|
||||
b, err := json.Marshal(struct {
|
||||
Kind string `json:"kind"`
|
||||
ModuleID *string `json:"module_id"`
|
||||
Meta map[string]any `json:"meta"`
|
||||
}{in.Kind, in.ModuleID, meta})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
h := sha256.Sum256(b)
|
||||
return hex.EncodeToString(h[:]), nil
|
||||
}
|
||||
|
||||
var (
|
||||
ErrJobNotFound = errors.New("store: job not found")
|
||||
ErrJobConflict = errors.New("store: idempotency key conflicts with another request")
|
||||
ErrJobQueueFull = errors.New("store: job queue is full")
|
||||
ErrJobLeaseLost = errors.New("store: job lease lost")
|
||||
)
|
||||
|
||||
// JobRecord is the shared, tenant-scoped job lifecycle record.
|
||||
type JobRecord struct {
|
||||
RequestHash string
|
||||
ID, TenantID, Kind, Status string
|
||||
IdempotencyKey, ModuleID *string
|
||||
Meta map[string]any
|
||||
CreatedAt time.Time
|
||||
StartedAt, FinishedAt *time.Time
|
||||
LeaseUntil *time.Time
|
||||
Error *string
|
||||
OwnerID string
|
||||
FencingToken int64
|
||||
RequestSequence int64
|
||||
Attempt int
|
||||
CancelRequested bool
|
||||
}
|
||||
|
||||
// JobEnqueue is the immutable request used for idempotent insertion.
|
||||
type JobEnqueue struct {
|
||||
TenantID, Kind string
|
||||
IdempotencyKey, ModuleID *string
|
||||
Meta map[string]any
|
||||
// Request optionally supplies the original selector before dynamic resolution.
|
||||
Request map[string]any
|
||||
}
|
||||
|
||||
// JobQueue provides durable operations; PostgreSQL is authoritative in production.
|
||||
type JobQueue interface {
|
||||
CountJobsByStatus(context.Context) (map[string]int64, error)
|
||||
FindJobByIdempotencyKey(context.Context, string, string) (JobRecord, error)
|
||||
EnqueueJob(context.Context, JobEnqueue) (JobRecord, bool, error)
|
||||
ClaimJobs(context.Context, string, int, time.Duration) ([]JobRecord, error)
|
||||
// ReapExpiredJobs marks orphaned cancellations and exhausted retries as terminal.
|
||||
// Production callers run it about every 15 seconds, not on every claim.
|
||||
ReapExpiredJobs(context.Context) error
|
||||
HeartbeatJob(context.Context, string, string, int64, time.Duration) (bool, error)
|
||||
CheckpointJob(context.Context, JobRecord) error
|
||||
FinishJob(context.Context, JobRecord) error
|
||||
ReleaseJob(context.Context, string, string, int64) error
|
||||
GetJob(context.Context, string, string) (JobRecord, error)
|
||||
ListJobs(context.Context, string, string, string, string, int) ([]JobRecord, string, bool, error)
|
||||
CancelJob(context.Context, string, string) (JobRecord, error)
|
||||
RetryJob(context.Context, string, string) (JobRecord, error)
|
||||
}
|
||||
+101
-30
@@ -21,7 +21,18 @@ var (
|
||||
|
||||
// Memory is a development-oriented in-memory backend for control-plane API handlers.
|
||||
type Memory struct {
|
||||
mu sync.RWMutex
|
||||
mu sync.RWMutex
|
||||
jobMu sync.Mutex
|
||||
jobTenantMu sync.Mutex
|
||||
jobTenantLocks map[string]*sync.Mutex
|
||||
jobSequence int64
|
||||
deploymentReservations map[string]deploymentReservation
|
||||
deployments map[string]DeploymentSpeaker
|
||||
deploymentCurrent map[string]DeploymentSpeaker
|
||||
jobs map[string]JobRecord
|
||||
jobHashes map[string]string
|
||||
jobRequests map[string]map[string]any
|
||||
telemetry map[string]telemetryMemoryRow
|
||||
|
||||
tenants map[string]*Tenant
|
||||
modules map[string]*Module
|
||||
@@ -35,26 +46,33 @@ type Memory struct {
|
||||
|
||||
peerDiscoveries map[string]*BGPPeerDiscovery
|
||||
|
||||
dohProfiles map[string]*DohProfile
|
||||
communities map[string]*Community
|
||||
cdnSources map[string]*CDNSource
|
||||
asEntries map[string]*ASEntry
|
||||
domainEnt map[string]*DomainEntry
|
||||
ipRanges map[string]*IPRangeEntry
|
||||
settings map[string]map[string]any // tenantID -> key -> JSON-compatible value
|
||||
revPrefixes map[string][]PrefixRow
|
||||
moduleSnapshots map[string]*moduleSnapshotRec
|
||||
snapshotLocks sync.Map // key -> *sync.Mutex (per-module snapshot RMW)
|
||||
asnPrefixCache map[int64]*ASNPrefixCacheEntry
|
||||
domainResolveCache map[string]*DomainResolveCacheEntry
|
||||
apiKeys map[string]*apiKeyRec
|
||||
firewallClients map[string]*firewallClientRec
|
||||
firewallRules map[string]*FirewallRule
|
||||
firewallHashIndex map[string]string // hex hash -> client id
|
||||
maintenancePolicies map[string]*MaintenancePolicy
|
||||
maintConfigAudit []*MaintenancePolicyConfigAudit
|
||||
runtimeLogCleanupAudit []*RuntimeLogCleanupAudit
|
||||
auditLog []*AuditEntry
|
||||
dohProfiles map[string]*DohProfile
|
||||
communities map[string]*Community
|
||||
cdnSources map[string]*CDNSource
|
||||
asEntries map[string]*ASEntry
|
||||
domainEnt map[string]*DomainEntry
|
||||
ipRanges map[string]*IPRangeEntry
|
||||
settings map[string]map[string]any // tenantID -> key -> JSON-compatible value
|
||||
revPrefixes map[string][]PrefixRow
|
||||
prefixSnapByHash map[string]string
|
||||
prefixSnapRows map[string][]PrefixRow
|
||||
revPrefixSnap map[string]string
|
||||
moduleSnapshots map[string]*moduleSnapshotRec
|
||||
nextModuleSnapshotVersion int64
|
||||
renderGeneration map[string]int64
|
||||
renderManifests map[string]*RenderManifestRecord
|
||||
snapshotLocksMu sync.Mutex
|
||||
snapshotLocks map[string]*snapshotLockEntry
|
||||
asnPrefixCache map[int64]*ASNPrefixCacheEntry
|
||||
domainResolveCache map[string]*DomainResolveCacheEntry
|
||||
apiKeys map[string]*apiKeyRec
|
||||
firewallClients map[string]*firewallClientRec
|
||||
firewallRules map[string]*FirewallRule
|
||||
firewallHashIndex map[string]string // hex hash -> client id
|
||||
maintenancePolicies map[string]*MaintenancePolicy
|
||||
maintConfigAudit []*MaintenancePolicyConfigAudit
|
||||
runtimeLogCleanupAudit []*RuntimeLogCleanupAudit
|
||||
auditLog []*AuditEntry
|
||||
|
||||
// DemoIDs valid after SeedDemo()
|
||||
demoTenantID string
|
||||
@@ -157,7 +175,12 @@ func NewMemory() *Memory {
|
||||
ipRanges: make(map[string]*IPRangeEntry),
|
||||
settings: make(map[string]map[string]any),
|
||||
revPrefixes: make(map[string][]PrefixRow),
|
||||
prefixSnapByHash: make(map[string]string),
|
||||
prefixSnapRows: make(map[string][]PrefixRow),
|
||||
revPrefixSnap: make(map[string]string),
|
||||
moduleSnapshots: make(map[string]*moduleSnapshotRec),
|
||||
renderGeneration: make(map[string]int64),
|
||||
renderManifests: make(map[string]*RenderManifestRecord),
|
||||
asnPrefixCache: make(map[int64]*ASNPrefixCacheEntry),
|
||||
domainResolveCache: make(map[string]*DomainResolveCacheEntry),
|
||||
apiKeys: make(map[string]*apiKeyRec),
|
||||
@@ -319,6 +342,31 @@ func (m *Memory) PeerSessionCountsByState() map[string]int {
|
||||
return out
|
||||
}
|
||||
|
||||
// ObserveStoreMetrics returns an error-aware snapshot of stored revision and peer inventory aggregates.
|
||||
func (m *Memory) ObserveStoreMetrics(ctx context.Context) (max, sum, peers int, states map[string]int, err error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, 0, 0, nil, err
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
for _, revision := range m.revisions {
|
||||
sum += revision.MaterializedPrefixCount
|
||||
if revision.MaterializedPrefixCount > max {
|
||||
max = revision.MaterializedPrefixCount
|
||||
}
|
||||
}
|
||||
states = make(map[string]int)
|
||||
for _, peer := range m.peers {
|
||||
peers++
|
||||
state := strings.TrimSpace(peer.SessionState)
|
||||
if state == "" {
|
||||
state = "unknown"
|
||||
}
|
||||
states[state]++
|
||||
}
|
||||
return max, sum, peers, states, nil
|
||||
}
|
||||
|
||||
// DemoIDs returns IDs from SeedDemo; empty strings if SeedDemo was not called.
|
||||
func (m *Memory) DemoIDs() (tenant, moduleCDN, moduleIP, revision, speaker string) {
|
||||
m.mu.RLock()
|
||||
@@ -349,13 +397,22 @@ func (m *Memory) ListTenantIDs() ([]string, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// CreateRenderRevision stores a new revision and its materialized prefixes.
|
||||
func (m *Memory) CreateRenderRevision(revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error {
|
||||
// CreateRenderRevision stores a new revision, its prefix snapshot, and the render manifest.
|
||||
func (m *Memory) CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(revisionID) == "" || strings.TrimSpace(moduleID) == "" || strings.TrimSpace(contentHash) == "" {
|
||||
return ErrInvalidInput
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := m.revisions[revisionID]; exists {
|
||||
return ErrInvalidInput
|
||||
}
|
||||
@@ -392,16 +449,30 @@ func (m *Memory) CreateRenderRevision(revisionID, tenantID, moduleID string, par
|
||||
MaterializedPrefixCount: len(prefixes),
|
||||
PreviewFragments: frag,
|
||||
}
|
||||
if len(prefixes) > 0 {
|
||||
cp := make([]PrefixRow, len(prefixes))
|
||||
copy(cp, prefixes)
|
||||
m.revPrefixes[revisionID] = cp
|
||||
} else {
|
||||
m.revPrefixes[revisionID] = nil
|
||||
}
|
||||
snapID := m.bindPrefixSnapshotLocked(prefixes)
|
||||
m.revPrefixSnap[revisionID] = snapID
|
||||
m.revPrefixes[revisionID] = m.prefixSnapRows[snapID]
|
||||
m.recordRenderManifestLocked(revisionID, tenantID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// bindPrefixSnapshotLocked reuses an existing prefix snapshot when the prefix hash matches.
|
||||
func (m *Memory) bindPrefixSnapshotLocked(prefixes []PrefixRow) string {
|
||||
if m.prefixSnapByHash == nil {
|
||||
m.prefixSnapByHash = make(map[string]string)
|
||||
m.prefixSnapRows = make(map[string][]PrefixRow)
|
||||
m.revPrefixSnap = make(map[string]string)
|
||||
}
|
||||
h := PrefixSnapshotHash(prefixes)
|
||||
if id, ok := m.prefixSnapByHash[h]; ok {
|
||||
return id
|
||||
}
|
||||
id := uuid.NewString()
|
||||
m.prefixSnapByHash[h] = id
|
||||
m.prefixSnapRows[id] = append([]PrefixRow(nil), prefixes...)
|
||||
return id
|
||||
}
|
||||
|
||||
// ListModules returns modules for a tenant (sorted by priority, then name).
|
||||
func (m *Memory) ListModules(tenantID string) []*Module {
|
||||
m.mu.RLock()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -29,6 +30,11 @@ func (m *Memory) SetASNPrefixCache(asn int64, holder string, prefixes []string)
|
||||
if m.asnPrefixCache == nil {
|
||||
m.asnPrefixCache = make(map[int64]*ASNPrefixCacheEntry)
|
||||
}
|
||||
if cur := m.asnPrefixCache[asn]; cur != nil && sameStringSet(cur.Prefixes, prefixes) {
|
||||
cur.Holder = holder
|
||||
cur.FetchedAt = time.Now().UTC()
|
||||
return nil
|
||||
}
|
||||
cp := append([]string(nil), prefixes...)
|
||||
m.asnPrefixCache[asn] = &ASNPrefixCacheEntry{
|
||||
ASN: asn,
|
||||
@@ -38,3 +44,19 @@ func (m *Memory) SetASNPrefixCache(asn int64, holder string, prefixes []string)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sameStringSet(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
aa := append([]string(nil), a...)
|
||||
bb := append([]string(nil), b...)
|
||||
sort.Strings(aa)
|
||||
sort.Strings(bb)
|
||||
for i := range aa {
|
||||
if aa[i] != bb[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -38,6 +38,7 @@ func (m *Memory) CreateModule(tenantID string, in *Module) (*Module, error) {
|
||||
}
|
||||
NormalizeModuleDoh(mod)
|
||||
m.modules[id] = mod
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
return cloneModule(mod), nil
|
||||
}
|
||||
|
||||
@@ -81,7 +82,10 @@ func (m *Memory) UpdateModule(tenantID, moduleID string, patch *ModulePatch) (*M
|
||||
t := patch.LastRefreshedAt.UTC()
|
||||
mod.LastRefreshedAt = &t
|
||||
}
|
||||
mod.InputHash = ""
|
||||
if patch.Name != nil || patch.Enabled != nil || patch.Priority != nil || patch.RefreshIntervalSec != nil || patch.CronExpr != nil || patch.DefaultCommunityID != nil || patch.DohProfileIDs != nil || patch.DohProfileID != nil || patch.DohResolverPolicy != nil {
|
||||
mod.InputHash = ""
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
}
|
||||
return cloneModule(mod), nil
|
||||
}
|
||||
|
||||
@@ -97,6 +101,7 @@ func (m *Memory) SoftDeleteModule(tenantID, moduleID string) error {
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
mod.DeletedAt = &now
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -154,6 +159,7 @@ func (m *Memory) CreateCDNSource(tenantID, moduleID string, in *CDNSource) (*CDN
|
||||
}
|
||||
m.cdnSources[id] = s
|
||||
mod.InputHash = ""
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
@@ -200,6 +206,7 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN
|
||||
}
|
||||
if CDNPatchAffectsInputHash(patch) {
|
||||
mod.InputHash = ""
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
@@ -216,6 +223,7 @@ func (m *Memory) DeleteCDNSource(tenantID, moduleID, sourceID string) error {
|
||||
}
|
||||
delete(m.cdnSources, sourceID)
|
||||
m.clearModuleInputHashLocked(moduleID)
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -255,6 +263,7 @@ func (m *Memory) CreateASEntry(tenantID, moduleID string, in *ASEntry) (*ASEntry
|
||||
e := &ASEntry{ID: id, ModuleID: moduleID, ASN: in.ASN, CommunityID: in.CommunityID}
|
||||
m.asEntries[id] = e
|
||||
mod.InputHash = ""
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
return e, nil
|
||||
}
|
||||
|
||||
@@ -293,6 +302,7 @@ func (m *Memory) UpdateASEntry(tenantID, moduleID, entryID string, patch *ASEntr
|
||||
e.ASNResolvedAt = nil
|
||||
}
|
||||
mod.InputHash = ""
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
return e, nil
|
||||
}
|
||||
|
||||
@@ -335,6 +345,7 @@ func (m *Memory) DeleteASEntry(tenantID, moduleID, entryID string) error {
|
||||
}
|
||||
delete(m.asEntries, entryID)
|
||||
m.clearModuleInputHashLocked(moduleID)
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -791,6 +802,7 @@ func (m *Memory) CreatePeer(tenantID string, in *BGPPeer) (*BGPPeer, error) {
|
||||
CreatedByUserID: strings.TrimSpace(in.CreatedByUserID),
|
||||
}
|
||||
m.peers[id] = p
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
@@ -804,6 +816,11 @@ func (m *Memory) UpdatePeer(tenantID, id string, patch *PeerPatch) (*BGPPeer, er
|
||||
if !ok || p.TenantID != tenantID {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
prevNeighbor, prevASN, prevEnabled, prevPolicies := p.Neighbor, p.RemoteASN, p.Enabled, p.PoliciesJSON
|
||||
prevSpeaker := ""
|
||||
if p.SpeakerID != nil {
|
||||
prevSpeaker = *p.SpeakerID
|
||||
}
|
||||
if patch.Neighbor != nil {
|
||||
n, ok := NormalizePeerNeighborString(*patch.Neighbor)
|
||||
if !ok {
|
||||
@@ -834,6 +851,13 @@ func (m *Memory) UpdatePeer(tenantID, id string, patch *PeerPatch) (*BGPPeer, er
|
||||
if patch.PoliciesJSON != nil {
|
||||
p.PoliciesJSON = *patch.PoliciesJSON
|
||||
}
|
||||
speaker := ""
|
||||
if p.SpeakerID != nil {
|
||||
speaker = *p.SpeakerID
|
||||
}
|
||||
if p.Neighbor != prevNeighbor || p.RemoteASN != prevASN || p.Enabled != prevEnabled || p.PoliciesJSON != prevPolicies || speaker != prevSpeaker {
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
@@ -846,6 +870,7 @@ func (m *Memory) DeletePeer(tenantID, id string) error {
|
||||
}
|
||||
delete(m.peers, id)
|
||||
_ = p
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
type deploymentReservation struct {
|
||||
Owner string
|
||||
Token int64
|
||||
Until time.Time
|
||||
}
|
||||
|
||||
func (m *Memory) deploymentFence(f JobFence) (JobRecord, error) {
|
||||
j, ok := m.jobs[f.JobID]
|
||||
if !ok || j.TenantID != f.TenantID || j.Status != "running" || j.OwnerID != f.OwnerID || j.FencingToken != f.Token || j.CancelRequested || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) {
|
||||
return JobRecord{}, ErrJobLeaseLost
|
||||
}
|
||||
return j, nil
|
||||
}
|
||||
|
||||
func (m *Memory) BeginDeploymentSpeaker(ctx context.Context, f JobFence, speaker, revision string) (DeploymentSpeaker, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return DeploymentSpeaker{}, err
|
||||
}
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
j, err := m.deploymentFence(f)
|
||||
if err != nil {
|
||||
return DeploymentSpeaker{}, err
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
sp, ok := m.speakers[speaker]
|
||||
if !ok || sp.TenantID != f.TenantID {
|
||||
return DeploymentSpeaker{}, ErrNotFound
|
||||
}
|
||||
r, ok := m.revisions[revision]
|
||||
if !ok || r.TenantID != f.TenantID {
|
||||
return DeploymentSpeaker{}, ErrNotFound
|
||||
}
|
||||
if m.deployments == nil {
|
||||
m.deployments = make(map[string]DeploymentSpeaker)
|
||||
m.deploymentCurrent = make(map[string]DeploymentSpeaker)
|
||||
}
|
||||
key := f.JobID + ":" + speaker
|
||||
cur := m.deploymentCurrent[speaker]
|
||||
if f.ExpectedGeneration != 0 && cur.JobID != f.JobID && (cur.Generation != f.ExpectedGeneration || cur.RevisionID != revision) {
|
||||
return DeploymentSpeaker{}, ErrJobLeaseLost
|
||||
}
|
||||
reserve := func() error {
|
||||
if m.deploymentReservations == nil {
|
||||
m.deploymentReservations = make(map[string]deploymentReservation)
|
||||
}
|
||||
if r := m.deploymentReservations[speaker]; r.Until.After(time.Now()) {
|
||||
return ErrDeploymentBusy
|
||||
}
|
||||
m.deploymentReservations[speaker] = deploymentReservation{Owner: f.OwnerID, Token: f.Token, Until: time.Now().Add(DeploymentDeliveryWindow)}
|
||||
return nil
|
||||
}
|
||||
if existing, ok := m.deployments[key]; ok {
|
||||
if cur.JobID != f.JobID || cur.Generation != existing.Generation || cur.RevisionID != revision {
|
||||
return DeploymentSpeaker{}, ErrJobLeaseLost
|
||||
}
|
||||
if existing.State != "applied" {
|
||||
if err := reserve(); err != nil {
|
||||
return DeploymentSpeaker{}, err
|
||||
}
|
||||
}
|
||||
return existing, nil
|
||||
}
|
||||
if f.ExpectedGeneration != 0 {
|
||||
j.RequestSequence = cur.RequestSequence
|
||||
}
|
||||
if cur.RequestSequence >= j.RequestSequence && cur.JobID != "" && f.ExpectedGeneration == 0 {
|
||||
return DeploymentSpeaker{}, ErrJobLeaseLost
|
||||
}
|
||||
if err := reserve(); err != nil {
|
||||
return DeploymentSpeaker{}, err
|
||||
}
|
||||
d := DeploymentSpeaker{JobID: f.JobID, TenantID: f.TenantID, SpeakerID: speaker, RevisionID: revision, Generation: cur.Generation + 1, RequestSequence: j.RequestSequence, State: "published", RetryRound: f.RetryRound}
|
||||
m.deployments[key] = d
|
||||
m.deploymentCurrent[speaker] = d
|
||||
m.publishedRevision[speaker] = publishedInfo{RevisionID: revision, PublishedAt: time.Now().UTC()}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (m *Memory) RecordDeploymentSpeaker(ctx context.Context, f JobFence, d DeploymentSpeaker, state, confirmation, message string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
if _, err := m.deploymentFence(f); err != nil {
|
||||
return err
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
cur := m.deploymentCurrent[d.SpeakerID]
|
||||
if d.JobID != f.JobID || d.TenantID != f.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID {
|
||||
return ErrJobLeaseLost
|
||||
}
|
||||
r := m.deploymentReservations[d.SpeakerID]
|
||||
if r.Owner != f.OwnerID || r.Token != f.Token || !r.Until.After(time.Now()) {
|
||||
return ErrJobLeaseLost
|
||||
}
|
||||
if state == "applied" && confirmation != "agent_reported" && confirmation != "local_applied" {
|
||||
return ErrInvalidInput
|
||||
}
|
||||
if state != "dispatched" && state != "unknown" {
|
||||
delete(m.deploymentReservations, d.SpeakerID)
|
||||
}
|
||||
cur.State, cur.Confirmation, cur.Error = state, confirmation, message
|
||||
if state == "dispatched" {
|
||||
cur.DispatchAttempts++
|
||||
}
|
||||
if state == "applied" {
|
||||
revision := d.RevisionID
|
||||
m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision
|
||||
}
|
||||
m.deploymentCurrent[d.SpeakerID] = cur
|
||||
m.deployments[d.JobID+":"+d.SpeakerID] = cur
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Memory) ListDeploymentSpeakers(ctx context.Context, tenant, job string) ([]DeploymentSpeaker, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
var out []DeploymentSpeaker
|
||||
for _, d := range m.deployments {
|
||||
if d.TenantID == tenant && d.JobID == job {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *Memory) ListPendingDeployments(ctx context.Context, limit int) ([]DeploymentSpeaker, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
var out []DeploymentSpeaker
|
||||
for speaker, d := range m.deploymentCurrent {
|
||||
j := m.jobs[d.JobID]
|
||||
if d.State != "applied" && d.RetryRound < 3 && !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) {
|
||||
out = append(out, d)
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *Memory) ListDeploymentObservations(ctx context.Context, limit int) ([]DeploymentSpeaker, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 10
|
||||
}
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var out []DeploymentSpeaker
|
||||
for speaker, d := range m.deploymentCurrent {
|
||||
j := m.jobs[d.JobID]
|
||||
if !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
a, b := out[i].HealthReceivedAt, out[j].HealthReceivedAt
|
||||
if a == nil && b == nil {
|
||||
return out[i].SpeakerID < out[j].SpeakerID
|
||||
}
|
||||
if a == nil {
|
||||
return true
|
||||
}
|
||||
if b == nil {
|
||||
return false
|
||||
}
|
||||
return a.Before(*b)
|
||||
})
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
for i := range out {
|
||||
out[i].ObservationToken++
|
||||
m.deploymentCurrent[out[i].SpeakerID] = out[i]
|
||||
m.deployments[out[i].JobID+":"+out[i].SpeakerID] = out[i]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *Memory) ObserveDeployment(ctx context.Context, d DeploymentSpeaker, o DeploymentObservation) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
cur := m.deploymentCurrent[d.SpeakerID]
|
||||
if cur.TenantID != d.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || cur.ObservationToken != d.ObservationToken || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID || m.deploymentReservations[d.SpeakerID].Until.After(time.Now()) {
|
||||
return ErrJobLeaseLost
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
cur.ObservationToken++
|
||||
cur.HealthReceivedAt = &now
|
||||
cur.HealthRevisionID = o.RevisionID
|
||||
cur.HealthSourceAt = o.SourceAt
|
||||
cur.HealthError = o.Error
|
||||
if o.Error == "" && o.RevisionID != "" {
|
||||
if o.RevisionID == d.RevisionID {
|
||||
cur.State = "applied"
|
||||
cur.Confirmation = "agent_reported"
|
||||
revision := d.RevisionID
|
||||
m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision
|
||||
} else {
|
||||
cur.State = "unknown"
|
||||
cur.Confirmation = "unconfirmed"
|
||||
}
|
||||
}
|
||||
m.deploymentCurrent[d.SpeakerID] = cur
|
||||
m.deployments[d.JobID+":"+d.SpeakerID] = cur
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func copyJob(j JobRecord) JobRecord {
|
||||
if j.IdempotencyKey != nil {
|
||||
v := *j.IdempotencyKey
|
||||
j.IdempotencyKey = &v
|
||||
}
|
||||
if j.ModuleID != nil {
|
||||
v := *j.ModuleID
|
||||
j.ModuleID = &v
|
||||
}
|
||||
if j.Error != nil {
|
||||
v := *j.Error
|
||||
j.Error = &v
|
||||
}
|
||||
if j.StartedAt != nil {
|
||||
v := *j.StartedAt
|
||||
j.StartedAt = &v
|
||||
}
|
||||
if j.FinishedAt != nil {
|
||||
v := *j.FinishedAt
|
||||
j.FinishedAt = &v
|
||||
}
|
||||
if j.LeaseUntil != nil {
|
||||
v := *j.LeaseUntil
|
||||
j.LeaseUntil = &v
|
||||
}
|
||||
if j.Meta != nil {
|
||||
raw, _ := json.Marshal(j.Meta)
|
||||
var meta map[string]any
|
||||
_ = json.Unmarshal(raw, &meta)
|
||||
j.Meta = meta
|
||||
}
|
||||
return j
|
||||
}
|
||||
|
||||
func (m *Memory) EnqueueJob(_ context.Context, in JobEnqueue) (JobRecord, bool, error) {
|
||||
h, err := JobRequestHash(in)
|
||||
if err != nil {
|
||||
return JobRecord{}, false, err
|
||||
}
|
||||
unlockTenant := m.lockJobEnqueueTenant(in.TenantID)
|
||||
defer unlockTenant()
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
if m.jobs == nil {
|
||||
m.jobs = make(map[string]JobRecord)
|
||||
m.jobHashes = make(map[string]string)
|
||||
m.jobRequests = make(map[string]map[string]any)
|
||||
}
|
||||
if in.IdempotencyKey != nil && *in.IdempotencyKey != "" {
|
||||
for _, j := range m.jobs {
|
||||
if j.TenantID == in.TenantID && j.IdempotencyKey != nil && *j.IdempotencyKey == *in.IdempotencyKey {
|
||||
if m.jobHashes[j.ID] != h {
|
||||
return JobRecord{}, false, ErrJobConflict
|
||||
}
|
||||
return copyJob(j), false, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
global, tenant := 0, 0
|
||||
for _, j := range m.jobs {
|
||||
if j.Status == "queued" || j.Status == "running" {
|
||||
global++
|
||||
if j.TenantID == in.TenantID {
|
||||
tenant++
|
||||
}
|
||||
}
|
||||
}
|
||||
if global >= jobLimitMemory("EVOBGP_JOB_MAX_PENDING", 10000) || tenant >= jobLimitMemory("EVOBGP_JOB_MAX_TENANT_PENDING", 1000) {
|
||||
return JobRecord{}, false, ErrJobQueueFull
|
||||
}
|
||||
j := JobRecord{ID: uuid.NewString(), TenantID: in.TenantID, Kind: in.Kind, Status: "queued", IdempotencyKey: in.IdempotencyKey, ModuleID: in.ModuleID, Meta: in.Meta, CreatedAt: time.Now().UTC()}
|
||||
j = copyJob(j)
|
||||
m.jobSequence++
|
||||
j.RequestSequence = m.jobSequence
|
||||
j.RequestHash = h
|
||||
m.jobs[j.ID] = j
|
||||
m.jobHashes[j.ID] = h
|
||||
m.jobRequests[j.ID] = copyJob(j).Meta
|
||||
return copyJob(j), true, nil
|
||||
}
|
||||
|
||||
func (m *Memory) ClaimJobs(_ context.Context, owner string, limit int, lease time.Duration) ([]JobRecord, error) {
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if limit > 64 {
|
||||
limit = 64
|
||||
}
|
||||
if lease <= 0 {
|
||||
lease = time.Minute
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
m.reapExpiredJobsLocked(now)
|
||||
var all []JobRecord
|
||||
for _, j := range m.jobs {
|
||||
if j.Attempt < 3 && (j.Status == "queued" || (j.Status == "running" && j.LeaseUntil != nil && now.After(*j.LeaseUntil))) {
|
||||
all = append(all, j)
|
||||
}
|
||||
}
|
||||
sort.Slice(all, func(i, j int) bool {
|
||||
if !all[i].CreatedAt.Equal(all[j].CreatedAt) {
|
||||
return all[i].CreatedAt.Before(all[j].CreatedAt)
|
||||
}
|
||||
if all[i].RequestSequence != all[j].RequestSequence {
|
||||
return all[i].RequestSequence < all[j].RequestSequence
|
||||
}
|
||||
return all[i].ID < all[j].ID
|
||||
})
|
||||
if len(all) > limit {
|
||||
all = all[:limit]
|
||||
}
|
||||
for i := range all {
|
||||
j := all[i]
|
||||
j.Status = "running"
|
||||
j.OwnerID = owner
|
||||
j.FencingToken++
|
||||
j.Attempt++
|
||||
if j.StartedAt == nil {
|
||||
j.StartedAt = &now
|
||||
}
|
||||
until := now.Add(lease)
|
||||
j.LeaseUntil = &until
|
||||
m.jobs[j.ID] = j
|
||||
all[i] = copyJob(j)
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// ReapExpiredJobs marks orphaned cancellations and exhausted retries terminal.
|
||||
func (m *Memory) ReapExpiredJobs(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
m.reapExpiredJobsLocked(time.Now().UTC())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Memory) reapExpiredJobsLocked(now time.Time) {
|
||||
for id, j := range m.jobs {
|
||||
if j.Status == "running" && j.LeaseUntil != nil && now.After(*j.LeaseUntil) && j.CancelRequested {
|
||||
j.Status = "cancelled"
|
||||
j.FinishedAt = &now
|
||||
j.OwnerID = ""
|
||||
j.LeaseUntil = nil
|
||||
m.jobs[id] = j
|
||||
continue
|
||||
}
|
||||
if (j.Status == "queued" || (j.Status == "running" && j.LeaseUntil != nil && now.After(*j.LeaseUntil))) && j.Attempt >= 3 {
|
||||
j.Status = "failed"
|
||||
msg := "retry limit reached"
|
||||
j.Error = &msg
|
||||
j.FinishedAt = &now
|
||||
j.OwnerID = ""
|
||||
j.LeaseUntil = nil
|
||||
m.jobs[id] = j
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Memory) HeartbeatJob(_ context.Context, id, owner string, token int64, lease time.Duration) (bool, error) {
|
||||
if lease <= 0 {
|
||||
lease = time.Minute
|
||||
}
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
j, ok := m.jobs[id]
|
||||
if !ok || j.OwnerID != owner || j.FencingToken != token || j.Status != "running" || j.LeaseUntil == nil || !time.Now().Before(*j.LeaseUntil) {
|
||||
return false, ErrJobLeaseLost
|
||||
}
|
||||
until := time.Now().Add(lease)
|
||||
j.LeaseUntil = &until
|
||||
m.jobs[id] = j
|
||||
return j.CancelRequested, nil
|
||||
}
|
||||
|
||||
func (m *Memory) FinishJob(_ context.Context, j JobRecord) error {
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
old, ok := m.jobs[j.ID]
|
||||
if !ok || old.OwnerID != j.OwnerID || old.FencingToken != j.FencingToken || old.Status != "running" || old.LeaseUntil == nil || !time.Now().Before(*old.LeaseUntil) {
|
||||
return ErrJobLeaseLost
|
||||
}
|
||||
j.CreatedAt = old.CreatedAt
|
||||
j.RequestSequence = old.RequestSequence
|
||||
j.RequestHash = old.RequestHash
|
||||
j.IdempotencyKey = old.IdempotencyKey
|
||||
j.ModuleID = old.ModuleID
|
||||
j.TenantID = old.TenantID
|
||||
j.Kind = old.Kind
|
||||
j.StartedAt = old.StartedAt
|
||||
j.Attempt = old.Attempt
|
||||
j.CancelRequested = old.CancelRequested
|
||||
j.OwnerID = ""
|
||||
j.LeaseUntil = nil
|
||||
now := time.Now().UTC()
|
||||
j.FinishedAt = &now
|
||||
if old.CancelRequested {
|
||||
j.Status = "cancelled"
|
||||
j.Error = nil
|
||||
}
|
||||
m.jobs[j.ID] = copyJob(j)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Memory) CheckpointJob(_ context.Context, in JobRecord) error {
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
j, ok := m.jobs[in.ID]
|
||||
if !ok || j.OwnerID != in.OwnerID || j.FencingToken != in.FencingToken || j.Status != "running" || j.LeaseUntil == nil || !time.Now().Before(*j.LeaseUntil) {
|
||||
return ErrJobLeaseLost
|
||||
}
|
||||
j.Meta = copyJob(in).Meta
|
||||
m.jobs[j.ID] = j
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Memory) ReleaseJob(_ context.Context, id, owner string, token int64) error {
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
j, ok := m.jobs[id]
|
||||
if !ok || j.OwnerID != owner || j.FencingToken != token || j.Status != "running" {
|
||||
return ErrJobLeaseLost
|
||||
}
|
||||
j.Status = "queued"
|
||||
if j.CancelRequested {
|
||||
j.Status = "cancelled"
|
||||
now := time.Now().UTC()
|
||||
j.FinishedAt = &now
|
||||
}
|
||||
j.OwnerID = ""
|
||||
j.LeaseUntil = nil
|
||||
if j.Attempt > 0 {
|
||||
j.Attempt--
|
||||
}
|
||||
m.jobs[id] = j
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Memory) GetJob(_ context.Context, tenant, id string) (JobRecord, error) {
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
j, ok := m.jobs[id]
|
||||
if !ok || j.TenantID != tenant {
|
||||
return JobRecord{}, ErrJobNotFound
|
||||
}
|
||||
return copyJob(j), nil
|
||||
}
|
||||
|
||||
func (m *Memory) ListJobs(_ context.Context, tenant, status, kind, cursor string, limit int) ([]JobRecord, string, bool, error) {
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
var all []JobRecord
|
||||
for _, j := range m.jobs {
|
||||
if j.TenantID == tenant && (status == "" || j.Status == status) && (kind == "" || j.Kind == kind) {
|
||||
all = append(all, copyJob(j))
|
||||
}
|
||||
}
|
||||
sort.Slice(all, func(i, j int) bool {
|
||||
if !all[i].CreatedAt.Equal(all[j].CreatedAt) {
|
||||
return all[i].CreatedAt.After(all[j].CreatedAt)
|
||||
}
|
||||
if all[i].RequestSequence != all[j].RequestSequence {
|
||||
return all[i].RequestSequence > all[j].RequestSequence
|
||||
}
|
||||
return all[i].ID > all[j].ID
|
||||
})
|
||||
off, _ := strconv.Atoi(cursor)
|
||||
if off < 0 {
|
||||
off = 0
|
||||
}
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 50
|
||||
}
|
||||
if off >= len(all) {
|
||||
return nil, "", false, nil
|
||||
}
|
||||
end := off + limit
|
||||
more := end < len(all)
|
||||
if end > len(all) {
|
||||
end = len(all)
|
||||
}
|
||||
next := ""
|
||||
if more {
|
||||
next = strconv.Itoa(end)
|
||||
}
|
||||
return all[off:end], next, more, nil
|
||||
}
|
||||
|
||||
func (m *Memory) CancelJob(_ context.Context, tenant, id string) (JobRecord, error) {
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
j, ok := m.jobs[id]
|
||||
if !ok || j.TenantID != tenant {
|
||||
return JobRecord{}, ErrJobNotFound
|
||||
}
|
||||
if j.Status == "queued" {
|
||||
now := time.Now().UTC()
|
||||
j.Status = "cancelled"
|
||||
j.FinishedAt = &now
|
||||
}
|
||||
if j.Status == "running" {
|
||||
j.CancelRequested = true
|
||||
}
|
||||
m.jobs[id] = j
|
||||
return copyJob(j), nil
|
||||
}
|
||||
|
||||
func (m *Memory) RetryJob(_ context.Context, tenant, id string) (JobRecord, error) {
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
j, ok := m.jobs[id]
|
||||
if !ok || j.TenantID != tenant {
|
||||
return JobRecord{}, ErrJobNotFound
|
||||
}
|
||||
if j.Status != "failed" {
|
||||
return JobRecord{}, ErrJobConflict
|
||||
}
|
||||
global, pending := 0, 0
|
||||
for _, item := range m.jobs {
|
||||
if item.Status == "queued" || item.Status == "running" {
|
||||
global++
|
||||
if item.TenantID == tenant {
|
||||
pending++
|
||||
}
|
||||
}
|
||||
}
|
||||
if global >= jobLimitMemory("EVOBGP_JOB_MAX_PENDING", 10000) || pending >= jobLimitMemory("EVOBGP_JOB_MAX_TENANT_PENDING", 1000) {
|
||||
return JobRecord{}, ErrJobQueueFull
|
||||
}
|
||||
j.Status = "queued"
|
||||
j.Error = nil
|
||||
j.FinishedAt = nil
|
||||
j.CancelRequested = false
|
||||
j.Attempt = 0
|
||||
j.StartedAt = nil
|
||||
j.OwnerID = ""
|
||||
j.LeaseUntil = nil
|
||||
j.Meta = copyJob(JobRecord{Meta: m.jobRequests[id]}).Meta
|
||||
m.jobs[id] = j
|
||||
return copyJob(j), nil
|
||||
}
|
||||
|
||||
// lockJobEnqueueTenant serializes idempotency and the tenant pending limit.
|
||||
// The global jobMu is still required to mutate the shared job map.
|
||||
func (m *Memory) lockJobEnqueueTenant(tenantID string) func() {
|
||||
m.jobTenantMu.Lock()
|
||||
if m.jobTenantLocks == nil {
|
||||
m.jobTenantLocks = make(map[string]*sync.Mutex)
|
||||
}
|
||||
mu := m.jobTenantLocks[tenantID]
|
||||
if mu == nil {
|
||||
mu = &sync.Mutex{}
|
||||
m.jobTenantLocks[tenantID] = mu
|
||||
}
|
||||
m.jobTenantMu.Unlock()
|
||||
mu.Lock()
|
||||
return mu.Unlock
|
||||
}
|
||||
|
||||
func jobLimitMemory(name string, fallback int) int {
|
||||
n, e := strconv.Atoi(os.Getenv(name))
|
||||
if e == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func (m *Memory) CountJobsByStatus(_ context.Context) (map[string]int64, error) {
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
out := map[string]int64{}
|
||||
for _, j := range m.jobs {
|
||||
out[j.Status]++
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *Memory) FindJobByIdempotencyKey(_ context.Context, tenant, key string) (JobRecord, error) {
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
for _, j := range m.jobs {
|
||||
if j.TenantID == tenant && j.IdempotencyKey != nil && *j.IdempotencyKey == key {
|
||||
return copyJob(j), nil
|
||||
}
|
||||
}
|
||||
return JobRecord{}, ErrJobNotFound
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestEnqueueJobTenantsDoNotShareLock(t *testing.T) {
|
||||
m := NewMemory()
|
||||
ctx := context.Background()
|
||||
unlock := m.lockJobEnqueueTenant("tenant-a")
|
||||
var once sync.Once
|
||||
release := func() { once.Do(unlock) }
|
||||
t.Cleanup(release)
|
||||
|
||||
otherDone := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: "tenant-b", Kind: "test"})
|
||||
otherDone <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-otherDone:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("other tenant waited on the held tenant lock")
|
||||
}
|
||||
|
||||
sameDone := make(chan struct{})
|
||||
go func() {
|
||||
_, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: "tenant-a", Kind: "test"})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
close(sameDone)
|
||||
}()
|
||||
select {
|
||||
case <-sameDone:
|
||||
t.Fatal("same tenant did not wait on its lock")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-sameDone:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("same tenant did not finish after unlock")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueueJobIdempotencyKeyConflicts(t *testing.T) {
|
||||
m := NewMemory()
|
||||
ctx := context.Background()
|
||||
key := "same-key"
|
||||
in := JobEnqueue{TenantID: "tenant-a", Kind: "refresh", IdempotencyKey: &key, Meta: map[string]any{"module": "a"}}
|
||||
first, created, err := m.EnqueueJob(ctx, in)
|
||||
if err != nil || !created || first.ID == "" {
|
||||
t.Fatalf("first=%+v created=%v err=%v", first, created, err)
|
||||
}
|
||||
in.Meta = map[string]any{"module": "b"}
|
||||
if _, _, err = m.EnqueueJob(ctx, in); !errors.Is(err, ErrJobConflict) {
|
||||
t.Fatalf("same key accepted: %v", err)
|
||||
}
|
||||
other := JobEnqueue{TenantID: "tenant-b", Kind: "refresh", IdempotencyKey: &key, Meta: map[string]any{"module": "b"}}
|
||||
if _, created, err = m.EnqueueJob(ctx, other); err != nil || !created {
|
||||
t.Fatalf("other tenant created=%v err=%v", created, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryJobQueueEqualTimestampOrdering(t *testing.T) {
|
||||
m := NewMemory()
|
||||
ctx := context.Background()
|
||||
stamp := time.Date(2026, 10, 26, 0, 0, 0, 0, time.UTC)
|
||||
var ids []string
|
||||
for range 32 {
|
||||
j, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: "tenant", Kind: "test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids = append(ids, j.ID)
|
||||
m.jobMu.Lock()
|
||||
saved := m.jobs[j.ID]
|
||||
saved.CreatedAt = stamp
|
||||
m.jobs[j.ID] = saved
|
||||
m.jobMu.Unlock()
|
||||
}
|
||||
// Repeated offset pages must use the same total order despite map iteration.
|
||||
for range 5 {
|
||||
cursor := ""
|
||||
var listed []string
|
||||
for {
|
||||
page, next, more, err := m.ListJobs(ctx, "tenant", "", "", cursor, 3)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, j := range page {
|
||||
listed = append(listed, j.ID)
|
||||
}
|
||||
if !more {
|
||||
break
|
||||
}
|
||||
cursor = next
|
||||
}
|
||||
if len(listed) != len(ids) {
|
||||
t.Fatalf("listed %d jobs", len(listed))
|
||||
}
|
||||
for i, id := range listed {
|
||||
if id != ids[len(ids)-1-i] {
|
||||
t.Fatalf("list[%d]=%s expected reverse enqueue sequence %s", i, id, ids[len(ids)-1-i])
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, want := range ids {
|
||||
claimed, err := m.ClaimJobs(ctx, "owner", 1, time.Minute)
|
||||
if err != nil || len(claimed) != 1 || claimed[0].ID != want {
|
||||
t.Fatalf("claim[%d]=%+v %v want %s", i, claimed, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -109,7 +110,13 @@ func (m *Memory) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (Rev
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Memory) PruneRevisionsBefore(tenantID string, cutoff time.Time) (int, error) {
|
||||
func (m *Memory) PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (int, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
res, err := m.PruneRevisionsWithStats(tenantID, cutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPruneRevisionsBeforeCancelledContext(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, rev, _ := m.DemoIDs()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := m.PruneRevisionsBefore(ctx, tenant, time.Now().Add(time.Hour)); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if _, err := m.GetRevision(tenant, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,43 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (m *Memory) GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error) {
|
||||
// ModuleSnapshotUpdate computes a replacement from the current snapshot; nil deletes it.
|
||||
// The callback must not perform store I/O or mutate its input.
|
||||
type ModuleSnapshotUpdate func(current *ModulePrefixSnapshot) (*ModulePrefixSnapshot, error)
|
||||
|
||||
// ListModuleSnapshotSourceRows returns persisted rows for one source key.
|
||||
func (m *Memory) ListModuleSnapshotSourceRows(ctx context.Context, tenantID, moduleID, source string) ([]PrefixRow, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
snap := m.moduleSnapshots[moduleSnapshotKey(tenantID, moduleID)]
|
||||
if snap == nil {
|
||||
return nil, nil
|
||||
}
|
||||
var out []PrefixRow
|
||||
for _, row := range snap.Prefixes {
|
||||
if row.Source == source {
|
||||
out = append(out, row)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *Memory) GetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
key := moduleSnapshotKey(tenantID, moduleID)
|
||||
@@ -19,42 +50,95 @@ func (m *Memory) GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePref
|
||||
return &ModulePrefixSnapshot{
|
||||
InputHash: snap.InputHash,
|
||||
CollectedAt: snap.CollectedAt,
|
||||
Version: snap.Version,
|
||||
Prefixes: cp,
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func (m *Memory) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error {
|
||||
func (m *Memory) SetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID, inputHash string, prefixes []PrefixRow) error {
|
||||
if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" || strings.TrimSpace(inputHash) == "" {
|
||||
return ErrInvalidInput
|
||||
}
|
||||
return m.UpdateModulePrefixSnapshot(ctx, tenantID, moduleID, func(*ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) {
|
||||
return &ModulePrefixSnapshot{InputHash: inputHash, Prefixes: prefixes}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Memory) DeleteModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) error {
|
||||
return m.UpdateModulePrefixSnapshot(ctx, tenantID, moduleID, func(*ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) {
|
||||
return nil, nil
|
||||
})
|
||||
}
|
||||
|
||||
// UpdateModulePrefixSnapshot serializes a pure RMW callback and commits snapshot plus input hash together.
|
||||
func (m *Memory) UpdateModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string, update ModuleSnapshotUpdate) error {
|
||||
if update == nil || strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" {
|
||||
return ErrInvalidInput
|
||||
}
|
||||
unlock, err := m.lockModuleSnapshot(ctx, tenantID, moduleID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
current, _, err := m.GetModulePrefixSnapshot(ctx, tenantID, moduleID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next, err := update(current)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if next != nil && strings.TrimSpace(next.InputHash) == "" {
|
||||
return ErrInvalidInput
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.moduleSnapshots == nil {
|
||||
m.moduleSnapshots = make(map[string]*moduleSnapshotRec)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
mod := m.modules[moduleID]
|
||||
if mod == nil || mod.DeletedAt != nil {
|
||||
return ErrNotFound
|
||||
}
|
||||
if mod.TenantID != tenantID {
|
||||
return ErrTenantScope
|
||||
}
|
||||
cp := make([]PrefixRow, len(prefixes))
|
||||
copy(cp, prefixes)
|
||||
key := moduleSnapshotKey(tenantID, moduleID)
|
||||
m.moduleSnapshots[key] = &moduleSnapshotRec{
|
||||
InputHash: inputHash,
|
||||
CollectedAt: time.Now().UTC(),
|
||||
Prefixes: cp,
|
||||
if next == nil {
|
||||
if m.moduleSnapshots[key] != nil {
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
}
|
||||
delete(m.moduleSnapshots, key)
|
||||
} else if !SameSnapshotContent(current, next) {
|
||||
m.bumpRenderGenerationLocked(tenantID)
|
||||
if m.moduleSnapshots == nil {
|
||||
m.moduleSnapshots = make(map[string]*moduleSnapshotRec)
|
||||
}
|
||||
m.nextModuleSnapshotVersion++
|
||||
m.moduleSnapshots[key] = &moduleSnapshotRec{InputHash: next.InputHash, CollectedAt: time.Now().UTC(), Version: m.nextModuleSnapshotVersion, Prefixes: append([]PrefixRow(nil), next.Prefixes...)}
|
||||
}
|
||||
if next == nil {
|
||||
mod.InputHash = ""
|
||||
} else {
|
||||
mod.InputHash = next.InputHash
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Memory) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.moduleSnapshots != nil {
|
||||
delete(m.moduleSnapshots, moduleSnapshotKey(tenantID, moduleID))
|
||||
func (m *Memory) SetModuleInputHash(ctx context.Context, tenantID, moduleID, hash string) error {
|
||||
unlock, err := m.lockModuleSnapshot(ctx, tenantID, moduleID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Memory) SetModuleInputHash(tenantID, moduleID, hash string) error {
|
||||
defer unlock()
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
mod, ok := m.modules[moduleID]
|
||||
if !ok || mod.DeletedAt != nil {
|
||||
return ErrNotFound
|
||||
@@ -66,20 +150,88 @@ func (m *Memory) SetModuleInputHash(tenantID, moduleID, hash string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Memory) LockModuleSnapshot(tenantID, moduleID string) func() {
|
||||
type snapshotLockEntry struct {
|
||||
gate chan struct{}
|
||||
refs int
|
||||
}
|
||||
|
||||
// lockModuleSnapshot waits until the caller deadline, or EVOBGP_SNAPSHOT_TIMEOUT when unset.
|
||||
func (m *Memory) lockModuleSnapshot(ctx context.Context, tenantID, moduleID string) (func(), error) {
|
||||
if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" {
|
||||
return nil, ErrInvalidInput
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
ctx, cancel, _ := SnapshotWriteBudget(ctx)
|
||||
defer cancel()
|
||||
key := moduleSnapshotKey(tenantID, moduleID)
|
||||
v, _ := m.snapshotLocks.LoadOrStore(key, &sync.Mutex{})
|
||||
mu := v.(*sync.Mutex)
|
||||
mu.Lock()
|
||||
return func() { mu.Unlock() }
|
||||
m.snapshotLocksMu.Lock()
|
||||
if m.snapshotLocks == nil {
|
||||
m.snapshotLocks = make(map[string]*snapshotLockEntry)
|
||||
}
|
||||
entry := m.snapshotLocks[key]
|
||||
if entry == nil {
|
||||
entry = &snapshotLockEntry{gate: make(chan struct{}, 1)}
|
||||
entry.gate <- struct{}{}
|
||||
m.snapshotLocks[key] = entry
|
||||
}
|
||||
entry.refs++
|
||||
m.snapshotLocksMu.Unlock()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
m.releaseSnapshotLockRef(key, entry)
|
||||
return nil, ctx.Err()
|
||||
case <-entry.gate:
|
||||
if err := ctx.Err(); err != nil {
|
||||
entry.gate <- struct{}{}
|
||||
m.releaseSnapshotLockRef(key, entry)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
entry.gate <- struct{}{}
|
||||
m.releaseSnapshotLockRef(key, entry)
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Memory) releaseSnapshotLockRef(key string, entry *snapshotLockEntry) {
|
||||
m.snapshotLocksMu.Lock()
|
||||
defer m.snapshotLocksMu.Unlock()
|
||||
entry.refs--
|
||||
if entry.refs == 0 {
|
||||
delete(m.snapshotLocks, key)
|
||||
}
|
||||
}
|
||||
|
||||
type moduleSnapshotRec struct {
|
||||
InputHash string
|
||||
CollectedAt time.Time
|
||||
Version int64
|
||||
Prefixes []PrefixRow
|
||||
}
|
||||
|
||||
// SameSnapshotContent compares persisted rows, ignoring timestamps, versions, and order.
|
||||
// Rows are matched by (prefix, community, source).
|
||||
func SameSnapshotContent(a, b *ModulePrefixSnapshot) bool {
|
||||
if a == nil || b == nil || a.InputHash != b.InputHash || len(a.Prefixes) != len(b.Prefixes) {
|
||||
return false
|
||||
}
|
||||
ap := append([]PrefixRow(nil), a.Prefixes...)
|
||||
bp := append([]PrefixRow(nil), b.Prefixes...)
|
||||
sortPrefixRows(ap)
|
||||
sortPrefixRows(bp)
|
||||
for i := range ap {
|
||||
if !samePrefixRow(ap[i], bp[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func moduleSnapshotKey(tenantID, moduleID string) string {
|
||||
return strings.TrimSpace(tenantID) + "\x00" + strings.TrimSpace(moduleID)
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMemoryModuleSnapshotAtomicConcurrentRMW(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
mod, err := m.CreateModule(tenant, &Module{Type: "CDN_CIDRS", Name: "atomic", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const writers = 12
|
||||
var wg sync.WaitGroup
|
||||
errCh := make(chan error, writers)
|
||||
for i := 0; i < writers; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
errCh <- m.UpdateModulePrefixSnapshot(context.Background(), tenant, mod.ID, func(current *ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) {
|
||||
var rows []PrefixRow
|
||||
if current != nil {
|
||||
rows = append(rows, current.Prefixes...)
|
||||
}
|
||||
rows = append(rows, PrefixRow{Prefix: fmt.Sprintf("192.0.2.%d/32", i)})
|
||||
return &ModulePrefixSnapshot{InputHash: "atomic", Prefixes: rows}, nil
|
||||
})
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errCh)
|
||||
for err := range errCh {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
snap, ok, err := m.GetModulePrefixSnapshot(context.Background(), tenant, mod.ID)
|
||||
if err != nil || !ok || len(snap.Prefixes) != writers {
|
||||
t.Fatalf("lost concurrent update: %+v ok=%v err=%v", snap, ok, err)
|
||||
}
|
||||
if got, err := m.GetModule(tenant, mod.ID); err != nil || got == nil || got.InputHash != snap.InputHash {
|
||||
t.Fatalf("module input hash differs from snapshot: %+v %+v", got, snap)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryModuleSnapshotAtomicCallbackErrorRollsBack(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
mod, err := m.CreateModule(tenant, &Module{Type: "CDN_CIDRS", Name: "rollback", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if err := m.SetModulePrefixSnapshot(ctx, tenant, mod.ID, "before", []PrefixRow{{Prefix: "192.0.2.0/24"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := errors.New("update failed")
|
||||
err = m.UpdateModulePrefixSnapshot(ctx, tenant, mod.ID, func(current *ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) {
|
||||
current.Prefixes[0].Prefix = "198.51.100.0/24"
|
||||
return nil, want
|
||||
})
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("callback error: %v", err)
|
||||
}
|
||||
snap, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, mod.ID)
|
||||
if err != nil || !ok || snap.InputHash != "before" || snap.Prefixes[0].Prefix != "192.0.2.0/24" {
|
||||
t.Fatalf("partial snapshot visible: %+v ok=%v err=%v", snap, ok, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMemorySnapshotMethodsRespectCancellation(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, module, _, _ := m.DemoIDs()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if snap, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, module); snap != nil || ok || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("get: %+v %v %v", snap, ok, err)
|
||||
}
|
||||
if err := m.SetModulePrefixSnapshot(ctx, tenant, module, "hash", []PrefixRow{{Prefix: "192.0.2.0/24"}}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("set: %v", err)
|
||||
}
|
||||
if err := m.DeleteModulePrefixSnapshot(ctx, tenant, module); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
if err := m.SetModuleInputHash(ctx, tenant, module, "hash"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("input hash: %v", err)
|
||||
}
|
||||
if rows, err := m.ListModuleSnapshotSourceRows(ctx, tenant, module, "ip_range"); rows != nil || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("source rows: %+v %v", rows, err)
|
||||
}
|
||||
if snap, ok, err := m.GetModulePrefixSnapshot(context.Background(), tenant, module); snap != nil || ok || err != nil {
|
||||
t.Fatalf("cancelled write committed: %+v %v %v", snap, ok, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMemoryModuleSnapshotLockCancellationAndCleanup(t *testing.T) {
|
||||
m := NewMemory()
|
||||
first, err := m.lockModuleSnapshot(context.Background(), "tenant-a", "module-a")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
|
||||
defer cancel()
|
||||
second, err := m.lockModuleSnapshot(ctx, "tenant-a", "module-a")
|
||||
if second != nil || !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("waiter not cancelled: unlock=%v err=%v", second != nil, err)
|
||||
}
|
||||
if len(m.snapshotLocks) != 1 || m.snapshotLocks[moduleSnapshotKey("tenant-a", "module-a")].refs != 1 {
|
||||
t.Fatalf("waiter leaked lock ref: %+v", m.snapshotLocks)
|
||||
}
|
||||
first()
|
||||
first()
|
||||
if len(m.snapshotLocks) != 0 {
|
||||
t.Fatalf("idle lock leaked: %+v", m.snapshotLocks)
|
||||
}
|
||||
third, err := m.lockModuleSnapshot(context.Background(), "tenant-a", "module-a")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
third()
|
||||
}
|
||||
|
||||
func TestMemoryModuleSnapshotLockScopesTenantAndModule(t *testing.T) {
|
||||
m := NewMemory()
|
||||
first, err := m.lockModuleSnapshot(context.Background(), "tenant-a", "module-x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer first()
|
||||
for _, pair := range [][2]string{{"tenant-b", "module-x"}, {"tenant-a", "module-y"}} {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
unlock, err := m.lockModuleSnapshot(ctx, pair[0], pair[1])
|
||||
cancel()
|
||||
if err != nil {
|
||||
t.Fatalf("unrelated key blocked %v: %v", pair, err)
|
||||
}
|
||||
unlock()
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
unlock, err := m.lockModuleSnapshot(ctx, "tenant-a", "module-x")
|
||||
if unlock != nil || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancelled context acquired: unlock=%v err=%v", unlock != nil, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type telemetryMemoryRow struct {
|
||||
snapshot TelemetrySnapshot
|
||||
owner string
|
||||
leaseUntil time.Time
|
||||
token int64
|
||||
lastRequested time.Time
|
||||
refreshPending bool
|
||||
}
|
||||
|
||||
func telemetryKey(tenantID, targetID string) string { return tenantID + "/" + targetID }
|
||||
|
||||
func (m *Memory) ensureTelemetryRows(localAuthority string) {
|
||||
if m.telemetry == nil {
|
||||
m.telemetry = make(map[string]telemetryMemoryRow)
|
||||
}
|
||||
for _, sp := range m.speakers {
|
||||
if sp == nil || !SpeakerNeedsRemoteDispatch(sp.Role, ParseSpeakerMeta(sp.MetaJSON)) {
|
||||
continue
|
||||
}
|
||||
key := telemetryKey(sp.TenantID, sp.ID)
|
||||
if _, ok := m.telemetry[key]; !ok {
|
||||
m.telemetry[key] = telemetryMemoryRow{snapshot: TelemetrySnapshot{TenantID: sp.TenantID, TargetID: sp.ID, SpeakerID: sp.ID, Kind: "remote_agent"}}
|
||||
}
|
||||
}
|
||||
if localAuthority != "" {
|
||||
for tenantID := range m.tenants {
|
||||
if tenantID == SystemJobTenantID {
|
||||
continue
|
||||
}
|
||||
targetID := TelemetryLocalTargetID(tenantID, localAuthority)
|
||||
key := telemetryKey(tenantID, targetID)
|
||||
if _, ok := m.telemetry[key]; !ok {
|
||||
m.telemetry[key] = telemetryMemoryRow{snapshot: TelemetrySnapshot{TenantID: tenantID, TargetID: targetID, Kind: "local_bird", LocalAuthority: localAuthority}}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ClaimTelemetry acquires at most limit due targets with a short fencing lease.
|
||||
func (m *Memory) ClaimTelemetry(_ context.Context, owner string, limit int, lease time.Duration, localAuthority string) ([]TelemetryClaim, error) {
|
||||
if strings.TrimSpace(owner) == "" || limit <= 0 {
|
||||
return nil, ErrInvalidInput
|
||||
}
|
||||
if limit > 64 {
|
||||
limit = 64
|
||||
}
|
||||
if lease <= 0 {
|
||||
lease = 15 * time.Second
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.ensureTelemetryRows(localAuthority)
|
||||
keys := make([]string, 0, len(m.telemetry))
|
||||
for key := range m.telemetry {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
a, b := m.telemetry[keys[i]], m.telemetry[keys[j]]
|
||||
if !a.snapshot.NextPollAt.Equal(b.snapshot.NextPollAt) {
|
||||
return a.snapshot.NextPollAt.Before(b.snapshot.NextPollAt)
|
||||
}
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
now := time.Now().UTC()
|
||||
out := make([]TelemetryClaim, 0, limit)
|
||||
for _, key := range keys {
|
||||
row := m.telemetry[key]
|
||||
if row.snapshot.Kind == "local_bird" && row.snapshot.LocalAuthority != localAuthority || row.snapshot.NextPollAt.After(now) || row.leaseUntil.After(now) {
|
||||
continue
|
||||
}
|
||||
var speaker *Speaker
|
||||
if row.snapshot.Kind == "remote_agent" {
|
||||
sp := m.speakers[row.snapshot.SpeakerID]
|
||||
if sp == nil || !SpeakerNeedsRemoteDispatch(sp.Role, ParseSpeakerMeta(sp.MetaJSON)) {
|
||||
delete(m.telemetry, key)
|
||||
continue
|
||||
}
|
||||
cp := *sp
|
||||
speaker = &cp
|
||||
} else {
|
||||
for _, id := range sortedSpeakerIDs(m.speakers) {
|
||||
sp := m.speakers[id]
|
||||
if sp.TenantID == row.snapshot.TenantID && strings.EqualFold(sp.Role, "master") {
|
||||
cp := *sp
|
||||
speaker = &cp
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
row.owner = owner
|
||||
row.token++
|
||||
row.leaseUntil = now.Add(lease)
|
||||
m.telemetry[key] = row
|
||||
claim := TelemetryClaim{TelemetrySnapshot: row.snapshot, Speaker: speaker, OwnerID: owner, FencingToken: row.token}
|
||||
if speaker != nil {
|
||||
claim.SpeakerID = speaker.ID
|
||||
}
|
||||
claim.ConfigFingerprint = TelemetrySpeakerFingerprint(speaker)
|
||||
out = append(out, claim)
|
||||
if len(out) == limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// CompleteTelemetry commits only the current live lease and preserves last success on failure.
|
||||
func (m *Memory) CompleteTelemetry(_ context.Context, claim TelemetryClaim, result TelemetryResult, nextPoll time.Time) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
key := telemetryKey(claim.TenantID, claim.TargetID)
|
||||
row, ok := m.telemetry[key]
|
||||
now := time.Now().UTC()
|
||||
if !ok || row.owner != claim.OwnerID || row.token != claim.FencingToken || !row.leaseUntil.After(now) {
|
||||
return ErrTelemetryLeaseLost
|
||||
}
|
||||
if claim.Speaker != nil {
|
||||
current := m.speakers[claim.Speaker.ID]
|
||||
if TelemetrySpeakerFingerprint(current) != claim.ConfigFingerprint {
|
||||
return ErrTelemetryLeaseLost
|
||||
}
|
||||
}
|
||||
if claim.Kind == "local_bird" {
|
||||
masterID := ""
|
||||
for _, id := range sortedSpeakerIDs(m.speakers) {
|
||||
sp := m.speakers[id]
|
||||
if sp.TenantID == claim.TenantID && strings.EqualFold(sp.Role, "master") {
|
||||
masterID = id
|
||||
break
|
||||
}
|
||||
}
|
||||
if masterID != claim.SpeakerID {
|
||||
return ErrTelemetryLeaseLost
|
||||
}
|
||||
}
|
||||
if row.snapshot.ConfigFingerprint != claim.ConfigFingerprint {
|
||||
row.snapshot.BGP = TelemetryBGP{}
|
||||
row.snapshot.Health = TelemetryHealth{}
|
||||
row.snapshot.DiscoveryCursor = 0
|
||||
}
|
||||
row.snapshot.ConfigFingerprint = claim.ConfigFingerprint
|
||||
row.snapshot.SpeakerID = claim.SpeakerID
|
||||
if result.BGP.Source != "" {
|
||||
if result.BGP.Error != "" {
|
||||
result.BGP.Sessions = nil
|
||||
result.BGP.LastSuccessAt = row.snapshot.BGP.LastSuccessAt
|
||||
} else {
|
||||
result.BGP.LastSuccessAt = result.BGP.ReceivedAt
|
||||
}
|
||||
row.snapshot.BGP = result.BGP
|
||||
}
|
||||
if result.Health.Source != "" {
|
||||
if !result.Health.OK {
|
||||
result.Health.LastSuccessAt = row.snapshot.Health.LastSuccessAt
|
||||
} else {
|
||||
result.Health.LastSuccessAt = result.Health.ReceivedAt
|
||||
}
|
||||
row.snapshot.Health = result.Health
|
||||
}
|
||||
if row.snapshot.BGP.Error != "" || row.snapshot.Health.Source != "" && !row.snapshot.Health.OK {
|
||||
row.snapshot.FailureCount++
|
||||
} else {
|
||||
row.snapshot.FailureCount = 0
|
||||
}
|
||||
row.snapshot.NextPollAt = nextPoll.UTC()
|
||||
if row.refreshPending {
|
||||
row.snapshot.NextPollAt = now
|
||||
row.refreshPending = false
|
||||
}
|
||||
row.owner = ""
|
||||
row.leaseUntil = time.Time{}
|
||||
if result.BGP.Source != "" && result.BGP.Error == "" {
|
||||
row.snapshot.DiscoveryCursor = m.writeTelemetryDiscoveriesLocked(claim, result.BGP, row.snapshot.DiscoveryCursor)
|
||||
}
|
||||
m.telemetry[key] = row
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortedSpeakerIDs(speakers map[string]*Speaker) []string {
|
||||
ids := make([]string, 0, len(speakers))
|
||||
for id := range speakers {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
func (m *Memory) writeTelemetryDiscoveriesLocked(claim TelemetryClaim, bgp TelemetryBGP, cursor int) int {
|
||||
sessions, next := TelemetryDiscoveryBatch(bgp.Sessions, cursor, 256)
|
||||
known := make(map[string]struct{})
|
||||
for _, p := range m.peers {
|
||||
if p == nil || p.TenantID != claim.TenantID {
|
||||
continue
|
||||
}
|
||||
if neighbor, ok := NormalizePeerNeighborString(p.Neighbor); ok {
|
||||
known[neighbor] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, sess := range sessions {
|
||||
neighbor, ok := NormalizePeerNeighborString(sess.Neighbor)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := known[neighbor]; ok {
|
||||
continue
|
||||
}
|
||||
neighborID := strings.TrimSpace(sess.NeighborID)
|
||||
d := m.findPeerDiscoveryLocked(claim.TenantID, neighborID, neighbor, sess.NeighborAS)
|
||||
if d == nil {
|
||||
id := uuid.NewString()
|
||||
m.peerDiscoveries[id] = &BGPPeerDiscovery{ID: id, TenantID: claim.TenantID, SpeakerID: claim.SpeakerID, NeighborID: neighborID, Neighbor: neighbor, RemoteASN: sess.NeighborAS, ProtocolName: sess.Name, SessionState: sess.State, Status: PeerDiscoveryPending, FirstSeenAt: bgp.ReceivedAt, LastSeenAt: bgp.ReceivedAt}
|
||||
continue
|
||||
}
|
||||
if d.Status == PeerDiscoveryRejected || d.LastSeenAt.After(bgp.ReceivedAt) {
|
||||
continue
|
||||
}
|
||||
d.ProtocolName = sess.Name
|
||||
d.SessionState = sess.State
|
||||
d.LastSeenAt = bgp.ReceivedAt
|
||||
if claim.SpeakerID != "" {
|
||||
d.SpeakerID = claim.SpeakerID
|
||||
}
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
// ListTelemetry reads the last committed snapshots without polling agents.
|
||||
func (m *Memory) ListTelemetry(_ context.Context, tenantID string) ([]TelemetrySnapshot, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
out := make([]TelemetrySnapshot, 0)
|
||||
for _, row := range m.telemetry {
|
||||
if row.snapshot.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if row.snapshot.Kind == "remote_agent" {
|
||||
sp := m.speakers[row.snapshot.SpeakerID]
|
||||
if sp == nil || !SpeakerNeedsRemoteDispatch(sp.Role, ParseSpeakerMeta(sp.MetaJSON)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
snap := row.snapshot
|
||||
snap.BGP.Sessions = append(snap.BGP.Sessions[:0:0], snap.BGP.Sessions...)
|
||||
out = append(out, snap)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].TargetID < out[j].TargetID })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// RequestTelemetryRefresh coalesces repeated requests per tenant within cooldown.
|
||||
func (m *Memory) RequestTelemetryRefresh(_ context.Context, tenantID string, cooldown time.Duration) (bool, error) {
|
||||
if cooldown <= 0 {
|
||||
cooldown = 10 * time.Second
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
now := time.Now().UTC()
|
||||
changed := false
|
||||
for key, row := range m.telemetry {
|
||||
if row.snapshot.TenantID != tenantID || now.Sub(row.lastRequested) < cooldown {
|
||||
continue
|
||||
}
|
||||
row.lastRequested = now
|
||||
if row.leaseUntil.After(now) {
|
||||
row.refreshPending = true
|
||||
} else {
|
||||
row.snapshot.NextPollAt = now
|
||||
}
|
||||
m.telemetry[key] = row
|
||||
changed = true
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
)
|
||||
|
||||
func TestMemoryTelemetryFencingRefreshAndDiscovery(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
sp, err := m.CreateSpeaker(tenant, &Speaker{Role: "replica", MetaJSON: `{"agent_domain":"node.test","agent_secret":"secret"}`})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
claims, err := m.ClaimTelemetry(ctx, "owner-a", 2, time.Minute, "")
|
||||
if err != nil || len(claims) != 1 || claims[0].SpeakerID != sp.ID {
|
||||
t.Fatalf("claims=%+v err=%v", claims, err)
|
||||
}
|
||||
first := claims[0]
|
||||
if snaps, err := m.ListTelemetry(ctx, tenant); err != nil || len(snaps) != 1 || snaps[0].ConfigFingerprint != "" {
|
||||
t.Fatalf("uncommitted config visible: %+v %v", snaps, err)
|
||||
}
|
||||
if other, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, ""); err != nil || len(other) != 0 {
|
||||
t.Fatalf("stole lease: %+v %v", other, err)
|
||||
}
|
||||
if requested, err := m.RequestTelemetryRefresh(ctx, tenant, time.Minute); err != nil || !requested {
|
||||
t.Fatalf("refresh=%v err=%v", requested, err)
|
||||
}
|
||||
if requested, err := m.RequestTelemetryRefresh(ctx, tenant, time.Minute); err != nil || requested {
|
||||
t.Fatalf("not coalesced: %v %v", requested, err)
|
||||
}
|
||||
at := time.Now().UTC()
|
||||
result := TelemetryResult{BGP: TelemetryBGP{TelemetryObservation: TelemetryObservation{Source: "agent_bird_protocols", ObservedAt: at, ReceivedAt: at}, Sessions: []birdfmt.BGPSession{{Name: birdfmt.DynamicPeerProtocolPrefix + "test", Neighbor: "192.0.2.1", NeighborAS: 64501, State: "Established"}}}, Health: TelemetryHealth{TelemetryObservation: TelemetryObservation{Source: "agent_health", ObservedAt: at, ReceivedAt: at}, OK: true}}
|
||||
if err := m.CompleteTelemetry(ctx, first, result, at.Add(time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.CompleteTelemetry(ctx, first, result, at.Add(time.Hour)); !errors.Is(err, ErrTelemetryLeaseLost) {
|
||||
t.Fatalf("reused lease: %v", err)
|
||||
}
|
||||
snaps, err := m.ListTelemetry(ctx, tenant)
|
||||
if err != nil || len(snaps) != 1 || snaps[0].ConfigFingerprint == "" || snaps[0].BGP.LastSuccessAt.IsZero() || snaps[0].NextPollAt.After(time.Now().Add(time.Second)) {
|
||||
t.Fatalf("snapshot=%+v err=%v", snaps, err)
|
||||
}
|
||||
discoveries, err := m.ListPeerDiscoveries(tenant, PeerDiscoveryPending)
|
||||
if err != nil || len(discoveries) != 1 {
|
||||
t.Fatalf("discovery=%+v err=%v", discoveries, err)
|
||||
}
|
||||
next, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, "")
|
||||
if err != nil || len(next) != 1 || next[0].FencingToken <= first.FencingToken {
|
||||
t.Fatalf("new claim=%+v %v", next, err)
|
||||
}
|
||||
meta := `{"agent_domain":"new.test","agent_secret":"secret"}`
|
||||
if _, err := m.UpdateSpeaker(tenant, sp.ID, &SpeakerPatch{MetaJSON: &meta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.CompleteTelemetry(ctx, next[0], result, at.Add(time.Hour)); !errors.Is(err, ErrTelemetryLeaseLost) {
|
||||
t.Fatalf("config changed, late result accepted: %v", err)
|
||||
}
|
||||
m.mu.Lock()
|
||||
row := m.telemetry[telemetryKey(tenant, sp.ID)]
|
||||
row.leaseUntil = time.Now().Add(-time.Second)
|
||||
m.telemetry[telemetryKey(tenant, sp.ID)] = row
|
||||
m.mu.Unlock()
|
||||
newConfig, err := m.ClaimTelemetry(ctx, "owner-c", 1, time.Minute, "")
|
||||
if err != nil || len(newConfig) != 1 {
|
||||
t.Fatalf("new config claim=%+v %v", newConfig, err)
|
||||
}
|
||||
failed := TelemetryResult{BGP: TelemetryBGP{TelemetryObservation: TelemetryObservation{Source: "agent_bird_protocols", Error: "timeout", ReceivedAt: time.Now().UTC()}}, Health: TelemetryHealth{TelemetryObservation: TelemetryObservation{Source: "agent_health", Error: "timeout", ReceivedAt: time.Now().UTC()}}}
|
||||
if err := m.CompleteTelemetry(ctx, newConfig[0], failed, time.Now().Add(time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snaps, err = m.ListTelemetry(ctx, tenant)
|
||||
if err != nil || len(snaps) != 1 || !snaps[0].BGP.LastSuccessAt.IsZero() || !snaps[0].Health.LastSuccessAt.IsZero() {
|
||||
t.Fatalf("old endpoint success survived config change: %+v %v", snaps, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryTelemetryLocalAuthority(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
sp, err := m.CreateSpeaker(tenant, &Speaker{Role: "master"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
first, err := m.ClaimTelemetry(ctx, "owner-a", 1, time.Minute, "site-a")
|
||||
if err != nil || len(first) != 1 || first[0].TargetID != TelemetryLocalTargetID(tenant, "site-a") || first[0].SpeakerID != sp.ID {
|
||||
t.Fatalf("local claim=%+v %v", first, err)
|
||||
}
|
||||
second, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, "site-b")
|
||||
if err != nil || len(second) != 1 || second[0].TargetID != TelemetryLocalTargetID(tenant, "site-b") {
|
||||
t.Fatalf("other authority=%+v %v", second, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryDiscoveryBatchRotatesPastStaticPrefix(t *testing.T) {
|
||||
sessions := make([]birdfmt.BGPSession, 300)
|
||||
for i := range sessions {
|
||||
sessions[i] = birdfmt.BGPSession{Name: fmt.Sprintf("static-%d", i)}
|
||||
}
|
||||
sessions[299] = birdfmt.BGPSession{Name: birdfmt.DynamicPeerProtocolPrefix + "last", Neighbor: "192.0.2.9"}
|
||||
batch, cursor := TelemetryDiscoveryBatch(sessions, 0, 256)
|
||||
if len(batch) != 1 || batch[0].Neighbor != "192.0.2.9" || cursor != 0 {
|
||||
t.Fatalf("batch=%+v cursor=%d", batch, cursor)
|
||||
}
|
||||
for i := range sessions {
|
||||
sessions[i].Name = birdfmt.DynamicPeerProtocolPrefix + fmt.Sprint(i)
|
||||
}
|
||||
first, cursor := TelemetryDiscoveryBatch(sessions, 0, 256)
|
||||
second, next := TelemetryDiscoveryBatch(sessions, cursor, 256)
|
||||
if len(first) != 256 || len(second) != 256 || second[0].Name != sessions[256].Name || next != 212 {
|
||||
t.Fatalf("rotation first=%d second=%d cursor=%d next=%d", len(first), len(second), cursor, next)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryFingerprintIgnoresDispatchStatus(t *testing.T) {
|
||||
sp := &Speaker{Role: "replica", Endpoint: "https://node.test", MetaJSON: `{"agent_domain":"node.test","agent_secret":"secret"}`}
|
||||
before := TelemetrySpeakerFingerprint(sp)
|
||||
sp.MetaJSON = `{"agent_domain":"node.test","agent_secret":"secret","last_dispatch_at":"2026-10-26T10:00:00Z","last_dispatch_status":"ok"}`
|
||||
if TelemetrySpeakerFingerprint(sp) != before {
|
||||
t.Fatal("dispatch status invalidated telemetry config")
|
||||
}
|
||||
sp.Role = "master"
|
||||
before = TelemetrySpeakerFingerprint(sp)
|
||||
sp.MetaJSON = `{"agent_domain":"other.test","agent_secret":"changed"}`
|
||||
if TelemetrySpeakerFingerprint(sp) != before {
|
||||
t.Fatal("agent metadata invalidated local BIRD config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryTelemetryExpiredClaimCannotDiscover(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
sp, err := m.CreateSpeaker(tenant, &Speaker{Role: "replica", MetaJSON: `{"agent_domain":"node.test","agent_secret":"secret"}`})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
first, err := m.ClaimTelemetry(ctx, "owner-a", 1, time.Minute, "")
|
||||
if err != nil || len(first) != 1 || first[0].SpeakerID != sp.ID {
|
||||
t.Fatalf("first=%+v %v", first, err)
|
||||
}
|
||||
m.mu.Lock()
|
||||
row := m.telemetry[telemetryKey(tenant, sp.ID)]
|
||||
row.leaseUntil = time.Now().Add(-time.Second)
|
||||
m.telemetry[telemetryKey(tenant, sp.ID)] = row
|
||||
m.mu.Unlock()
|
||||
second, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, "")
|
||||
if err != nil || len(second) != 1 {
|
||||
t.Fatalf("second=%+v %v", second, err)
|
||||
}
|
||||
at := time.Now().UTC()
|
||||
result := TelemetryResult{BGP: TelemetryBGP{TelemetryObservation: TelemetryObservation{Source: "agent_bird_protocols", ReceivedAt: at}, Sessions: []birdfmt.BGPSession{{Name: birdfmt.DynamicPeerProtocolPrefix + "late", Neighbor: "192.0.2.13"}}}}
|
||||
if err := m.CompleteTelemetry(ctx, first[0], result, at.Add(time.Hour)); !errors.Is(err, ErrTelemetryLeaseLost) {
|
||||
t.Fatalf("old claim accepted: %v", err)
|
||||
}
|
||||
if list, err := m.ListPeerDiscoveries(tenant, ""); err != nil || len(list) != 0 {
|
||||
t.Fatalf("stale discovery persisted: %+v %v", list, err)
|
||||
}
|
||||
if err := m.CompleteTelemetry(ctx, second[0], result, at.Add(time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ComputeModuleInputHash fingerprints module config and child entries so snapshots
|
||||
@@ -109,7 +111,9 @@ func TouchModuleInputHash(st Backend, tenantID, moduleID string) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = st.SetModuleInputHash(tenantID, moduleID, h)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = st.SetModuleInputHash(ctx, tenantID, moduleID, h)
|
||||
}
|
||||
|
||||
// CDNPatchAffectsInputHash reports whether a CDN source patch changes ingest fingerprint
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// PrefixSnapshotHash is the content key of prefix_snapshot.
|
||||
// It fingerprints prefix rows only. Peer configuration stays on
|
||||
// config_revision.content_hash so a peer-only change can reuse the snapshot.
|
||||
// The result is 64 lowercase hex characters. Row order does not affect the hash.
|
||||
func PrefixSnapshotHash(rows []PrefixRow) string {
|
||||
cp := append([]PrefixRow(nil), rows...)
|
||||
sortPrefixRows(cp)
|
||||
h := sha256.New()
|
||||
for _, row := range cp {
|
||||
_, _ = h.Write([]byte(row.Prefix))
|
||||
_, _ = h.Write([]byte{1})
|
||||
if row.CommunityID == nil {
|
||||
_, _ = h.Write([]byte{0})
|
||||
} else {
|
||||
_, _ = h.Write([]byte{1})
|
||||
_, _ = h.Write([]byte(*row.CommunityID))
|
||||
}
|
||||
_, _ = h.Write([]byte{1})
|
||||
_, _ = h.Write([]byte(row.Source))
|
||||
_, _ = h.Write([]byte{0})
|
||||
}
|
||||
sum := h.Sum(nil)
|
||||
return hex.EncodeToString(sum)
|
||||
}
|
||||
|
||||
func sortPrefixRows(rows []PrefixRow) {
|
||||
sort.Slice(rows, func(i, j int) bool {
|
||||
if rows[i].Prefix != rows[j].Prefix {
|
||||
return rows[i].Prefix < rows[j].Prefix
|
||||
}
|
||||
ni, vi := communitySortKey(rows[i].CommunityID)
|
||||
nj, vj := communitySortKey(rows[j].CommunityID)
|
||||
if ni != nj {
|
||||
return ni
|
||||
}
|
||||
if vi != vj {
|
||||
return vi < vj
|
||||
}
|
||||
return rows[i].Source < rows[j].Source
|
||||
})
|
||||
}
|
||||
|
||||
func communitySortKey(id *string) (isNil bool, val string) {
|
||||
if id == nil {
|
||||
return true, ""
|
||||
}
|
||||
return false, *id
|
||||
}
|
||||
|
||||
func samePrefixRow(a, b PrefixRow) bool {
|
||||
if a.Prefix != b.Prefix || a.Source != b.Source {
|
||||
return false
|
||||
}
|
||||
if (a.CommunityID == nil) != (b.CommunityID == nil) {
|
||||
return false
|
||||
}
|
||||
if a.CommunityID != nil && *a.CommunityID != *b.CommunityID {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestSameSnapshotContentIgnoresOrder(t *testing.T) {
|
||||
comm := "c1"
|
||||
a := &ModulePrefixSnapshot{InputHash: "h", Prefixes: []PrefixRow{
|
||||
{Prefix: "203.0.113.0/24", CommunityID: &comm, Source: "cdn:a"},
|
||||
{Prefix: "198.51.100.0/24", Source: "cdn:b"},
|
||||
}}
|
||||
b := &ModulePrefixSnapshot{InputHash: "h", Prefixes: []PrefixRow{
|
||||
{Prefix: "198.51.100.0/24", Source: "cdn:b"},
|
||||
{Prefix: "203.0.113.0/24", CommunityID: &comm, Source: "cdn:a"},
|
||||
}}
|
||||
if !SameSnapshotContent(a, b) {
|
||||
t.Fatal("same set in another order was treated as new")
|
||||
}
|
||||
b.Prefixes[0].Prefix = "192.0.2.0/24"
|
||||
if SameSnapshotContent(a, b) {
|
||||
t.Fatal("different prefix compared equal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefixSnapshotHashIgnoresOrder(t *testing.T) {
|
||||
rows := []PrefixRow{
|
||||
{Prefix: "203.0.113.0/24", Source: "ip_range"},
|
||||
{Prefix: "198.51.100.0/24", Source: "cdn:a"},
|
||||
}
|
||||
rev := []PrefixRow{rows[1], rows[0]}
|
||||
if PrefixSnapshotHash(rows) != PrefixSnapshotHash(rev) {
|
||||
t.Fatal("prefix hash depends on order")
|
||||
}
|
||||
rev = append(rev, PrefixRow{Prefix: "2001:db8::/32", Source: "ip_range"})
|
||||
if PrefixSnapshotHash(rows) == PrefixSnapshotHash(rev) {
|
||||
t.Fatal("prefix change kept the same hash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateRenderRevisionReusesPrefixSnapshot(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, mod, _, _ := m.DemoIDs()
|
||||
ctx := context.Background()
|
||||
rows := []PrefixRow{
|
||||
{Prefix: "203.0.113.0/24", Source: "ip_range"},
|
||||
{Prefix: "198.51.100.0/24", Source: "cdn:a"},
|
||||
}
|
||||
rev1 := uuid.NewString()
|
||||
if err := m.CreateRenderRevision(ctx, rev1, tenant, mod, nil, "hash-peer-a", map[string]string{"bird.conf": "a"}, rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rev2 := uuid.NewString()
|
||||
reversed := []PrefixRow{rows[1], rows[0]}
|
||||
if err := m.CreateRenderRevision(ctx, rev2, tenant, mod, nil, "hash-peer-b", map[string]string{"bird.conf": "b"}, reversed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.revPrefixSnap[rev1] == "" || m.revPrefixSnap[rev1] != m.revPrefixSnap[rev2] {
|
||||
t.Fatalf("peer-only change copied prefix rows: %q vs %q", m.revPrefixSnap[rev1], m.revPrefixSnap[rev2])
|
||||
}
|
||||
if len(m.prefixSnapRows) != 1 {
|
||||
t.Fatalf("stored prefix snapshots: %d", len(m.prefixSnapRows))
|
||||
}
|
||||
page, _, _ := m.ListRevisionPrefixes(tenant, rev2, "", 10)
|
||||
if len(page) != 2 {
|
||||
t.Fatalf("reused snapshot not readable: %+v", page)
|
||||
}
|
||||
changed := append(append([]PrefixRow(nil), rows...), PrefixRow{Prefix: "2001:db8::/32", Source: "ip_range"})
|
||||
rev3 := uuid.NewString()
|
||||
if err := m.CreateRenderRevision(ctx, rev3, tenant, mod, nil, "hash-prefix", nil, changed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.revPrefixSnap[rev3] == m.revPrefixSnap[rev1] || len(m.prefixSnapRows) != 2 {
|
||||
t.Fatalf("prefix change did not allocate a snapshot: id=%s count=%d", m.revPrefixSnap[rev3], len(m.prefixSnapRows))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateModulePrefixSnapshotCancelledContext(t *testing.T) {
|
||||
m := NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, mod, _, _, _ := m.DemoIDs()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
err := m.UpdateModulePrefixSnapshot(ctx, tenant, mod, func(*ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) {
|
||||
t.Fatal("snapshot update ran after cancel")
|
||||
return nil, nil
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemorySnapshotLockUsesSnapshotTimeout(t *testing.T) {
|
||||
t.Setenv("EVOBGP_SNAPSHOT_TIMEOUT", "80ms")
|
||||
m := NewMemory()
|
||||
hold, err := m.lockModuleSnapshot(context.Background(), "tenant", "module")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer hold()
|
||||
start := time.Now()
|
||||
_, err = m.lockModuleSnapshot(context.Background(), "tenant", "module")
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if time.Since(start) > time.Second {
|
||||
t.Fatalf("lock wait ignored snapshot timeout: %s", time.Since(start))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotWriteBudgetKeepsCallerDeadline(t *testing.T) {
|
||||
parent, cancel := context.WithTimeout(context.Background(), time.Hour)
|
||||
defer cancel()
|
||||
got, stop, installed := SnapshotWriteBudget(parent)
|
||||
defer stop()
|
||||
if installed {
|
||||
t.Fatal("caller deadline was replaced")
|
||||
}
|
||||
dl, ok := got.Deadline()
|
||||
if !ok || time.Until(dl) < 30*time.Minute {
|
||||
t.Fatal("caller deadline was shortened")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetASNPrefixCacheKeepsRowsWhenSetUnchanged(t *testing.T) {
|
||||
m := NewMemory()
|
||||
if err := m.SetASNPrefixCache(64500, "holder-a", []string{"203.0.113.0/24", "198.51.100.0/24"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := m.asnPrefixCache[64500]
|
||||
ptr := &before.Prefixes[0]
|
||||
fetched := before.FetchedAt
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
if err := m.SetASNPrefixCache(64500, "holder-b", []string{"198.51.100.0/24", "203.0.113.0/24"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := m.asnPrefixCache[64500]
|
||||
if &after.Prefixes[0] != ptr {
|
||||
t.Fatal("unchanged ASN prefixes were rewritten")
|
||||
}
|
||||
if !after.FetchedAt.After(fetched) {
|
||||
t.Fatal("fetched_at was not updated")
|
||||
}
|
||||
if after.Holder != "holder-b" {
|
||||
t.Fatalf("holder=%q", after.Holder)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (m *Memory) GetRenderGeneration(ctx context.Context, tenantID string) (int64, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
if m.tenants[tenantID] == nil {
|
||||
return 0, ErrNotFound
|
||||
}
|
||||
gen := m.renderGeneration[tenantID]
|
||||
if gen == 0 {
|
||||
gen = 1
|
||||
}
|
||||
return gen, nil
|
||||
}
|
||||
|
||||
func (m *Memory) ListModuleSnapshotHeaders(ctx context.Context, tenantID string) ([]ModuleSnapshotUse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return append([]ModuleSnapshotUse(nil), m.snapshotHeadersLocked(tenantID)...), nil
|
||||
}
|
||||
|
||||
func (m *Memory) snapshotHeadersLocked(tenantID string) []ModuleSnapshotUse {
|
||||
prefix := moduleSnapshotKey(tenantID, "")
|
||||
var out []ModuleSnapshotUse
|
||||
for key, snap := range m.moduleSnapshots {
|
||||
if snap == nil || !strings.HasPrefix(key, prefix) {
|
||||
continue
|
||||
}
|
||||
out = append(out, ModuleSnapshotUse{
|
||||
ModuleID: strings.TrimPrefix(key, prefix),
|
||||
InputHash: snap.InputHash,
|
||||
SnapshotVersion: snap.Version,
|
||||
})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ModuleID < out[j].ModuleID })
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Memory) LatestRenderManifest(ctx context.Context, tenantID string) (*RenderManifestRecord, bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
var latest *Revision
|
||||
for _, rev := range m.revisions {
|
||||
if rev == nil || rev.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if latest == nil || rev.CreatedAt.After(latest.CreatedAt) || (rev.CreatedAt.Equal(latest.CreatedAt) && rev.ID > latest.ID) {
|
||||
latest = rev
|
||||
}
|
||||
}
|
||||
if latest == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
rec := m.renderManifests[latest.ID]
|
||||
if rec == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
return cloneRenderManifest(rec), true, nil
|
||||
}
|
||||
|
||||
func (m *Memory) recordRenderManifestLocked(revisionID, tenantID string) {
|
||||
if m.renderManifests == nil {
|
||||
m.renderManifests = make(map[string]*RenderManifestRecord)
|
||||
}
|
||||
gen := m.renderGeneration[tenantID]
|
||||
if gen == 0 {
|
||||
gen = 1
|
||||
}
|
||||
m.renderManifests[revisionID] = &RenderManifestRecord{
|
||||
RevisionID: revisionID,
|
||||
Generation: gen,
|
||||
Snapshots: m.snapshotHeadersLocked(tenantID),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneRenderManifest(rec *RenderManifestRecord) *RenderManifestRecord {
|
||||
if rec == nil {
|
||||
return nil
|
||||
}
|
||||
out := *rec
|
||||
out.Snapshots = append([]ModuleSnapshotUse(nil), rec.Snapshots...)
|
||||
return &out
|
||||
}
|
||||
|
||||
// bumpRenderGenerationLocked is called only while holding m.mu for a semantic write.
|
||||
func (m *Memory) bumpRenderGenerationLocked(tenantID string) {
|
||||
if m.renderGeneration == nil {
|
||||
m.renderGeneration = make(map[string]int64)
|
||||
}
|
||||
if m.renderGeneration[tenantID] == 0 {
|
||||
m.renderGeneration[tenantID] = 1
|
||||
}
|
||||
m.renderGeneration[tenantID]++
|
||||
}
|
||||
|
||||
// RenderInputs is the complete, immutable configuration read by tenant rendering.
|
||||
// Fetch observations such as CDN ETags, resolve metadata and dispatch status are
|
||||
// retained in the captured rows where the ingest path needs them, but excluded
|
||||
// from SemanticHash. Callers must not mutate a captured value.
|
||||
type RenderInputs struct {
|
||||
TenantID string `json:"tenant_id"`
|
||||
Generation int64 `json:"generation"`
|
||||
Modules []RenderModule `json:"modules"`
|
||||
Profiles []*DohProfile `json:"profiles"`
|
||||
Peers []*BGPPeer `json:"peers"`
|
||||
Communities []*Community `json:"communities"`
|
||||
Settings map[string]any `json:"settings"`
|
||||
Speakers []RenderSpeaker `json:"speakers"`
|
||||
}
|
||||
|
||||
// RenderModule contains all ingestion inputs for one module in one capture.
|
||||
type RenderModule struct {
|
||||
Module *Module `json:"module"`
|
||||
CDNSources []*CDNSource `json:"cdn_sources"`
|
||||
ASEntries []*ASEntry `json:"as_entries"`
|
||||
DomainEntries []*DomainEntry `json:"domain_entries"`
|
||||
IPRanges []*IPRangeEntry `json:"ip_ranges"`
|
||||
}
|
||||
|
||||
// RenderSpeaker holds only values that affect speaker-specific BIRD fragments.
|
||||
type RenderSpeaker struct {
|
||||
ID string `json:"id"`
|
||||
NodeIPv4 string `json:"node_ipv4"`
|
||||
BirdBgpSourceIPv4 string `json:"bird_bgp_source_ipv4"`
|
||||
BirdBgpSourceIPv6 string `json:"bird_bgp_source_ipv6"`
|
||||
}
|
||||
|
||||
// ModuleSnapshotUse identifies the exact module materialization used by a render.
|
||||
type ModuleSnapshotUse struct {
|
||||
ModuleID string `json:"module_id"`
|
||||
InputHash string `json:"input_hash"`
|
||||
SnapshotVersion int64 `json:"snapshot_version"`
|
||||
}
|
||||
|
||||
// RenderManifestRecord is the admission barrier stored with a config revision.
|
||||
// Generation and snapshot versions let a later render return the revision
|
||||
// without loading prefix rows when nothing material changed.
|
||||
type RenderManifestRecord struct {
|
||||
RevisionID string
|
||||
Generation int64
|
||||
Snapshots []ModuleSnapshotUse
|
||||
}
|
||||
|
||||
// RenderManifest is persisted with a revision so new publish admission can
|
||||
// distinguish a current render from a historical rollback or durable replay.
|
||||
type RenderManifest struct {
|
||||
Inputs RenderInputs `json:"inputs"`
|
||||
ConfigHash string `json:"config_hash"`
|
||||
Snapshots []ModuleSnapshotUse `json:"snapshots"`
|
||||
OverlayBySpeaker map[string]map[string]string `json:"overlay_by_speaker"`
|
||||
}
|
||||
|
||||
// SemanticHash fingerprints captured render settings and ingest configuration.
|
||||
// It deliberately excludes fetch timestamps, ETags, health and dispatch state.
|
||||
func (in *RenderInputs) SemanticHash() (string, error) {
|
||||
if in == nil || strings.TrimSpace(in.TenantID) == "" {
|
||||
return "", ErrInvalidInput
|
||||
}
|
||||
semantic := struct {
|
||||
TenantID string `json:"tenant_id"`
|
||||
Modules []any `json:"modules"`
|
||||
Profiles []any `json:"profiles"`
|
||||
Peers []any `json:"peers"`
|
||||
Communities []any `json:"communities"`
|
||||
Settings map[string]any `json:"settings"`
|
||||
Speakers []RenderSpeaker `json:"speakers"`
|
||||
}{TenantID: in.TenantID, Settings: in.Settings, Speakers: append([]RenderSpeaker(nil), in.Speakers...)}
|
||||
modules := append([]RenderModule(nil), in.Modules...)
|
||||
for _, item := range modules {
|
||||
if item.Module == nil {
|
||||
return "", fmt.Errorf("store: render inputs: nil module")
|
||||
}
|
||||
}
|
||||
sort.Slice(modules, func(i, j int) bool { return modules[i].Module.ID < modules[j].Module.ID })
|
||||
for _, item := range modules {
|
||||
m := item.Module
|
||||
semantic.Modules = append(semantic.Modules, struct {
|
||||
ID string
|
||||
Type string
|
||||
Enabled bool
|
||||
DefaultCommunityID *string
|
||||
DohResolverPolicy string
|
||||
DohProfileIDs []string
|
||||
CDNSources []any
|
||||
ASEntries []any
|
||||
DomainEntries []any
|
||||
IPRanges []any
|
||||
}{ID: m.ID, Type: m.Type, Enabled: m.Enabled, DefaultCommunityID: m.DefaultCommunityID, DohResolverPolicy: m.DohResolverPolicy,
|
||||
DohProfileIDs: append([]string(nil), m.EffectiveDohProfileIDs()...),
|
||||
CDNSources: semanticCDNSources(item.CDNSources), ASEntries: semanticASEntries(item.ASEntries), DomainEntries: semanticDomainEntries(item.DomainEntries), IPRanges: semanticIPRanges(item.IPRanges)})
|
||||
}
|
||||
profiles := append([]*DohProfile(nil), in.Profiles...)
|
||||
sort.Slice(profiles, func(i, j int) bool {
|
||||
if profiles[i] == nil {
|
||||
return false
|
||||
}
|
||||
if profiles[j] == nil {
|
||||
return true
|
||||
}
|
||||
return profiles[i].ID < profiles[j].ID
|
||||
})
|
||||
for _, p := range profiles {
|
||||
if p != nil {
|
||||
semantic.Profiles = append(semantic.Profiles, struct {
|
||||
ID, URL string
|
||||
TimeoutMs *int
|
||||
SecretRef *string
|
||||
}{p.ID, p.URL, p.TimeoutMs, p.SecretRef})
|
||||
}
|
||||
}
|
||||
peers := append([]*BGPPeer(nil), in.Peers...)
|
||||
sort.Slice(peers, func(i, j int) bool {
|
||||
if peers[i] == nil {
|
||||
return false
|
||||
}
|
||||
if peers[j] == nil {
|
||||
return true
|
||||
}
|
||||
return peers[i].ID < peers[j].ID
|
||||
})
|
||||
for _, p := range peers {
|
||||
if p != nil {
|
||||
semantic.Peers = append(semantic.Peers, struct {
|
||||
ID, Neighbor, PoliciesJSON string
|
||||
RemoteASN int64
|
||||
Enabled bool
|
||||
SpeakerID *string
|
||||
}{p.ID, p.Neighbor, p.PoliciesJSON, p.RemoteASN, p.Enabled, p.SpeakerID})
|
||||
}
|
||||
}
|
||||
communities := append([]*Community(nil), in.Communities...)
|
||||
sort.Slice(communities, func(i, j int) bool {
|
||||
if communities[i] == nil {
|
||||
return false
|
||||
}
|
||||
if communities[j] == nil {
|
||||
return true
|
||||
}
|
||||
return communities[i].ID < communities[j].ID
|
||||
})
|
||||
for _, c := range communities {
|
||||
if c != nil {
|
||||
semantic.Communities = append(semantic.Communities, struct{ ID, Community, ValueJSON string }{c.ID, c.Community, c.ValueJSON})
|
||||
}
|
||||
}
|
||||
sort.Slice(semantic.Speakers, func(i, j int) bool { return semantic.Speakers[i].ID < semantic.Speakers[j].ID })
|
||||
data, err := json.Marshal(semantic)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return "sha256:" + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func semanticCDNSources(items []*CDNSource) []any {
|
||||
items = append([]*CDNSource(nil), items...)
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
if items[i] == nil {
|
||||
return false
|
||||
}
|
||||
if items[j] == nil {
|
||||
return true
|
||||
}
|
||||
return items[i].ID < items[j].ID
|
||||
})
|
||||
out := make([]any, 0, len(items))
|
||||
for _, s := range items {
|
||||
if s != nil {
|
||||
out = append(out, struct {
|
||||
ID, SourceKind, URL, PrefixPath string
|
||||
RefreshIntervalSec *int
|
||||
CommunityID *string
|
||||
}{s.ID, s.SourceKind, s.URL, s.PrefixPath, s.RefreshIntervalSec, s.CommunityID})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
func semanticASEntries(items []*ASEntry) []any {
|
||||
items = append([]*ASEntry(nil), items...)
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
if items[i] == nil {
|
||||
return false
|
||||
}
|
||||
if items[j] == nil {
|
||||
return true
|
||||
}
|
||||
return items[i].ID < items[j].ID
|
||||
})
|
||||
out := make([]any, 0, len(items))
|
||||
for _, s := range items {
|
||||
if s != nil {
|
||||
out = append(out, struct {
|
||||
ID string
|
||||
ASN int64
|
||||
CommunityID *string
|
||||
}{s.ID, s.ASN, s.CommunityID})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
func semanticDomainEntries(items []*DomainEntry) []any {
|
||||
items = append([]*DomainEntry(nil), items...)
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
if items[i] == nil {
|
||||
return false
|
||||
}
|
||||
if items[j] == nil {
|
||||
return true
|
||||
}
|
||||
return items[i].ID < items[j].ID
|
||||
})
|
||||
out := make([]any, 0, len(items))
|
||||
for _, s := range items {
|
||||
if s != nil {
|
||||
out = append(out, struct {
|
||||
ID, FQDN string
|
||||
CommunityID *string
|
||||
}{s.ID, s.FQDN, s.CommunityID})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
func semanticIPRanges(items []*IPRangeEntry) []any {
|
||||
items = append([]*IPRangeEntry(nil), items...)
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
if items[i] == nil {
|
||||
return false
|
||||
}
|
||||
if items[j] == nil {
|
||||
return true
|
||||
}
|
||||
return items[i].ID < items[j].ID
|
||||
})
|
||||
out := make([]any, 0, len(items))
|
||||
for _, s := range items {
|
||||
if s != nil {
|
||||
out = append(out, struct {
|
||||
ID, Prefix string
|
||||
CommunityID *string
|
||||
}{s.ID, s.Prefix, s.CommunityID})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRenderInputsSemanticHashCanonicalAndIgnoresFetchState(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
a := &RenderInputs{TenantID: "tenant", Modules: []RenderModule{
|
||||
{Module: &Module{ID: "b", Type: "CDN_CIDRS", Enabled: true}, CDNSources: []*CDNSource{{ID: "s2", URL: "https://b.test", Etag: "old"}, {ID: "s1", URL: "https://a.test", LastRefreshedAt: &now}}},
|
||||
{Module: &Module{ID: "a", Type: "IP_RANGES", Enabled: true}, IPRanges: []*IPRangeEntry{{ID: "r2", Prefix: "192.0.2.0/24"}, {ID: "r1", Prefix: "198.51.100.0/24"}}},
|
||||
}}
|
||||
b := &RenderInputs{TenantID: "tenant", Generation: 99, Modules: []RenderModule{
|
||||
{Module: &Module{ID: "a", Type: "IP_RANGES", Enabled: true}, IPRanges: []*IPRangeEntry{{ID: "r1", Prefix: "198.51.100.0/24"}, {ID: "r2", Prefix: "192.0.2.0/24"}}},
|
||||
{Module: &Module{ID: "b", Type: "CDN_CIDRS", Enabled: true}, CDNSources: []*CDNSource{{ID: "s1", URL: "https://a.test"}, {ID: "s2", URL: "https://b.test", Etag: "new"}}},
|
||||
}}
|
||||
ha, err := a.SemanticHash()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hb, err := b.SemanticHash()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ha != hb {
|
||||
t.Fatalf("same semantic inputs have different hashes: %s %s", ha, hb)
|
||||
}
|
||||
b.Modules[1].CDNSources[0].URL = "https://changed.test"
|
||||
hb, err = b.SemanticHash()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ha == hb {
|
||||
t.Fatal("source URL change did not change hash")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultSnapshotTimeout = 2 * time.Minute
|
||||
|
||||
// SnapshotWriteBudget keeps a caller deadline. When ctx has none, it applies
|
||||
// EVOBGP_SNAPSHOT_TIMEOUT (default 2 minutes). installed is true when this
|
||||
// function created the deadline; callers use that to set statement_timeout.
|
||||
func SnapshotWriteBudget(ctx context.Context) (context.Context, context.CancelFunc, bool) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if _, ok := ctx.Deadline(); ok {
|
||||
return ctx, func() {}, false
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, snapshotTimeout())
|
||||
return ctx, cancel, true
|
||||
}
|
||||
|
||||
// SnapshotStatementTimeout is the PostgreSQL statement_timeout that matches
|
||||
// SnapshotWriteBudget. A caller deadline wins; otherwise the env default applies.
|
||||
func SnapshotStatementTimeout(ctx context.Context) time.Duration {
|
||||
if ctx != nil {
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
d := time.Until(dl)
|
||||
if d < time.Millisecond {
|
||||
return time.Millisecond
|
||||
}
|
||||
return d
|
||||
}
|
||||
}
|
||||
return snapshotTimeout()
|
||||
}
|
||||
|
||||
func snapshotTimeout() time.Duration {
|
||||
s := strings.TrimSpace(os.Getenv("EVOBGP_SNAPSHOT_TIMEOUT"))
|
||||
if s == "" {
|
||||
return defaultSnapshotTimeout
|
||||
}
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil || d <= 0 {
|
||||
return defaultSnapshotTimeout
|
||||
}
|
||||
return d
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
)
|
||||
|
||||
// ErrTelemetryLeaseLost means a collector result belongs to an expired or replaced lease.
|
||||
var ErrTelemetryLeaseLost = errors.New("store: telemetry lease lost")
|
||||
|
||||
// TelemetryLocalTargetID identifies the synthetic local BIRD target of a tenant.
|
||||
func TelemetryLocalTargetID(tenantID, authority string) string {
|
||||
return "cp-local/" + tenantID + "/" + authority
|
||||
}
|
||||
|
||||
// TelemetryObservation is the most recent attempt and last successful reading of one source.
|
||||
type TelemetryObservation struct {
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
ReceivedAt time.Time `json:"received_at"`
|
||||
LastSuccessAt time.Time `json:"last_success_at"`
|
||||
Source string `json:"source"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// TelemetryBGP is one BIRD protocols measurement.
|
||||
type TelemetryBGP struct {
|
||||
TelemetryObservation
|
||||
Sessions []birdfmt.BGPSession `json:"sessions,omitempty"`
|
||||
}
|
||||
|
||||
// TelemetryHealth is one legacy agent health measurement.
|
||||
type TelemetryHealth struct {
|
||||
TelemetryObservation
|
||||
OK bool `json:"ok"`
|
||||
LastSyncAt string `json:"last_sync_at,omitempty"`
|
||||
LastAppliedRevisionID string `json:"last_applied_revision_id,omitempty"`
|
||||
}
|
||||
|
||||
// TelemetrySnapshot contains independent BGP and agent health readings for one target.
|
||||
type TelemetrySnapshot struct {
|
||||
TenantID, TargetID, SpeakerID, Kind string
|
||||
LocalAuthority string
|
||||
ConfigFingerprint string
|
||||
BGP TelemetryBGP
|
||||
Health TelemetryHealth
|
||||
NextPollAt time.Time
|
||||
FailureCount int
|
||||
DiscoveryCursor int
|
||||
}
|
||||
|
||||
// TelemetrySpeakerFingerprint binds a snapshot to the configuration it observed.
|
||||
func TelemetrySpeakerFingerprint(sp *Speaker) string {
|
||||
if sp == nil {
|
||||
return ""
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
|
||||
sum := sha256.Sum256([]byte("master\x00" + sp.Endpoint))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
meta := ParseSpeakerMeta(sp.MetaJSON)
|
||||
sum := sha256.Sum256([]byte(sp.Role + "\x00" + sp.Endpoint + "\x00" + meta.AgentDomain + "\x00" + meta.AgentSecret + "\x00" + strconv.Itoa(meta.AgentPort)))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// TelemetryClaim fences a bounded collector attempt. Speaker is the current config.
|
||||
type TelemetryClaim struct {
|
||||
TelemetrySnapshot
|
||||
Speaker *Speaker
|
||||
OwnerID string
|
||||
FencingToken int64
|
||||
}
|
||||
|
||||
// TelemetryResult is the atomic pair of results of one collector attempt.
|
||||
type TelemetryResult struct {
|
||||
BGP TelemetryBGP
|
||||
Health TelemetryHealth
|
||||
}
|
||||
|
||||
// TelemetryDiscoveryBatch rotates over long protocol lists without starving later entries.
|
||||
func TelemetryDiscoveryBatch(sessions []birdfmt.BGPSession, cursor, limit int) ([]birdfmt.BGPSession, int) {
|
||||
if len(sessions) == 0 || limit <= 0 {
|
||||
return nil, 0
|
||||
}
|
||||
if cursor < 0 || cursor >= len(sessions) {
|
||||
cursor = 0
|
||||
}
|
||||
out := make([]birdfmt.BGPSession, 0, limit)
|
||||
for visited := 0; visited < len(sessions); visited++ {
|
||||
idx := (cursor + visited) % len(sessions)
|
||||
if birdfmt.IsDynamicDiscoverySession(sessions[idx].Name) {
|
||||
out = append(out, sessions[idx])
|
||||
if len(out) == limit {
|
||||
return out, (idx + 1) % len(sessions)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, 0
|
||||
}
|
||||
|
||||
// TelemetryStore coordinates polling and read-only snapshots across API replicas.
|
||||
type TelemetryStore interface {
|
||||
ClaimTelemetry(context.Context, string, int, time.Duration, string) ([]TelemetryClaim, error)
|
||||
CompleteTelemetry(context.Context, TelemetryClaim, TelemetryResult, time.Time) error
|
||||
ListTelemetry(context.Context, string) ([]TelemetrySnapshot, error)
|
||||
RequestTelemetryRefresh(context.Context, string, time.Duration) (bool, error)
|
||||
}
|
||||
Reference in new issue
Block a user