feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed 2026-11-02 17:23:59 +07:00
1 parent 924f111e48
commit f8e0520817
197 files changed
+18655 -1416

No files matched your search

+29 -8
View File
@@ -8,6 +8,11 @@ import (
// Backend is the persistence abstraction for the control plane (memory, PostgreSQL, SQLite).
type Backend interface {
JobQueue
Deployments
TelemetryStore
// ObserveStoreMetrics returns historical revision and peer inventory aggregates or an error.
ObserveStoreMetrics(ctx context.Context) (max, sum, peers int, states map[string]int, err error)
MaterializedPrefixStats() (max int, sum int)
PeerCount() int
PeerSessionCountsByState() map[string]int
@@ -16,6 +21,14 @@ type Backend interface {
// ListTenantIDs returns distinct tenant identifiers (for background workers).
ListTenantIDs() ([]string, error)
// GetRenderGeneration returns the tenant's monotonic render-input barrier.
GetRenderGeneration(ctx context.Context, tenantID string) (int64, error)
// ListModuleSnapshotHeaders returns snapshot versions for the tenant, ordered by module id.
// It does not load prefix rows.
ListModuleSnapshotHeaders(ctx context.Context, tenantID string) ([]ModuleSnapshotUse, error)
// LatestRenderManifest returns the manifest stored with the newest config revision.
// ok is false when that revision has no manifest yet.
LatestRenderManifest(ctx context.Context, tenantID string) (*RenderManifestRecord, bool, error)
// ListModules returns all modules for a tenant (control plane may paginate in httpapi).
ListModules(tenantID string) []*Module
@@ -91,11 +104,15 @@ type Backend interface {
ListRevisionPrefixes(tenantID, revisionID string, cursor string, limit int) (prefixes []PrefixRow, next string, more bool)
CreateRollbackRevision(tenantID, sourceRevisionID string) (newID string, err error)
// CreateRenderRevision inserts a new config_revision (revID must be unique) with materialized prefixes and preview fragments.
CreateRenderRevision(revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error
// The same transaction records config_generation and snapshot versions in render_revision_manifest.
// ctx cancellation aborts the write. prefix_snapshot is keyed by a hash of prefixes only.
CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error
RevisionDiff(tenantID, aID, bID string) (map[string]any, error)
EstimateRevisionPrune(tenantID string, cutoff time.Time, retentionMinutes int) (RevisionPruneEstimate, error)
PruneRevisionsWithStats(tenantID string, cutoff time.Time) (RevisionPruneResult, error)
PruneRevisionsBefore(tenantID string, cutoff time.Time) (deleted int, err error)
// PruneRevisionsBefore deletes revisions older than cutoff, keeping the newest
// revision and any revision a speaker still references. ctx bounds the delete.
PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (deleted int, err error)
SetLastAppliedRevision(tenantID, speakerID, revisionID string) error
PublishRevisionForSpeaker(speakerID, revisionID string) error
LatestPublishedRevision(speakerID string) (revisionID string, publishedAt time.Time, err error)
@@ -113,13 +130,16 @@ type Backend interface {
TouchAPIKeyLastUsed(id string) error
// Module prefix snapshots cache last successful collect per module (pipeline ingest/render).
GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error)
SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error
DeleteModulePrefixSnapshot(tenantID, moduleID string) error
GetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error)
// ListModuleSnapshotSourceRows returns persisted rows for one source key.
// It does not load the rest of the module snapshot.
ListModuleSnapshotSourceRows(ctx context.Context, tenantID, moduleID, source string) ([]PrefixRow, error)
SetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID, inputHash string, prefixes []PrefixRow) error
DeleteModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) error
// UpdateModulePrefixSnapshot runs a pure mutation against the latest snapshot atomically.
UpdateModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string, update ModuleSnapshotUpdate) error
// SetModuleInputHash stores the ingest fingerprint on the module row (O(1) snapshot check).
SetModuleInputHash(tenantID, moduleID, hash string) error
// LockModuleSnapshot serializes read-modify-write of one module snapshot; unlock must be called.
LockModuleSnapshot(tenantID, moduleID string) (unlock func())
SetModuleInputHash(ctx context.Context, tenantID, moduleID, hash string) error
// ASNPrefixCache stores RIPEstat announced-prefixes per ASN (global TTL cache).
GetASNPrefixCache(asn int64) (*ASNPrefixCacheEntry, bool, error)
@@ -197,6 +217,7 @@ type DomainResolveCacheEntry struct {
type ModulePrefixSnapshot struct {
InputHash string
CollectedAt time.Time
Version int64
Prefixes []PrefixRow
}
+50
View File
@@ -0,0 +1,50 @@
package store
import (
"context"
"errors"
"time"
)
// ErrDeploymentBusy means a previous legacy request may still be executing.
var ErrDeploymentBusy = errors.New("store: speaker delivery reservation is active")
// DeploymentDeliveryWindow retains delivery ownership after an ambiguous response.
const DeploymentDeliveryWindow = 2 * time.Minute
// JobFence identifies the currently leased deployment attempt.
type JobFence struct {
JobID, TenantID, OwnerID string
Token int64
ExpectedGeneration int64
RetryRound int
}
// DeploymentSpeaker preserves delivery progress independently from job result metadata.
// Generation is monotonic per speaker, including across different jobs.
type DeploymentSpeaker struct {
ObservationToken int64
JobID, TenantID, SpeakerID, RevisionID string
Generation int64
RequestSequence int64
State, Confirmation, Error string
DispatchAttempts int
RetryRound int
HealthReceivedAt *time.Time
HealthRevisionID, HealthSourceAt, HealthError string
}
// DeploymentObservation is a compatible authenticated health report. SourceAt may
// be absent on legacy agents; ReceivedAt is generated by the control-plane store.
type DeploymentObservation struct{ RevisionID, SourceAt, Error string }
// Deployments atomically fences publication and acknowledgement against the job
// lease and the speaker's current deployment generation.
type Deployments interface {
BeginDeploymentSpeaker(context.Context, JobFence, string, string) (DeploymentSpeaker, error)
RecordDeploymentSpeaker(context.Context, JobFence, DeploymentSpeaker, string, string, string) error
ListDeploymentSpeakers(context.Context, string, string) ([]DeploymentSpeaker, error)
ListPendingDeployments(context.Context, int) ([]DeploymentSpeaker, error)
ListDeploymentObservations(context.Context, int) ([]DeploymentSpeaker, error)
ObserveDeployment(context.Context, DeploymentSpeaker, DeploymentObservation) error
}
+170
View File
@@ -0,0 +1,170 @@
package store
import (
"context"
"errors"
"testing"
"time"
)
func TestDeploymentLeaseGenerationAndReplay(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, _, _, revision, speaker := m.DemoIDs()
ctx := context.Background()
enqueue := func() JobRecord {
t.Helper()
j, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: tenant, Kind: "deploy_apply"})
if err != nil {
t.Fatal(err)
}
return j
}
claim := func(owner string) JobRecord {
t.Helper()
rows, err := m.ClaimJobs(ctx, owner, 1, time.Minute)
if err != nil || len(rows) != 1 {
t.Fatalf("claim: %v %v", rows, err)
}
return rows[0]
}
fence := func(j JobRecord) JobFence {
return JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken}
}
first := enqueue()
first = claim("one")
d, err := m.BeginDeploymentSpeaker(ctx, fence(first), speaker, revision)
if err != nil {
t.Fatal(err)
}
if err = m.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "verified", ""); !errors.Is(err, ErrInvalidInput) {
t.Fatalf("unproved verified: %v", err)
}
if err = m.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "agent_reported", ""); err != nil {
t.Fatal(err)
}
replayed, err := m.BeginDeploymentSpeaker(ctx, fence(first), speaker, revision)
if err != nil || replayed.Generation != d.Generation || replayed.State != "applied" {
t.Fatalf("replay: %+v %v", replayed, err)
}
if err = m.ReleaseJob(ctx, first.ID, first.OwnerID, first.FencingToken); err != nil {
t.Fatal(err)
}
reclaimed := claim("two")
if err = m.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "agent_reported", ""); !errors.Is(err, ErrJobLeaseLost) {
t.Fatalf("old owner ACK: %v", err)
}
if _, err = m.BeginDeploymentSpeaker(ctx, fence(reclaimed), speaker, revision); err != nil {
t.Fatal(err)
}
second := enqueue()
second = claim("three")
d2, err := m.BeginDeploymentSpeaker(ctx, fence(second), speaker, revision)
if err != nil {
t.Fatal(err)
}
if d2.Generation <= d.Generation {
t.Fatal("generation failed to advance across independent job fencing tokens")
}
if err = m.RecordDeploymentSpeaker(ctx, fence(reclaimed), d, "applied", "agent_reported", ""); !errors.Is(err, ErrJobLeaseLost) {
t.Fatalf("old generation ACK: %v", err)
}
if _, err = m.BeginDeploymentSpeaker(ctx, fence(reclaimed), speaker, revision); !errors.Is(err, ErrJobLeaseLost) {
t.Fatalf("old generation republish: %v", err)
}
rows, err := m.ListDeploymentSpeakers(ctx, tenant, first.ID)
if err != nil || len(rows) != 1 {
t.Fatalf("durable progress: %v %v", rows, err)
}
if err = m.RecordDeploymentSpeaker(ctx, fence(second), d2, "unknown", "unconfirmed", "response lost"); err != nil {
t.Fatal(err)
}
}
func TestDeploymentObservationTokenRejectsLateHealth(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, _, _, revision, speaker := m.DemoIDs()
ctx := context.Background()
_, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: tenant, Kind: "deploy_apply"})
if err != nil {
t.Fatal(err)
}
jobs, err := m.ClaimJobs(ctx, "owner", 1, time.Minute)
if err != nil || len(jobs) != 1 {
t.Fatal(err)
}
j := jobs[0]
f := JobFence{JobID: j.ID, TenantID: tenant, OwnerID: j.OwnerID, Token: j.FencingToken}
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
if err != nil {
t.Fatal(err)
}
if err = m.RecordDeploymentSpeaker(ctx, f, d, "applied", "agent_reported", ""); err != nil {
t.Fatal(err)
}
j.Status = "succeeded"
if err = m.FinishJob(ctx, j); err != nil {
t.Fatal(err)
}
old, err := m.ListDeploymentObservations(ctx, 1)
if err != nil || len(old) != 1 {
t.Fatal(err)
}
latest, err := m.ListDeploymentObservations(ctx, 1)
if err != nil || len(latest) != 1 {
t.Fatal(err)
}
if err = m.ObserveDeployment(ctx, latest[0], DeploymentObservation{Error: "timeout"}); err != nil {
t.Fatal(err)
}
if err = m.ObserveDeployment(ctx, old[0], DeploymentObservation{RevisionID: "older"}); !errors.Is(err, ErrJobLeaseLost) {
t.Fatalf("late report accepted: %v", err)
}
rows, err := m.ListDeploymentSpeakers(ctx, tenant, j.ID)
if err != nil || rows[0].State != "applied" || rows[0].Confirmation != "agent_reported" || rows[0].HealthError != "timeout" {
t.Fatalf("error downgraded confirmation: %+v %v", rows, err)
}
}
func TestDeploymentReconciliationRetryBudget(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, _, _, revision, speaker := m.DemoIDs()
ctx := context.Background()
var previous DeploymentSpeaker
for round := 0; round <= 3; round++ {
_, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: tenant, Kind: "deploy_apply"})
if err != nil {
t.Fatal(err)
}
jobs, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
if err != nil || len(jobs) != 1 {
t.Fatal(err)
}
j := jobs[0]
f := JobFence{JobID: j.ID, TenantID: tenant, OwnerID: j.OwnerID, Token: j.FencingToken, RetryRound: round}
if round > 0 {
f.ExpectedGeneration = previous.Generation
}
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
if err != nil {
t.Fatal(err)
}
if err = m.RecordDeploymentSpeaker(ctx, f, d, "failed", "unconfirmed", "unreachable"); err != nil {
t.Fatal(err)
}
j.Status = "failed"
if err = m.FinishJob(ctx, j); err != nil {
t.Fatal(err)
}
pending, err := m.ListPendingDeployments(ctx, 100)
if err != nil {
t.Fatal(err)
}
if (round < 3 && len(pending) != 1) || (round == 3 && len(pending) != 0) {
t.Fatalf("round %d pending=%d", round, len(pending))
}
previous = d
}
}
+83
View File
@@ -0,0 +1,83 @@
package store
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"time"
)
// SystemJobTenantID owns global maintenance jobs outside user tenant scope.
const SystemJobTenantID = "00000000-0000-0000-0000-000000000000"
// JobRequestHash identifies the immutable requested operation, excluding result metadata.
func JobRequestHash(in JobEnqueue) (string, error) {
meta := in.Meta
if in.Request != nil {
meta = in.Request
}
b, err := json.Marshal(struct {
Kind string `json:"kind"`
ModuleID *string `json:"module_id"`
Meta map[string]any `json:"meta"`
}{in.Kind, in.ModuleID, meta})
if err != nil {
return "", err
}
h := sha256.Sum256(b)
return hex.EncodeToString(h[:]), nil
}
var (
ErrJobNotFound = errors.New("store: job not found")
ErrJobConflict = errors.New("store: idempotency key conflicts with another request")
ErrJobQueueFull = errors.New("store: job queue is full")
ErrJobLeaseLost = errors.New("store: job lease lost")
)
// JobRecord is the shared, tenant-scoped job lifecycle record.
type JobRecord struct {
RequestHash string
ID, TenantID, Kind, Status string
IdempotencyKey, ModuleID *string
Meta map[string]any
CreatedAt time.Time
StartedAt, FinishedAt *time.Time
LeaseUntil *time.Time
Error *string
OwnerID string
FencingToken int64
RequestSequence int64
Attempt int
CancelRequested bool
}
// JobEnqueue is the immutable request used for idempotent insertion.
type JobEnqueue struct {
TenantID, Kind string
IdempotencyKey, ModuleID *string
Meta map[string]any
// Request optionally supplies the original selector before dynamic resolution.
Request map[string]any
}
// JobQueue provides durable operations; PostgreSQL is authoritative in production.
type JobQueue interface {
CountJobsByStatus(context.Context) (map[string]int64, error)
FindJobByIdempotencyKey(context.Context, string, string) (JobRecord, error)
EnqueueJob(context.Context, JobEnqueue) (JobRecord, bool, error)
ClaimJobs(context.Context, string, int, time.Duration) ([]JobRecord, error)
// ReapExpiredJobs marks orphaned cancellations and exhausted retries as terminal.
// Production callers run it about every 15 seconds, not on every claim.
ReapExpiredJobs(context.Context) error
HeartbeatJob(context.Context, string, string, int64, time.Duration) (bool, error)
CheckpointJob(context.Context, JobRecord) error
FinishJob(context.Context, JobRecord) error
ReleaseJob(context.Context, string, string, int64) error
GetJob(context.Context, string, string) (JobRecord, error)
ListJobs(context.Context, string, string, string, string, int) ([]JobRecord, string, bool, error)
CancelJob(context.Context, string, string) (JobRecord, error)
RetryJob(context.Context, string, string) (JobRecord, error)
}
+101 -30
View File
@@ -21,7 +21,18 @@ var (
// Memory is a development-oriented in-memory backend for control-plane API handlers.
type Memory struct {
mu sync.RWMutex
mu sync.RWMutex
jobMu sync.Mutex
jobTenantMu sync.Mutex
jobTenantLocks map[string]*sync.Mutex
jobSequence int64
deploymentReservations map[string]deploymentReservation
deployments map[string]DeploymentSpeaker
deploymentCurrent map[string]DeploymentSpeaker
jobs map[string]JobRecord
jobHashes map[string]string
jobRequests map[string]map[string]any
telemetry map[string]telemetryMemoryRow
tenants map[string]*Tenant
modules map[string]*Module
@@ -35,26 +46,33 @@ type Memory struct {
peerDiscoveries map[string]*BGPPeerDiscovery
dohProfiles map[string]*DohProfile
communities map[string]*Community
cdnSources map[string]*CDNSource
asEntries map[string]*ASEntry
domainEnt map[string]*DomainEntry
ipRanges map[string]*IPRangeEntry
settings map[string]map[string]any // tenantID -> key -> JSON-compatible value
revPrefixes map[string][]PrefixRow
moduleSnapshots map[string]*moduleSnapshotRec
snapshotLocks sync.Map // key -> *sync.Mutex (per-module snapshot RMW)
asnPrefixCache map[int64]*ASNPrefixCacheEntry
domainResolveCache map[string]*DomainResolveCacheEntry
apiKeys map[string]*apiKeyRec
firewallClients map[string]*firewallClientRec
firewallRules map[string]*FirewallRule
firewallHashIndex map[string]string // hex hash -> client id
maintenancePolicies map[string]*MaintenancePolicy
maintConfigAudit []*MaintenancePolicyConfigAudit
runtimeLogCleanupAudit []*RuntimeLogCleanupAudit
auditLog []*AuditEntry
dohProfiles map[string]*DohProfile
communities map[string]*Community
cdnSources map[string]*CDNSource
asEntries map[string]*ASEntry
domainEnt map[string]*DomainEntry
ipRanges map[string]*IPRangeEntry
settings map[string]map[string]any // tenantID -> key -> JSON-compatible value
revPrefixes map[string][]PrefixRow
prefixSnapByHash map[string]string
prefixSnapRows map[string][]PrefixRow
revPrefixSnap map[string]string
moduleSnapshots map[string]*moduleSnapshotRec
nextModuleSnapshotVersion int64
renderGeneration map[string]int64
renderManifests map[string]*RenderManifestRecord
snapshotLocksMu sync.Mutex
snapshotLocks map[string]*snapshotLockEntry
asnPrefixCache map[int64]*ASNPrefixCacheEntry
domainResolveCache map[string]*DomainResolveCacheEntry
apiKeys map[string]*apiKeyRec
firewallClients map[string]*firewallClientRec
firewallRules map[string]*FirewallRule
firewallHashIndex map[string]string // hex hash -> client id
maintenancePolicies map[string]*MaintenancePolicy
maintConfigAudit []*MaintenancePolicyConfigAudit
runtimeLogCleanupAudit []*RuntimeLogCleanupAudit
auditLog []*AuditEntry
// DemoIDs valid after SeedDemo()
demoTenantID string
@@ -157,7 +175,12 @@ func NewMemory() *Memory {
ipRanges: make(map[string]*IPRangeEntry),
settings: make(map[string]map[string]any),
revPrefixes: make(map[string][]PrefixRow),
prefixSnapByHash: make(map[string]string),
prefixSnapRows: make(map[string][]PrefixRow),
revPrefixSnap: make(map[string]string),
moduleSnapshots: make(map[string]*moduleSnapshotRec),
renderGeneration: make(map[string]int64),
renderManifests: make(map[string]*RenderManifestRecord),
asnPrefixCache: make(map[int64]*ASNPrefixCacheEntry),
domainResolveCache: make(map[string]*DomainResolveCacheEntry),
apiKeys: make(map[string]*apiKeyRec),
@@ -319,6 +342,31 @@ func (m *Memory) PeerSessionCountsByState() map[string]int {
return out
}
// ObserveStoreMetrics returns an error-aware snapshot of stored revision and peer inventory aggregates.
func (m *Memory) ObserveStoreMetrics(ctx context.Context) (max, sum, peers int, states map[string]int, err error) {
if err := ctx.Err(); err != nil {
return 0, 0, 0, nil, err
}
m.mu.RLock()
defer m.mu.RUnlock()
for _, revision := range m.revisions {
sum += revision.MaterializedPrefixCount
if revision.MaterializedPrefixCount > max {
max = revision.MaterializedPrefixCount
}
}
states = make(map[string]int)
for _, peer := range m.peers {
peers++
state := strings.TrimSpace(peer.SessionState)
if state == "" {
state = "unknown"
}
states[state]++
}
return max, sum, peers, states, nil
}
// DemoIDs returns IDs from SeedDemo; empty strings if SeedDemo was not called.
func (m *Memory) DemoIDs() (tenant, moduleCDN, moduleIP, revision, speaker string) {
m.mu.RLock()
@@ -349,13 +397,22 @@ func (m *Memory) ListTenantIDs() ([]string, error) {
return out, nil
}
// CreateRenderRevision stores a new revision and its materialized prefixes.
func (m *Memory) CreateRenderRevision(revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error {
// CreateRenderRevision stores a new revision, its prefix snapshot, and the render manifest.
func (m *Memory) CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error {
if ctx == nil {
ctx = context.Background()
}
if err := ctx.Err(); err != nil {
return err
}
if strings.TrimSpace(revisionID) == "" || strings.TrimSpace(moduleID) == "" || strings.TrimSpace(contentHash) == "" {
return ErrInvalidInput
}
m.mu.Lock()
defer m.mu.Unlock()
if err := ctx.Err(); err != nil {
return err
}
if _, exists := m.revisions[revisionID]; exists {
return ErrInvalidInput
}
@@ -392,16 +449,30 @@ func (m *Memory) CreateRenderRevision(revisionID, tenantID, moduleID string, par
MaterializedPrefixCount: len(prefixes),
PreviewFragments: frag,
}
if len(prefixes) > 0 {
cp := make([]PrefixRow, len(prefixes))
copy(cp, prefixes)
m.revPrefixes[revisionID] = cp
} else {
m.revPrefixes[revisionID] = nil
}
snapID := m.bindPrefixSnapshotLocked(prefixes)
m.revPrefixSnap[revisionID] = snapID
m.revPrefixes[revisionID] = m.prefixSnapRows[snapID]
m.recordRenderManifestLocked(revisionID, tenantID)
return nil
}
// bindPrefixSnapshotLocked reuses an existing prefix snapshot when the prefix hash matches.
func (m *Memory) bindPrefixSnapshotLocked(prefixes []PrefixRow) string {
if m.prefixSnapByHash == nil {
m.prefixSnapByHash = make(map[string]string)
m.prefixSnapRows = make(map[string][]PrefixRow)
m.revPrefixSnap = make(map[string]string)
}
h := PrefixSnapshotHash(prefixes)
if id, ok := m.prefixSnapByHash[h]; ok {
return id
}
id := uuid.NewString()
m.prefixSnapByHash[h] = id
m.prefixSnapRows[id] = append([]PrefixRow(nil), prefixes...)
return id
}
// ListModules returns modules for a tenant (sorted by priority, then name).
func (m *Memory) ListModules(tenantID string) []*Module {
m.mu.RLock()
+22
View File
@@ -1,6 +1,7 @@
package store
import (
"sort"
"time"
)
@@ -29,6 +30,11 @@ func (m *Memory) SetASNPrefixCache(asn int64, holder string, prefixes []string)
if m.asnPrefixCache == nil {
m.asnPrefixCache = make(map[int64]*ASNPrefixCacheEntry)
}
if cur := m.asnPrefixCache[asn]; cur != nil && sameStringSet(cur.Prefixes, prefixes) {
cur.Holder = holder
cur.FetchedAt = time.Now().UTC()
return nil
}
cp := append([]string(nil), prefixes...)
m.asnPrefixCache[asn] = &ASNPrefixCacheEntry{
ASN: asn,
@@ -38,3 +44,19 @@ func (m *Memory) SetASNPrefixCache(asn int64, holder string, prefixes []string)
}
return nil
}
func sameStringSet(a, b []string) bool {
if len(a) != len(b) {
return false
}
aa := append([]string(nil), a...)
bb := append([]string(nil), b...)
sort.Strings(aa)
sort.Strings(bb)
for i := range aa {
if aa[i] != bb[i] {
return false
}
}
return true
}
+26 -1
View File
@@ -38,6 +38,7 @@ func (m *Memory) CreateModule(tenantID string, in *Module) (*Module, error) {
}
NormalizeModuleDoh(mod)
m.modules[id] = mod
m.bumpRenderGenerationLocked(tenantID)
return cloneModule(mod), nil
}
@@ -81,7 +82,10 @@ func (m *Memory) UpdateModule(tenantID, moduleID string, patch *ModulePatch) (*M
t := patch.LastRefreshedAt.UTC()
mod.LastRefreshedAt = &t
}
mod.InputHash = ""
if patch.Name != nil || patch.Enabled != nil || patch.Priority != nil || patch.RefreshIntervalSec != nil || patch.CronExpr != nil || patch.DefaultCommunityID != nil || patch.DohProfileIDs != nil || patch.DohProfileID != nil || patch.DohResolverPolicy != nil {
mod.InputHash = ""
m.bumpRenderGenerationLocked(tenantID)
}
return cloneModule(mod), nil
}
@@ -97,6 +101,7 @@ func (m *Memory) SoftDeleteModule(tenantID, moduleID string) error {
}
now := time.Now().UTC()
mod.DeletedAt = &now
m.bumpRenderGenerationLocked(tenantID)
return nil
}
@@ -154,6 +159,7 @@ func (m *Memory) CreateCDNSource(tenantID, moduleID string, in *CDNSource) (*CDN
}
m.cdnSources[id] = s
mod.InputHash = ""
m.bumpRenderGenerationLocked(tenantID)
return s, nil
}
@@ -200,6 +206,7 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN
}
if CDNPatchAffectsInputHash(patch) {
mod.InputHash = ""
m.bumpRenderGenerationLocked(tenantID)
}
return s, nil
}
@@ -216,6 +223,7 @@ func (m *Memory) DeleteCDNSource(tenantID, moduleID, sourceID string) error {
}
delete(m.cdnSources, sourceID)
m.clearModuleInputHashLocked(moduleID)
m.bumpRenderGenerationLocked(tenantID)
return nil
}
@@ -255,6 +263,7 @@ func (m *Memory) CreateASEntry(tenantID, moduleID string, in *ASEntry) (*ASEntry
e := &ASEntry{ID: id, ModuleID: moduleID, ASN: in.ASN, CommunityID: in.CommunityID}
m.asEntries[id] = e
mod.InputHash = ""
m.bumpRenderGenerationLocked(tenantID)
return e, nil
}
@@ -293,6 +302,7 @@ func (m *Memory) UpdateASEntry(tenantID, moduleID, entryID string, patch *ASEntr
e.ASNResolvedAt = nil
}
mod.InputHash = ""
m.bumpRenderGenerationLocked(tenantID)
return e, nil
}
@@ -335,6 +345,7 @@ func (m *Memory) DeleteASEntry(tenantID, moduleID, entryID string) error {
}
delete(m.asEntries, entryID)
m.clearModuleInputHashLocked(moduleID)
m.bumpRenderGenerationLocked(tenantID)
return nil
}
@@ -791,6 +802,7 @@ func (m *Memory) CreatePeer(tenantID string, in *BGPPeer) (*BGPPeer, error) {
CreatedByUserID: strings.TrimSpace(in.CreatedByUserID),
}
m.peers[id] = p
m.bumpRenderGenerationLocked(tenantID)
return p, nil
}
@@ -804,6 +816,11 @@ func (m *Memory) UpdatePeer(tenantID, id string, patch *PeerPatch) (*BGPPeer, er
if !ok || p.TenantID != tenantID {
return nil, ErrNotFound
}
prevNeighbor, prevASN, prevEnabled, prevPolicies := p.Neighbor, p.RemoteASN, p.Enabled, p.PoliciesJSON
prevSpeaker := ""
if p.SpeakerID != nil {
prevSpeaker = *p.SpeakerID
}
if patch.Neighbor != nil {
n, ok := NormalizePeerNeighborString(*patch.Neighbor)
if !ok {
@@ -834,6 +851,13 @@ func (m *Memory) UpdatePeer(tenantID, id string, patch *PeerPatch) (*BGPPeer, er
if patch.PoliciesJSON != nil {
p.PoliciesJSON = *patch.PoliciesJSON
}
speaker := ""
if p.SpeakerID != nil {
speaker = *p.SpeakerID
}
if p.Neighbor != prevNeighbor || p.RemoteASN != prevASN || p.Enabled != prevEnabled || p.PoliciesJSON != prevPolicies || speaker != prevSpeaker {
m.bumpRenderGenerationLocked(tenantID)
}
return p, nil
}
@@ -846,6 +870,7 @@ func (m *Memory) DeletePeer(tenantID, id string) error {
}
delete(m.peers, id)
_ = p
m.bumpRenderGenerationLocked(tenantID)
return nil
}
+235
View File
@@ -0,0 +1,235 @@
package store
import (
"context"
"sort"
"time"
)
type deploymentReservation struct {
Owner string
Token int64
Until time.Time
}
func (m *Memory) deploymentFence(f JobFence) (JobRecord, error) {
j, ok := m.jobs[f.JobID]
if !ok || j.TenantID != f.TenantID || j.Status != "running" || j.OwnerID != f.OwnerID || j.FencingToken != f.Token || j.CancelRequested || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) {
return JobRecord{}, ErrJobLeaseLost
}
return j, nil
}
func (m *Memory) BeginDeploymentSpeaker(ctx context.Context, f JobFence, speaker, revision string) (DeploymentSpeaker, error) {
if err := ctx.Err(); err != nil {
return DeploymentSpeaker{}, err
}
m.jobMu.Lock()
defer m.jobMu.Unlock()
j, err := m.deploymentFence(f)
if err != nil {
return DeploymentSpeaker{}, err
}
m.mu.Lock()
defer m.mu.Unlock()
sp, ok := m.speakers[speaker]
if !ok || sp.TenantID != f.TenantID {
return DeploymentSpeaker{}, ErrNotFound
}
r, ok := m.revisions[revision]
if !ok || r.TenantID != f.TenantID {
return DeploymentSpeaker{}, ErrNotFound
}
if m.deployments == nil {
m.deployments = make(map[string]DeploymentSpeaker)
m.deploymentCurrent = make(map[string]DeploymentSpeaker)
}
key := f.JobID + ":" + speaker
cur := m.deploymentCurrent[speaker]
if f.ExpectedGeneration != 0 && cur.JobID != f.JobID && (cur.Generation != f.ExpectedGeneration || cur.RevisionID != revision) {
return DeploymentSpeaker{}, ErrJobLeaseLost
}
reserve := func() error {
if m.deploymentReservations == nil {
m.deploymentReservations = make(map[string]deploymentReservation)
}
if r := m.deploymentReservations[speaker]; r.Until.After(time.Now()) {
return ErrDeploymentBusy
}
m.deploymentReservations[speaker] = deploymentReservation{Owner: f.OwnerID, Token: f.Token, Until: time.Now().Add(DeploymentDeliveryWindow)}
return nil
}
if existing, ok := m.deployments[key]; ok {
if cur.JobID != f.JobID || cur.Generation != existing.Generation || cur.RevisionID != revision {
return DeploymentSpeaker{}, ErrJobLeaseLost
}
if existing.State != "applied" {
if err := reserve(); err != nil {
return DeploymentSpeaker{}, err
}
}
return existing, nil
}
if f.ExpectedGeneration != 0 {
j.RequestSequence = cur.RequestSequence
}
if cur.RequestSequence >= j.RequestSequence && cur.JobID != "" && f.ExpectedGeneration == 0 {
return DeploymentSpeaker{}, ErrJobLeaseLost
}
if err := reserve(); err != nil {
return DeploymentSpeaker{}, err
}
d := DeploymentSpeaker{JobID: f.JobID, TenantID: f.TenantID, SpeakerID: speaker, RevisionID: revision, Generation: cur.Generation + 1, RequestSequence: j.RequestSequence, State: "published", RetryRound: f.RetryRound}
m.deployments[key] = d
m.deploymentCurrent[speaker] = d
m.publishedRevision[speaker] = publishedInfo{RevisionID: revision, PublishedAt: time.Now().UTC()}
return d, nil
}
func (m *Memory) RecordDeploymentSpeaker(ctx context.Context, f JobFence, d DeploymentSpeaker, state, confirmation, message string) error {
if err := ctx.Err(); err != nil {
return err
}
m.jobMu.Lock()
defer m.jobMu.Unlock()
if _, err := m.deploymentFence(f); err != nil {
return err
}
m.mu.Lock()
defer m.mu.Unlock()
cur := m.deploymentCurrent[d.SpeakerID]
if d.JobID != f.JobID || d.TenantID != f.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID {
return ErrJobLeaseLost
}
r := m.deploymentReservations[d.SpeakerID]
if r.Owner != f.OwnerID || r.Token != f.Token || !r.Until.After(time.Now()) {
return ErrJobLeaseLost
}
if state == "applied" && confirmation != "agent_reported" && confirmation != "local_applied" {
return ErrInvalidInput
}
if state != "dispatched" && state != "unknown" {
delete(m.deploymentReservations, d.SpeakerID)
}
cur.State, cur.Confirmation, cur.Error = state, confirmation, message
if state == "dispatched" {
cur.DispatchAttempts++
}
if state == "applied" {
revision := d.RevisionID
m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision
}
m.deploymentCurrent[d.SpeakerID] = cur
m.deployments[d.JobID+":"+d.SpeakerID] = cur
return nil
}
func (m *Memory) ListDeploymentSpeakers(ctx context.Context, tenant, job string) ([]DeploymentSpeaker, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
m.mu.RLock()
defer m.mu.RUnlock()
var out []DeploymentSpeaker
for _, d := range m.deployments {
if d.TenantID == tenant && d.JobID == job {
out = append(out, d)
}
}
return out, nil
}
func (m *Memory) ListPendingDeployments(ctx context.Context, limit int) ([]DeploymentSpeaker, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
m.jobMu.Lock()
defer m.jobMu.Unlock()
m.mu.RLock()
defer m.mu.RUnlock()
var out []DeploymentSpeaker
for speaker, d := range m.deploymentCurrent {
j := m.jobs[d.JobID]
if d.State != "applied" && d.RetryRound < 3 && !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) {
out = append(out, d)
if len(out) >= limit {
break
}
}
}
return out, nil
}
func (m *Memory) ListDeploymentObservations(ctx context.Context, limit int) ([]DeploymentSpeaker, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if limit <= 0 || limit > 100 {
limit = 10
}
m.jobMu.Lock()
defer m.jobMu.Unlock()
m.mu.Lock()
defer m.mu.Unlock()
var out []DeploymentSpeaker
for speaker, d := range m.deploymentCurrent {
j := m.jobs[d.JobID]
if !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) {
out = append(out, d)
}
}
sort.Slice(out, func(i, j int) bool {
a, b := out[i].HealthReceivedAt, out[j].HealthReceivedAt
if a == nil && b == nil {
return out[i].SpeakerID < out[j].SpeakerID
}
if a == nil {
return true
}
if b == nil {
return false
}
return a.Before(*b)
})
if len(out) > limit {
out = out[:limit]
}
for i := range out {
out[i].ObservationToken++
m.deploymentCurrent[out[i].SpeakerID] = out[i]
m.deployments[out[i].JobID+":"+out[i].SpeakerID] = out[i]
}
return out, nil
}
func (m *Memory) ObserveDeployment(ctx context.Context, d DeploymentSpeaker, o DeploymentObservation) error {
if err := ctx.Err(); err != nil {
return err
}
m.mu.Lock()
defer m.mu.Unlock()
cur := m.deploymentCurrent[d.SpeakerID]
if cur.TenantID != d.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || cur.ObservationToken != d.ObservationToken || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID || m.deploymentReservations[d.SpeakerID].Until.After(time.Now()) {
return ErrJobLeaseLost
}
now := time.Now().UTC()
cur.ObservationToken++
cur.HealthReceivedAt = &now
cur.HealthRevisionID = o.RevisionID
cur.HealthSourceAt = o.SourceAt
cur.HealthError = o.Error
if o.Error == "" && o.RevisionID != "" {
if o.RevisionID == d.RevisionID {
cur.State = "applied"
cur.Confirmation = "agent_reported"
revision := d.RevisionID
m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision
} else {
cur.State = "unknown"
cur.Confirmation = "unconfirmed"
}
}
m.deploymentCurrent[d.SpeakerID] = cur
m.deployments[d.JobID+":"+d.SpeakerID] = cur
return nil
}
+405
View File
@@ -0,0 +1,405 @@
package store
import (
"context"
"encoding/json"
"os"
"sort"
"strconv"
"sync"
"time"
"github.com/google/uuid"
)
func copyJob(j JobRecord) JobRecord {
if j.IdempotencyKey != nil {
v := *j.IdempotencyKey
j.IdempotencyKey = &v
}
if j.ModuleID != nil {
v := *j.ModuleID
j.ModuleID = &v
}
if j.Error != nil {
v := *j.Error
j.Error = &v
}
if j.StartedAt != nil {
v := *j.StartedAt
j.StartedAt = &v
}
if j.FinishedAt != nil {
v := *j.FinishedAt
j.FinishedAt = &v
}
if j.LeaseUntil != nil {
v := *j.LeaseUntil
j.LeaseUntil = &v
}
if j.Meta != nil {
raw, _ := json.Marshal(j.Meta)
var meta map[string]any
_ = json.Unmarshal(raw, &meta)
j.Meta = meta
}
return j
}
func (m *Memory) EnqueueJob(_ context.Context, in JobEnqueue) (JobRecord, bool, error) {
h, err := JobRequestHash(in)
if err != nil {
return JobRecord{}, false, err
}
unlockTenant := m.lockJobEnqueueTenant(in.TenantID)
defer unlockTenant()
m.jobMu.Lock()
defer m.jobMu.Unlock()
if m.jobs == nil {
m.jobs = make(map[string]JobRecord)
m.jobHashes = make(map[string]string)
m.jobRequests = make(map[string]map[string]any)
}
if in.IdempotencyKey != nil && *in.IdempotencyKey != "" {
for _, j := range m.jobs {
if j.TenantID == in.TenantID && j.IdempotencyKey != nil && *j.IdempotencyKey == *in.IdempotencyKey {
if m.jobHashes[j.ID] != h {
return JobRecord{}, false, ErrJobConflict
}
return copyJob(j), false, nil
}
}
}
global, tenant := 0, 0
for _, j := range m.jobs {
if j.Status == "queued" || j.Status == "running" {
global++
if j.TenantID == in.TenantID {
tenant++
}
}
}
if global >= jobLimitMemory("EVOBGP_JOB_MAX_PENDING", 10000) || tenant >= jobLimitMemory("EVOBGP_JOB_MAX_TENANT_PENDING", 1000) {
return JobRecord{}, false, ErrJobQueueFull
}
j := JobRecord{ID: uuid.NewString(), TenantID: in.TenantID, Kind: in.Kind, Status: "queued", IdempotencyKey: in.IdempotencyKey, ModuleID: in.ModuleID, Meta: in.Meta, CreatedAt: time.Now().UTC()}
j = copyJob(j)
m.jobSequence++
j.RequestSequence = m.jobSequence
j.RequestHash = h
m.jobs[j.ID] = j
m.jobHashes[j.ID] = h
m.jobRequests[j.ID] = copyJob(j).Meta
return copyJob(j), true, nil
}
func (m *Memory) ClaimJobs(_ context.Context, owner string, limit int, lease time.Duration) ([]JobRecord, error) {
m.jobMu.Lock()
defer m.jobMu.Unlock()
if limit <= 0 {
return nil, nil
}
if limit > 64 {
limit = 64
}
if lease <= 0 {
lease = time.Minute
}
now := time.Now().UTC()
m.reapExpiredJobsLocked(now)
var all []JobRecord
for _, j := range m.jobs {
if j.Attempt < 3 && (j.Status == "queued" || (j.Status == "running" && j.LeaseUntil != nil && now.After(*j.LeaseUntil))) {
all = append(all, j)
}
}
sort.Slice(all, func(i, j int) bool {
if !all[i].CreatedAt.Equal(all[j].CreatedAt) {
return all[i].CreatedAt.Before(all[j].CreatedAt)
}
if all[i].RequestSequence != all[j].RequestSequence {
return all[i].RequestSequence < all[j].RequestSequence
}
return all[i].ID < all[j].ID
})
if len(all) > limit {
all = all[:limit]
}
for i := range all {
j := all[i]
j.Status = "running"
j.OwnerID = owner
j.FencingToken++
j.Attempt++
if j.StartedAt == nil {
j.StartedAt = &now
}
until := now.Add(lease)
j.LeaseUntil = &until
m.jobs[j.ID] = j
all[i] = copyJob(j)
}
return all, nil
}
// ReapExpiredJobs marks orphaned cancellations and exhausted retries terminal.
func (m *Memory) ReapExpiredJobs(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
m.jobMu.Lock()
defer m.jobMu.Unlock()
m.reapExpiredJobsLocked(time.Now().UTC())
return nil
}
func (m *Memory) reapExpiredJobsLocked(now time.Time) {
for id, j := range m.jobs {
if j.Status == "running" && j.LeaseUntil != nil && now.After(*j.LeaseUntil) && j.CancelRequested {
j.Status = "cancelled"
j.FinishedAt = &now
j.OwnerID = ""
j.LeaseUntil = nil
m.jobs[id] = j
continue
}
if (j.Status == "queued" || (j.Status == "running" && j.LeaseUntil != nil && now.After(*j.LeaseUntil))) && j.Attempt >= 3 {
j.Status = "failed"
msg := "retry limit reached"
j.Error = &msg
j.FinishedAt = &now
j.OwnerID = ""
j.LeaseUntil = nil
m.jobs[id] = j
}
}
}
func (m *Memory) HeartbeatJob(_ context.Context, id, owner string, token int64, lease time.Duration) (bool, error) {
if lease <= 0 {
lease = time.Minute
}
m.jobMu.Lock()
defer m.jobMu.Unlock()
j, ok := m.jobs[id]
if !ok || j.OwnerID != owner || j.FencingToken != token || j.Status != "running" || j.LeaseUntil == nil || !time.Now().Before(*j.LeaseUntil) {
return false, ErrJobLeaseLost
}
until := time.Now().Add(lease)
j.LeaseUntil = &until
m.jobs[id] = j
return j.CancelRequested, nil
}
func (m *Memory) FinishJob(_ context.Context, j JobRecord) error {
m.jobMu.Lock()
defer m.jobMu.Unlock()
old, ok := m.jobs[j.ID]
if !ok || old.OwnerID != j.OwnerID || old.FencingToken != j.FencingToken || old.Status != "running" || old.LeaseUntil == nil || !time.Now().Before(*old.LeaseUntil) {
return ErrJobLeaseLost
}
j.CreatedAt = old.CreatedAt
j.RequestSequence = old.RequestSequence
j.RequestHash = old.RequestHash
j.IdempotencyKey = old.IdempotencyKey
j.ModuleID = old.ModuleID
j.TenantID = old.TenantID
j.Kind = old.Kind
j.StartedAt = old.StartedAt
j.Attempt = old.Attempt
j.CancelRequested = old.CancelRequested
j.OwnerID = ""
j.LeaseUntil = nil
now := time.Now().UTC()
j.FinishedAt = &now
if old.CancelRequested {
j.Status = "cancelled"
j.Error = nil
}
m.jobs[j.ID] = copyJob(j)
return nil
}
func (m *Memory) CheckpointJob(_ context.Context, in JobRecord) error {
m.jobMu.Lock()
defer m.jobMu.Unlock()
j, ok := m.jobs[in.ID]
if !ok || j.OwnerID != in.OwnerID || j.FencingToken != in.FencingToken || j.Status != "running" || j.LeaseUntil == nil || !time.Now().Before(*j.LeaseUntil) {
return ErrJobLeaseLost
}
j.Meta = copyJob(in).Meta
m.jobs[j.ID] = j
return nil
}
func (m *Memory) ReleaseJob(_ context.Context, id, owner string, token int64) error {
m.jobMu.Lock()
defer m.jobMu.Unlock()
j, ok := m.jobs[id]
if !ok || j.OwnerID != owner || j.FencingToken != token || j.Status != "running" {
return ErrJobLeaseLost
}
j.Status = "queued"
if j.CancelRequested {
j.Status = "cancelled"
now := time.Now().UTC()
j.FinishedAt = &now
}
j.OwnerID = ""
j.LeaseUntil = nil
if j.Attempt > 0 {
j.Attempt--
}
m.jobs[id] = j
return nil
}
func (m *Memory) GetJob(_ context.Context, tenant, id string) (JobRecord, error) {
m.jobMu.Lock()
defer m.jobMu.Unlock()
j, ok := m.jobs[id]
if !ok || j.TenantID != tenant {
return JobRecord{}, ErrJobNotFound
}
return copyJob(j), nil
}
func (m *Memory) ListJobs(_ context.Context, tenant, status, kind, cursor string, limit int) ([]JobRecord, string, bool, error) {
m.jobMu.Lock()
defer m.jobMu.Unlock()
var all []JobRecord
for _, j := range m.jobs {
if j.TenantID == tenant && (status == "" || j.Status == status) && (kind == "" || j.Kind == kind) {
all = append(all, copyJob(j))
}
}
sort.Slice(all, func(i, j int) bool {
if !all[i].CreatedAt.Equal(all[j].CreatedAt) {
return all[i].CreatedAt.After(all[j].CreatedAt)
}
if all[i].RequestSequence != all[j].RequestSequence {
return all[i].RequestSequence > all[j].RequestSequence
}
return all[i].ID > all[j].ID
})
off, _ := strconv.Atoi(cursor)
if off < 0 {
off = 0
}
if limit <= 0 || limit > 100 {
limit = 50
}
if off >= len(all) {
return nil, "", false, nil
}
end := off + limit
more := end < len(all)
if end > len(all) {
end = len(all)
}
next := ""
if more {
next = strconv.Itoa(end)
}
return all[off:end], next, more, nil
}
func (m *Memory) CancelJob(_ context.Context, tenant, id string) (JobRecord, error) {
m.jobMu.Lock()
defer m.jobMu.Unlock()
j, ok := m.jobs[id]
if !ok || j.TenantID != tenant {
return JobRecord{}, ErrJobNotFound
}
if j.Status == "queued" {
now := time.Now().UTC()
j.Status = "cancelled"
j.FinishedAt = &now
}
if j.Status == "running" {
j.CancelRequested = true
}
m.jobs[id] = j
return copyJob(j), nil
}
func (m *Memory) RetryJob(_ context.Context, tenant, id string) (JobRecord, error) {
m.jobMu.Lock()
defer m.jobMu.Unlock()
j, ok := m.jobs[id]
if !ok || j.TenantID != tenant {
return JobRecord{}, ErrJobNotFound
}
if j.Status != "failed" {
return JobRecord{}, ErrJobConflict
}
global, pending := 0, 0
for _, item := range m.jobs {
if item.Status == "queued" || item.Status == "running" {
global++
if item.TenantID == tenant {
pending++
}
}
}
if global >= jobLimitMemory("EVOBGP_JOB_MAX_PENDING", 10000) || pending >= jobLimitMemory("EVOBGP_JOB_MAX_TENANT_PENDING", 1000) {
return JobRecord{}, ErrJobQueueFull
}
j.Status = "queued"
j.Error = nil
j.FinishedAt = nil
j.CancelRequested = false
j.Attempt = 0
j.StartedAt = nil
j.OwnerID = ""
j.LeaseUntil = nil
j.Meta = copyJob(JobRecord{Meta: m.jobRequests[id]}).Meta
m.jobs[id] = j
return copyJob(j), nil
}
// lockJobEnqueueTenant serializes idempotency and the tenant pending limit.
// The global jobMu is still required to mutate the shared job map.
func (m *Memory) lockJobEnqueueTenant(tenantID string) func() {
m.jobTenantMu.Lock()
if m.jobTenantLocks == nil {
m.jobTenantLocks = make(map[string]*sync.Mutex)
}
mu := m.jobTenantLocks[tenantID]
if mu == nil {
mu = &sync.Mutex{}
m.jobTenantLocks[tenantID] = mu
}
m.jobTenantMu.Unlock()
mu.Lock()
return mu.Unlock
}
func jobLimitMemory(name string, fallback int) int {
n, e := strconv.Atoi(os.Getenv(name))
if e == nil && n > 0 {
return n
}
return fallback
}
func (m *Memory) CountJobsByStatus(_ context.Context) (map[string]int64, error) {
m.jobMu.Lock()
defer m.jobMu.Unlock()
out := map[string]int64{}
for _, j := range m.jobs {
out[j.Status]++
}
return out, nil
}
func (m *Memory) FindJobByIdempotencyKey(_ context.Context, tenant, key string) (JobRecord, error) {
m.jobMu.Lock()
defer m.jobMu.Unlock()
for _, j := range m.jobs {
if j.TenantID == tenant && j.IdempotencyKey != nil && *j.IdempotencyKey == key {
return copyJob(j), nil
}
}
return JobRecord{}, ErrJobNotFound
}
+122
View File
@@ -0,0 +1,122 @@
package store
import (
"context"
"errors"
"sync"
"testing"
"time"
)
func TestEnqueueJobTenantsDoNotShareLock(t *testing.T) {
m := NewMemory()
ctx := context.Background()
unlock := m.lockJobEnqueueTenant("tenant-a")
var once sync.Once
release := func() { once.Do(unlock) }
t.Cleanup(release)
otherDone := make(chan error, 1)
go func() {
_, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: "tenant-b", Kind: "test"})
otherDone <- err
}()
select {
case err := <-otherDone:
if err != nil {
t.Fatal(err)
}
case <-time.After(2 * time.Second):
t.Fatal("other tenant waited on the held tenant lock")
}
sameDone := make(chan struct{})
go func() {
_, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: "tenant-a", Kind: "test"})
if err != nil {
t.Error(err)
}
close(sameDone)
}()
select {
case <-sameDone:
t.Fatal("same tenant did not wait on its lock")
case <-time.After(200 * time.Millisecond):
}
release()
select {
case <-sameDone:
case <-time.After(2 * time.Second):
t.Fatal("same tenant did not finish after unlock")
}
}
func TestEnqueueJobIdempotencyKeyConflicts(t *testing.T) {
m := NewMemory()
ctx := context.Background()
key := "same-key"
in := JobEnqueue{TenantID: "tenant-a", Kind: "refresh", IdempotencyKey: &key, Meta: map[string]any{"module": "a"}}
first, created, err := m.EnqueueJob(ctx, in)
if err != nil || !created || first.ID == "" {
t.Fatalf("first=%+v created=%v err=%v", first, created, err)
}
in.Meta = map[string]any{"module": "b"}
if _, _, err = m.EnqueueJob(ctx, in); !errors.Is(err, ErrJobConflict) {
t.Fatalf("same key accepted: %v", err)
}
other := JobEnqueue{TenantID: "tenant-b", Kind: "refresh", IdempotencyKey: &key, Meta: map[string]any{"module": "b"}}
if _, created, err = m.EnqueueJob(ctx, other); err != nil || !created {
t.Fatalf("other tenant created=%v err=%v", created, err)
}
}
func TestMemoryJobQueueEqualTimestampOrdering(t *testing.T) {
m := NewMemory()
ctx := context.Background()
stamp := time.Date(2026, 10, 26, 0, 0, 0, 0, time.UTC)
var ids []string
for range 32 {
j, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: "tenant", Kind: "test"})
if err != nil {
t.Fatal(err)
}
ids = append(ids, j.ID)
m.jobMu.Lock()
saved := m.jobs[j.ID]
saved.CreatedAt = stamp
m.jobs[j.ID] = saved
m.jobMu.Unlock()
}
// Repeated offset pages must use the same total order despite map iteration.
for range 5 {
cursor := ""
var listed []string
for {
page, next, more, err := m.ListJobs(ctx, "tenant", "", "", cursor, 3)
if err != nil {
t.Fatal(err)
}
for _, j := range page {
listed = append(listed, j.ID)
}
if !more {
break
}
cursor = next
}
if len(listed) != len(ids) {
t.Fatalf("listed %d jobs", len(listed))
}
for i, id := range listed {
if id != ids[len(ids)-1-i] {
t.Fatalf("list[%d]=%s expected reverse enqueue sequence %s", i, id, ids[len(ids)-1-i])
}
}
}
for i, want := range ids {
claimed, err := m.ClaimJobs(ctx, "owner", 1, time.Minute)
if err != nil || len(claimed) != 1 || claimed[0].ID != want {
t.Fatalf("claim[%d]=%+v %v want %s", i, claimed, err, want)
}
}
}
+8 -1
View File
@@ -1,6 +1,7 @@
package store
import (
"context"
"strings"
"time"
)
@@ -109,7 +110,13 @@ func (m *Memory) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (Rev
}, nil
}
func (m *Memory) PruneRevisionsBefore(tenantID string, cutoff time.Time) (int, error) {
func (m *Memory) PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (int, error) {
if ctx == nil {
ctx = context.Background()
}
if err := ctx.Err(); err != nil {
return 0, err
}
res, err := m.PruneRevisionsWithStats(tenantID, cutoff)
if err != nil {
return 0, err
@@ -0,0 +1,22 @@
package store
import (
"context"
"errors"
"testing"
"time"
)
func TestPruneRevisionsBeforeCancelledContext(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, _, _, rev, _ := m.DemoIDs()
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := m.PruneRevisionsBefore(ctx, tenant, time.Now().Add(time.Hour)); !errors.Is(err, context.Canceled) {
t.Fatalf("err=%v", err)
}
if _, err := m.GetRevision(tenant, rev); err != nil {
t.Fatal(err)
}
}
+176 -24
View File
@@ -1,12 +1,43 @@
package store
import (
"context"
"strings"
"sync"
"time"
)
func (m *Memory) GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error) {
// ModuleSnapshotUpdate computes a replacement from the current snapshot; nil deletes it.
// The callback must not perform store I/O or mutate its input.
type ModuleSnapshotUpdate func(current *ModulePrefixSnapshot) (*ModulePrefixSnapshot, error)
// ListModuleSnapshotSourceRows returns persisted rows for one source key.
func (m *Memory) ListModuleSnapshotSourceRows(ctx context.Context, tenantID, moduleID, source string) ([]PrefixRow, error) {
if ctx == nil {
ctx = context.Background()
}
if err := ctx.Err(); err != nil {
return nil, err
}
m.mu.RLock()
defer m.mu.RUnlock()
snap := m.moduleSnapshots[moduleSnapshotKey(tenantID, moduleID)]
if snap == nil {
return nil, nil
}
var out []PrefixRow
for _, row := range snap.Prefixes {
if row.Source == source {
out = append(out, row)
}
}
return out, nil
}
func (m *Memory) GetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error) {
if err := ctx.Err(); err != nil {
return nil, false, err
}
m.mu.RLock()
defer m.mu.RUnlock()
key := moduleSnapshotKey(tenantID, moduleID)
@@ -19,42 +50,95 @@ func (m *Memory) GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePref
return &ModulePrefixSnapshot{
InputHash: snap.InputHash,
CollectedAt: snap.CollectedAt,
Version: snap.Version,
Prefixes: cp,
}, true, nil
}
func (m *Memory) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error {
func (m *Memory) SetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID, inputHash string, prefixes []PrefixRow) error {
if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" || strings.TrimSpace(inputHash) == "" {
return ErrInvalidInput
}
return m.UpdateModulePrefixSnapshot(ctx, tenantID, moduleID, func(*ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) {
return &ModulePrefixSnapshot{InputHash: inputHash, Prefixes: prefixes}, nil
})
}
func (m *Memory) DeleteModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) error {
return m.UpdateModulePrefixSnapshot(ctx, tenantID, moduleID, func(*ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) {
return nil, nil
})
}
// UpdateModulePrefixSnapshot serializes a pure RMW callback and commits snapshot plus input hash together.
func (m *Memory) UpdateModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string, update ModuleSnapshotUpdate) error {
if update == nil || strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" {
return ErrInvalidInput
}
unlock, err := m.lockModuleSnapshot(ctx, tenantID, moduleID)
if err != nil {
return err
}
defer unlock()
current, _, err := m.GetModulePrefixSnapshot(ctx, tenantID, moduleID)
if err != nil {
return err
}
next, err := update(current)
if err != nil {
return err
}
if next != nil && strings.TrimSpace(next.InputHash) == "" {
return ErrInvalidInput
}
if err := ctx.Err(); err != nil {
return err
}
m.mu.Lock()
defer m.mu.Unlock()
if m.moduleSnapshots == nil {
m.moduleSnapshots = make(map[string]*moduleSnapshotRec)
if err := ctx.Err(); err != nil {
return err
}
mod := m.modules[moduleID]
if mod == nil || mod.DeletedAt != nil {
return ErrNotFound
}
if mod.TenantID != tenantID {
return ErrTenantScope
}
cp := make([]PrefixRow, len(prefixes))
copy(cp, prefixes)
key := moduleSnapshotKey(tenantID, moduleID)
m.moduleSnapshots[key] = &moduleSnapshotRec{
InputHash: inputHash,
CollectedAt: time.Now().UTC(),
Prefixes: cp,
if next == nil {
if m.moduleSnapshots[key] != nil {
m.bumpRenderGenerationLocked(tenantID)
}
delete(m.moduleSnapshots, key)
} else if !SameSnapshotContent(current, next) {
m.bumpRenderGenerationLocked(tenantID)
if m.moduleSnapshots == nil {
m.moduleSnapshots = make(map[string]*moduleSnapshotRec)
}
m.nextModuleSnapshotVersion++
m.moduleSnapshots[key] = &moduleSnapshotRec{InputHash: next.InputHash, CollectedAt: time.Now().UTC(), Version: m.nextModuleSnapshotVersion, Prefixes: append([]PrefixRow(nil), next.Prefixes...)}
}
if next == nil {
mod.InputHash = ""
} else {
mod.InputHash = next.InputHash
}
return nil
}
func (m *Memory) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
m.mu.Lock()
defer m.mu.Unlock()
if m.moduleSnapshots != nil {
delete(m.moduleSnapshots, moduleSnapshotKey(tenantID, moduleID))
func (m *Memory) SetModuleInputHash(ctx context.Context, tenantID, moduleID, hash string) error {
unlock, err := m.lockModuleSnapshot(ctx, tenantID, moduleID)
if err != nil {
return err
}
return nil
}
func (m *Memory) SetModuleInputHash(tenantID, moduleID, hash string) error {
defer unlock()
m.mu.Lock()
defer m.mu.Unlock()
if err := ctx.Err(); err != nil {
return err
}
mod, ok := m.modules[moduleID]
if !ok || mod.DeletedAt != nil {
return ErrNotFound
@@ -66,20 +150,88 @@ func (m *Memory) SetModuleInputHash(tenantID, moduleID, hash string) error {
return nil
}
func (m *Memory) LockModuleSnapshot(tenantID, moduleID string) func() {
type snapshotLockEntry struct {
gate chan struct{}
refs int
}
// lockModuleSnapshot waits until the caller deadline, or EVOBGP_SNAPSHOT_TIMEOUT when unset.
func (m *Memory) lockModuleSnapshot(ctx context.Context, tenantID, moduleID string) (func(), error) {
if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" {
return nil, ErrInvalidInput
}
if ctx == nil {
ctx = context.Background()
}
ctx, cancel, _ := SnapshotWriteBudget(ctx)
defer cancel()
key := moduleSnapshotKey(tenantID, moduleID)
v, _ := m.snapshotLocks.LoadOrStore(key, &sync.Mutex{})
mu := v.(*sync.Mutex)
mu.Lock()
return func() { mu.Unlock() }
m.snapshotLocksMu.Lock()
if m.snapshotLocks == nil {
m.snapshotLocks = make(map[string]*snapshotLockEntry)
}
entry := m.snapshotLocks[key]
if entry == nil {
entry = &snapshotLockEntry{gate: make(chan struct{}, 1)}
entry.gate <- struct{}{}
m.snapshotLocks[key] = entry
}
entry.refs++
m.snapshotLocksMu.Unlock()
select {
case <-ctx.Done():
m.releaseSnapshotLockRef(key, entry)
return nil, ctx.Err()
case <-entry.gate:
if err := ctx.Err(); err != nil {
entry.gate <- struct{}{}
m.releaseSnapshotLockRef(key, entry)
return nil, err
}
}
var once sync.Once
return func() {
once.Do(func() {
entry.gate <- struct{}{}
m.releaseSnapshotLockRef(key, entry)
})
}, nil
}
func (m *Memory) releaseSnapshotLockRef(key string, entry *snapshotLockEntry) {
m.snapshotLocksMu.Lock()
defer m.snapshotLocksMu.Unlock()
entry.refs--
if entry.refs == 0 {
delete(m.snapshotLocks, key)
}
}
type moduleSnapshotRec struct {
InputHash string
CollectedAt time.Time
Version int64
Prefixes []PrefixRow
}
// SameSnapshotContent compares persisted rows, ignoring timestamps, versions, and order.
// Rows are matched by (prefix, community, source).
func SameSnapshotContent(a, b *ModulePrefixSnapshot) bool {
if a == nil || b == nil || a.InputHash != b.InputHash || len(a.Prefixes) != len(b.Prefixes) {
return false
}
ap := append([]PrefixRow(nil), a.Prefixes...)
bp := append([]PrefixRow(nil), b.Prefixes...)
sortPrefixRows(ap)
sortPrefixRows(bp)
for i := range ap {
if !samePrefixRow(ap[i], bp[i]) {
return false
}
}
return true
}
func moduleSnapshotKey(tenantID, moduleID string) string {
return strings.TrimSpace(tenantID) + "\x00" + strings.TrimSpace(moduleID)
}
@@ -0,0 +1,76 @@
package store
import (
"context"
"errors"
"fmt"
"sync"
"testing"
)
func TestMemoryModuleSnapshotAtomicConcurrentRMW(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, _, _, _, _ := m.DemoIDs()
mod, err := m.CreateModule(tenant, &Module{Type: "CDN_CIDRS", Name: "atomic", Enabled: true})
if err != nil {
t.Fatal(err)
}
const writers = 12
var wg sync.WaitGroup
errCh := make(chan error, writers)
for i := 0; i < writers; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
errCh <- m.UpdateModulePrefixSnapshot(context.Background(), tenant, mod.ID, func(current *ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) {
var rows []PrefixRow
if current != nil {
rows = append(rows, current.Prefixes...)
}
rows = append(rows, PrefixRow{Prefix: fmt.Sprintf("192.0.2.%d/32", i)})
return &ModulePrefixSnapshot{InputHash: "atomic", Prefixes: rows}, nil
})
}(i)
}
wg.Wait()
close(errCh)
for err := range errCh {
if err != nil {
t.Fatal(err)
}
}
snap, ok, err := m.GetModulePrefixSnapshot(context.Background(), tenant, mod.ID)
if err != nil || !ok || len(snap.Prefixes) != writers {
t.Fatalf("lost concurrent update: %+v ok=%v err=%v", snap, ok, err)
}
if got, err := m.GetModule(tenant, mod.ID); err != nil || got == nil || got.InputHash != snap.InputHash {
t.Fatalf("module input hash differs from snapshot: %+v %+v", got, snap)
}
}
func TestMemoryModuleSnapshotAtomicCallbackErrorRollsBack(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, _, _, _, _ := m.DemoIDs()
mod, err := m.CreateModule(tenant, &Module{Type: "CDN_CIDRS", Name: "rollback", Enabled: true})
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
if err := m.SetModulePrefixSnapshot(ctx, tenant, mod.ID, "before", []PrefixRow{{Prefix: "192.0.2.0/24"}}); err != nil {
t.Fatal(err)
}
want := errors.New("update failed")
err = m.UpdateModulePrefixSnapshot(ctx, tenant, mod.ID, func(current *ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) {
current.Prefixes[0].Prefix = "198.51.100.0/24"
return nil, want
})
if !errors.Is(err, want) {
t.Fatalf("callback error: %v", err)
}
snap, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, mod.ID)
if err != nil || !ok || snap.InputHash != "before" || snap.Prefixes[0].Prefix != "192.0.2.0/24" {
t.Fatalf("partial snapshot visible: %+v ok=%v err=%v", snap, ok, err)
}
}
@@ -0,0 +1,33 @@
package store
import (
"context"
"errors"
"testing"
)
func TestMemorySnapshotMethodsRespectCancellation(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, _, module, _, _ := m.DemoIDs()
ctx, cancel := context.WithCancel(context.Background())
cancel()
if snap, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, module); snap != nil || ok || !errors.Is(err, context.Canceled) {
t.Fatalf("get: %+v %v %v", snap, ok, err)
}
if err := m.SetModulePrefixSnapshot(ctx, tenant, module, "hash", []PrefixRow{{Prefix: "192.0.2.0/24"}}); !errors.Is(err, context.Canceled) {
t.Fatalf("set: %v", err)
}
if err := m.DeleteModulePrefixSnapshot(ctx, tenant, module); !errors.Is(err, context.Canceled) {
t.Fatalf("delete: %v", err)
}
if err := m.SetModuleInputHash(ctx, tenant, module, "hash"); !errors.Is(err, context.Canceled) {
t.Fatalf("input hash: %v", err)
}
if rows, err := m.ListModuleSnapshotSourceRows(ctx, tenant, module, "ip_range"); rows != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("source rows: %+v %v", rows, err)
}
if snap, ok, err := m.GetModulePrefixSnapshot(context.Background(), tenant, module); snap != nil || ok || err != nil {
t.Fatalf("cancelled write committed: %+v %v %v", snap, ok, err)
}
}
@@ -0,0 +1,59 @@
package store
import (
"context"
"errors"
"testing"
"time"
)
func TestMemoryModuleSnapshotLockCancellationAndCleanup(t *testing.T) {
m := NewMemory()
first, err := m.lockModuleSnapshot(context.Background(), "tenant-a", "module-a")
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancel()
second, err := m.lockModuleSnapshot(ctx, "tenant-a", "module-a")
if second != nil || !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("waiter not cancelled: unlock=%v err=%v", second != nil, err)
}
if len(m.snapshotLocks) != 1 || m.snapshotLocks[moduleSnapshotKey("tenant-a", "module-a")].refs != 1 {
t.Fatalf("waiter leaked lock ref: %+v", m.snapshotLocks)
}
first()
first()
if len(m.snapshotLocks) != 0 {
t.Fatalf("idle lock leaked: %+v", m.snapshotLocks)
}
third, err := m.lockModuleSnapshot(context.Background(), "tenant-a", "module-a")
if err != nil {
t.Fatal(err)
}
third()
}
func TestMemoryModuleSnapshotLockScopesTenantAndModule(t *testing.T) {
m := NewMemory()
first, err := m.lockModuleSnapshot(context.Background(), "tenant-a", "module-x")
if err != nil {
t.Fatal(err)
}
defer first()
for _, pair := range [][2]string{{"tenant-b", "module-x"}, {"tenant-a", "module-y"}} {
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
unlock, err := m.lockModuleSnapshot(ctx, pair[0], pair[1])
cancel()
if err != nil {
t.Fatalf("unrelated key blocked %v: %v", pair, err)
}
unlock()
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
unlock, err := m.lockModuleSnapshot(ctx, "tenant-a", "module-x")
if unlock != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("cancelled context acquired: unlock=%v err=%v", unlock != nil, err)
}
}
+284
View File
@@ -0,0 +1,284 @@
package store
import (
"context"
"sort"
"strings"
"time"
"github.com/google/uuid"
)
type telemetryMemoryRow struct {
snapshot TelemetrySnapshot
owner string
leaseUntil time.Time
token int64
lastRequested time.Time
refreshPending bool
}
func telemetryKey(tenantID, targetID string) string { return tenantID + "/" + targetID }
func (m *Memory) ensureTelemetryRows(localAuthority string) {
if m.telemetry == nil {
m.telemetry = make(map[string]telemetryMemoryRow)
}
for _, sp := range m.speakers {
if sp == nil || !SpeakerNeedsRemoteDispatch(sp.Role, ParseSpeakerMeta(sp.MetaJSON)) {
continue
}
key := telemetryKey(sp.TenantID, sp.ID)
if _, ok := m.telemetry[key]; !ok {
m.telemetry[key] = telemetryMemoryRow{snapshot: TelemetrySnapshot{TenantID: sp.TenantID, TargetID: sp.ID, SpeakerID: sp.ID, Kind: "remote_agent"}}
}
}
if localAuthority != "" {
for tenantID := range m.tenants {
if tenantID == SystemJobTenantID {
continue
}
targetID := TelemetryLocalTargetID(tenantID, localAuthority)
key := telemetryKey(tenantID, targetID)
if _, ok := m.telemetry[key]; !ok {
m.telemetry[key] = telemetryMemoryRow{snapshot: TelemetrySnapshot{TenantID: tenantID, TargetID: targetID, Kind: "local_bird", LocalAuthority: localAuthority}}
}
}
}
}
// ClaimTelemetry acquires at most limit due targets with a short fencing lease.
func (m *Memory) ClaimTelemetry(_ context.Context, owner string, limit int, lease time.Duration, localAuthority string) ([]TelemetryClaim, error) {
if strings.TrimSpace(owner) == "" || limit <= 0 {
return nil, ErrInvalidInput
}
if limit > 64 {
limit = 64
}
if lease <= 0 {
lease = 15 * time.Second
}
m.mu.Lock()
defer m.mu.Unlock()
m.ensureTelemetryRows(localAuthority)
keys := make([]string, 0, len(m.telemetry))
for key := range m.telemetry {
keys = append(keys, key)
}
sort.Slice(keys, func(i, j int) bool {
a, b := m.telemetry[keys[i]], m.telemetry[keys[j]]
if !a.snapshot.NextPollAt.Equal(b.snapshot.NextPollAt) {
return a.snapshot.NextPollAt.Before(b.snapshot.NextPollAt)
}
return keys[i] < keys[j]
})
now := time.Now().UTC()
out := make([]TelemetryClaim, 0, limit)
for _, key := range keys {
row := m.telemetry[key]
if row.snapshot.Kind == "local_bird" && row.snapshot.LocalAuthority != localAuthority || row.snapshot.NextPollAt.After(now) || row.leaseUntil.After(now) {
continue
}
var speaker *Speaker
if row.snapshot.Kind == "remote_agent" {
sp := m.speakers[row.snapshot.SpeakerID]
if sp == nil || !SpeakerNeedsRemoteDispatch(sp.Role, ParseSpeakerMeta(sp.MetaJSON)) {
delete(m.telemetry, key)
continue
}
cp := *sp
speaker = &cp
} else {
for _, id := range sortedSpeakerIDs(m.speakers) {
sp := m.speakers[id]
if sp.TenantID == row.snapshot.TenantID && strings.EqualFold(sp.Role, "master") {
cp := *sp
speaker = &cp
break
}
}
}
row.owner = owner
row.token++
row.leaseUntil = now.Add(lease)
m.telemetry[key] = row
claim := TelemetryClaim{TelemetrySnapshot: row.snapshot, Speaker: speaker, OwnerID: owner, FencingToken: row.token}
if speaker != nil {
claim.SpeakerID = speaker.ID
}
claim.ConfigFingerprint = TelemetrySpeakerFingerprint(speaker)
out = append(out, claim)
if len(out) == limit {
break
}
}
return out, nil
}
// CompleteTelemetry commits only the current live lease and preserves last success on failure.
func (m *Memory) CompleteTelemetry(_ context.Context, claim TelemetryClaim, result TelemetryResult, nextPoll time.Time) error {
m.mu.Lock()
defer m.mu.Unlock()
key := telemetryKey(claim.TenantID, claim.TargetID)
row, ok := m.telemetry[key]
now := time.Now().UTC()
if !ok || row.owner != claim.OwnerID || row.token != claim.FencingToken || !row.leaseUntil.After(now) {
return ErrTelemetryLeaseLost
}
if claim.Speaker != nil {
current := m.speakers[claim.Speaker.ID]
if TelemetrySpeakerFingerprint(current) != claim.ConfigFingerprint {
return ErrTelemetryLeaseLost
}
}
if claim.Kind == "local_bird" {
masterID := ""
for _, id := range sortedSpeakerIDs(m.speakers) {
sp := m.speakers[id]
if sp.TenantID == claim.TenantID && strings.EqualFold(sp.Role, "master") {
masterID = id
break
}
}
if masterID != claim.SpeakerID {
return ErrTelemetryLeaseLost
}
}
if row.snapshot.ConfigFingerprint != claim.ConfigFingerprint {
row.snapshot.BGP = TelemetryBGP{}
row.snapshot.Health = TelemetryHealth{}
row.snapshot.DiscoveryCursor = 0
}
row.snapshot.ConfigFingerprint = claim.ConfigFingerprint
row.snapshot.SpeakerID = claim.SpeakerID
if result.BGP.Source != "" {
if result.BGP.Error != "" {
result.BGP.Sessions = nil
result.BGP.LastSuccessAt = row.snapshot.BGP.LastSuccessAt
} else {
result.BGP.LastSuccessAt = result.BGP.ReceivedAt
}
row.snapshot.BGP = result.BGP
}
if result.Health.Source != "" {
if !result.Health.OK {
result.Health.LastSuccessAt = row.snapshot.Health.LastSuccessAt
} else {
result.Health.LastSuccessAt = result.Health.ReceivedAt
}
row.snapshot.Health = result.Health
}
if row.snapshot.BGP.Error != "" || row.snapshot.Health.Source != "" && !row.snapshot.Health.OK {
row.snapshot.FailureCount++
} else {
row.snapshot.FailureCount = 0
}
row.snapshot.NextPollAt = nextPoll.UTC()
if row.refreshPending {
row.snapshot.NextPollAt = now
row.refreshPending = false
}
row.owner = ""
row.leaseUntil = time.Time{}
if result.BGP.Source != "" && result.BGP.Error == "" {
row.snapshot.DiscoveryCursor = m.writeTelemetryDiscoveriesLocked(claim, result.BGP, row.snapshot.DiscoveryCursor)
}
m.telemetry[key] = row
return nil
}
func sortedSpeakerIDs(speakers map[string]*Speaker) []string {
ids := make([]string, 0, len(speakers))
for id := range speakers {
ids = append(ids, id)
}
sort.Strings(ids)
return ids
}
func (m *Memory) writeTelemetryDiscoveriesLocked(claim TelemetryClaim, bgp TelemetryBGP, cursor int) int {
sessions, next := TelemetryDiscoveryBatch(bgp.Sessions, cursor, 256)
known := make(map[string]struct{})
for _, p := range m.peers {
if p == nil || p.TenantID != claim.TenantID {
continue
}
if neighbor, ok := NormalizePeerNeighborString(p.Neighbor); ok {
known[neighbor] = struct{}{}
}
}
for _, sess := range sessions {
neighbor, ok := NormalizePeerNeighborString(sess.Neighbor)
if !ok {
continue
}
if _, ok := known[neighbor]; ok {
continue
}
neighborID := strings.TrimSpace(sess.NeighborID)
d := m.findPeerDiscoveryLocked(claim.TenantID, neighborID, neighbor, sess.NeighborAS)
if d == nil {
id := uuid.NewString()
m.peerDiscoveries[id] = &BGPPeerDiscovery{ID: id, TenantID: claim.TenantID, SpeakerID: claim.SpeakerID, NeighborID: neighborID, Neighbor: neighbor, RemoteASN: sess.NeighborAS, ProtocolName: sess.Name, SessionState: sess.State, Status: PeerDiscoveryPending, FirstSeenAt: bgp.ReceivedAt, LastSeenAt: bgp.ReceivedAt}
continue
}
if d.Status == PeerDiscoveryRejected || d.LastSeenAt.After(bgp.ReceivedAt) {
continue
}
d.ProtocolName = sess.Name
d.SessionState = sess.State
d.LastSeenAt = bgp.ReceivedAt
if claim.SpeakerID != "" {
d.SpeakerID = claim.SpeakerID
}
}
return next
}
// ListTelemetry reads the last committed snapshots without polling agents.
func (m *Memory) ListTelemetry(_ context.Context, tenantID string) ([]TelemetrySnapshot, error) {
m.mu.RLock()
defer m.mu.RUnlock()
out := make([]TelemetrySnapshot, 0)
for _, row := range m.telemetry {
if row.snapshot.TenantID != tenantID {
continue
}
if row.snapshot.Kind == "remote_agent" {
sp := m.speakers[row.snapshot.SpeakerID]
if sp == nil || !SpeakerNeedsRemoteDispatch(sp.Role, ParseSpeakerMeta(sp.MetaJSON)) {
continue
}
}
snap := row.snapshot
snap.BGP.Sessions = append(snap.BGP.Sessions[:0:0], snap.BGP.Sessions...)
out = append(out, snap)
}
sort.Slice(out, func(i, j int) bool { return out[i].TargetID < out[j].TargetID })
return out, nil
}
// RequestTelemetryRefresh coalesces repeated requests per tenant within cooldown.
func (m *Memory) RequestTelemetryRefresh(_ context.Context, tenantID string, cooldown time.Duration) (bool, error) {
if cooldown <= 0 {
cooldown = 10 * time.Second
}
m.mu.Lock()
defer m.mu.Unlock()
now := time.Now().UTC()
changed := false
for key, row := range m.telemetry {
if row.snapshot.TenantID != tenantID || now.Sub(row.lastRequested) < cooldown {
continue
}
row.lastRequested = now
if row.leaseUntil.After(now) {
row.refreshPending = true
} else {
row.snapshot.NextPollAt = now
}
m.telemetry[key] = row
changed = true
}
return changed, nil
}
+172
View File
@@ -0,0 +1,172 @@
package store
import (
"context"
"errors"
"fmt"
"testing"
"time"
"evobgp/internal/birdfmt"
)
func TestMemoryTelemetryFencingRefreshAndDiscovery(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, _, _, _, _ := m.DemoIDs()
sp, err := m.CreateSpeaker(tenant, &Speaker{Role: "replica", MetaJSON: `{"agent_domain":"node.test","agent_secret":"secret"}`})
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
claims, err := m.ClaimTelemetry(ctx, "owner-a", 2, time.Minute, "")
if err != nil || len(claims) != 1 || claims[0].SpeakerID != sp.ID {
t.Fatalf("claims=%+v err=%v", claims, err)
}
first := claims[0]
if snaps, err := m.ListTelemetry(ctx, tenant); err != nil || len(snaps) != 1 || snaps[0].ConfigFingerprint != "" {
t.Fatalf("uncommitted config visible: %+v %v", snaps, err)
}
if other, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, ""); err != nil || len(other) != 0 {
t.Fatalf("stole lease: %+v %v", other, err)
}
if requested, err := m.RequestTelemetryRefresh(ctx, tenant, time.Minute); err != nil || !requested {
t.Fatalf("refresh=%v err=%v", requested, err)
}
if requested, err := m.RequestTelemetryRefresh(ctx, tenant, time.Minute); err != nil || requested {
t.Fatalf("not coalesced: %v %v", requested, err)
}
at := time.Now().UTC()
result := TelemetryResult{BGP: TelemetryBGP{TelemetryObservation: TelemetryObservation{Source: "agent_bird_protocols", ObservedAt: at, ReceivedAt: at}, Sessions: []birdfmt.BGPSession{{Name: birdfmt.DynamicPeerProtocolPrefix + "test", Neighbor: "192.0.2.1", NeighborAS: 64501, State: "Established"}}}, Health: TelemetryHealth{TelemetryObservation: TelemetryObservation{Source: "agent_health", ObservedAt: at, ReceivedAt: at}, OK: true}}
if err := m.CompleteTelemetry(ctx, first, result, at.Add(time.Hour)); err != nil {
t.Fatal(err)
}
if err := m.CompleteTelemetry(ctx, first, result, at.Add(time.Hour)); !errors.Is(err, ErrTelemetryLeaseLost) {
t.Fatalf("reused lease: %v", err)
}
snaps, err := m.ListTelemetry(ctx, tenant)
if err != nil || len(snaps) != 1 || snaps[0].ConfigFingerprint == "" || snaps[0].BGP.LastSuccessAt.IsZero() || snaps[0].NextPollAt.After(time.Now().Add(time.Second)) {
t.Fatalf("snapshot=%+v err=%v", snaps, err)
}
discoveries, err := m.ListPeerDiscoveries(tenant, PeerDiscoveryPending)
if err != nil || len(discoveries) != 1 {
t.Fatalf("discovery=%+v err=%v", discoveries, err)
}
next, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, "")
if err != nil || len(next) != 1 || next[0].FencingToken <= first.FencingToken {
t.Fatalf("new claim=%+v %v", next, err)
}
meta := `{"agent_domain":"new.test","agent_secret":"secret"}`
if _, err := m.UpdateSpeaker(tenant, sp.ID, &SpeakerPatch{MetaJSON: &meta}); err != nil {
t.Fatal(err)
}
if err := m.CompleteTelemetry(ctx, next[0], result, at.Add(time.Hour)); !errors.Is(err, ErrTelemetryLeaseLost) {
t.Fatalf("config changed, late result accepted: %v", err)
}
m.mu.Lock()
row := m.telemetry[telemetryKey(tenant, sp.ID)]
row.leaseUntil = time.Now().Add(-time.Second)
m.telemetry[telemetryKey(tenant, sp.ID)] = row
m.mu.Unlock()
newConfig, err := m.ClaimTelemetry(ctx, "owner-c", 1, time.Minute, "")
if err != nil || len(newConfig) != 1 {
t.Fatalf("new config claim=%+v %v", newConfig, err)
}
failed := TelemetryResult{BGP: TelemetryBGP{TelemetryObservation: TelemetryObservation{Source: "agent_bird_protocols", Error: "timeout", ReceivedAt: time.Now().UTC()}}, Health: TelemetryHealth{TelemetryObservation: TelemetryObservation{Source: "agent_health", Error: "timeout", ReceivedAt: time.Now().UTC()}}}
if err := m.CompleteTelemetry(ctx, newConfig[0], failed, time.Now().Add(time.Hour)); err != nil {
t.Fatal(err)
}
snaps, err = m.ListTelemetry(ctx, tenant)
if err != nil || len(snaps) != 1 || !snaps[0].BGP.LastSuccessAt.IsZero() || !snaps[0].Health.LastSuccessAt.IsZero() {
t.Fatalf("old endpoint success survived config change: %+v %v", snaps, err)
}
}
func TestMemoryTelemetryLocalAuthority(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, _, _, _, _ := m.DemoIDs()
sp, err := m.CreateSpeaker(tenant, &Speaker{Role: "master"})
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
first, err := m.ClaimTelemetry(ctx, "owner-a", 1, time.Minute, "site-a")
if err != nil || len(first) != 1 || first[0].TargetID != TelemetryLocalTargetID(tenant, "site-a") || first[0].SpeakerID != sp.ID {
t.Fatalf("local claim=%+v %v", first, err)
}
second, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, "site-b")
if err != nil || len(second) != 1 || second[0].TargetID != TelemetryLocalTargetID(tenant, "site-b") {
t.Fatalf("other authority=%+v %v", second, err)
}
}
func TestTelemetryDiscoveryBatchRotatesPastStaticPrefix(t *testing.T) {
sessions := make([]birdfmt.BGPSession, 300)
for i := range sessions {
sessions[i] = birdfmt.BGPSession{Name: fmt.Sprintf("static-%d", i)}
}
sessions[299] = birdfmt.BGPSession{Name: birdfmt.DynamicPeerProtocolPrefix + "last", Neighbor: "192.0.2.9"}
batch, cursor := TelemetryDiscoveryBatch(sessions, 0, 256)
if len(batch) != 1 || batch[0].Neighbor != "192.0.2.9" || cursor != 0 {
t.Fatalf("batch=%+v cursor=%d", batch, cursor)
}
for i := range sessions {
sessions[i].Name = birdfmt.DynamicPeerProtocolPrefix + fmt.Sprint(i)
}
first, cursor := TelemetryDiscoveryBatch(sessions, 0, 256)
second, next := TelemetryDiscoveryBatch(sessions, cursor, 256)
if len(first) != 256 || len(second) != 256 || second[0].Name != sessions[256].Name || next != 212 {
t.Fatalf("rotation first=%d second=%d cursor=%d next=%d", len(first), len(second), cursor, next)
}
}
func TestTelemetryFingerprintIgnoresDispatchStatus(t *testing.T) {
sp := &Speaker{Role: "replica", Endpoint: "https://node.test", MetaJSON: `{"agent_domain":"node.test","agent_secret":"secret"}`}
before := TelemetrySpeakerFingerprint(sp)
sp.MetaJSON = `{"agent_domain":"node.test","agent_secret":"secret","last_dispatch_at":"2026-10-26T10:00:00Z","last_dispatch_status":"ok"}`
if TelemetrySpeakerFingerprint(sp) != before {
t.Fatal("dispatch status invalidated telemetry config")
}
sp.Role = "master"
before = TelemetrySpeakerFingerprint(sp)
sp.MetaJSON = `{"agent_domain":"other.test","agent_secret":"changed"}`
if TelemetrySpeakerFingerprint(sp) != before {
t.Fatal("agent metadata invalidated local BIRD config")
}
}
func TestMemoryTelemetryExpiredClaimCannotDiscover(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, _, _, _, _ := m.DemoIDs()
sp, err := m.CreateSpeaker(tenant, &Speaker{Role: "replica", MetaJSON: `{"agent_domain":"node.test","agent_secret":"secret"}`})
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
first, err := m.ClaimTelemetry(ctx, "owner-a", 1, time.Minute, "")
if err != nil || len(first) != 1 || first[0].SpeakerID != sp.ID {
t.Fatalf("first=%+v %v", first, err)
}
m.mu.Lock()
row := m.telemetry[telemetryKey(tenant, sp.ID)]
row.leaseUntil = time.Now().Add(-time.Second)
m.telemetry[telemetryKey(tenant, sp.ID)] = row
m.mu.Unlock()
second, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, "")
if err != nil || len(second) != 1 {
t.Fatalf("second=%+v %v", second, err)
}
at := time.Now().UTC()
result := TelemetryResult{BGP: TelemetryBGP{TelemetryObservation: TelemetryObservation{Source: "agent_bird_protocols", ReceivedAt: at}, Sessions: []birdfmt.BGPSession{{Name: birdfmt.DynamicPeerProtocolPrefix + "late", Neighbor: "192.0.2.13"}}}}
if err := m.CompleteTelemetry(ctx, first[0], result, at.Add(time.Hour)); !errors.Is(err, ErrTelemetryLeaseLost) {
t.Fatalf("old claim accepted: %v", err)
}
if list, err := m.ListPeerDiscoveries(tenant, ""); err != nil || len(list) != 0 {
t.Fatalf("stale discovery persisted: %+v %v", list, err)
}
if err := m.CompleteTelemetry(ctx, second[0], result, at.Add(time.Hour)); err != nil {
t.Fatal(err)
}
}
+5 -1
View File
@@ -1,11 +1,13 @@
package store
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
"time"
)
// ComputeModuleInputHash fingerprints module config and child entries so snapshots
@@ -109,7 +111,9 @@ func TouchModuleInputHash(st Backend, tenantID, moduleID string) {
if err != nil {
return
}
_ = st.SetModuleInputHash(tenantID, moduleID, h)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = st.SetModuleInputHash(ctx, tenantID, moduleID, h)
}
// CDNPatchAffectsInputHash reports whether a CDN source patch changes ingest fingerprint
+69
View File
@@ -0,0 +1,69 @@
package store
import (
"crypto/sha256"
"encoding/hex"
"sort"
)
// PrefixSnapshotHash is the content key of prefix_snapshot.
// It fingerprints prefix rows only. Peer configuration stays on
// config_revision.content_hash so a peer-only change can reuse the snapshot.
// The result is 64 lowercase hex characters. Row order does not affect the hash.
func PrefixSnapshotHash(rows []PrefixRow) string {
cp := append([]PrefixRow(nil), rows...)
sortPrefixRows(cp)
h := sha256.New()
for _, row := range cp {
_, _ = h.Write([]byte(row.Prefix))
_, _ = h.Write([]byte{1})
if row.CommunityID == nil {
_, _ = h.Write([]byte{0})
} else {
_, _ = h.Write([]byte{1})
_, _ = h.Write([]byte(*row.CommunityID))
}
_, _ = h.Write([]byte{1})
_, _ = h.Write([]byte(row.Source))
_, _ = h.Write([]byte{0})
}
sum := h.Sum(nil)
return hex.EncodeToString(sum)
}
func sortPrefixRows(rows []PrefixRow) {
sort.Slice(rows, func(i, j int) bool {
if rows[i].Prefix != rows[j].Prefix {
return rows[i].Prefix < rows[j].Prefix
}
ni, vi := communitySortKey(rows[i].CommunityID)
nj, vj := communitySortKey(rows[j].CommunityID)
if ni != nj {
return ni
}
if vi != vj {
return vi < vj
}
return rows[i].Source < rows[j].Source
})
}
func communitySortKey(id *string) (isNil bool, val string) {
if id == nil {
return true, ""
}
return false, *id
}
func samePrefixRow(a, b PrefixRow) bool {
if a.Prefix != b.Prefix || a.Source != b.Source {
return false
}
if (a.CommunityID == nil) != (b.CommunityID == nil) {
return false
}
if a.CommunityID != nil && *a.CommunityID != *b.CommunityID {
return false
}
return true
}
@@ -0,0 +1,153 @@
package store
import (
"context"
"errors"
"testing"
"time"
"github.com/google/uuid"
)
func TestSameSnapshotContentIgnoresOrder(t *testing.T) {
comm := "c1"
a := &ModulePrefixSnapshot{InputHash: "h", Prefixes: []PrefixRow{
{Prefix: "203.0.113.0/24", CommunityID: &comm, Source: "cdn:a"},
{Prefix: "198.51.100.0/24", Source: "cdn:b"},
}}
b := &ModulePrefixSnapshot{InputHash: "h", Prefixes: []PrefixRow{
{Prefix: "198.51.100.0/24", Source: "cdn:b"},
{Prefix: "203.0.113.0/24", CommunityID: &comm, Source: "cdn:a"},
}}
if !SameSnapshotContent(a, b) {
t.Fatal("same set in another order was treated as new")
}
b.Prefixes[0].Prefix = "192.0.2.0/24"
if SameSnapshotContent(a, b) {
t.Fatal("different prefix compared equal")
}
}
func TestPrefixSnapshotHashIgnoresOrder(t *testing.T) {
rows := []PrefixRow{
{Prefix: "203.0.113.0/24", Source: "ip_range"},
{Prefix: "198.51.100.0/24", Source: "cdn:a"},
}
rev := []PrefixRow{rows[1], rows[0]}
if PrefixSnapshotHash(rows) != PrefixSnapshotHash(rev) {
t.Fatal("prefix hash depends on order")
}
rev = append(rev, PrefixRow{Prefix: "2001:db8::/32", Source: "ip_range"})
if PrefixSnapshotHash(rows) == PrefixSnapshotHash(rev) {
t.Fatal("prefix change kept the same hash")
}
}
func TestCreateRenderRevisionReusesPrefixSnapshot(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, _, mod, _, _ := m.DemoIDs()
ctx := context.Background()
rows := []PrefixRow{
{Prefix: "203.0.113.0/24", Source: "ip_range"},
{Prefix: "198.51.100.0/24", Source: "cdn:a"},
}
rev1 := uuid.NewString()
if err := m.CreateRenderRevision(ctx, rev1, tenant, mod, nil, "hash-peer-a", map[string]string{"bird.conf": "a"}, rows); err != nil {
t.Fatal(err)
}
rev2 := uuid.NewString()
reversed := []PrefixRow{rows[1], rows[0]}
if err := m.CreateRenderRevision(ctx, rev2, tenant, mod, nil, "hash-peer-b", map[string]string{"bird.conf": "b"}, reversed); err != nil {
t.Fatal(err)
}
if m.revPrefixSnap[rev1] == "" || m.revPrefixSnap[rev1] != m.revPrefixSnap[rev2] {
t.Fatalf("peer-only change copied prefix rows: %q vs %q", m.revPrefixSnap[rev1], m.revPrefixSnap[rev2])
}
if len(m.prefixSnapRows) != 1 {
t.Fatalf("stored prefix snapshots: %d", len(m.prefixSnapRows))
}
page, _, _ := m.ListRevisionPrefixes(tenant, rev2, "", 10)
if len(page) != 2 {
t.Fatalf("reused snapshot not readable: %+v", page)
}
changed := append(append([]PrefixRow(nil), rows...), PrefixRow{Prefix: "2001:db8::/32", Source: "ip_range"})
rev3 := uuid.NewString()
if err := m.CreateRenderRevision(ctx, rev3, tenant, mod, nil, "hash-prefix", nil, changed); err != nil {
t.Fatal(err)
}
if m.revPrefixSnap[rev3] == m.revPrefixSnap[rev1] || len(m.prefixSnapRows) != 2 {
t.Fatalf("prefix change did not allocate a snapshot: id=%s count=%d", m.revPrefixSnap[rev3], len(m.prefixSnapRows))
}
}
func TestUpdateModulePrefixSnapshotCancelledContext(t *testing.T) {
m := NewMemory()
m.SeedDemo()
tenant, mod, _, _, _ := m.DemoIDs()
ctx, cancel := context.WithCancel(context.Background())
cancel()
err := m.UpdateModulePrefixSnapshot(ctx, tenant, mod, func(*ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) {
t.Fatal("snapshot update ran after cancel")
return nil, nil
})
if !errors.Is(err, context.Canceled) {
t.Fatalf("err=%v", err)
}
}
func TestMemorySnapshotLockUsesSnapshotTimeout(t *testing.T) {
t.Setenv("EVOBGP_SNAPSHOT_TIMEOUT", "80ms")
m := NewMemory()
hold, err := m.lockModuleSnapshot(context.Background(), "tenant", "module")
if err != nil {
t.Fatal(err)
}
defer hold()
start := time.Now()
_, err = m.lockModuleSnapshot(context.Background(), "tenant", "module")
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("err=%v", err)
}
if time.Since(start) > time.Second {
t.Fatalf("lock wait ignored snapshot timeout: %s", time.Since(start))
}
}
func TestSnapshotWriteBudgetKeepsCallerDeadline(t *testing.T) {
parent, cancel := context.WithTimeout(context.Background(), time.Hour)
defer cancel()
got, stop, installed := SnapshotWriteBudget(parent)
defer stop()
if installed {
t.Fatal("caller deadline was replaced")
}
dl, ok := got.Deadline()
if !ok || time.Until(dl) < 30*time.Minute {
t.Fatal("caller deadline was shortened")
}
}
func TestSetASNPrefixCacheKeepsRowsWhenSetUnchanged(t *testing.T) {
m := NewMemory()
if err := m.SetASNPrefixCache(64500, "holder-a", []string{"203.0.113.0/24", "198.51.100.0/24"}); err != nil {
t.Fatal(err)
}
before := m.asnPrefixCache[64500]
ptr := &before.Prefixes[0]
fetched := before.FetchedAt
time.Sleep(5 * time.Millisecond)
if err := m.SetASNPrefixCache(64500, "holder-b", []string{"198.51.100.0/24", "203.0.113.0/24"}); err != nil {
t.Fatal(err)
}
after := m.asnPrefixCache[64500]
if &after.Prefixes[0] != ptr {
t.Fatal("unchanged ASN prefixes were rewritten")
}
if !after.FetchedAt.After(fetched) {
t.Fatal("fetched_at was not updated")
}
if after.Holder != "holder-b" {
t.Fatalf("holder=%q", after.Holder)
}
}
+363
View File
@@ -0,0 +1,363 @@
package store
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"sort"
"strings"
)
func (m *Memory) GetRenderGeneration(ctx context.Context, tenantID string) (int64, error) {
if err := ctx.Err(); err != nil {
return 0, err
}
m.mu.RLock()
defer m.mu.RUnlock()
if m.tenants[tenantID] == nil {
return 0, ErrNotFound
}
gen := m.renderGeneration[tenantID]
if gen == 0 {
gen = 1
}
return gen, nil
}
func (m *Memory) ListModuleSnapshotHeaders(ctx context.Context, tenantID string) ([]ModuleSnapshotUse, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
m.mu.RLock()
defer m.mu.RUnlock()
return append([]ModuleSnapshotUse(nil), m.snapshotHeadersLocked(tenantID)...), nil
}
func (m *Memory) snapshotHeadersLocked(tenantID string) []ModuleSnapshotUse {
prefix := moduleSnapshotKey(tenantID, "")
var out []ModuleSnapshotUse
for key, snap := range m.moduleSnapshots {
if snap == nil || !strings.HasPrefix(key, prefix) {
continue
}
out = append(out, ModuleSnapshotUse{
ModuleID: strings.TrimPrefix(key, prefix),
InputHash: snap.InputHash,
SnapshotVersion: snap.Version,
})
}
sort.Slice(out, func(i, j int) bool { return out[i].ModuleID < out[j].ModuleID })
return out
}
func (m *Memory) LatestRenderManifest(ctx context.Context, tenantID string) (*RenderManifestRecord, bool, error) {
if err := ctx.Err(); err != nil {
return nil, false, err
}
m.mu.RLock()
defer m.mu.RUnlock()
var latest *Revision
for _, rev := range m.revisions {
if rev == nil || rev.TenantID != tenantID {
continue
}
if latest == nil || rev.CreatedAt.After(latest.CreatedAt) || (rev.CreatedAt.Equal(latest.CreatedAt) && rev.ID > latest.ID) {
latest = rev
}
}
if latest == nil {
return nil, false, nil
}
rec := m.renderManifests[latest.ID]
if rec == nil {
return nil, false, nil
}
return cloneRenderManifest(rec), true, nil
}
func (m *Memory) recordRenderManifestLocked(revisionID, tenantID string) {
if m.renderManifests == nil {
m.renderManifests = make(map[string]*RenderManifestRecord)
}
gen := m.renderGeneration[tenantID]
if gen == 0 {
gen = 1
}
m.renderManifests[revisionID] = &RenderManifestRecord{
RevisionID: revisionID,
Generation: gen,
Snapshots: m.snapshotHeadersLocked(tenantID),
}
}
func cloneRenderManifest(rec *RenderManifestRecord) *RenderManifestRecord {
if rec == nil {
return nil
}
out := *rec
out.Snapshots = append([]ModuleSnapshotUse(nil), rec.Snapshots...)
return &out
}
// bumpRenderGenerationLocked is called only while holding m.mu for a semantic write.
func (m *Memory) bumpRenderGenerationLocked(tenantID string) {
if m.renderGeneration == nil {
m.renderGeneration = make(map[string]int64)
}
if m.renderGeneration[tenantID] == 0 {
m.renderGeneration[tenantID] = 1
}
m.renderGeneration[tenantID]++
}
// RenderInputs is the complete, immutable configuration read by tenant rendering.
// Fetch observations such as CDN ETags, resolve metadata and dispatch status are
// retained in the captured rows where the ingest path needs them, but excluded
// from SemanticHash. Callers must not mutate a captured value.
type RenderInputs struct {
TenantID string `json:"tenant_id"`
Generation int64 `json:"generation"`
Modules []RenderModule `json:"modules"`
Profiles []*DohProfile `json:"profiles"`
Peers []*BGPPeer `json:"peers"`
Communities []*Community `json:"communities"`
Settings map[string]any `json:"settings"`
Speakers []RenderSpeaker `json:"speakers"`
}
// RenderModule contains all ingestion inputs for one module in one capture.
type RenderModule struct {
Module *Module `json:"module"`
CDNSources []*CDNSource `json:"cdn_sources"`
ASEntries []*ASEntry `json:"as_entries"`
DomainEntries []*DomainEntry `json:"domain_entries"`
IPRanges []*IPRangeEntry `json:"ip_ranges"`
}
// RenderSpeaker holds only values that affect speaker-specific BIRD fragments.
type RenderSpeaker struct {
ID string `json:"id"`
NodeIPv4 string `json:"node_ipv4"`
BirdBgpSourceIPv4 string `json:"bird_bgp_source_ipv4"`
BirdBgpSourceIPv6 string `json:"bird_bgp_source_ipv6"`
}
// ModuleSnapshotUse identifies the exact module materialization used by a render.
type ModuleSnapshotUse struct {
ModuleID string `json:"module_id"`
InputHash string `json:"input_hash"`
SnapshotVersion int64 `json:"snapshot_version"`
}
// RenderManifestRecord is the admission barrier stored with a config revision.
// Generation and snapshot versions let a later render return the revision
// without loading prefix rows when nothing material changed.
type RenderManifestRecord struct {
RevisionID string
Generation int64
Snapshots []ModuleSnapshotUse
}
// RenderManifest is persisted with a revision so new publish admission can
// distinguish a current render from a historical rollback or durable replay.
type RenderManifest struct {
Inputs RenderInputs `json:"inputs"`
ConfigHash string `json:"config_hash"`
Snapshots []ModuleSnapshotUse `json:"snapshots"`
OverlayBySpeaker map[string]map[string]string `json:"overlay_by_speaker"`
}
// SemanticHash fingerprints captured render settings and ingest configuration.
// It deliberately excludes fetch timestamps, ETags, health and dispatch state.
func (in *RenderInputs) SemanticHash() (string, error) {
if in == nil || strings.TrimSpace(in.TenantID) == "" {
return "", ErrInvalidInput
}
semantic := struct {
TenantID string `json:"tenant_id"`
Modules []any `json:"modules"`
Profiles []any `json:"profiles"`
Peers []any `json:"peers"`
Communities []any `json:"communities"`
Settings map[string]any `json:"settings"`
Speakers []RenderSpeaker `json:"speakers"`
}{TenantID: in.TenantID, Settings: in.Settings, Speakers: append([]RenderSpeaker(nil), in.Speakers...)}
modules := append([]RenderModule(nil), in.Modules...)
for _, item := range modules {
if item.Module == nil {
return "", fmt.Errorf("store: render inputs: nil module")
}
}
sort.Slice(modules, func(i, j int) bool { return modules[i].Module.ID < modules[j].Module.ID })
for _, item := range modules {
m := item.Module
semantic.Modules = append(semantic.Modules, struct {
ID string
Type string
Enabled bool
DefaultCommunityID *string
DohResolverPolicy string
DohProfileIDs []string
CDNSources []any
ASEntries []any
DomainEntries []any
IPRanges []any
}{ID: m.ID, Type: m.Type, Enabled: m.Enabled, DefaultCommunityID: m.DefaultCommunityID, DohResolverPolicy: m.DohResolverPolicy,
DohProfileIDs: append([]string(nil), m.EffectiveDohProfileIDs()...),
CDNSources: semanticCDNSources(item.CDNSources), ASEntries: semanticASEntries(item.ASEntries), DomainEntries: semanticDomainEntries(item.DomainEntries), IPRanges: semanticIPRanges(item.IPRanges)})
}
profiles := append([]*DohProfile(nil), in.Profiles...)
sort.Slice(profiles, func(i, j int) bool {
if profiles[i] == nil {
return false
}
if profiles[j] == nil {
return true
}
return profiles[i].ID < profiles[j].ID
})
for _, p := range profiles {
if p != nil {
semantic.Profiles = append(semantic.Profiles, struct {
ID, URL string
TimeoutMs *int
SecretRef *string
}{p.ID, p.URL, p.TimeoutMs, p.SecretRef})
}
}
peers := append([]*BGPPeer(nil), in.Peers...)
sort.Slice(peers, func(i, j int) bool {
if peers[i] == nil {
return false
}
if peers[j] == nil {
return true
}
return peers[i].ID < peers[j].ID
})
for _, p := range peers {
if p != nil {
semantic.Peers = append(semantic.Peers, struct {
ID, Neighbor, PoliciesJSON string
RemoteASN int64
Enabled bool
SpeakerID *string
}{p.ID, p.Neighbor, p.PoliciesJSON, p.RemoteASN, p.Enabled, p.SpeakerID})
}
}
communities := append([]*Community(nil), in.Communities...)
sort.Slice(communities, func(i, j int) bool {
if communities[i] == nil {
return false
}
if communities[j] == nil {
return true
}
return communities[i].ID < communities[j].ID
})
for _, c := range communities {
if c != nil {
semantic.Communities = append(semantic.Communities, struct{ ID, Community, ValueJSON string }{c.ID, c.Community, c.ValueJSON})
}
}
sort.Slice(semantic.Speakers, func(i, j int) bool { return semantic.Speakers[i].ID < semantic.Speakers[j].ID })
data, err := json.Marshal(semantic)
if err != nil {
return "", err
}
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
func semanticCDNSources(items []*CDNSource) []any {
items = append([]*CDNSource(nil), items...)
sort.Slice(items, func(i, j int) bool {
if items[i] == nil {
return false
}
if items[j] == nil {
return true
}
return items[i].ID < items[j].ID
})
out := make([]any, 0, len(items))
for _, s := range items {
if s != nil {
out = append(out, struct {
ID, SourceKind, URL, PrefixPath string
RefreshIntervalSec *int
CommunityID *string
}{s.ID, s.SourceKind, s.URL, s.PrefixPath, s.RefreshIntervalSec, s.CommunityID})
}
}
return out
}
func semanticASEntries(items []*ASEntry) []any {
items = append([]*ASEntry(nil), items...)
sort.Slice(items, func(i, j int) bool {
if items[i] == nil {
return false
}
if items[j] == nil {
return true
}
return items[i].ID < items[j].ID
})
out := make([]any, 0, len(items))
for _, s := range items {
if s != nil {
out = append(out, struct {
ID string
ASN int64
CommunityID *string
}{s.ID, s.ASN, s.CommunityID})
}
}
return out
}
func semanticDomainEntries(items []*DomainEntry) []any {
items = append([]*DomainEntry(nil), items...)
sort.Slice(items, func(i, j int) bool {
if items[i] == nil {
return false
}
if items[j] == nil {
return true
}
return items[i].ID < items[j].ID
})
out := make([]any, 0, len(items))
for _, s := range items {
if s != nil {
out = append(out, struct {
ID, FQDN string
CommunityID *string
}{s.ID, s.FQDN, s.CommunityID})
}
}
return out
}
func semanticIPRanges(items []*IPRangeEntry) []any {
items = append([]*IPRangeEntry(nil), items...)
sort.Slice(items, func(i, j int) bool {
if items[i] == nil {
return false
}
if items[j] == nil {
return true
}
return items[i].ID < items[j].ID
})
out := make([]any, 0, len(items))
for _, s := range items {
if s != nil {
out = append(out, struct {
ID, Prefix string
CommunityID *string
}{s.ID, s.Prefix, s.CommunityID})
}
}
return out
}
+37
View File
@@ -0,0 +1,37 @@
package store
import (
"testing"
"time"
)
func TestRenderInputsSemanticHashCanonicalAndIgnoresFetchState(t *testing.T) {
now := time.Now().UTC()
a := &RenderInputs{TenantID: "tenant", Modules: []RenderModule{
{Module: &Module{ID: "b", Type: "CDN_CIDRS", Enabled: true}, CDNSources: []*CDNSource{{ID: "s2", URL: "https://b.test", Etag: "old"}, {ID: "s1", URL: "https://a.test", LastRefreshedAt: &now}}},
{Module: &Module{ID: "a", Type: "IP_RANGES", Enabled: true}, IPRanges: []*IPRangeEntry{{ID: "r2", Prefix: "192.0.2.0/24"}, {ID: "r1", Prefix: "198.51.100.0/24"}}},
}}
b := &RenderInputs{TenantID: "tenant", Generation: 99, Modules: []RenderModule{
{Module: &Module{ID: "a", Type: "IP_RANGES", Enabled: true}, IPRanges: []*IPRangeEntry{{ID: "r1", Prefix: "198.51.100.0/24"}, {ID: "r2", Prefix: "192.0.2.0/24"}}},
{Module: &Module{ID: "b", Type: "CDN_CIDRS", Enabled: true}, CDNSources: []*CDNSource{{ID: "s1", URL: "https://a.test"}, {ID: "s2", URL: "https://b.test", Etag: "new"}}},
}}
ha, err := a.SemanticHash()
if err != nil {
t.Fatal(err)
}
hb, err := b.SemanticHash()
if err != nil {
t.Fatal(err)
}
if ha != hb {
t.Fatalf("same semantic inputs have different hashes: %s %s", ha, hb)
}
b.Modules[1].CDNSources[0].URL = "https://changed.test"
hb, err = b.SemanticHash()
if err != nil {
t.Fatal(err)
}
if ha == hb {
t.Fatal("source URL change did not change hash")
}
}
+51
View File
@@ -0,0 +1,51 @@
package store
import (
"context"
"os"
"strings"
"time"
)
const defaultSnapshotTimeout = 2 * time.Minute
// SnapshotWriteBudget keeps a caller deadline. When ctx has none, it applies
// EVOBGP_SNAPSHOT_TIMEOUT (default 2 minutes). installed is true when this
// function created the deadline; callers use that to set statement_timeout.
func SnapshotWriteBudget(ctx context.Context) (context.Context, context.CancelFunc, bool) {
if ctx == nil {
ctx = context.Background()
}
if _, ok := ctx.Deadline(); ok {
return ctx, func() {}, false
}
ctx, cancel := context.WithTimeout(ctx, snapshotTimeout())
return ctx, cancel, true
}
// SnapshotStatementTimeout is the PostgreSQL statement_timeout that matches
// SnapshotWriteBudget. A caller deadline wins; otherwise the env default applies.
func SnapshotStatementTimeout(ctx context.Context) time.Duration {
if ctx != nil {
if dl, ok := ctx.Deadline(); ok {
d := time.Until(dl)
if d < time.Millisecond {
return time.Millisecond
}
return d
}
}
return snapshotTimeout()
}
func snapshotTimeout() time.Duration {
s := strings.TrimSpace(os.Getenv("EVOBGP_SNAPSHOT_TIMEOUT"))
if s == "" {
return defaultSnapshotTimeout
}
d, err := time.ParseDuration(s)
if err != nil || d <= 0 {
return defaultSnapshotTimeout
}
return d
}
+113
View File
@@ -0,0 +1,113 @@
package store
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"strconv"
"strings"
"time"
"evobgp/internal/birdfmt"
)
// ErrTelemetryLeaseLost means a collector result belongs to an expired or replaced lease.
var ErrTelemetryLeaseLost = errors.New("store: telemetry lease lost")
// TelemetryLocalTargetID identifies the synthetic local BIRD target of a tenant.
func TelemetryLocalTargetID(tenantID, authority string) string {
return "cp-local/" + tenantID + "/" + authority
}
// TelemetryObservation is the most recent attempt and last successful reading of one source.
type TelemetryObservation struct {
ObservedAt time.Time `json:"observed_at"`
ReceivedAt time.Time `json:"received_at"`
LastSuccessAt time.Time `json:"last_success_at"`
Source string `json:"source"`
Error string `json:"error,omitempty"`
}
// TelemetryBGP is one BIRD protocols measurement.
type TelemetryBGP struct {
TelemetryObservation
Sessions []birdfmt.BGPSession `json:"sessions,omitempty"`
}
// TelemetryHealth is one legacy agent health measurement.
type TelemetryHealth struct {
TelemetryObservation
OK bool `json:"ok"`
LastSyncAt string `json:"last_sync_at,omitempty"`
LastAppliedRevisionID string `json:"last_applied_revision_id,omitempty"`
}
// TelemetrySnapshot contains independent BGP and agent health readings for one target.
type TelemetrySnapshot struct {
TenantID, TargetID, SpeakerID, Kind string
LocalAuthority string
ConfigFingerprint string
BGP TelemetryBGP
Health TelemetryHealth
NextPollAt time.Time
FailureCount int
DiscoveryCursor int
}
// TelemetrySpeakerFingerprint binds a snapshot to the configuration it observed.
func TelemetrySpeakerFingerprint(sp *Speaker) string {
if sp == nil {
return ""
}
if strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
sum := sha256.Sum256([]byte("master\x00" + sp.Endpoint))
return hex.EncodeToString(sum[:])
}
meta := ParseSpeakerMeta(sp.MetaJSON)
sum := sha256.Sum256([]byte(sp.Role + "\x00" + sp.Endpoint + "\x00" + meta.AgentDomain + "\x00" + meta.AgentSecret + "\x00" + strconv.Itoa(meta.AgentPort)))
return hex.EncodeToString(sum[:])
}
// TelemetryClaim fences a bounded collector attempt. Speaker is the current config.
type TelemetryClaim struct {
TelemetrySnapshot
Speaker *Speaker
OwnerID string
FencingToken int64
}
// TelemetryResult is the atomic pair of results of one collector attempt.
type TelemetryResult struct {
BGP TelemetryBGP
Health TelemetryHealth
}
// TelemetryDiscoveryBatch rotates over long protocol lists without starving later entries.
func TelemetryDiscoveryBatch(sessions []birdfmt.BGPSession, cursor, limit int) ([]birdfmt.BGPSession, int) {
if len(sessions) == 0 || limit <= 0 {
return nil, 0
}
if cursor < 0 || cursor >= len(sessions) {
cursor = 0
}
out := make([]birdfmt.BGPSession, 0, limit)
for visited := 0; visited < len(sessions); visited++ {
idx := (cursor + visited) % len(sessions)
if birdfmt.IsDynamicDiscoverySession(sessions[idx].Name) {
out = append(out, sessions[idx])
if len(out) == limit {
return out, (idx + 1) % len(sessions)
}
}
}
return out, 0
}
// TelemetryStore coordinates polling and read-only snapshots across API replicas.
type TelemetryStore interface {
ClaimTelemetry(context.Context, string, int, time.Duration, string) ([]TelemetryClaim, error)
CompleteTelemetry(context.Context, TelemetryClaim, TelemetryResult, time.Time) error
ListTelemetry(context.Context, string) ([]TelemetrySnapshot, error)
RequestTelemetryRefresh(context.Context, string, time.Duration) (bool, error)
}