package jobs import ( "context" "errors" "fmt" "net/http" "strconv" "time" "evobgp/internal/logging" "evobgp/internal/nodedispatch" "evobgp/internal/store" ) func deploymentFenceForJob(j *Job) store.JobFence { generation, _ := strconv.ParseInt(j.metaString("reconcile_generation"), 10, 64) round, _ := strconv.Atoi(j.metaString("reconcile_round")) return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken, ExpectedGeneration: generation, RetryRound: round} } func beginDeployment(ctx context.Context, st store.Deployments, f store.JobFence, speaker, revision string) (store.DeploymentSpeaker, error) { for { d, err := st.BeginDeploymentSpeaker(ctx, f, speaker, revision) if !errors.Is(err, store.ErrDeploymentBusy) { return d, err } timer := time.NewTimer(time.Second) select { case <-ctx.Done(): timer.Stop() return store.DeploymentSpeaker{}, ctx.Err() case <-timer.C: } } } // ReconcileDeployments schedules at most three recovery rounds for current desired // generations. Stable keys coalesce API replicas; the worker CAS checks generation // again before publishing, and preserves the original desired request sequence. func ReconcileDeployments(ctx context.Context, r *Registry, st store.Deployments) error { rows, err := st.ListPendingDeployments(ctx, 100) if err != nil { return err } for _, d := range rows { key := fmt.Sprintf("deployment_reconcile:%s:%s:%d", d.JobID, d.SpeakerID, d.RetryRound+1) _, _, err = r.EnqueueContext(ctx, d.TenantID, KindDeployApply, &key, nil, map[string]any{"revision_id": d.RevisionID, "speaker_id": d.SpeakerID, "reconcile_generation": strconv.FormatInt(d.Generation, 10), "reconcile_round": strconv.Itoa(d.RetryRound + 1), "source_deployment_job_id": d.JobID}) if err != nil { return err } } return nil } // StartDeploymentReconciliationLoop belongs to the API server lifecycle. Static // ingest/deploy/scheduler processes must not start this job-producing loop. func StartDeploymentReconciliationLoop(ctx context.Context, r *Registry, st store.Backend) { if r == nil || r.queue == nil || st == nil { return } r.durableMu.Lock() if r.durableStop { r.durableMu.Unlock() return } r.durableLoopWG.Add(1) r.durableMu.Unlock() go func() { defer r.durableLoopWG.Done() ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: } r.durableMu.Lock() stopped := r.durableStop r.durableMu.Unlock() if stopped { return } qctx, cancel := context.WithTimeout(ctx, 40*time.Second) err := ObserveDeployments(qctx, st, nil) if err == nil { err = ReconcileDeployments(qctx, r, st) } cancel() if err != nil && ctx.Err() == nil { logging.With("component", "jobs.deployment_reconcile").Warn("reconciliation failed", "err", err) } } }() } // ObserveDeployments samples compatible health before reconciling drift. The CP // receipt time and optional agent source time are stored separately; missing // legacy revision fields never manufacture an applied/verified confirmation. func ObserveDeployments(ctx context.Context, st store.Backend, client *http.Client) error { rows, err := st.ListDeploymentObservations(ctx, 10) if err != nil { return err } for _, d := range rows { sp, e := st.GetSpeaker(d.TenantID, d.SpeakerID) if e != nil { return e } observation := store.DeploymentObservation{Error: "remote health unavailable for this speaker"} if nodedispatch.Enabled() && store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON)) { hctx, cancel := context.WithTimeout(ctx, 3*time.Second) h := nodedispatch.FetchAgentHealth(hctx, sp, nodedispatch.Options{HTTPClient: client, Timeout: 3 * time.Second}) cancel() observation = store.DeploymentObservation{RevisionID: h.LastAppliedRevisionID, SourceAt: h.LastSyncAt, Error: h.Error} if h.OK && observation.RevisionID == "" { observation.Error = "legacy health did not report an applied revision" } } if err = st.ObserveDeployment(ctx, d, observation); err != nil && !errors.Is(err, store.ErrJobLeaseLost) { return err } } return nil }