feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -0,0 +1,235 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
type deploymentReservation struct {
|
||||
Owner string
|
||||
Token int64
|
||||
Until time.Time
|
||||
}
|
||||
|
||||
func (m *Memory) deploymentFence(f JobFence) (JobRecord, error) {
|
||||
j, ok := m.jobs[f.JobID]
|
||||
if !ok || j.TenantID != f.TenantID || j.Status != "running" || j.OwnerID != f.OwnerID || j.FencingToken != f.Token || j.CancelRequested || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) {
|
||||
return JobRecord{}, ErrJobLeaseLost
|
||||
}
|
||||
return j, nil
|
||||
}
|
||||
|
||||
func (m *Memory) BeginDeploymentSpeaker(ctx context.Context, f JobFence, speaker, revision string) (DeploymentSpeaker, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return DeploymentSpeaker{}, err
|
||||
}
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
j, err := m.deploymentFence(f)
|
||||
if err != nil {
|
||||
return DeploymentSpeaker{}, err
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
sp, ok := m.speakers[speaker]
|
||||
if !ok || sp.TenantID != f.TenantID {
|
||||
return DeploymentSpeaker{}, ErrNotFound
|
||||
}
|
||||
r, ok := m.revisions[revision]
|
||||
if !ok || r.TenantID != f.TenantID {
|
||||
return DeploymentSpeaker{}, ErrNotFound
|
||||
}
|
||||
if m.deployments == nil {
|
||||
m.deployments = make(map[string]DeploymentSpeaker)
|
||||
m.deploymentCurrent = make(map[string]DeploymentSpeaker)
|
||||
}
|
||||
key := f.JobID + ":" + speaker
|
||||
cur := m.deploymentCurrent[speaker]
|
||||
if f.ExpectedGeneration != 0 && cur.JobID != f.JobID && (cur.Generation != f.ExpectedGeneration || cur.RevisionID != revision) {
|
||||
return DeploymentSpeaker{}, ErrJobLeaseLost
|
||||
}
|
||||
reserve := func() error {
|
||||
if m.deploymentReservations == nil {
|
||||
m.deploymentReservations = make(map[string]deploymentReservation)
|
||||
}
|
||||
if r := m.deploymentReservations[speaker]; r.Until.After(time.Now()) {
|
||||
return ErrDeploymentBusy
|
||||
}
|
||||
m.deploymentReservations[speaker] = deploymentReservation{Owner: f.OwnerID, Token: f.Token, Until: time.Now().Add(DeploymentDeliveryWindow)}
|
||||
return nil
|
||||
}
|
||||
if existing, ok := m.deployments[key]; ok {
|
||||
if cur.JobID != f.JobID || cur.Generation != existing.Generation || cur.RevisionID != revision {
|
||||
return DeploymentSpeaker{}, ErrJobLeaseLost
|
||||
}
|
||||
if existing.State != "applied" {
|
||||
if err := reserve(); err != nil {
|
||||
return DeploymentSpeaker{}, err
|
||||
}
|
||||
}
|
||||
return existing, nil
|
||||
}
|
||||
if f.ExpectedGeneration != 0 {
|
||||
j.RequestSequence = cur.RequestSequence
|
||||
}
|
||||
if cur.RequestSequence >= j.RequestSequence && cur.JobID != "" && f.ExpectedGeneration == 0 {
|
||||
return DeploymentSpeaker{}, ErrJobLeaseLost
|
||||
}
|
||||
if err := reserve(); err != nil {
|
||||
return DeploymentSpeaker{}, err
|
||||
}
|
||||
d := DeploymentSpeaker{JobID: f.JobID, TenantID: f.TenantID, SpeakerID: speaker, RevisionID: revision, Generation: cur.Generation + 1, RequestSequence: j.RequestSequence, State: "published", RetryRound: f.RetryRound}
|
||||
m.deployments[key] = d
|
||||
m.deploymentCurrent[speaker] = d
|
||||
m.publishedRevision[speaker] = publishedInfo{RevisionID: revision, PublishedAt: time.Now().UTC()}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (m *Memory) RecordDeploymentSpeaker(ctx context.Context, f JobFence, d DeploymentSpeaker, state, confirmation, message string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
if _, err := m.deploymentFence(f); err != nil {
|
||||
return err
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
cur := m.deploymentCurrent[d.SpeakerID]
|
||||
if d.JobID != f.JobID || d.TenantID != f.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID {
|
||||
return ErrJobLeaseLost
|
||||
}
|
||||
r := m.deploymentReservations[d.SpeakerID]
|
||||
if r.Owner != f.OwnerID || r.Token != f.Token || !r.Until.After(time.Now()) {
|
||||
return ErrJobLeaseLost
|
||||
}
|
||||
if state == "applied" && confirmation != "agent_reported" && confirmation != "local_applied" {
|
||||
return ErrInvalidInput
|
||||
}
|
||||
if state != "dispatched" && state != "unknown" {
|
||||
delete(m.deploymentReservations, d.SpeakerID)
|
||||
}
|
||||
cur.State, cur.Confirmation, cur.Error = state, confirmation, message
|
||||
if state == "dispatched" {
|
||||
cur.DispatchAttempts++
|
||||
}
|
||||
if state == "applied" {
|
||||
revision := d.RevisionID
|
||||
m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision
|
||||
}
|
||||
m.deploymentCurrent[d.SpeakerID] = cur
|
||||
m.deployments[d.JobID+":"+d.SpeakerID] = cur
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Memory) ListDeploymentSpeakers(ctx context.Context, tenant, job string) ([]DeploymentSpeaker, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
var out []DeploymentSpeaker
|
||||
for _, d := range m.deployments {
|
||||
if d.TenantID == tenant && d.JobID == job {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *Memory) ListPendingDeployments(ctx context.Context, limit int) ([]DeploymentSpeaker, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
var out []DeploymentSpeaker
|
||||
for speaker, d := range m.deploymentCurrent {
|
||||
j := m.jobs[d.JobID]
|
||||
if d.State != "applied" && d.RetryRound < 3 && !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) {
|
||||
out = append(out, d)
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *Memory) ListDeploymentObservations(ctx context.Context, limit int) ([]DeploymentSpeaker, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 10
|
||||
}
|
||||
m.jobMu.Lock()
|
||||
defer m.jobMu.Unlock()
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var out []DeploymentSpeaker
|
||||
for speaker, d := range m.deploymentCurrent {
|
||||
j := m.jobs[d.JobID]
|
||||
if !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
a, b := out[i].HealthReceivedAt, out[j].HealthReceivedAt
|
||||
if a == nil && b == nil {
|
||||
return out[i].SpeakerID < out[j].SpeakerID
|
||||
}
|
||||
if a == nil {
|
||||
return true
|
||||
}
|
||||
if b == nil {
|
||||
return false
|
||||
}
|
||||
return a.Before(*b)
|
||||
})
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
for i := range out {
|
||||
out[i].ObservationToken++
|
||||
m.deploymentCurrent[out[i].SpeakerID] = out[i]
|
||||
m.deployments[out[i].JobID+":"+out[i].SpeakerID] = out[i]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *Memory) ObserveDeployment(ctx context.Context, d DeploymentSpeaker, o DeploymentObservation) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
cur := m.deploymentCurrent[d.SpeakerID]
|
||||
if cur.TenantID != d.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || cur.ObservationToken != d.ObservationToken || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID || m.deploymentReservations[d.SpeakerID].Until.After(time.Now()) {
|
||||
return ErrJobLeaseLost
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
cur.ObservationToken++
|
||||
cur.HealthReceivedAt = &now
|
||||
cur.HealthRevisionID = o.RevisionID
|
||||
cur.HealthSourceAt = o.SourceAt
|
||||
cur.HealthError = o.Error
|
||||
if o.Error == "" && o.RevisionID != "" {
|
||||
if o.RevisionID == d.RevisionID {
|
||||
cur.State = "applied"
|
||||
cur.Confirmation = "agent_reported"
|
||||
revision := d.RevisionID
|
||||
m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision
|
||||
} else {
|
||||
cur.State = "unknown"
|
||||
cur.Confirmation = "unconfirmed"
|
||||
}
|
||||
}
|
||||
m.deploymentCurrent[d.SpeakerID] = cur
|
||||
m.deployments[d.JobID+":"+d.SpeakerID] = cur
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user