feat(jobs): add durable PG queue reclaim, slog, and richer metrics

JSON slog в ключевых пакетах; Prometheus path_group, job_audit_depth, upstream breaker; job_audit ClaimQueued/ReclaimStaleRunning + Adopt loop для HA после рестарта.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DenozordecandCursor committed 2026-07-31 12:26:18 +07:00
1 parent 13e3d21ce2
commit 8fe74c1d3b
18 files changed
+448 -51

No files matched your search

+18
View File
@@ -49,6 +49,24 @@ func (b *hostBreaker) recordFailure() {
}
}
// SnapshotBreakers returns whether each known host breaker is currently open.
func SnapshotBreakers() map[string]bool {
out := map[string]bool{}
hostBreakers.Range(func(key, value any) bool {
host, _ := key.(string)
b, _ := value.(*hostBreaker)
if b == nil {
return true
}
b.mu.Lock()
open := time.Now().Before(b.openUntil)
b.mu.Unlock()
out[host] = open
return true
})
return out
}
// ResetHostBreakers clears all circuit breakers (tests only).
func ResetHostBreakers() {
hostBreakers = sync.Map{}
+9 -2
View File
@@ -7,6 +7,8 @@ import (
"io"
"net/http"
"time"
"evobgp/internal/observability"
)
const DefaultTimeout = 45 * time.Second
@@ -69,19 +71,24 @@ func DoWithBreaker(ctx context.Context, hc *http.Client, req *http.Request, maxA
if req == nil || req.URL == nil {
return nil, fmt.Errorf("httpclient: nil request")
}
br := breakerForHost(req.URL.Hostname())
host := req.URL.Hostname()
br := breakerForHost(host)
if !br.allow() {
return nil, fmt.Errorf("httpclient: circuit open for %s", req.URL.Hostname())
observability.SetUpstreamBreakerOpen(host, true)
return nil, fmt.Errorf("httpclient: circuit open for %s", host)
}
resp, err := DoWithRetry(ctx, hc, req, maxAttempts)
if err != nil {
br.recordFailure()
observability.SetUpstreamBreakerOpen(host, !br.allow())
return nil, err
}
if resp.StatusCode >= 500 {
br.recordFailure()
observability.SetUpstreamBreakerOpen(host, !br.allow())
return resp, nil
}
br.recordSuccess()
observability.SetUpstreamBreakerOpen(host, false)
return resp, nil
}