feat(jobs): add durable PG queue reclaim, slog, and richer metrics
JSON slog в ключевых пакетах; Prometheus path_group, job_audit_depth, upstream breaker; job_audit ClaimQueued/ReclaimStaleRunning + Adopt loop для HA после рестарта. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
13e3d21ce2
commit
8fe74c1d3b
18 files changed
+448
-51
No files matched your search
@@ -49,6 +49,24 @@ func (b *hostBreaker) recordFailure() {
|
||||
}
|
||||
}
|
||||
|
||||
// SnapshotBreakers returns whether each known host breaker is currently open.
|
||||
func SnapshotBreakers() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
hostBreakers.Range(func(key, value any) bool {
|
||||
host, _ := key.(string)
|
||||
b, _ := value.(*hostBreaker)
|
||||
if b == nil {
|
||||
return true
|
||||
}
|
||||
b.mu.Lock()
|
||||
open := time.Now().Before(b.openUntil)
|
||||
b.mu.Unlock()
|
||||
out[host] = open
|
||||
return true
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// ResetHostBreakers clears all circuit breakers (tests only).
|
||||
func ResetHostBreakers() {
|
||||
hostBreakers = sync.Map{}
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/observability"
|
||||
)
|
||||
|
||||
const DefaultTimeout = 45 * time.Second
|
||||
@@ -69,19 +71,24 @@ func DoWithBreaker(ctx context.Context, hc *http.Client, req *http.Request, maxA
|
||||
if req == nil || req.URL == nil {
|
||||
return nil, fmt.Errorf("httpclient: nil request")
|
||||
}
|
||||
br := breakerForHost(req.URL.Hostname())
|
||||
host := req.URL.Hostname()
|
||||
br := breakerForHost(host)
|
||||
if !br.allow() {
|
||||
return nil, fmt.Errorf("httpclient: circuit open for %s", req.URL.Hostname())
|
||||
observability.SetUpstreamBreakerOpen(host, true)
|
||||
return nil, fmt.Errorf("httpclient: circuit open for %s", host)
|
||||
}
|
||||
resp, err := DoWithRetry(ctx, hc, req, maxAttempts)
|
||||
if err != nil {
|
||||
br.recordFailure()
|
||||
observability.SetUpstreamBreakerOpen(host, !br.allow())
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode >= 500 {
|
||||
br.recordFailure()
|
||||
observability.SetUpstreamBreakerOpen(host, !br.allow())
|
||||
return resp, nil
|
||||
}
|
||||
br.recordSuccess()
|
||||
observability.SetUpstreamBreakerOpen(host, false)
|
||||
return resp, nil
|
||||
}
|
||||
Reference in new issue
Block a user