CI / changes (push) Successful in 8s
CI / openapi (push) Has been skipped
CI / go (push) Successful in 40s
CI / docker-web (deploy/docker/evobgp-web/Dockerfile, , evobgp-web) (push) Has been skipped
CI / docker-web (deploy/docker/evobgp-web/Dockerfile, evobgp-all, evobgp-web-all) (push) Has been skipped
CI / docker-bird (push) Has been skipped
CI / bird2 (push) Successful in 15s
CI / docker-go-prime (push) Successful in 24s
CI / docker-go (deploy/docker/evobgp-agent/Dockerfile, , evobgp-agent) (push) Successful in 59s
CI / docker-go (evobgp-all, 1, deploy/docker/gobinary/Dockerfile, , evobgp-all) (push) Successful in 2m18s
CI / docker-go (evobgp-api, 1, deploy/docker/gobinary/Dockerfile, , evobgp-api) (push) Successful in 1m23s
CI / docker-go (evobgp-deploy, 0, deploy/docker/gobinary/Dockerfile, , evobgp-deploy) (push) Successful in 1m20s
CI / docker-go (evobgp-ingest, 0, deploy/docker/gobinary/Dockerfile, , evobgp-ingest) (push) Successful in 1m24s
CI / docker-go (evobgp-node, 0, deploy/docker/gobinary/Dockerfile, , evobgp-node) (push) Successful in 1m9s
CI / docker-go (evobgp-render, 0, deploy/docker/gobinary/Dockerfile, , evobgp-render) (push) Successful in 1m22s
CI / docker-go (evobgp-scheduler, 0, deploy/docker/gobinary/Dockerfile, , evobgp-scheduler) (push) Successful in 1m24s
Updated the `startBirdMetricsPoller` function to accept a context parameter, allowing for better control over the polling lifecycle. This change was applied in both `evobgp-all` and `evobgp-api` main files. Additionally, modified the `StartBirdProtocolsPoller` function to handle context cancellation, ensuring graceful shutdown of the polling routine. Introduced a new service in the Docker Compose configuration for logging runtime service outputs, improving observability during deployment.
243 lines
6.7 KiB
Go
243 lines
6.7 KiB
Go
// Package observability registers Prometheus metrics for the control plane (prefix aggregates, BGP peers, jobs, HTTP).
|
|
package observability
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"strconv"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
|
|
"evobgp/internal/store"
|
|
)
|
|
|
|
const namespace = "evobgp"
|
|
|
|
var (
|
|
metricsStore atomic.Value // store.Backend
|
|
registerCollectorOnce sync.Once
|
|
)
|
|
|
|
var (
|
|
httpRequests = promauto.NewCounterVec(
|
|
prometheus.CounterOpts{
|
|
Namespace: namespace,
|
|
Name: "http_requests_total",
|
|
Help: "HTTP requests handled by the API mux (excludes /metrics).",
|
|
},
|
|
[]string{"method", "code"},
|
|
)
|
|
|
|
jobsFinished = promauto.NewCounterVec(
|
|
prometheus.CounterOpts{
|
|
Namespace: namespace,
|
|
Name: "jobs_finished_total",
|
|
Help: "Async jobs that reached a terminal state.",
|
|
},
|
|
[]string{"kind", "status"},
|
|
)
|
|
|
|
birdBGPEstablished = promauto.NewGauge(prometheus.GaugeOpts{
|
|
Namespace: namespace,
|
|
Name: "bird_bgp_sessions_established",
|
|
Help: "BGP sessions in Established state from birdc show protocols (0 if scrape disabled or failed).",
|
|
})
|
|
|
|
birdProtocolsScrapeSuccess = promauto.NewGauge(prometheus.GaugeOpts{
|
|
Namespace: namespace,
|
|
Name: "bird_protocols_scrape_success",
|
|
Help: "1 if the last birdc protocols scrape succeeded, else 0.",
|
|
})
|
|
|
|
buildInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
|
Namespace: namespace,
|
|
Name: "build_info",
|
|
Help: "Build metadata (value always 1).",
|
|
}, []string{"version", "git_sha"})
|
|
)
|
|
|
|
// RecordJobTerminal increments jobs_finished_total for terminal statuses.
|
|
func RecordJobTerminal(kind, status string) {
|
|
switch status {
|
|
case "succeeded", "failed", "cancelled":
|
|
jobsFinished.WithLabelValues(kind, status).Inc()
|
|
default:
|
|
return
|
|
}
|
|
}
|
|
|
|
// SetBuildInfo sets evobgp_build_info gauge (idempotent labels).
|
|
func SetBuildInfo(version, gitSHA string) {
|
|
if version == "" {
|
|
version = "unknown"
|
|
}
|
|
if gitSHA == "" {
|
|
gitSHA = "unknown"
|
|
}
|
|
buildInfo.WithLabelValues(version, gitSHA).Set(1)
|
|
}
|
|
|
|
type memoryStoreCollector struct {
|
|
prefixMaxDesc *prometheus.Desc
|
|
prefixSumDesc *prometheus.Desc
|
|
peersDesc *prometheus.Desc
|
|
peerStateDesc *prometheus.Desc
|
|
}
|
|
|
|
func newMemoryStoreCollector() *memoryStoreCollector {
|
|
return &memoryStoreCollector{
|
|
prefixMaxDesc: prometheus.NewDesc(
|
|
prometheus.BuildFQName(namespace, "", "materialized_prefixes_max"),
|
|
"Maximum materialized_prefix_count among all revisions in the store.",
|
|
nil, nil,
|
|
),
|
|
prefixSumDesc: prometheus.NewDesc(
|
|
prometheus.BuildFQName(namespace, "", "materialized_prefixes_sum"),
|
|
"Sum of materialized_prefix_count over revisions (development aggregate).",
|
|
nil, nil,
|
|
),
|
|
peersDesc: prometheus.NewDesc(
|
|
prometheus.BuildFQName(namespace, "", "bgp_peers_configured_total"),
|
|
"BGP peers configured in the control-plane store.",
|
|
nil, nil,
|
|
),
|
|
peerStateDesc: prometheus.NewDesc(
|
|
prometheus.BuildFQName(namespace, "", "bgp_peer_sessions"),
|
|
"Configured BGP peers in the store by session_state (intent / last known, not live BIRD).",
|
|
[]string{"state"}, nil,
|
|
),
|
|
}
|
|
}
|
|
|
|
func (c *memoryStoreCollector) Describe(ch chan<- *prometheus.Desc) {
|
|
ch <- c.prefixMaxDesc
|
|
ch <- c.prefixSumDesc
|
|
ch <- c.peersDesc
|
|
ch <- c.peerStateDesc
|
|
}
|
|
|
|
func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
|
|
v := metricsStore.Load()
|
|
if v == nil {
|
|
return
|
|
}
|
|
b, ok := v.(store.Backend)
|
|
if !ok || b == nil {
|
|
return
|
|
}
|
|
maxN, sumN := b.MaterializedPrefixStats()
|
|
ch <- prometheus.MustNewConstMetric(c.prefixMaxDesc, prometheus.GaugeValue, float64(maxN))
|
|
ch <- prometheus.MustNewConstMetric(c.prefixSumDesc, prometheus.GaugeValue, float64(sumN))
|
|
ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(b.PeerCount()))
|
|
for state, n := range b.PeerSessionCountsByState() {
|
|
if state == "" {
|
|
state = "unknown"
|
|
}
|
|
ch <- prometheus.MustNewConstMetric(c.peerStateDesc, prometheus.GaugeValue, float64(n), state)
|
|
}
|
|
}
|
|
|
|
// RegisterStoreBackend points Prometheus collectors at any store.Backend (last call wins; safe for tests).
|
|
func RegisterStoreBackend(b store.Backend) {
|
|
if b == nil {
|
|
return
|
|
}
|
|
metricsStore.Store(b)
|
|
registerCollectorOnce.Do(func() {
|
|
prometheus.DefaultRegisterer.MustRegister(newMemoryStoreCollector())
|
|
})
|
|
}
|
|
|
|
// RegisterStoreMetrics is a deprecated alias for RegisterStoreBackend (memory-only callers).
|
|
func RegisterStoreMetrics(mem *store.Memory) {
|
|
if mem == nil {
|
|
return
|
|
}
|
|
RegisterStoreBackend(mem)
|
|
}
|
|
|
|
// SetBirdSessionMetrics updates gauges from an optional birdc scrape.
|
|
func SetBirdSessionMetrics(established int, scrapeOK bool) {
|
|
birdBGPEstablished.Set(float64(established))
|
|
if scrapeOK {
|
|
birdProtocolsScrapeSuccess.Set(1)
|
|
} else {
|
|
birdProtocolsScrapeSuccess.Set(0)
|
|
}
|
|
}
|
|
|
|
// MetricsHandler returns the Prometheus scrape handler.
|
|
func MetricsHandler() http.Handler {
|
|
return promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})
|
|
}
|
|
|
|
// HTTPMiddleware records method and status code for all wrapped requests.
|
|
func HTTPMiddleware(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
sw := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
|
|
next.ServeHTTP(sw, r)
|
|
httpRequests.WithLabelValues(r.Method, strconv.Itoa(sw.status)).Inc()
|
|
})
|
|
}
|
|
|
|
type statusRecorder struct {
|
|
http.ResponseWriter
|
|
status int
|
|
}
|
|
|
|
func (s *statusRecorder) WriteHeader(code int) {
|
|
s.status = code
|
|
s.ResponseWriter.WriteHeader(code)
|
|
}
|
|
|
|
// StartBirdProtocolsPoller runs birdc "show protocols" on interval when socket is non-empty.
|
|
// Горутина завершается при отмене ctx (корректное завершение вместе с процессом API).
|
|
func StartBirdProtocolsPoller(ctx context.Context, socket string, birdcPath string, interval time.Duration, showFn func(ctx context.Context, socket, birdcBin string) (string, error), countFn func(output string) int) {
|
|
socket = trimSpace(socket)
|
|
if ctx == nil || socket == "" || interval <= 0 || showFn == nil || countFn == nil {
|
|
return
|
|
}
|
|
scrape := func() {
|
|
sctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
out, err := showFn(sctx, socket, birdcPath)
|
|
cancel()
|
|
if err != nil {
|
|
SetBirdSessionMetrics(0, false)
|
|
return
|
|
}
|
|
SetBirdSessionMetrics(countFn(out), true)
|
|
}
|
|
go func() {
|
|
scrape()
|
|
t := time.NewTicker(interval)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
scrape()
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
func trimSpace(s string) string {
|
|
for len(s) > 0 && (s[0] == ' ' || s[0] == '\t') {
|
|
s = s[1:]
|
|
}
|
|
for len(s) > 0 {
|
|
last := s[len(s)-1]
|
|
if last != ' ' && last != '\t' {
|
|
break
|
|
}
|
|
s = s[:len(s)-1]
|
|
}
|
|
return s
|
|
}
|