diff --git a/internal/aggregate/bad_baseline.go b/internal/aggregate/bad_baseline.go new file mode 100644 index 0000000..e63d49c --- /dev/null +++ b/internal/aggregate/bad_baseline.go @@ -0,0 +1,69 @@ +package aggregate + +import ( + "context" + "net/http" + "time" +) + +// effectiveConnectionsBad возвращает значение для порогов инцидентов: прирост от последнего сброса. +func (h *Handler) effectiveConnectionsBad(alias string, current uint64) uint64 { + h.badConnBaselineMu.Lock() + defer h.badConnBaselineMu.Unlock() + b, ok := h.badConnBaseline[alias] + if !ok { + return current + } + if current < b { + // Счётчик на ноде уменьшился (перезапуск и т.п.) — старая отметка неприменима. + delete(h.badConnBaseline, alias) + return current + } + return current - b +} + +func (h *Handler) setBadConnectionBaseline(alias string, snapshot uint64) { + h.badConnBaselineMu.Lock() + defer h.badConnBaselineMu.Unlock() + if h.badConnBaseline == nil { + h.badConnBaseline = make(map[string]uint64) + } + h.badConnBaseline[alias] = snapshot +} + +// HandleResetBadConnectionsBaseline — POST /api/agg/bad-connections/reset +// Сохраняет текущий connections_bad_total по каждой выбранной ноде как нулевую отметку для алертов. +func (h *Handler) HandleResetBadConnectionsBaseline(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + aliases, err := h.resolveAliases(r) + if err != nil { + writeBadRequest(w, err) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + defer cancel() + + type row struct { + Alias string `json:"alias"` + OK bool `json:"ok"` + Baseline uint64 `json:"baseline,omitempty"` + Error string `json:"error,omitempty"` + } + rows := make([]row, 0, len(aliases)) + partial := false + for _, alias := range aliases { + data, meta := FetchTelemtGET[statsSummaryLite](ctx, h.Client, h.Parsed, alias, "stats/summary") + if !meta.OK { + partial = true + rows = append(rows, row{Alias: alias, OK: false, Error: meta.Error}) + continue + } + h.setBadConnectionBaseline(alias, data.ConnectionsBadTotal) + rows = append(rows, row{Alias: alias, OK: true, Baseline: data.ConnectionsBadTotal}) + } + writeAggOK(w, partial, map[string]any{"servers": rows}) +} diff --git a/internal/aggregate/bad_baseline_test.go b/internal/aggregate/bad_baseline_test.go new file mode 100644 index 0000000..0725211 --- /dev/null +++ b/internal/aggregate/bad_baseline_test.go @@ -0,0 +1,24 @@ +package aggregate + +import "testing" + +func TestEffectiveConnectionsBad(t *testing.T) { + h := &Handler{} + + if got := h.effectiveConnectionsBad("a", 500); got != 500 { + t.Fatalf("without baseline want 500, got %d", got) + } + + h.setBadConnectionBaseline("a", 400) + if got := h.effectiveConnectionsBad("a", 900); got != 500 { + t.Fatalf("with baseline 400 want delta 500, got %d", got) + } + + h.setBadConnectionBaseline("b", 1000) + if got := h.effectiveConnectionsBad("b", 500); got != 500 { + t.Fatalf("after counter drop want raw value 500, got %d", got) + } + if _, ok := h.badConnBaseline["b"]; ok { + t.Fatal("baseline for b should be cleared when current < baseline") + } +} diff --git a/internal/aggregate/handlers.go b/internal/aggregate/handlers.go index b18d0a4..0227eac 100644 --- a/internal/aggregate/handlers.go +++ b/internal/aggregate/handlers.go @@ -33,6 +33,11 @@ type Handler struct { cacheMu sync.Mutex cache map[string]cacheEntry + + // badConnBaseline хранит снимок connections_bad_total на момент сброса в UI; + // инциденты bad_connections_* считаются по приросту от этой отметки (в памяти процесса шлюза). + badConnBaselineMu sync.Mutex + badConnBaseline map[string]uint64 } // ResolveAliasesForLive resolves aliases for external endpoints (SSE/live). diff --git a/internal/aggregate/incidents.go b/internal/aggregate/incidents.go index d1b8623..ba33900 100644 --- a/internal/aggregate/incidents.go +++ b/internal/aggregate/incidents.go @@ -65,7 +65,8 @@ func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (Incident partial = true continue } - if data.ConnectionsBadTotal >= 10000 { + badEff := h.effectiveConnectionsBad(alias, data.ConnectionsBadTotal) + if badEff >= 10000 { out.Items = append(out.Items, IncidentItem{ ID: fmt.Sprintf("bad_connections_high:%s", alias), Kind: "bad_connections_high", @@ -75,13 +76,13 @@ func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (Incident Summary: "Резкий рост ошибок клиентских соединений", AffectedAliases: []string{alias}, MetricName: "connections_bad_total", - MetricValue: float64(data.ConnectionsBadTotal), + MetricValue: float64(badEff), MetricThreshold: 10000, Actions: []IncidentAction{ {Label: "Node dashboard", Href: "/servers/" + alias}, }, }) - } else if data.ConnectionsBadTotal >= 1000 { + } else if badEff >= 1000 { out.Items = append(out.Items, IncidentItem{ ID: fmt.Sprintf("bad_connections_warn:%s", alias), Kind: "bad_connections_warn", @@ -91,7 +92,7 @@ func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (Incident Summary: "Наблюдается рост ошибок клиентских соединений", AffectedAliases: []string{alias}, MetricName: "connections_bad_total", - MetricValue: float64(data.ConnectionsBadTotal), + MetricValue: float64(badEff), MetricThreshold: 1000, Actions: []IncidentAction{ {Label: "Node dashboard", Href: "/servers/" + alias}, diff --git a/internal/server/gateway.go b/internal/server/gateway.go index 21d23d9..2ec0b67 100644 --- a/internal/server/gateway.go +++ b/internal/server/gateway.go @@ -308,6 +308,10 @@ func (g *Gateway) serve(w http.ResponseWriter, r *http.Request) { proxy.NormalizeRequestURLPath(r) } const prefix = "/api/" + if r.Method == http.MethodPost && r.URL.Path == "/api/agg/bad-connections/reset" { + g.agg.HandleResetBadConnectionsBaseline(w, r) + return + } if r.URL.Path == "/api/agg" || strings.HasPrefix(r.URL.Path, "/api/agg/") { g.agg.ServeHTTP(w, r) return diff --git a/web/src/lib/api/client.ts b/web/src/lib/api/client.ts index 65380eb..c29bbd2 100644 --- a/web/src/lib/api/client.ts +++ b/web/src/lib/api/client.ts @@ -282,6 +282,35 @@ export async function fetchAggIncidents(params?: { aliases?: string }): Promise< return body as AggEnvelope; } +export type ResetBadConnectionsRow = { + alias: string; + ok: boolean; + baseline?: number; + error?: string; +}; + +export type ResetBadConnectionsData = { + servers: ResetBadConnectionsRow[]; +}; + +/** POST: зафиксировать текущий connections_bad_total как «ноль» для алертов bad connections на шлюзе. */ +export async function postAggResetBadConnectionsBaseline(params?: { + aliases?: string; +}): Promise> { + const q = new URLSearchParams(); + if (params?.aliases) q.set('aliases', params.aliases); + const url = `${gatewayBase()}/api/agg/bad-connections/reset${q.toString() ? `?${q}` : ''}`; + const res = await fetch(url, { method: 'POST' }); + const body = (await parseJson(res)) as Record | null; + if (!res.ok) { + throw new ApiError(`bad-connections/reset HTTP ${res.status}`, res.status, body); + } + if (!body || body.ok !== true) { + throw new ApiError('bad-connections/reset: ok !== true', res.status, body); + } + return body as AggEnvelope; +} + export function liveEventsUrl(params?: { aliases?: string }): string { const q = new URLSearchParams(); if (params?.aliases) q.set('aliases', params.aliases); diff --git a/web/src/routes/incidents/+page.svelte b/web/src/routes/incidents/+page.svelte index 3061cff..73b6d7b 100644 --- a/web/src/routes/incidents/+page.svelte +++ b/web/src/routes/incidents/+page.svelte @@ -6,9 +6,11 @@ ApiError, fetchAggIncidents, fetchAggSummary, + postAggResetBadConnectionsBaseline, type IncidentItem, type IncidentsData } from '$lib/api/client.js'; + import { toast } from 'svelte-sonner'; import * as Card from '$lib/components/ui/card/index.js'; import KpiStatCard from '$lib/components/kpi-stat-card.svelte'; import DataTableCard from '$lib/components/data-table/data-table-card.svelte'; @@ -33,6 +35,7 @@ import SirenIcon from '@lucide/svelte/icons/siren'; import TriangleAlertIcon from '@lucide/svelte/icons/triangle-alert'; import InfoIconBadge from '@lucide/svelte/icons/info'; + import EraserIcon from '@lucide/svelte/icons/eraser'; type TriageState = { ack?: boolean; @@ -61,6 +64,7 @@ let incSearch = $state(''); let incSortKey = $state<'sev' | 'title' | null>(null); let incSortDir = $state(null); + let resetBadBusy = $state(false); function parseRefresh(raw: string | null): number { if (raw == null || raw.trim() === '') return 30; @@ -274,6 +278,31 @@ incSortKey = next === null ? null : key; incSortDir = next; } + + async function resetBadConnectionsBaseline() { + if (resetBadBusy) return; + resetBadBusy = true; + try { + const env = await postAggResetBadConnectionsBaseline({ + aliases: aliasFilter === 'all' ? undefined : aliasFilter + }); + const failed = env.data.servers.filter((s) => !s.ok); + if (failed.length > 0) { + toast.warning( + `Не на всех нодах удалось прочитать stats/summary: ${failed.map((f) => `${f.alias}: ${f.error ?? 'ошибка'}`).join('; ')}` + ); + } else { + toast.success( + 'Для алертов bad connections зафиксированы текущие значения счётчиков (отметка на шлюзе).' + ); + } + await load(); + } catch (e) { + toast.error(e instanceof ApiError ? e.message : String(e)); + } finally { + resetBadBusy = false; + } + }
@@ -286,6 +315,25 @@
+ + + + + +

+ Запомнить текущий connections_bad_total на выбранных нодах как ноль для + инцидентов на этом шлюзе (память процесса). Сырые счётчики на нодах Telemt не меняются. +

+ + + + +

+ Запомнить текущий connections_bad_total на выбранных нодах как ноль для + инцидентов на этом шлюзе (память процесса). Сырые счётчики на нодах Telemt не меняются. +

+ +