Implement bad connections baseline reset functionality
Publish telemt-api gateway Docker image / test (push) Successful in 9s
Publish telemt-api gateway Docker image / build-and-push (push) Successful in 2m1s

- Added a new field `badConnBaseline` in the `Handler` struct to store the baseline for bad connections.
- Updated the `BuildIncidents` function to utilize the effective bad connections count for incident generation.
- Introduced a new API endpoint `/api/agg/bad-connections/reset` to reset the bad connections baseline.
- Enhanced the frontend with a button to trigger the reset action, providing user feedback through toast notifications.
- Updated Svelte components to handle the reset functionality and display appropriate messages based on the operation's success or failure.
This commit is contained in:
Denozordec
2026-04-12 13:30:56 +07:00
parent ea2ecb44d2
commit d1095081bc
8 changed files with 233 additions and 4 deletions
+69
View File
@@ -0,0 +1,69 @@
package aggregate
import (
"context"
"net/http"
"time"
)
// effectiveConnectionsBad возвращает значение для порогов инцидентов: прирост от последнего сброса.
func (h *Handler) effectiveConnectionsBad(alias string, current uint64) uint64 {
h.badConnBaselineMu.Lock()
defer h.badConnBaselineMu.Unlock()
b, ok := h.badConnBaseline[alias]
if !ok {
return current
}
if current < b {
// Счётчик на ноде уменьшился (перезапуск и т.п.) — старая отметка неприменима.
delete(h.badConnBaseline, alias)
return current
}
return current - b
}
func (h *Handler) setBadConnectionBaseline(alias string, snapshot uint64) {
h.badConnBaselineMu.Lock()
defer h.badConnBaselineMu.Unlock()
if h.badConnBaseline == nil {
h.badConnBaseline = make(map[string]uint64)
}
h.badConnBaseline[alias] = snapshot
}
// HandleResetBadConnectionsBaseline — POST /api/agg/bad-connections/reset
// Сохраняет текущий connections_bad_total по каждой выбранной ноде как нулевую отметку для алертов.
func (h *Handler) HandleResetBadConnectionsBaseline(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
w.Header().Set("Allow", http.MethodPost)
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
aliases, err := h.resolveAliases(r)
if err != nil {
writeBadRequest(w, err)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
defer cancel()
type row struct {
Alias string `json:"alias"`
OK bool `json:"ok"`
Baseline uint64 `json:"baseline,omitempty"`
Error string `json:"error,omitempty"`
}
rows := make([]row, 0, len(aliases))
partial := false
for _, alias := range aliases {
data, meta := FetchTelemtGET[statsSummaryLite](ctx, h.Client, h.Parsed, alias, "stats/summary")
if !meta.OK {
partial = true
rows = append(rows, row{Alias: alias, OK: false, Error: meta.Error})
continue
}
h.setBadConnectionBaseline(alias, data.ConnectionsBadTotal)
rows = append(rows, row{Alias: alias, OK: true, Baseline: data.ConnectionsBadTotal})
}
writeAggOK(w, partial, map[string]any{"servers": rows})
}
+24
View File
@@ -0,0 +1,24 @@
package aggregate
import "testing"
func TestEffectiveConnectionsBad(t *testing.T) {
h := &Handler{}
if got := h.effectiveConnectionsBad("a", 500); got != 500 {
t.Fatalf("without baseline want 500, got %d", got)
}
h.setBadConnectionBaseline("a", 400)
if got := h.effectiveConnectionsBad("a", 900); got != 500 {
t.Fatalf("with baseline 400 want delta 500, got %d", got)
}
h.setBadConnectionBaseline("b", 1000)
if got := h.effectiveConnectionsBad("b", 500); got != 500 {
t.Fatalf("after counter drop want raw value 500, got %d", got)
}
if _, ok := h.badConnBaseline["b"]; ok {
t.Fatal("baseline for b should be cleared when current < baseline")
}
}
+5
View File
@@ -33,6 +33,11 @@ type Handler struct {
cacheMu sync.Mutex
cache map[string]cacheEntry
// badConnBaseline хранит снимок connections_bad_total на момент сброса в UI;
// инциденты bad_connections_* считаются по приросту от этой отметки (в памяти процесса шлюза).
badConnBaselineMu sync.Mutex
badConnBaseline map[string]uint64
}
// ResolveAliasesForLive resolves aliases for external endpoints (SSE/live).
+5 -4
View File
@@ -65,7 +65,8 @@ func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (Incident
partial = true
continue
}
if data.ConnectionsBadTotal >= 10000 {
badEff := h.effectiveConnectionsBad(alias, data.ConnectionsBadTotal)
if badEff >= 10000 {
out.Items = append(out.Items, IncidentItem{
ID: fmt.Sprintf("bad_connections_high:%s", alias),
Kind: "bad_connections_high",
@@ -75,13 +76,13 @@ func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (Incident
Summary: "Резкий рост ошибок клиентских соединений",
AffectedAliases: []string{alias},
MetricName: "connections_bad_total",
MetricValue: float64(data.ConnectionsBadTotal),
MetricValue: float64(badEff),
MetricThreshold: 10000,
Actions: []IncidentAction{
{Label: "Node dashboard", Href: "/servers/" + alias},
},
})
} else if data.ConnectionsBadTotal >= 1000 {
} else if badEff >= 1000 {
out.Items = append(out.Items, IncidentItem{
ID: fmt.Sprintf("bad_connections_warn:%s", alias),
Kind: "bad_connections_warn",
@@ -91,7 +92,7 @@ func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (Incident
Summary: "Наблюдается рост ошибок клиентских соединений",
AffectedAliases: []string{alias},
MetricName: "connections_bad_total",
MetricValue: float64(data.ConnectionsBadTotal),
MetricValue: float64(badEff),
MetricThreshold: 1000,
Actions: []IncidentAction{
{Label: "Node dashboard", Href: "/servers/" + alias},
+4
View File
@@ -308,6 +308,10 @@ func (g *Gateway) serve(w http.ResponseWriter, r *http.Request) {
proxy.NormalizeRequestURLPath(r)
}
const prefix = "/api/"
if r.Method == http.MethodPost && r.URL.Path == "/api/agg/bad-connections/reset" {
g.agg.HandleResetBadConnectionsBaseline(w, r)
return
}
if r.URL.Path == "/api/agg" || strings.HasPrefix(r.URL.Path, "/api/agg/") {
g.agg.ServeHTTP(w, r)
return
+29
View File
@@ -282,6 +282,35 @@ export async function fetchAggIncidents(params?: { aliases?: string }): Promise<
return body as AggEnvelope<IncidentsData>;
}
export type ResetBadConnectionsRow = {
alias: string;
ok: boolean;
baseline?: number;
error?: string;
};
export type ResetBadConnectionsData = {
servers: ResetBadConnectionsRow[];
};
/** POST: зафиксировать текущий connections_bad_total как «ноль» для алертов bad connections на шлюзе. */
export async function postAggResetBadConnectionsBaseline(params?: {
aliases?: string;
}): Promise<AggEnvelope<ResetBadConnectionsData>> {
const q = new URLSearchParams();
if (params?.aliases) q.set('aliases', params.aliases);
const url = `${gatewayBase()}/api/agg/bad-connections/reset${q.toString() ? `?${q}` : ''}`;
const res = await fetch(url, { method: 'POST' });
const body = (await parseJson(res)) as Record<string, unknown> | null;
if (!res.ok) {
throw new ApiError(`bad-connections/reset HTTP ${res.status}`, res.status, body);
}
if (!body || body.ok !== true) {
throw new ApiError('bad-connections/reset: ok !== true', res.status, body);
}
return body as AggEnvelope<ResetBadConnectionsData>;
}
export function liveEventsUrl(params?: { aliases?: string }): string {
const q = new URLSearchParams();
if (params?.aliases) q.set('aliases', params.aliases);
+48
View File
@@ -6,9 +6,11 @@
ApiError,
fetchAggIncidents,
fetchAggSummary,
postAggResetBadConnectionsBaseline,
type IncidentItem,
type IncidentsData
} from '$lib/api/client.js';
import { toast } from 'svelte-sonner';
import * as Card from '$lib/components/ui/card/index.js';
import KpiStatCard from '$lib/components/kpi-stat-card.svelte';
import DataTableCard from '$lib/components/data-table/data-table-card.svelte';
@@ -33,6 +35,7 @@
import SirenIcon from '@lucide/svelte/icons/siren';
import TriangleAlertIcon from '@lucide/svelte/icons/triangle-alert';
import InfoIconBadge from '@lucide/svelte/icons/info';
import EraserIcon from '@lucide/svelte/icons/eraser';
type TriageState = {
ack?: boolean;
@@ -61,6 +64,7 @@
let incSearch = $state('');
let incSortKey = $state<'sev' | 'title' | null>(null);
let incSortDir = $state<SortDir | null>(null);
let resetBadBusy = $state(false);
function parseRefresh(raw: string | null): number {
if (raw == null || raw.trim() === '') return 30;
@@ -274,6 +278,31 @@
incSortKey = next === null ? null : key;
incSortDir = next;
}
async function resetBadConnectionsBaseline() {
if (resetBadBusy) return;
resetBadBusy = true;
try {
const env = await postAggResetBadConnectionsBaseline({
aliases: aliasFilter === 'all' ? undefined : aliasFilter
});
const failed = env.data.servers.filter((s) => !s.ok);
if (failed.length > 0) {
toast.warning(
`Не на всех нодах удалось прочитать stats/summary: ${failed.map((f) => `${f.alias}: ${f.error ?? 'ошибка'}`).join('; ')}`
);
} else {
toast.success(
'Для алертов bad connections зафиксированы текущие значения счётчиков (отметка на шлюзе).'
);
}
await load();
} catch (e) {
toast.error(e instanceof ApiError ? e.message : String(e));
} finally {
resetBadBusy = false;
}
}
</script>
<div class="mb-6 flex flex-wrap items-center justify-between gap-4">
@@ -286,6 +315,25 @@
</div>
<div class="flex items-center gap-2">
<FleetLiveStaleBadge {isStale} {staleSeconds} refreshSeconds={refreshSeconds} />
<Tooltip.Root>
<Tooltip.Trigger class="inline-flex">
<Button
variant="outline"
size="sm"
disabled={resetBadBusy || loading}
onclick={() => void resetBadConnectionsBaseline()}
>
<EraserIcon class="mr-1 size-4 {resetBadBusy ? 'animate-pulse' : ''}" />
Сброс bad connections
</Button>
</Tooltip.Trigger>
<Tooltip.Content class="max-w-xs">
<p class="text-xs">
Запомнить текущий <span class="font-mono">connections_bad_total</span> на выбранных нодах как ноль для
инцидентов на этом шлюзе (память процесса). Сырые счётчики на нодах Telemt не меняются.
</p>
</Tooltip.Content>
</Tooltip.Root>
<Button variant="outline" size="sm" onclick={() => void load()} disabled={loading}>
<RefreshCwIcon class="mr-1 size-4 {loading ? 'animate-spin' : ''}" />
Обновить
+49
View File
@@ -5,10 +5,13 @@
import {
liveEventsUrl,
fetchAggSummary,
postAggResetBadConnectionsBaseline,
ApiError,
type IncidentItem,
type IncidentSeverity,
type IncidentStatus
} from '$lib/api/client.js';
import { toast } from 'svelte-sonner';
import KpiStatCard from '$lib/components/kpi-stat-card.svelte';
import DataTableCard from '$lib/components/data-table/data-table-card.svelte';
import DataTableToolbar from '$lib/components/data-table/data-table-toolbar.svelte';
@@ -30,6 +33,7 @@
import FleetLiveStaleBadge from '$lib/components/fleet/fleet-live-stale-badge.svelte';
import RefreshCwIcon from '@lucide/svelte/icons/refresh-cw';
import InfoIcon from '@lucide/svelte/icons/info';
import EraserIcon from '@lucide/svelte/icons/eraser';
type LiveStatus = 'healthy' | 'degraded' | 'critical';
@@ -64,6 +68,7 @@
let liveIncSearch = $state('');
let liveSortKey = $state<'sev' | 'title' | null>(null);
let liveSortDir = $state<SortDir | null>(null);
let resetBadBusy = $state(false);
function liveSevRank(s: IncidentSeverity): number {
if (s === 'critical') return 3;
@@ -94,6 +99,31 @@
return out;
});
async function resetBadConnectionsBaseline() {
if (resetBadBusy) return;
resetBadBusy = true;
try {
const env = await postAggResetBadConnectionsBaseline({
aliases: aliasFilter === 'all' ? undefined : aliasFilter
});
const failed = env.data.servers.filter((s) => !s.ok);
if (failed.length > 0) {
toast.warning(
`Не на всех нодах удалось прочитать stats/summary: ${failed.map((f) => `${f.alias}: ${f.error ?? 'ошибка'}`).join('; ')}`
);
} else {
toast.success(
'Для алертов bad connections зафиксированы текущие значения счётчиков (отметка на шлюзе). Следующий снимок потока обновит список.'
);
}
connectStream();
} catch (e) {
toast.error(e instanceof ApiError ? e.message : String(e));
} finally {
resetBadBusy = false;
}
}
function toggleLiveSort(key: 'sev' | 'title') {
const next = nextSortDir(liveSortDir, key, liveSortKey);
liveSortKey = next === null ? null : key;
@@ -339,6 +369,25 @@
? 'Нет событий SSE дольше 12 с — снимок считается устаревшим.'
: `Последнее событие потока: ${staleSeconds ?? 0} с назад.`}
/>
<Tooltip.Root>
<Tooltip.Trigger class="inline-flex">
<Button
variant="outline"
size="sm"
disabled={resetBadBusy}
onclick={() => void resetBadConnectionsBaseline()}
>
<EraserIcon class="mr-1 size-4 {resetBadBusy ? 'animate-pulse' : ''}" />
Сброс bad connections
</Button>
</Tooltip.Trigger>
<Tooltip.Content class="max-w-xs">
<p class="text-xs">
Запомнить текущий <span class="font-mono">connections_bad_total</span> на выбранных нодах как ноль для
инцидентов на этом шлюзе (память процесса). Сырые счётчики на нодах Telemt не меняются.
</p>
</Tooltip.Content>
</Tooltip.Root>
<Button variant="outline" size="sm" onclick={() => connectStream()}>
<RefreshCwIcon class="mr-1 size-4" />
Переподключить