diff --git a/apps/web/src/components/failover-timeline.tsx b/apps/web/src/components/failover-timeline.tsx index 14b4b0b..f7da948 100644 --- a/apps/web/src/components/failover-timeline.tsx +++ b/apps/web/src/components/failover-timeline.tsx @@ -16,8 +16,8 @@ import { formatDate, formatRelative, sqliteUtcToIso } from '@/lib/format' import { failoverEventCopy, type FailoverEvent, + type FailoverHistoryItem, } from '@/lib/failover-events' -import type { FailoverLogEntry } from '@/lib/schemas' import { cn } from '@cfdm/ui/lib/utils' import type { ComponentProps } from 'react' @@ -45,12 +45,6 @@ function separatorClass(tone: 'down' | 'added' | 'removed'): string { return 'bg-destructive/25' } -function failoverHistoryCopy(item: FailoverLogEntry): string { - return item.action === 'removed' - ? `${item.ip} убрана с ${item.fqdn}` - : `${item.ip} добавлена на ${item.fqdn}` -} - /** * Failover как sibling «Смены статуса»: ReUI Timeline + Badge. * Preview: https://reui.io/preview/base/components/c-timeline-10 @@ -63,14 +57,14 @@ export function FailoverTimeline({ history = [], }: { events: FailoverEvent[] - history?: readonly FailoverLogEntry[] + history?: readonly FailoverHistoryItem[] }) { if (events.length === 0 && history.length === 0) { return ( @@ -142,6 +136,10 @@ export function FailoverTimeline({ {item.ip} + {item.fqdn} @@ -151,9 +149,7 @@ export function FailoverTimeline({ -

- {failoverHistoryCopy(item)} -

+

{item.copy}

 ) diff --git a/apps/web/src/components/reui-kit/service-failover-panel.tsx b/apps/web/src/components/reui-kit/service-failover-panel.tsx index 7cfb1d2..f31867f 100644 --- a/apps/web/src/components/reui-kit/service-failover-panel.tsx +++ b/apps/web/src/components/reui-kit/service-failover-panel.tsx @@ -2,10 +2,17 @@ import { UnplugIcon } from 'lucide-react' import { FailoverTimeline } from '@/components/failover-timeline' import { + mergeFailoverHistory, toFailoverEvents, type FailoverBindingPool, type FailoverHealthInput, } from '@/lib/failover-events' +import { + latestHealthByIp, + resolveIpDisplayHealth, + type HealthLogProbe, + type HealthLogStatus, +} from '@/lib/health-log' import type { FailoverLogEntry } from '@/lib/schemas' import { Badge } from '@/components/reui/badge' import { @@ -32,7 +39,7 @@ function failoverCountLabel(count: number): string { } /** - * Failover — текущие Down + журнал add/remove по FQDN. + * Failover — текущие Down + кто вышел из пула и кто вернулся. * Preview: https://reui.io/preview/base/components/c-timeline-10 * Preview: https://reui.io/preview/base/empty-state-12 * Docs: https://reui.io/docs/components/base/frame @@ -44,12 +51,32 @@ export function ServiceFailoverPanel({ ipHealth, bindings, history, + probes = [], }: { ipHealth: readonly FailoverHealthInput[] bindings: readonly FailoverBindingPool[] history: readonly FailoverLogEntry[] + probes?: readonly HealthLogProbe[] }) { - const events = toFailoverEvents(ipHealth, bindings) + const liveByIp = latestHealthByIp(probes) + const overlayHealth = ipHealth.map((row) => { + const live = liveByIp.get(row.ip) + return { + ...row, + status: resolveIpDisplayHealth( + row.status as HealthLogStatus, + live?.status, + ), + last_error: + live && live.status !== 'unknown' ? live.last_error : row.last_error, + last_checked_at: + live && live.status !== 'unknown' + ? live.last_checked_at + : row.last_checked_at, + } + }) + const events = toFailoverEvents(overlayHealth, bindings) + const mergedHistory = mergeFailoverHistory(history, probes, bindings) const removedCount = events.filter((event) => event.kind === 'removed').length return ( @@ -69,7 +96,7 @@ export function ServiceFailoverPanel({ )} - Текущие Down и история A-записей по FQDN + Текущие Down и история: кто вышел из пула и кто вернулся @@ -85,7 +112,7 @@ export function ServiceFailoverPanel({ ) : null} - + ) diff --git a/apps/web/src/components/services/service-detail-grid.tsx b/apps/web/src/components/services/service-detail-grid.tsx index 620d3c4..3abf4cd 100644 --- a/apps/web/src/components/services/service-detail-grid.tsx +++ b/apps/web/src/components/services/service-detail-grid.tsx @@ -18,6 +18,11 @@ import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-colu import { IconTile } from '@/components/reui/icon-tile' import { createFilter, type Filter, type FilterFieldConfig } from '@/components/reui/filters' import { ResourcePage } from '@/components/reui-kit' +import { + latestHealthByIp, + resolveIpDisplayHealth, + type HealthLogProbe, +} from '@/lib/health-log' import { certRelativeBadge } from '@/components/columns/certificates-columns' import { certMonitoringOptions } from '@/lib/cert-monitoring' import { formatDate } from '@/lib/format' @@ -99,8 +104,12 @@ function mapNodeHealth(status: string): HealthStatus { return 'unknown' } -function buildIpRows(service: ServiceView): ServiceIpRow[] { +function buildIpRows( + service: ServiceView, + probes: readonly HealthLogProbe[] = [], +): ServiceIpRow[] { const healthByIp = new Map(service.ip_health.map((row) => [row.ip, row])) + const liveByIp = latestHealthByIp(probes) const weights = Object.assign( {}, ...service.domains.map((domain) => domain.target_ip_weights ?? {}), @@ -113,19 +122,25 @@ function buildIpRows(service: ServiceView): ServiceIpRow[] { return service.ips.map((ip) => { const health = healthByIp.get(ip) + const live = liveByIp.get(ip) + const status = resolveIpDisplayHealth(health?.status, live?.status) + const extras = live && live.status !== 'unknown' ? live : health return { id: ip, ip, - status: health?.status ?? 'unknown', + status, enabled: service.ip_enabled[ip] !== false, active: activeSet.has(ip), weight: weights[ip] ?? 1, priority: priorities[ip] ?? 1, - latency_ms: health?.latency_ms ?? null, - last_checked_at: health?.last_checked_at ?? null, - last_error: health?.last_error ?? null, - colo: health?.colo ?? null, - provider: health?.provider ?? null, + latency_ms: extras?.latency_ms ?? null, + last_checked_at: extras?.last_checked_at ?? null, + last_error: + live && live.status !== 'unknown' + ? live.last_error + : (health?.last_error ?? null), + colo: extras?.colo ?? null, + provider: extras?.provider ?? null, } }) } @@ -151,17 +166,23 @@ function buildNodeRows( weight: number priority: number }>, + probes: readonly HealthLogProbe[] = [], ): ServiceNodeRow[] { - return nodes.map((node) => ({ - id: String(node.id), - nodeId: node.id, - address: node.address, - protocol: node.protocol, - port: node.port, - health_status: mapNodeHealth(node.health_status), - weight: node.weight, - priority: node.priority, - })) + const liveByIp = latestHealthByIp(probes) + return nodes.map((node) => { + const stored = mapNodeHealth(node.health_status) + const live = liveByIp.get(node.address) + return { + id: String(node.id), + nodeId: node.id, + address: node.address, + protocol: node.protocol, + port: node.port, + health_status: resolveIpDisplayHealth(stored, live?.status), + weight: node.weight, + priority: node.priority, + } + }) } function NameCell({ @@ -199,6 +220,7 @@ interface ServiceDetailGridProps { weight: number priority: number }> + probes?: readonly HealthLogProbe[] togglingIp: string | null onToggleIp: (ip: string, enabled: boolean) => void onChangeIp: (row: ServiceFqdnRow) => void @@ -211,6 +233,7 @@ interface ServiceDetailGridProps { export function ServiceDetailGrid({ service, nodes, + probes = [], togglingIp, onToggleIp, onChangeIp, @@ -283,9 +306,9 @@ export function ServiceDetailGrid({ }, }) - const ipRows = useMemo(() => buildIpRows(service), [service]) + const ipRows = useMemo(() => buildIpRows(service, probes), [service, probes]) const fqdnRows = useMemo(() => buildFqdnRows(service), [service]) - const nodeRows = useMemo(() => buildNodeRows(nodes), [nodes]) + const nodeRows = useMemo(() => buildNodeRows(nodes, probes), [nodes, probes]) const markActive = service.lb_mode === 'failover' || service.lb_mode === 'weighted' const tabs = TABS.map((entry) => ({ diff --git a/apps/web/src/lib/failover-events.test.ts b/apps/web/src/lib/failover-events.test.ts index 65733d2..3384379 100644 --- a/apps/web/src/lib/failover-events.test.ts +++ b/apps/web/src/lib/failover-events.test.ts @@ -3,10 +3,14 @@ import { describe, expect, it } from 'vitest' import { failoverEventCopy, isFailoverEventStatus, + mergeFailoverHistory, toFailoverEvents, + toIpAliveTransitions, type FailoverBindingPool, type FailoverHealthInput, } from '@/lib/failover-events' +import type { FailoverLogEntry } from '@/lib/schemas' +import type { HealthLogProbe } from '@/lib/health-log' function row( overrides: Partial & Pick, @@ -186,3 +190,105 @@ describe('toFailoverEvents', () => { ]) }) }) + +function probe( + overrides: Partial & Pick, +): HealthLogProbe { + return { + ip: '130.49.213.153', + provider: 'local', + ok: overrides.status === 'up', + latency_ms: 12, + colo: null, + error: null, + ...overrides, + } +} + +function dns( + overrides: Partial & Pick, +): FailoverLogEntry { + return { + service_id: 13, + binding_id: 1, + fqdn: 'gt.rkns.top', + ip: '130.49.213.153', + ...overrides, + } +} + +describe('toIpAliveTransitions', () => { + it('emits leave then return, not the initial up', () => { + const transitions = toIpAliveTransitions([ + probe({ id: 1, status: 'up', checked_at: '2026-08-20T09:00:00Z' }), + probe({ id: 2, status: 'up', checked_at: '2026-08-20T09:01:00Z' }), + probe({ id: 3, status: 'down', checked_at: '2026-08-20T09:02:00Z' }), + probe({ id: 4, status: 'down', checked_at: '2026-08-20T09:03:00Z' }), + probe({ id: 5, status: 'up', checked_at: '2026-08-20T09:04:00Z' }), + ]) + expect(transitions).toEqual([ + { id: 'probe:3', ip: '130.49.213.153', alive: false, at: '2026-08-20T09:02:00Z' }, + { id: 'probe:5', ip: '130.49.213.153', alive: true, at: '2026-08-20T09:04:00Z' }, + ]) + }) + + it('any-up across providers: leave only when every source is down', () => { + const transitions = toIpAliveTransitions([ + probe({ + id: 1, + provider: 'local', + status: 'up', + checked_at: '2026-08-20T09:00:00Z', + }), + probe({ + id: 2, + provider: 'cloudflare', + status: 'down', + checked_at: '2026-08-20T09:01:00Z', + }), + probe({ + id: 3, + provider: 'local', + status: 'down', + checked_at: '2026-08-20T09:02:00Z', + }), + probe({ + id: 4, + provider: 'local', + status: 'up', + checked_at: '2026-08-20T09:03:00Z', + }), + ]) + expect(transitions.map((item) => item.id)).toEqual(['probe:3', 'probe:4']) + }) +}) + +describe('mergeFailoverHistory', () => { + it('fills leave/return from probes when DNS has only the add', () => { + const history = mergeFailoverHistory( + [dns({ id: 10, action: 'added', created_at: '2026-08-20 09:26:00' })], + [ + probe({ id: 1, status: 'up', checked_at: '2026-08-20T09:00:00Z' }), + probe({ id: 2, status: 'down', checked_at: '2026-08-20T09:10:00Z' }), + probe({ id: 3, status: 'up', checked_at: '2026-08-20T09:26:30Z' }), + ], + mskHip, + ) + expect(history.map((item) => `${item.action}:${item.source}`)).toEqual([ + 'added:dns', + 'removed:probe', + ]) + expect(history[0]?.copy).toBe('130.49.213.153 добавлена на gt.rkns.top') + expect(history[1]?.copy).toContain('вышла из пула') + }) + + it('keeps DNS over a probe return in the same 2-minute window', () => { + const history = mergeFailoverHistory( + [dns({ id: 10, action: 'added', created_at: '2026-08-20 09:26:00' })], + [probe({ id: 3, status: 'up', checked_at: '2026-08-20T09:26:30Z' })], + mskHip, + ) + expect(history).toHaveLength(1) + expect(history[0]?.source).toBe('dns') + }) +}) diff --git a/apps/web/src/lib/failover-events.ts b/apps/web/src/lib/failover-events.ts index e628053..7880d11 100644 --- a/apps/web/src/lib/failover-events.ts +++ b/apps/web/src/lib/failover-events.ts @@ -1,5 +1,23 @@ +import { + bestAliveHealthStatus, + probeTime, + type HealthLogProbe, + type HealthLogStatus, +} from '@/lib/health-log' +import type { FailoverLogEntry } from '@/lib/schemas' + export type FailoverEventKind = 'removed' | 'last-resort' +export interface FailoverHistoryItem { + id: string + ip: string + fqdn: string + action: 'added' | 'removed' + created_at: string + copy: string + source: 'dns' | 'probe' +} + export interface FailoverEvent { id: string address: string @@ -73,3 +91,141 @@ export function toFailoverEvents( } }) } + +const DEDUPE_WINDOW_MS = 2 * 60 * 1000 + +function fqdnsForIp( + ip: string, + bindings: readonly FailoverBindingPool[], +): string[] { + return bindings.filter((binding) => binding.configured.includes(ip)).map((binding) => binding.fqdn) +} + +function fqdnLabel(fqdns: string[]): string { + return fqdns.join(', ') || 'пул' +} + +function isAliveStatus(status: HealthLogStatus): boolean { + return status === 'up' || status === 'degraded' +} + +/** + * Per-IP any-up flips: down → вышла из пула, up после down → вернулась. + * Initial state is not an event. + */ +export function toIpAliveTransitions( + probes: readonly HealthLogProbe[], +): Array<{ id: string; ip: string; alive: boolean; at: string }> { + const byIp = new Map() + for (const item of probes) { + const list = byIp.get(item.ip) + if (list) list.push(item) + else byIp.set(item.ip, [item]) + } + + const out: Array<{ id: string; ip: string; alive: boolean; at: string }> = [] + for (const [ip, list] of byIp) { + list.sort( + (a, b) => probeTime(a.checked_at) - probeTime(b.checked_at) || a.id - b.id, + ) + const latestByProvider = new Map() + let prevAlive: boolean | undefined + for (const probe of list) { + latestByProvider.set(probe.provider, probe) + const status = bestAliveHealthStatus( + [...latestByProvider.values()].map((item) => item.status), + ) + if (status === 'unknown') continue + const alive = isAliveStatus(status) + if (prevAlive !== undefined && alive !== prevAlive) { + out.push({ + id: `probe:${probe.id}`, + ip, + alive, + at: probe.checked_at, + }) + } + prevAlive = alive + } + } + return out +} + +function dnsHistoryItem(item: FailoverLogEntry): FailoverHistoryItem { + return { + id: `dns:${item.id}`, + ip: item.ip, + fqdn: item.fqdn, + action: item.action, + created_at: item.created_at, + copy: + item.action === 'removed' + ? `${item.ip} убрана с ${item.fqdn}` + : `${item.ip} добавлена на ${item.fqdn}`, + source: 'dns', + } +} + +function probeHistoryItem( + transition: { id: string; ip: string; alive: boolean; at: string }, + bindings: readonly FailoverBindingPool[], +): FailoverHistoryItem { + const fqdns = fqdnsForIp(transition.ip, bindings) + const fqdn = fqdnLabel(fqdns) + const action = transition.alive ? 'added' : 'removed' + return { + id: transition.id, + ip: transition.ip, + fqdn, + action, + created_at: transition.at, + copy: + action === 'removed' + ? `${transition.ip} вышла из пула (${fqdn})` + : `${transition.ip} вернулась в пул (${fqdn})`, + source: 'probe', + } +} + +function eventTime(value: string): number { + return probeTime(value) +} + +/** + * DNS add/remove + health leave/return, newest first. + * Same IP+action within 2 minutes: keep the DNS row (it has a concrete FQDN). + */ +export function mergeFailoverHistory( + dns: readonly FailoverLogEntry[], + probes: readonly HealthLogProbe[], + bindings: readonly FailoverBindingPool[] = [], +): FailoverHistoryItem[] { + const merged = [ + ...dns.map(dnsHistoryItem), + ...toIpAliveTransitions(probes).map((transition) => + probeHistoryItem(transition, bindings), + ), + ] + merged.sort( + (a, b) => eventTime(b.created_at) - eventTime(a.created_at) || a.id.localeCompare(b.id), + ) + + const kept: FailoverHistoryItem[] = [] + for (const item of merged) { + const duplicate = kept.find( + (other) => + other.ip === item.ip && + other.action === item.action && + Math.abs(eventTime(other.created_at) - eventTime(item.created_at)) <= + DEDUPE_WINDOW_MS, + ) + if (!duplicate) { + kept.push(item) + continue + } + if (duplicate.source === 'probe' && item.source === 'dns') { + kept[kept.indexOf(duplicate)] = item + } + } + return kept +} diff --git a/apps/web/src/lib/health-log.test.ts b/apps/web/src/lib/health-log.test.ts index 255feeb..97d61db 100644 --- a/apps/web/src/lib/health-log.test.ts +++ b/apps/web/src/lib/health-log.test.ts @@ -4,7 +4,9 @@ import { bestAliveHealthStatus, collapseStatusChanges, enabledHealthProviders, + latestHealthByIp, providerHealthStatuses, + resolveIpDisplayHealth, worstHealthStatus, type HealthLogProbe, } from '@/lib/health-log' @@ -105,3 +107,42 @@ describe('bestAliveHealthStatus', () => { expect(bestAliveHealthStatus([])).toBe('unknown') }) }) + +describe('latestHealthByIp', () => { + it('any-up among latest-per-provider probes', () => { + const items = [ + probe({ + id: 1, + ip: '130.49.213.153', + provider: 'local', + status: 'up', + checked_at: '2026-08-20T09:00:00Z', + }), + probe({ + id: 2, + ip: '130.49.213.153', + provider: 'cloudflare', + status: 'down', + checked_at: '2026-08-20T09:00:01Z', + }), + probe({ + id: 3, + ip: '93.115.203.183', + status: 'unknown', + checked_at: '2026-08-20T08:59:00Z', + }), + ] + const byIp = latestHealthByIp(items) + expect(byIp.get('130.49.213.153')?.status).toBe('up') + expect(byIp.get('93.115.203.183')?.status).toBe('unknown') + }) +}) + +describe('resolveIpDisplayHealth', () => { + it('prefers a live probe over stored unknown', () => { + expect(resolveIpDisplayHealth('unknown', 'up')).toBe('up') + expect(resolveIpDisplayHealth('down', 'up')).toBe('up') + expect(resolveIpDisplayHealth('up', 'unknown')).toBe('up') + expect(resolveIpDisplayHealth('unknown', undefined)).toBe('unknown') + }) +}) diff --git a/apps/web/src/lib/health-log.ts b/apps/web/src/lib/health-log.ts index e795b18..a74e8cd 100644 --- a/apps/web/src/lib/health-log.ts +++ b/apps/web/src/lib/health-log.ts @@ -136,3 +136,61 @@ export function providerHealthStatuses( return result } + +export interface IpDisplayHealth { + status: HealthLogStatus + latency_ms: number | null + last_checked_at: string + last_error: string | null + colo: string | null + provider: HealthCheckProvider +} + +/** + * Latest probe per provider+IP, then any-up among those providers. + * Used by the IP table so hysteresis `unknown` in ip_health does not hide a live OK. + */ +export function latestHealthByIp( + items: readonly HealthLogProbe[], +): Map { + const latest = new Map() + const sorted = [...items].sort( + (a, b) => probeTime(b.checked_at) - probeTime(a.checked_at) || b.id - a.id, + ) + for (const item of sorted) { + const key = `${item.provider}\0${item.ip}` + if (!latest.has(key)) latest.set(key, item) + } + + const byIp = new Map() + for (const item of latest.values()) { + const list = byIp.get(item.ip) + if (list) list.push(item) + else byIp.set(item.ip, [item]) + } + + const result = new Map() + for (const [ip, probes] of byIp) { + const status = bestAliveHealthStatus(probes.map((probe) => probe.status)) + const preferred = + probes.find((probe) => probe.status === status) ?? probes[0]! + result.set(ip, { + status, + latency_ms: preferred.latency_ms, + last_checked_at: preferred.checked_at, + last_error: preferred.error, + colo: preferred.colo, + provider: preferred.provider, + }) + } + return result +} + +/** Prefer a concrete live probe over stored hysteresis `unknown`. */ +export function resolveIpDisplayHealth( + stored: HealthLogStatus | undefined, + live: HealthLogStatus | undefined, +): HealthLogStatus { + if (live && live !== 'unknown') return live + return stored ?? live ?? 'unknown' +} diff --git a/apps/web/src/routes/_auth/services/$serviceId/index.tsx b/apps/web/src/routes/_auth/services/$serviceId/index.tsx index 5708076..9f0e042 100644 --- a/apps/web/src/routes/_auth/services/$serviceId/index.tsx +++ b/apps/web/src/routes/_auth/services/$serviceId/index.tsx @@ -331,6 +331,7 @@ function ServiceDetailPage() { ipHealth={service.ip_health} bindings={failoverBindings} history={failoverHistory} + probes={logItems} /> @@ -344,6 +345,7 @@ function ServiceDetailPage() { { setTogglingIp(ip)