fix(services): показывать живой health и историю выхода из пула

Таблица IP брала hysteresis unknown вместо последней пробы. Failover теперь показывает выход и возврат в пул, а не только DNS add.

Co-authored-by: Cursor <cursoragent@cursor.com>

apps/web/src/components/failover-timeline.tsx

@@ -16,8 +16,8 @@ import { formatDate, formatRelative, sqliteUtcToIso } from '@/lib/format'
 import {
   failoverEventCopy,
   type FailoverEvent,
+  type FailoverHistoryItem,
 } from '@/lib/failover-events'
-import type { FailoverLogEntry } from '@/lib/schemas'
 import { cn } from '@cfdm/ui/lib/utils'
 import type { ComponentProps } from 'react'
 
@@ -45,12 +45,6 @@ function separatorClass(tone: 'down' | 'added' | 'removed'): string {
   return 'bg-destructive/25'
 }
 
-function failoverHistoryCopy(item: FailoverLogEntry): string {
-  return item.action === 'removed'
-    ? `${item.ip} убрана с ${item.fqdn}`
-    : `${item.ip} добавлена на ${item.fqdn}`
-}
-
 /**
  * Failover как sibling «Смены статуса»: ReUI Timeline + Badge.
  * Preview: https://reui.io/preview/base/components/c-timeline-10
@@ -63,14 +57,14 @@ export function FailoverTimeline({
   history = [],
 }: {
   events: FailoverEvent[]
-  history?: readonly FailoverLogEntry[]
+  history?: readonly FailoverHistoryItem[]
 }) {
   if (events.length === 0 && history.length === 0) {
     return (
       <EmptyState
         icon={ShieldCheckIcon}
         title="Нет инцидентов Failover"
-        description="Нет Down и нет смен A-записей по FQDN"
+        description="Нет Down и нет выходов из пула"
         stackedIcon={false}
         centered={false}
       />
@@ -142,6 +136,10 @@ export function FailoverTimeline({
                 <TimelineHeader>
                   <TimelineTitle className="flex flex-wrap items-center gap-2">
                     <span className="font-mono text-sm">{item.ip}</span>
+                    <HealthCheckBadge
+                      status={item.action === 'added' ? 'up' : 'down'}
+                      size="xs"
+                    />
                     <span className="text-muted-foreground text-xs">
                       {item.fqdn}
                     </span>
@@ -151,9 +149,7 @@ export function FailoverTimeline({
                   </TimelineDate>
                 </TimelineHeader>
                 <TimelineContent>
-                  <p className="text-foreground text-sm">
-                    {failoverHistoryCopy(item)}
-                  </p>
+                  <p className="text-foreground text-sm">{item.copy}</p>
                 </TimelineContent>
               </TimelineItem>
             )

apps/web/src/components/reui-kit/service-failover-panel.tsx

@@ -2,10 +2,17 @@ import { UnplugIcon } from 'lucide-react'
 
 import { FailoverTimeline } from '@/components/failover-timeline'
 import {
+  mergeFailoverHistory,
   toFailoverEvents,
   type FailoverBindingPool,
   type FailoverHealthInput,
 } from '@/lib/failover-events'
+import {
+  latestHealthByIp,
+  resolveIpDisplayHealth,
+  type HealthLogProbe,
+  type HealthLogStatus,
+} from '@/lib/health-log'
 import type { FailoverLogEntry } from '@/lib/schemas'
 import { Badge } from '@/components/reui/badge'
 import {
@@ -32,7 +39,7 @@ function failoverCountLabel(count: number): string {
 }
 
 /**
- * Failover — текущие Down + журнал add/remove по FQDN.
+ * Failover — текущие Down + кто вышел из пула и кто вернулся.
  * Preview: https://reui.io/preview/base/components/c-timeline-10
  * Preview: https://reui.io/preview/base/empty-state-12
  * Docs: https://reui.io/docs/components/base/frame
@@ -44,12 +51,32 @@ export function ServiceFailoverPanel({
   ipHealth,
   bindings,
   history,
+  probes = [],
 }: {
   ipHealth: readonly FailoverHealthInput[]
   bindings: readonly FailoverBindingPool[]
   history: readonly FailoverLogEntry[]
+  probes?: readonly HealthLogProbe[]
 }) {
-  const events = toFailoverEvents(ipHealth, bindings)
+  const liveByIp = latestHealthByIp(probes)
+  const overlayHealth = ipHealth.map((row) => {
+    const live = liveByIp.get(row.ip)
+    return {
+      ...row,
+      status: resolveIpDisplayHealth(
+        row.status as HealthLogStatus,
+        live?.status,
+      ),
+      last_error:
+        live && live.status !== 'unknown' ? live.last_error : row.last_error,
+      last_checked_at:
+        live && live.status !== 'unknown'
+          ? live.last_checked_at
+          : row.last_checked_at,
+    }
+  })
+  const events = toFailoverEvents(overlayHealth, bindings)
+  const mergedHistory = mergeFailoverHistory(history, probes, bindings)
   const removedCount = events.filter((event) => event.kind === 'removed').length
 
   return (
@@ -69,7 +96,7 @@ export function ServiceFailoverPanel({
             )}
           </FrameTitle>
           <FrameDescription>
-            Текущие Down и история A-записей по FQDN
+            Текущие Down и история: кто вышел из пула и кто вернулся
           </FrameDescription>
         </FrameHeader>
 
@@ -85,7 +112,7 @@ export function ServiceFailoverPanel({
           </Alert>
         ) : null}
 
-        <FailoverTimeline events={events} history={history} />
+        <FailoverTimeline events={events} history={mergedHistory} />
       </FramePanel>
     </Frame>
   )

apps/web/src/components/services/service-detail-grid.tsx

@@ -18,6 +18,11 @@ import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-colu
 import { IconTile } from '@/components/reui/icon-tile'
 import { createFilter, type Filter, type FilterFieldConfig } from '@/components/reui/filters'
 import { ResourcePage } from '@/components/reui-kit'
+import {
+  latestHealthByIp,
+  resolveIpDisplayHealth,
+  type HealthLogProbe,
+} from '@/lib/health-log'
 import { certRelativeBadge } from '@/components/columns/certificates-columns'
 import { certMonitoringOptions } from '@/lib/cert-monitoring'
 import { formatDate } from '@/lib/format'
@@ -99,8 +104,12 @@ function mapNodeHealth(status: string): HealthStatus {
   return 'unknown'
 }
 
-function buildIpRows(service: ServiceView): ServiceIpRow[] {
+function buildIpRows(
+  service: ServiceView,
+  probes: readonly HealthLogProbe[] = [],
+): ServiceIpRow[] {
   const healthByIp = new Map(service.ip_health.map((row) => [row.ip, row]))
+  const liveByIp = latestHealthByIp(probes)
   const weights = Object.assign(
     {},
     ...service.domains.map((domain) => domain.target_ip_weights ?? {}),
@@ -113,19 +122,25 @@ function buildIpRows(service: ServiceView): ServiceIpRow[] {
 
   return service.ips.map((ip) => {
     const health = healthByIp.get(ip)
+    const live = liveByIp.get(ip)
+    const status = resolveIpDisplayHealth(health?.status, live?.status)
+    const extras = live && live.status !== 'unknown' ? live : health
     return {
       id: ip,
       ip,
-      status: health?.status ?? 'unknown',
+      status,
       enabled: service.ip_enabled[ip] !== false,
       active: activeSet.has(ip),
       weight: weights[ip] ?? 1,
       priority: priorities[ip] ?? 1,
-      latency_ms: health?.latency_ms ?? null,
-      last_checked_at: health?.last_checked_at ?? null,
-      last_error: health?.last_error ?? null,
-      colo: health?.colo ?? null,
-      provider: health?.provider ?? null,
+      latency_ms: extras?.latency_ms ?? null,
+      last_checked_at: extras?.last_checked_at ?? null,
+      last_error:
+        live && live.status !== 'unknown'
+          ? live.last_error
+          : (health?.last_error ?? null),
+      colo: extras?.colo ?? null,
+      provider: extras?.provider ?? null,
     }
   })
 }
@@ -151,17 +166,23 @@ function buildNodeRows(
     weight: number
     priority: number
   }>,
+  probes: readonly HealthLogProbe[] = [],
 ): ServiceNodeRow[] {
-  return nodes.map((node) => ({
-    id: String(node.id),
-    nodeId: node.id,
-    address: node.address,
-    protocol: node.protocol,
-    port: node.port,
-    health_status: mapNodeHealth(node.health_status),
-    weight: node.weight,
-    priority: node.priority,
-  }))
+  const liveByIp = latestHealthByIp(probes)
+  return nodes.map((node) => {
+    const stored = mapNodeHealth(node.health_status)
+    const live = liveByIp.get(node.address)
+    return {
+      id: String(node.id),
+      nodeId: node.id,
+      address: node.address,
+      protocol: node.protocol,
+      port: node.port,
+      health_status: resolveIpDisplayHealth(stored, live?.status),
+      weight: node.weight,
+      priority: node.priority,
+    }
+  })
 }
 
 function NameCell({
@@ -199,6 +220,7 @@ interface ServiceDetailGridProps {
     weight: number
     priority: number
   }>
+  probes?: readonly HealthLogProbe[]
   togglingIp: string | null
   onToggleIp: (ip: string, enabled: boolean) => void
   onChangeIp: (row: ServiceFqdnRow) => void
@@ -211,6 +233,7 @@ interface ServiceDetailGridProps {
 export function ServiceDetailGrid({
   service,
   nodes,
+  probes = [],
   togglingIp,
   onToggleIp,
   onChangeIp,
@@ -283,9 +306,9 @@ export function ServiceDetailGrid({
     },
   })
 
-  const ipRows = useMemo(() => buildIpRows(service), [service])
+  const ipRows = useMemo(() => buildIpRows(service, probes), [service, probes])
   const fqdnRows = useMemo(() => buildFqdnRows(service), [service])
-  const nodeRows = useMemo(() => buildNodeRows(nodes), [nodes])
+  const nodeRows = useMemo(() => buildNodeRows(nodes, probes), [nodes, probes])
   const markActive = service.lb_mode === 'failover' || service.lb_mode === 'weighted'
 
   const tabs = TABS.map((entry) => ({

apps/web/src/lib/failover-events.test.ts

@@ -3,10 +3,14 @@ import { describe, expect, it } from 'vitest'
 import {
   failoverEventCopy,
   isFailoverEventStatus,
+  mergeFailoverHistory,
   toFailoverEvents,
+  toIpAliveTransitions,
   type FailoverBindingPool,
   type FailoverHealthInput,
 } from '@/lib/failover-events'
+import type { FailoverLogEntry } from '@/lib/schemas'
+import type { HealthLogProbe } from '@/lib/health-log'
 
 function row(
   overrides: Partial<FailoverHealthInput> & Pick<FailoverHealthInput, 'ip'>,
@@ -186,3 +190,105 @@ describe('toFailoverEvents', () => {
     ])
   })
 })
+
+function probe(
+  overrides: Partial<HealthLogProbe> & Pick<HealthLogProbe, 'id' | 'status' | 'checked_at'>,
+): HealthLogProbe {
+  return {
+    ip: '130.49.213.153',
+    provider: 'local',
+    ok: overrides.status === 'up',
+    latency_ms: 12,
+    colo: null,
+    error: null,
+    ...overrides,
+  }
+}
+
+function dns(
+  overrides: Partial<FailoverLogEntry> & Pick<FailoverLogEntry, 'id' | 'action' | 'created_at'>,
+): FailoverLogEntry {
+  return {
+    service_id: 13,
+    binding_id: 1,
+    fqdn: 'gt.rkns.top',
+    ip: '130.49.213.153',
+    ...overrides,
+  }
+}
+
+describe('toIpAliveTransitions', () => {
+  it('emits leave then return, not the initial up', () => {
+    const transitions = toIpAliveTransitions([
+      probe({ id: 1, status: 'up', checked_at: '2026-08-20T09:00:00Z' }),
+      probe({ id: 2, status: 'up', checked_at: '2026-08-20T09:01:00Z' }),
+      probe({ id: 3, status: 'down', checked_at: '2026-08-20T09:02:00Z' }),
+      probe({ id: 4, status: 'down', checked_at: '2026-08-20T09:03:00Z' }),
+      probe({ id: 5, status: 'up', checked_at: '2026-08-20T09:04:00Z' }),
+    ])
+    expect(transitions).toEqual([
+      { id: 'probe:3', ip: '130.49.213.153', alive: false, at: '2026-08-20T09:02:00Z' },
+      { id: 'probe:5', ip: '130.49.213.153', alive: true, at: '2026-08-20T09:04:00Z' },
+    ])
+  })
+
+  it('any-up across providers: leave only when every source is down', () => {
+    const transitions = toIpAliveTransitions([
+      probe({
+        id: 1,
+        provider: 'local',
+        status: 'up',
+        checked_at: '2026-08-20T09:00:00Z',
+      }),
+      probe({
+        id: 2,
+        provider: 'cloudflare',
+        status: 'down',
+        checked_at: '2026-08-20T09:01:00Z',
+      }),
+      probe({
+        id: 3,
+        provider: 'local',
+        status: 'down',
+        checked_at: '2026-08-20T09:02:00Z',
+      }),
+      probe({
+        id: 4,
+        provider: 'local',
+        status: 'up',
+        checked_at: '2026-08-20T09:03:00Z',
+      }),
+    ])
+    expect(transitions.map((item) => item.id)).toEqual(['probe:3', 'probe:4'])
+  })
+})
+
+describe('mergeFailoverHistory', () => {
+  it('fills leave/return from probes when DNS has only the add', () => {
+    const history = mergeFailoverHistory(
+      [dns({ id: 10, action: 'added', created_at: '2026-08-20 09:26:00' })],
+      [
+        probe({ id: 1, status: 'up', checked_at: '2026-08-20T09:00:00Z' }),
+        probe({ id: 2, status: 'down', checked_at: '2026-08-20T09:10:00Z' }),
+        probe({ id: 3, status: 'up', checked_at: '2026-08-20T09:26:30Z' }),
+      ],
+      mskHip,
+    )
+    expect(history.map((item) => `${item.action}:${item.source}`)).toEqual([
+      'added:dns',
+      'removed:probe',
+    ])
+    expect(history[0]?.copy).toBe('130.49.213.153 добавлена на gt.rkns.top')
+    expect(history[1]?.copy).toContain('вышла из пула')
+  })
+
+  it('keeps DNS over a probe return in the same 2-minute window', () => {
+    const history = mergeFailoverHistory(
+      [dns({ id: 10, action: 'added', created_at: '2026-08-20 09:26:00' })],
+      [probe({ id: 3, status: 'up', checked_at: '2026-08-20T09:26:30Z' })],
+      mskHip,
+    )
+    expect(history).toHaveLength(1)
+    expect(history[0]?.source).toBe('dns')
+  })
+})

apps/web/src/lib/failover-events.ts

@@ -1,5 +1,23 @@
+import {
+  bestAliveHealthStatus,
+  probeTime,
+  type HealthLogProbe,
+  type HealthLogStatus,
+} from '@/lib/health-log'
+import type { FailoverLogEntry } from '@/lib/schemas'
+
 export type FailoverEventKind = 'removed' | 'last-resort'
 
+export interface FailoverHistoryItem {
+  id: string
+  ip: string
+  fqdn: string
+  action: 'added' | 'removed'
+  created_at: string
+  copy: string
+  source: 'dns' | 'probe'
+}
+
 export interface FailoverEvent {
   id: string
   address: string
@@ -73,3 +91,141 @@ export function toFailoverEvents(
     }
   })
 }
+
+const DEDUPE_WINDOW_MS = 2 * 60 * 1000
+
+function fqdnsForIp(
+  ip: string,
+  bindings: readonly FailoverBindingPool[],
+): string[] {
+  return bindings.filter((binding) => binding.configured.includes(ip)).map((binding) => binding.fqdn)
+}
+
+function fqdnLabel(fqdns: string[]): string {
+  return fqdns.join(', ') || 'пул'
+}
+
+function isAliveStatus(status: HealthLogStatus): boolean {
+  return status === 'up' || status === 'degraded'
+}
+
+/**
+ * Per-IP any-up flips: down → вышла из пула, up после down → вернулась.
+ * Initial state is not an event.
+ */
+export function toIpAliveTransitions(
+  probes: readonly HealthLogProbe[],
+): Array<{ id: string; ip: string; alive: boolean; at: string }> {
+  const byIp = new Map<string, HealthLogProbe[]>()
+  for (const item of probes) {
+    const list = byIp.get(item.ip)
+    if (list) list.push(item)
+    else byIp.set(item.ip, [item])
+  }
+
+  const out: Array<{ id: string; ip: string; alive: boolean; at: string }> = []
+  for (const [ip, list] of byIp) {
+    list.sort(
+      (a, b) => probeTime(a.checked_at) - probeTime(b.checked_at) || a.id - b.id,
+    )
+    const latestByProvider = new Map<string, HealthLogProbe>()
+    let prevAlive: boolean | undefined
+    for (const probe of list) {
+      latestByProvider.set(probe.provider, probe)
+      const status = bestAliveHealthStatus(
+        [...latestByProvider.values()].map((item) => item.status),
+      )
+      if (status === 'unknown') continue
+      const alive = isAliveStatus(status)
+      if (prevAlive !== undefined && alive !== prevAlive) {
+        out.push({
+          id: `probe:${probe.id}`,
+          ip,
+          alive,
+          at: probe.checked_at,
+        })
+      }
+      prevAlive = alive
+    }
+  }
+  return out
+}
+
+function dnsHistoryItem(item: FailoverLogEntry): FailoverHistoryItem {
+  return {
+    id: `dns:${item.id}`,
+    ip: item.ip,
+    fqdn: item.fqdn,
+    action: item.action,
+    created_at: item.created_at,
+    copy:
+      item.action === 'removed'
+        ? `${item.ip} убрана с ${item.fqdn}`
+        : `${item.ip} добавлена на ${item.fqdn}`,
+    source: 'dns',
+  }
+}
+
+function probeHistoryItem(
+  transition: { id: string; ip: string; alive: boolean; at: string },
+  bindings: readonly FailoverBindingPool[],
+): FailoverHistoryItem {
+  const fqdns = fqdnsForIp(transition.ip, bindings)
+  const fqdn = fqdnLabel(fqdns)
+  const action = transition.alive ? 'added' : 'removed'
+  return {
+    id: transition.id,
+    ip: transition.ip,
+    fqdn,
+    action,
+    created_at: transition.at,
+    copy:
+      action === 'removed'
+        ? `${transition.ip} вышла из пула (${fqdn})`
+        : `${transition.ip} вернулась в пул (${fqdn})`,
+    source: 'probe',
+  }
+}
+
+function eventTime(value: string): number {
+  return probeTime(value)
+}
+
+/**
+ * DNS add/remove + health leave/return, newest first.
+ * Same IP+action within 2 minutes: keep the DNS row (it has a concrete FQDN).
+ */
+export function mergeFailoverHistory(
+  dns: readonly FailoverLogEntry[],
+  probes: readonly HealthLogProbe[],
+  bindings: readonly FailoverBindingPool[] = [],
+): FailoverHistoryItem[] {
+  const merged = [
+    ...dns.map(dnsHistoryItem),
+    ...toIpAliveTransitions(probes).map((transition) =>
+      probeHistoryItem(transition, bindings),
+    ),
+  ]
+  merged.sort(
+    (a, b) => eventTime(b.created_at) - eventTime(a.created_at) || a.id.localeCompare(b.id),
+  )
+
+  const kept: FailoverHistoryItem[] = []
+  for (const item of merged) {
+    const duplicate = kept.find(
+      (other) =>
+        other.ip === item.ip &&
+        other.action === item.action &&
+        Math.abs(eventTime(other.created_at) - eventTime(item.created_at)) <=
+          DEDUPE_WINDOW_MS,
+    )
+    if (!duplicate) {
+      kept.push(item)
+      continue
+    }
+    if (duplicate.source === 'probe' && item.source === 'dns') {
+      kept[kept.indexOf(duplicate)] = item
+    }
+  }
+  return kept
+}

apps/web/src/lib/health-log.test.ts

@@ -4,7 +4,9 @@ import {
   bestAliveHealthStatus,
   collapseStatusChanges,
   enabledHealthProviders,
+  latestHealthByIp,
   providerHealthStatuses,
+  resolveIpDisplayHealth,
   worstHealthStatus,
   type HealthLogProbe,
 } from '@/lib/health-log'
@@ -105,3 +107,42 @@ describe('bestAliveHealthStatus', () => {
     expect(bestAliveHealthStatus([])).toBe('unknown')
   })
 })
+
+describe('latestHealthByIp', () => {
+  it('any-up among latest-per-provider probes', () => {
+    const items = [
+      probe({
+        id: 1,
+        ip: '130.49.213.153',
+        provider: 'local',
+        status: 'up',
+        checked_at: '2026-08-20T09:00:00Z',
+      }),
+      probe({
+        id: 2,
+        ip: '130.49.213.153',
+        provider: 'cloudflare',
+        status: 'down',
+        checked_at: '2026-08-20T09:00:01Z',
+      }),
+      probe({
+        id: 3,
+        ip: '93.115.203.183',
+        status: 'unknown',
+        checked_at: '2026-08-20T08:59:00Z',
+      }),
+    ]
+    const byIp = latestHealthByIp(items)
+    expect(byIp.get('130.49.213.153')?.status).toBe('up')
+    expect(byIp.get('93.115.203.183')?.status).toBe('unknown')
+  })
+})
+
+describe('resolveIpDisplayHealth', () => {
+  it('prefers a live probe over stored unknown', () => {
+    expect(resolveIpDisplayHealth('unknown', 'up')).toBe('up')
+    expect(resolveIpDisplayHealth('down', 'up')).toBe('up')
+    expect(resolveIpDisplayHealth('up', 'unknown')).toBe('up')
+    expect(resolveIpDisplayHealth('unknown', undefined)).toBe('unknown')
+  })
+})

apps/web/src/lib/health-log.ts

@@ -136,3 +136,61 @@ export function providerHealthStatuses(
 
   return result
 }
+
+export interface IpDisplayHealth {
+  status: HealthLogStatus
+  latency_ms: number | null
+  last_checked_at: string
+  last_error: string | null
+  colo: string | null
+  provider: HealthCheckProvider
+}
+
+/**
+ * Latest probe per provider+IP, then any-up among those providers.
+ * Used by the IP table so hysteresis `unknown` in ip_health does not hide a live OK.
+ */
+export function latestHealthByIp(
+  items: readonly HealthLogProbe[],
+): Map<string, IpDisplayHealth> {
+  const latest = new Map<string, HealthLogProbe>()
+  const sorted = [...items].sort(
+    (a, b) => probeTime(b.checked_at) - probeTime(a.checked_at) || b.id - a.id,
+  )
+  for (const item of sorted) {
+    const key = `${item.provider}\0${item.ip}`
+    if (!latest.has(key)) latest.set(key, item)
+  }
+
+  const byIp = new Map<string, HealthLogProbe[]>()
+  for (const item of latest.values()) {
+    const list = byIp.get(item.ip)
+    if (list) list.push(item)
+    else byIp.set(item.ip, [item])
+  }
+
+  const result = new Map<string, IpDisplayHealth>()
+  for (const [ip, probes] of byIp) {
+    const status = bestAliveHealthStatus(probes.map((probe) => probe.status))
+    const preferred =
+      probes.find((probe) => probe.status === status) ?? probes[0]!
+    result.set(ip, {
+      status,
+      latency_ms: preferred.latency_ms,
+      last_checked_at: preferred.checked_at,
+      last_error: preferred.error,
+      colo: preferred.colo,
+      provider: preferred.provider,
+    })
+  }
+  return result
+}
+
+/** Prefer a concrete live probe over stored hysteresis `unknown`. */
+export function resolveIpDisplayHealth(
+  stored: HealthLogStatus | undefined,
+  live: HealthLogStatus | undefined,
+): HealthLogStatus {
+  if (live && live !== 'unknown') return live
+  return stored ?? live ?? 'unknown'
+}

apps/web/src/routes/_auth/services/$serviceId/index.tsx

@@ -331,6 +331,7 @@ function ServiceDetailPage() {
               ipHealth={service.ip_health}
               bindings={failoverBindings}
               history={failoverHistory}
+              probes={logItems}
             />
           </section>
 
@@ -344,6 +345,7 @@ function ServiceDetailPage() {
             <ServiceDetailGrid
               service={service}
               nodes={nodes}
+              probes={logItems}
               togglingIp={togglingIp}
               onToggleIp={(ip, enabled) => {
                 setTogglingIp(ip)