diff --git a/apps/api/src/services/health-check-service.ts b/apps/api/src/services/health-check-service.ts index bb45e4c..8b2ef14 100644 --- a/apps/api/src/services/health-check-service.ts +++ b/apps/api/src/services/health-check-service.ts @@ -386,7 +386,8 @@ function applyAggregatedStatus( { colo, provider: statusProvider }, ); const matchedNode = repos.findNodeByIp(db, target.ip); - if (matchedNode && matchedNode.enabled) { + // Binding-scope only: group apply must not clobber node with its own fetch failed. + if (matchedNode && matchedNode.enabled && target.scope === "binding") { repos.updateNode(db, matchedNode.id, { health_status: node, consecutive_failures: failures, diff --git a/apps/api/test/health-check.test.ts b/apps/api/test/health-check.test.ts index ca0f9aa..9d2b29c 100644 --- a/apps/api/test/health-check.test.ts +++ b/apps/api/test/health-check.test.ts @@ -291,6 +291,77 @@ describe("health-check state derivation via runAllChecks", () => { expect(targets[0]?.ip).toBe("2.59.161.102"); expect(targets[0]?.hostname).toBe("s.rkns.top"); }); + + it("does not mark node unhealthy when binding majority is OK and group local fails", async () => { + const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db"); + const { db, sqlite } = createMemoryDb(); + runMigrations(sqlite); + + const tcp = await startTcpServer(); + try { + const domain = repos.createDomain(db, null, "example.com", "zone-id"); + const group = repos.createServiceGroup( + db, + "VPN", + "vpn", + null, + "vpn.example.com", + { + health_check_enabled: true, + health_check_type: "http", + health_check_port: 1, + health_check_timeout_ms: 200, + health_check_path: "/", + }, + ); + const service = repos.createService(db, "Svc", "svc"); + repos.setServiceGroup(db, service.id, group.id); + repos.setServiceEnabled(db, service.id, true); + const binding = repos.insertBinding(db, domain.id, service.id, "@", null); + repos.updateBindingLbConfig(db, binding.id, { + health_check_enabled: true, + health_check_type: "tcp", + health_check_port: tcp.port, + health_check_timeout_ms: 500, + }); + repos.replaceBindingIpsWithMeta(db, binding.id, [ + { ip: "127.0.0.1", weight: 1, priority: 1 }, + ]); + const node = repos.findNodeByIp(db, "127.0.0.1"); + expect(node).not.toBeNull(); + + await healthCheckService.runAllChecks(db, { + probeGapMs: 0, + thresholds: { + degradedFailures: 1, + downFailures: 1, + latencyWarnMs: 1000, + }, + }); + + const bindingHealth = repos.getIpHealthStatusRow( + db, + "binding", + binding.id, + "127.0.0.1", + ); + const groupHealth = repos.getIpHealthStatusRow( + db, + "group", + group.id, + "127.0.0.1", + ); + const after = repos.getNode(db, node!.id); + + expect(bindingHealth?.status).toBe("up"); + expect(groupHealth?.status).toBe("down"); + expect(after.health_status).toBe("healthy"); + expect(after.consecutive_failures).toBe(0); + expect(after.last_failure_reason).toBeNull(); + } finally { + await new Promise((resolve) => tcp.server.close(() => resolve())); + } + }); }); describe("CNAME health mapped onto service IPs", () => { diff --git a/apps/web/src/components/failover-timeline.tsx b/apps/web/src/components/failover-timeline.tsx index d9c166d..9718d76 100644 --- a/apps/web/src/components/failover-timeline.tsx +++ b/apps/web/src/components/failover-timeline.tsx @@ -54,7 +54,7 @@ export function FailoverTimeline({ events }: { events: FailoverEvent[] }) { @@ -84,9 +84,11 @@ export function FailoverTimeline({ events }: { events: FailoverEvent[] }) { /> - {failStreakLabel(event.consecutiveFailures)} + {event.consecutiveFailures > 0 + ? failStreakLabel(event.consecutiveFailures) + : null} {checkedIso - ? ` · ${formatRelative(checkedIso)} · ${formatDate(checkedIso)}` + ? `${event.consecutiveFailures > 0 ? ' · ' : ''}${formatRelative(checkedIso)} · ${formatDate(checkedIso)}` : null} diff --git a/apps/web/src/components/reui-kit/service-failover-panel.tsx b/apps/web/src/components/reui-kit/service-failover-panel.tsx index 4ff2a9f..0dfb845 100644 --- a/apps/web/src/components/reui-kit/service-failover-panel.tsx +++ b/apps/web/src/components/reui-kit/service-failover-panel.tsx @@ -3,7 +3,7 @@ import { UnplugIcon } from 'lucide-react' import { FailoverTimeline } from '@/components/failover-timeline' import { toFailoverEvents, - type FailoverNodeInput, + type FailoverHealthInput, } from '@/lib/failover-events' import { Badge } from '@/components/reui/badge' import { @@ -39,13 +39,13 @@ function failoverCountLabel(count: number): string { * Docs: https://reui.io/docs/components/base/alert */ export function ServiceFailoverPanel({ - nodes, + ipHealth, activeAddresses, }: { - nodes: readonly FailoverNodeInput[] + ipHealth: readonly FailoverHealthInput[] activeAddresses: readonly string[] }) { - const events = toFailoverEvents(nodes, activeAddresses) + const events = toFailoverEvents(ipHealth, activeAddresses) return ( @@ -64,7 +64,7 @@ export function ServiceFailoverPanel({ )} - Только адреса, которых нет в DNS-пуле · не статус строки ноды + Только down из IP Health вне DNS-пула · как таблица активов diff --git a/apps/web/src/lib/failover-events.test.ts b/apps/web/src/lib/failover-events.test.ts index 89d1e73..e66334f 100644 --- a/apps/web/src/lib/failover-events.test.ts +++ b/apps/web/src/lib/failover-events.test.ts @@ -3,76 +3,90 @@ import { describe, expect, it } from 'vitest' import { isFailoverEventStatus, toFailoverEvents, - type FailoverNodeInput, + type FailoverHealthInput, } from '@/lib/failover-events' -function node( - overrides: Partial & Pick, -): FailoverNodeInput { +function row( + overrides: Partial & Pick, +): FailoverHealthInput { return { - id: overrides.id ?? overrides.address, - health_status: 'healthy', + status: 'up', consecutive_failures: 0, - last_failure_reason: null, - last_check_at: null, + last_error: null, + last_checked_at: null, ...overrides, } } describe('toFailoverEvents', () => { - it('не считает healthy failover-событием', () => { - expect(isFailoverEventStatus('healthy')).toBe(false) - expect(isFailoverEventStatus('unhealthy')).toBe(true) + it('инцидент только при binding down, не при unhealthy ноды', () => { + expect(isFailoverEventStatus('down')).toBe(true) + expect(isFailoverEventStatus('up')).toBe(false) + expect(isFailoverEventStatus('degraded')).toBe(false) + expect(isFailoverEventStatus('unknown')).toBe(false) + expect(isFailoverEventStatus('unhealthy')).toBe(false) }) it('без DNS-пула ничего не показывает — нельзя врать про вывод', () => { const events = toFailoverEvents([ - node({ - address: '130.49.213.153', - health_status: 'unhealthy', + row({ + ip: '130.49.213.153', + status: 'down', consecutive_failures: 9, - last_failure_reason: 'fetch failed', + last_error: 'fetch failed', }), ]) expect(events).toEqual([]) }) - it('нездоровый адрес в DNS-пуле не инцидент (last-resort / ещё в A-записи)', () => { + it('OK вне пула не инцидент (standby)', () => { const events = toFailoverEvents( [ - node({ - address: '130.49.213.153', - health_status: 'unhealthy', - consecutive_failures: 9, - last_failure_reason: 'fetch failed', - }), - node({ address: '10.0.0.3', health_status: 'checking' }), - ], - ['130.49.213.153', '10.0.0.2'], - ) - expect(events.map((event) => event.address)).toEqual(['10.0.0.3']) - }) - - it('здоровый standby вне пула не инцидент', () => { - const events = toFailoverEvents( - [ - node({ address: '10.0.0.1', health_status: 'healthy' }), - node({ address: '10.0.0.2', health_status: 'healthy' }), + row({ ip: '10.0.0.1', status: 'up' }), + row({ ip: '130.49.213.153', status: 'up' }), ], ['10.0.0.1'], ) expect(events).toEqual([]) }) - it('нездоровый адрес без A-записи — реальный вывод из пула', () => { + it('unknown и degraded вне пула не инцидент', () => { const events = toFailoverEvents( [ - node({ - address: '130.49.213.153', - health_status: 'unhealthy', + row({ ip: '10.0.0.1', status: 'up' }), + row({ ip: '10.0.0.2', status: 'unknown' }), + row({ ip: '10.0.0.3', status: 'degraded' }), + ], + ['10.0.0.1'], + ) + expect(events).toEqual([]) + }) + + it('down в DNS-пуле не инцидент (last-resort / ещё в A-записи)', () => { + const events = toFailoverEvents( + [ + row({ + ip: '130.49.213.153', + status: 'down', consecutive_failures: 9, - last_failure_reason: 'fetch failed', - last_check_at: '2026-08-20 07:00:00', + last_error: 'fetch failed', + }), + row({ ip: '10.0.0.3', status: 'down' }), + ], + ['130.49.213.153', '10.0.0.2'], + ) + expect(events.map((event) => event.address)).toEqual(['10.0.0.3']) + }) + + it('down без A-записи — реальный вывод из пула', () => { + const events = toFailoverEvents( + [ + row({ + ip: '130.49.213.153', + status: 'down', + consecutive_failures: 9, + last_error: 'fetch failed', + last_checked_at: '2026-08-20 07:00:00', }), ], ['10.0.0.1'], @@ -81,7 +95,7 @@ describe('toFailoverEvents', () => { { id: '130.49.213.153', address: '130.49.213.153', - status: 'unhealthy', + status: 'down', consecutiveFailures: 9, lastFailureReason: 'fetch failed', lastCheckAt: '2026-08-20 07:00:00', diff --git a/apps/web/src/lib/failover-events.ts b/apps/web/src/lib/failover-events.ts index 8d833d5..17a5969 100644 --- a/apps/web/src/lib/failover-events.ts +++ b/apps/web/src/lib/failover-events.ts @@ -7,40 +7,40 @@ export interface FailoverEvent { lastCheckAt?: string | null } -export interface FailoverNodeInput { - id?: number | string - address: string - health_status: string - consecutive_failures: number - last_failure_reason: string | null - last_check_at?: string | null +/** Binding IP health — тот же контур, что таблица активов. */ +export interface FailoverHealthInput { + ip: string + status: string + consecutive_failures?: number + last_error?: string | null + last_checked_at?: string | null } -const FAILOVER_STATUSES = new Set(['unhealthy', 'down', 'checking']) - +/** Инцидент только при down. up / degraded / unknown — не вывод из пула. */ export function isFailoverEventStatus(status: string): boolean { - return FAILOVER_STATUSES.has(status) + return status === 'down' } /** - * Инцидент failover = нездоровый адрес, которого нет в DNS-пуле. - * Нода в activeAddresses (в т.ч. last-resort) — не «выведена». + * Инцидент failover = binding health down и адреса нет в DNS-пуле. + * OK / unknown / degraded вне пула — standby, не инцидент. + * Down в activeAddresses (last-resort) — не «снята с DNS». */ export function toFailoverEvents( - nodes: readonly FailoverNodeInput[], + ipHealth: readonly FailoverHealthInput[], activeAddresses: readonly string[] = [], ): FailoverEvent[] { const pool = new Set(activeAddresses) if (pool.size === 0) return [] - return nodes - .filter((node) => isFailoverEventStatus(node.health_status) && !pool.has(node.address)) - .map((node) => ({ - id: String(node.id ?? node.address), - address: node.address, - status: node.health_status, - consecutiveFailures: node.consecutive_failures, - lastFailureReason: node.last_failure_reason, - lastCheckAt: node.last_check_at ?? null, + return ipHealth + .filter((row) => isFailoverEventStatus(row.status) && !pool.has(row.ip)) + .map((row) => ({ + id: row.ip, + address: row.ip, + status: row.status, + consecutiveFailures: row.consecutive_failures ?? 0, + lastFailureReason: row.last_error ?? null, + lastCheckAt: row.last_checked_at ?? null, })) } diff --git a/apps/web/src/routes/_auth/services/$serviceId/index.tsx b/apps/web/src/routes/_auth/services/$serviceId/index.tsx index 3213e14..d98be62 100644 --- a/apps/web/src/routes/_auth/services/$serviceId/index.tsx +++ b/apps/web/src/routes/_auth/services/$serviceId/index.tsx @@ -203,8 +203,6 @@ function ServiceDetailPage() { const isError = viewQuery.isError || overviewQuery.isError const error = viewQuery.error ?? overviewQuery.error - const failoverNodes = nodes.length > 0 ? nodes : (overview?.nodes ?? []) - const enabledProviders = useMemo( () => enabledHealthProviders(service?.domains ?? []), [service], @@ -314,7 +312,7 @@ function ServiceDetailPage() { isLoading={logQuery.isLoading} />