diff --git a/apps/api/src/services/health-check-service.ts b/apps/api/src/services/health-check-service.ts index 8b2ef14..397d0af 100644 --- a/apps/api/src/services/health-check-service.ts +++ b/apps/api/src/services/health-check-service.ts @@ -290,7 +290,7 @@ export interface RunAllChecksOptions { target: HealthCheckTarget, prevState: IpHealthState | null, nextState: IpHealthState, - ) => void; + ) => void | Promise; } function sleep(ms: number): Promise { @@ -326,7 +326,7 @@ function logSourceResult( }); } -function applyAggregatedStatus( +async function applyAggregatedStatus( db: Db, target: HealthCheckTarget, sources: Array<{ provider: HealthCheckProvider; result: ProbeResult }>, @@ -397,7 +397,7 @@ function applyAggregatedStatus( }); } if (prevState !== state) { - options.onStatusChange?.(target, prevState, state); + await options.onStatusChange?.(target, prevState, state); } } @@ -513,7 +513,7 @@ export async function runAllChecks( for (const source of sources) { logSourceResult(db, target, source.provider, source.result); } - applyAggregatedStatus(db, target, sources, options); + await applyAggregatedStatus(db, target, sources, options); } } diff --git a/apps/api/src/services/routing/failover.ts b/apps/api/src/services/routing/failover.ts index efbdad3..e6747b4 100644 --- a/apps/api/src/services/routing/failover.ts +++ b/apps/api/src/services/routing/failover.ts @@ -1,16 +1,16 @@ import type { LbIpRow } from "./types.js"; -import { isHealthy } from "./health.js"; +import { isPoolMember } from "./health.js"; export function failoverDesired(rows: LbIpRow[]): string[] { if (rows.length === 0) return []; - const healthy = rows.filter((r) => isHealthy(r.health)); - const pool = healthy.length > 0 ? healthy : rows; + const live = rows.filter((r) => isPoolMember(r.health)); + const pool = live.length > 0 ? live : rows; const sorted = [...pool].sort( (a, b) => a.priority - b.priority || a.weight - b.weight, ); const minPriority = sorted[0]!.priority; const primaries = sorted.filter((r) => r.priority === minPriority); - if (healthy.length > 0) { + if (live.length > 0) { return primaries.map((r) => r.ip); } return [sorted[0]!.ip]; diff --git a/apps/api/src/services/routing/health.ts b/apps/api/src/services/routing/health.ts index 554118f..af02592 100644 --- a/apps/api/src/services/routing/health.ts +++ b/apps/api/src/services/routing/health.ts @@ -3,3 +3,12 @@ import type { IpHealthState, NodeHealthState } from "@cfdm/shared"; export function isHealthy(state: IpHealthState | NodeHealthState | string): boolean { return state === "up" || state === "healthy"; } + +export function isDown(state: IpHealthState | NodeHealthState | string): boolean { + return state === "down" || state === "unhealthy"; +} + +/** A-pool membership: only Down is drained. Recovering (unknown/checking) and Slow return immediately. */ +export function isPoolMember(state: IpHealthState | NodeHealthState | string): boolean { + return !isDown(state); +} diff --git a/apps/api/src/services/routing/index.ts b/apps/api/src/services/routing/index.ts index 0028888..1b60ec9 100644 --- a/apps/api/src/services/routing/index.ts +++ b/apps/api/src/services/routing/index.ts @@ -6,7 +6,7 @@ import type { LbIpRow, LbTargetConfig } from "./types.js"; import { weightedDesired } from "./weighted.js"; export type { LbIpRow, LbTargetConfig } from "./types.js"; -export { isHealthy } from "./health.js"; +export { isDown, isHealthy, isPoolMember } from "./health.js"; export { withBindingLock } from "./binding-lock.js"; export { canApplyLb, diff --git a/apps/api/src/services/routing/round-robin.ts b/apps/api/src/services/routing/round-robin.ts index 345c70c..9c7500e 100644 --- a/apps/api/src/services/routing/round-robin.ts +++ b/apps/api/src/services/routing/round-robin.ts @@ -1,8 +1,8 @@ import type { LbIpRow } from "./types.js"; -import { isHealthy } from "./health.js"; +import { isPoolMember } from "./health.js"; export function roundRobinDesired(rows: LbIpRow[]): string[] { - const healthy = rows.filter((r) => isHealthy(r.health)); - const pool = healthy.length > 0 ? healthy : rows; + const live = rows.filter((r) => isPoolMember(r.health)); + const pool = live.length > 0 ? live : rows; return pool.map((r) => r.ip); } diff --git a/apps/api/src/services/routing/weighted.ts b/apps/api/src/services/routing/weighted.ts index 831ef83..362de24 100644 --- a/apps/api/src/services/routing/weighted.ts +++ b/apps/api/src/services/routing/weighted.ts @@ -1,5 +1,5 @@ import type { LbIpRow } from "./types.js"; -import { isHealthy } from "./health.js"; +import { isPoolMember } from "./health.js"; /** Slot length for time-sliced weighted DNS (one A at a time). */ export const WEIGHTED_SLOT_MS = 60_000; @@ -9,8 +9,8 @@ export const WEIGHTED_DNS_TTL = 60; export function weightedDesired(rows: LbIpRow[], nowMs = Date.now()): string[] { if (rows.length === 0) return []; - const healthy = rows.filter((r) => isHealthy(r.health)); - const pool = healthy.length > 0 ? healthy : rows; + const live = rows.filter((r) => isPoolMember(r.health)); + const pool = live.length > 0 ? live : rows; if (pool.length === 1) return [pool[0]!.ip]; const sorted = [...pool].sort((a, b) => a.ip.localeCompare(b.ip)); diff --git a/apps/api/src/services/service-config-service.ts b/apps/api/src/services/service-config-service.ts index 6592283..abdf7e5 100644 --- a/apps/api/src/services/service-config-service.ts +++ b/apps/api/src/services/service-config-service.ts @@ -30,7 +30,7 @@ import { syncServiceToVpsTracker } from "./vps-tracker-sync.js"; import { fireEnsureHealthWorker, DEFAULT_HEALTH_FALLBACKS } from "./health/health-worker-deploy.js"; import { canApplyLb, - isHealthy, + isPoolMember, isSharedPool, resolveDesiredAIps, selectActiveIpsByMode, @@ -287,7 +287,7 @@ function getGroupLbState( } else { existing.weight += weight; existing.priority = Math.min(existing.priority, priority); - if (isHealthy(existing.health) && status && !isHealthy(status.status as IpHealthState)) { + if (isPoolMember(existing.health) && status && !isPoolMember(status.status as IpHealthState)) { existing.health = status.status as IpHealthState; } } diff --git a/apps/api/test/lb-reconcile.test.ts b/apps/api/test/lb-reconcile.test.ts index 89d128b..c010c91 100644 --- a/apps/api/test/lb-reconcile.test.ts +++ b/apps/api/test/lb-reconcile.test.ts @@ -70,6 +70,18 @@ describe("selectActiveIpsByMode", () => { expect(selectActiveIpsByMode(config, rows)).toEqual(["1.1.1.1"]); }); + it("failover returns recovering unknown primary immediately", () => { + const config: LbTargetConfig = { + lb_mode: "failover", + health_check_enabled: true, + }; + const rows = [ + row("1.1.1.1", { priority: 1, health: "unknown" }), + row("2.2.2.2", { priority: 2, health: "up" }), + ]; + expect(selectActiveIpsByMode(config, rows)).toEqual(["1.1.1.1"]); + }); + it("failover falls back to min-priority ip among all when none healthy", () => { const config: LbTargetConfig = { lb_mode: "failover", @@ -105,6 +117,17 @@ describe("selectActiveIpsByMode", () => { ).toEqual(["1.1.1.1"]); }); + it("weighted includes recovering unknown in the cycle", () => { + const rows = [ + row("1.1.1.1", { weight: 1, health: "up" }), + row("2.2.2.2", { weight: 3, health: "unknown" }), + ]; + expect(selectActiveIpsByMode(weightedConfig, rows, 0)).toEqual(["1.1.1.1"]); + expect( + selectActiveIpsByMode(weightedConfig, rows, WEIGHTED_SLOT_MS), + ).toEqual(["2.2.2.2"]); + }); + it("weighted with one ip always returns that ip", () => { expect( selectActiveIpsByMode(weightedConfig, [row("1.1.1.1", { weight: 5 })], 0), @@ -126,7 +149,7 @@ describe("selectActiveIpsByMode", () => { expect(selectActiveIpsByMode(weightedConfig, [], 0)).toEqual([]); }); - it("round_robin excludes unknown when another ip is up", () => { + it("round_robin puts recovering unknown back with live ips", () => { const config: LbTargetConfig = { lb_mode: "round_robin", health_check_enabled: true, @@ -135,7 +158,25 @@ describe("selectActiveIpsByMode", () => { row("1.1.1.1", { health: "up" }), row("2.2.2.2", { health: "unknown" }), ]; - expect(selectActiveIpsByMode(config, rows)).toEqual(["1.1.1.1"]); + expect(selectActiveIpsByMode(config, rows).sort()).toEqual([ + "1.1.1.1", + "2.2.2.2", + ]); + }); + + it("round_robin keeps degraded in the pool with live ips", () => { + const config: LbTargetConfig = { + lb_mode: "round_robin", + health_check_enabled: true, + }; + const rows = [ + row("1.1.1.1", { health: "up" }), + row("2.2.2.2", { health: "degraded" }), + ]; + expect(selectActiveIpsByMode(config, rows).sort()).toEqual([ + "1.1.1.1", + "2.2.2.2", + ]); }); it("returns empty array for no rows", () => { diff --git a/docs/Home.md b/docs/Home.md index 6537f49..f9379e4 100644 --- a/docs/Home.md +++ b/docs/Home.md @@ -44,9 +44,10 @@ health-check работают на двух уровнях: - **Change Domain** — перенос привязок между зонами `POST /api/v1/services/:id/change-domain`. Режимы LB: `round_robin`, `failover`, `weighted`. В Cloudflare free `weighted` -работает как `round_robin` (одна A на IP). `unknown` **не** считается healthy и -не попадает в пул, пока нет успешных проб; восстановление — `UNHEALTHY → CHECKING → HEALTHY` -после `HEALTH_SUCCESS_RECOVERIES` (default 2). Пороги и cron движка задаются в +работает как `round_robin` (одна A на IP). В A-пул попадает всё, кроме **Down** +(`unknown` / checking / Slow возвращаются в DNS на первой успешной пробе). +Бейдж Healthy — `UNHEALTHY → CHECKING → HEALTHY` после `HEALTH_SUCCESS_RECOVERIES` +(default 2). Пороги и cron движка задаются в **Настройки → Health-check** (env — fallback, пока значения не сохранены в UI). ### Источники проб: Local, Cloudflare Worker, Globalping