Compare commits
4
Commits
3d0ea33baf
...
v1.21.3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5cf39880f8 | ||
|
|
f1443f1db5 | ||
|
|
a9a84fabac | ||
|
|
2b150fa3a6 |
No files matched your search
@@ -290,7 +290,7 @@ export interface RunAllChecksOptions {
|
|||||||
target: HealthCheckTarget,
|
target: HealthCheckTarget,
|
||||||
prevState: IpHealthState | null,
|
prevState: IpHealthState | null,
|
||||||
nextState: IpHealthState,
|
nextState: IpHealthState,
|
||||||
) => void;
|
) => void | Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
function sleep(ms: number): Promise<void> {
|
function sleep(ms: number): Promise<void> {
|
||||||
@@ -326,7 +326,7 @@ function logSourceResult(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyAggregatedStatus(
|
async function applyAggregatedStatus(
|
||||||
db: Db,
|
db: Db,
|
||||||
target: HealthCheckTarget,
|
target: HealthCheckTarget,
|
||||||
sources: Array<{ provider: HealthCheckProvider; result: ProbeResult }>,
|
sources: Array<{ provider: HealthCheckProvider; result: ProbeResult }>,
|
||||||
@@ -397,7 +397,7 @@ function applyAggregatedStatus(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (prevState !== state) {
|
if (prevState !== state) {
|
||||||
options.onStatusChange?.(target, prevState, state);
|
await options.onStatusChange?.(target, prevState, state);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -513,7 +513,7 @@ export async function runAllChecks(
|
|||||||
for (const source of sources) {
|
for (const source of sources) {
|
||||||
logSourceResult(db, target, source.provider, source.result);
|
logSourceResult(db, target, source.provider, source.result);
|
||||||
}
|
}
|
||||||
applyAggregatedStatus(db, target, sources, options);
|
await applyAggregatedStatus(db, target, sources, options);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,10 @@ import type {
|
|||||||
import { AppError } from "../errors.js";
|
import { AppError } from "../errors.js";
|
||||||
import { isValidIpv4 } from "../lib/validators.js";
|
import { isValidIpv4 } from "../lib/validators.js";
|
||||||
import { getView } from "./service-config-service.js";
|
import { getView } from "./service-config-service.js";
|
||||||
import { selectActiveIpsByMode } from "./routing/index.js";
|
import {
|
||||||
|
isSharedPool,
|
||||||
|
resolveDesiredAIps,
|
||||||
|
} from "./routing/index.js";
|
||||||
|
|
||||||
function assertAddress(address: string): void {
|
function assertAddress(address: string): void {
|
||||||
if (!isValidIpv4(address)) {
|
if (!isValidIpv4(address)) {
|
||||||
@@ -89,8 +92,12 @@ export async function getOverview(
|
|||||||
: null;
|
: null;
|
||||||
|
|
||||||
const active = new Set<string>();
|
const active = new Set<string>();
|
||||||
|
const serviceIps = service.ips.filter(
|
||||||
|
(ip) => service.ip_enabled[ip] !== false,
|
||||||
|
);
|
||||||
for (const binding of bindings) {
|
for (const binding of bindings) {
|
||||||
const metas = repos.listBindingIpsWithMeta(db, binding.id);
|
const metas = repos.listBindingIpsWithMeta(db, binding.id);
|
||||||
|
const targetIps = metas.map((entry) => entry.ip);
|
||||||
const rows = metas.map((entry) => {
|
const rows = metas.map((entry) => {
|
||||||
const status = repos.getIpHealthStatusRow(db, "binding", binding.id, entry.ip);
|
const status = repos.getIpHealthStatusRow(db, "binding", binding.id, entry.ip);
|
||||||
return {
|
return {
|
||||||
@@ -100,12 +107,15 @@ export async function getOverview(
|
|||||||
health: status ? status.status : ("unknown" as const),
|
health: status ? status.status : ("unknown" as const),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
for (const ip of selectActiveIpsByMode(
|
for (const ip of resolveDesiredAIps(
|
||||||
{
|
{
|
||||||
lb_mode: binding.lb_mode,
|
lb_mode: binding.lb_mode,
|
||||||
health_check_enabled: binding.health_check_enabled,
|
health_check_enabled: binding.health_check_enabled,
|
||||||
},
|
},
|
||||||
rows,
|
rows,
|
||||||
|
targetIps,
|
||||||
|
Date.now(),
|
||||||
|
serviceIps,
|
||||||
)) {
|
)) {
|
||||||
active.add(ip);
|
active.add(ip);
|
||||||
}
|
}
|
||||||
@@ -134,10 +144,13 @@ export function opsSummary(db: Db) {
|
|||||||
if (binding.lb_mode !== "failover" || !binding.health_check_enabled) {
|
if (binding.lb_mode !== "failover" || !binding.health_check_enabled) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return binding.target_ips.some((ip) => {
|
return (
|
||||||
const row = repos.getIpHealthStatusRow(db, "binding", binding.id, ip);
|
isSharedPool(binding.target_ips) &&
|
||||||
return row?.status === "down";
|
binding.target_ips.some((ip) => {
|
||||||
});
|
const row = repos.getIpHealthStatusRow(db, "binding", binding.id, ip);
|
||||||
|
return row?.status === "down";
|
||||||
|
})
|
||||||
|
);
|
||||||
}).length;
|
}).length;
|
||||||
return {
|
return {
|
||||||
domains: domains.length,
|
domains: domains.length,
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
import type { LbIpRow } from "./types.js";
|
import type { LbIpRow } from "./types.js";
|
||||||
import { isHealthy } from "./health.js";
|
import { isPoolMember } from "./health.js";
|
||||||
|
|
||||||
export function failoverDesired(rows: LbIpRow[]): string[] {
|
export function failoverDesired(rows: LbIpRow[]): string[] {
|
||||||
if (rows.length === 0) return [];
|
if (rows.length === 0) return [];
|
||||||
const healthy = rows.filter((r) => isHealthy(r.health));
|
const live = rows.filter((r) => isPoolMember(r.health));
|
||||||
const pool = healthy.length > 0 ? healthy : rows;
|
const pool = live.length > 0 ? live : rows;
|
||||||
const sorted = [...pool].sort(
|
const sorted = [...pool].sort(
|
||||||
(a, b) => a.priority - b.priority || a.weight - b.weight,
|
(a, b) => a.priority - b.priority || a.weight - b.weight,
|
||||||
);
|
);
|
||||||
const minPriority = sorted[0]!.priority;
|
const minPriority = sorted[0]!.priority;
|
||||||
const primaries = sorted.filter((r) => r.priority === minPriority);
|
const primaries = sorted.filter((r) => r.priority === minPriority);
|
||||||
if (healthy.length > 0) {
|
if (live.length > 0) {
|
||||||
return primaries.map((r) => r.ip);
|
return primaries.map((r) => r.ip);
|
||||||
}
|
}
|
||||||
return [sorted[0]!.ip];
|
return [sorted[0]!.ip];
|
||||||
|
|||||||
@@ -3,3 +3,12 @@ import type { IpHealthState, NodeHealthState } from "@cfdm/shared";
|
|||||||
export function isHealthy(state: IpHealthState | NodeHealthState | string): boolean {
|
export function isHealthy(state: IpHealthState | NodeHealthState | string): boolean {
|
||||||
return state === "up" || state === "healthy";
|
return state === "up" || state === "healthy";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isDown(state: IpHealthState | NodeHealthState | string): boolean {
|
||||||
|
return state === "down" || state === "unhealthy";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A-pool membership: only Down is drained. Recovering (unknown/checking) and Slow return immediately. */
|
||||||
|
export function isPoolMember(state: IpHealthState | NodeHealthState | string): boolean {
|
||||||
|
return !isDown(state);
|
||||||
|
}
|
||||||
@@ -1,12 +1,19 @@
|
|||||||
import type { LbMode } from "@cfdm/shared";
|
import type { LbMode } from "@cfdm/shared";
|
||||||
import { failoverDesired } from "./failover.js";
|
import { failoverDesired } from "./failover.js";
|
||||||
|
import { canApplyLb, isSharedPool } from "./pool.js";
|
||||||
import { roundRobinDesired } from "./round-robin.js";
|
import { roundRobinDesired } from "./round-robin.js";
|
||||||
import type { LbIpRow, LbTargetConfig } from "./types.js";
|
import type { LbIpRow, LbTargetConfig } from "./types.js";
|
||||||
import { weightedDesired } from "./weighted.js";
|
import { weightedDesired } from "./weighted.js";
|
||||||
|
|
||||||
export type { LbIpRow, LbTargetConfig } from "./types.js";
|
export type { LbIpRow, LbTargetConfig } from "./types.js";
|
||||||
export { isHealthy } from "./health.js";
|
export { isDown, isHealthy, isPoolMember } from "./health.js";
|
||||||
export { withBindingLock } from "./binding-lock.js";
|
export { withBindingLock } from "./binding-lock.js";
|
||||||
|
export {
|
||||||
|
canApplyLb,
|
||||||
|
isSharedPool,
|
||||||
|
shouldRecordFailoverDnsDiff,
|
||||||
|
uniqueIpCount,
|
||||||
|
} from "./pool.js";
|
||||||
export { WEIGHTED_DNS_TTL, WEIGHTED_SLOT_MS, weightedDesired } from "./weighted.js";
|
export { WEIGHTED_DNS_TTL, WEIGHTED_SLOT_MS, weightedDesired } from "./weighted.js";
|
||||||
|
|
||||||
export function selectActiveIpsByMode(
|
export function selectActiveIpsByMode(
|
||||||
@@ -24,6 +31,23 @@ export function selectActiveIpsByMode(
|
|||||||
return roundRobinDesired(rows);
|
return roundRobinDesired(rows);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function resolveDesiredAIps(
|
||||||
|
config: LbTargetConfig,
|
||||||
|
rows: LbIpRow[],
|
||||||
|
fallbackIps: readonly string[],
|
||||||
|
nowMs = Date.now(),
|
||||||
|
serviceIps: readonly string[] = fallbackIps,
|
||||||
|
): string[] {
|
||||||
|
const fallback = [...fallbackIps];
|
||||||
|
if (!canApplyLb(serviceIps, fallback)) return fallback;
|
||||||
|
if (!isSharedPool(rows.map((row) => row.ip))) return fallback;
|
||||||
|
if (config.lb_mode === "weighted" || config.health_check_enabled) {
|
||||||
|
const activeIps = selectActiveIpsByMode(config, rows, nowMs);
|
||||||
|
if (activeIps.length > 0) return activeIps;
|
||||||
|
}
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
export function strategyLabel(mode: LbMode): string {
|
export function strategyLabel(mode: LbMode): string {
|
||||||
if (mode === "failover") return "Failover";
|
if (mode === "failover") return "Failover";
|
||||||
if (mode === "weighted") return "Weighted";
|
if (mode === "weighted") return "Weighted";
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
import type { LbMode } from "@cfdm/shared";
|
||||||
|
|
||||||
|
export function uniqueIpCount(ips: readonly string[]): number {
|
||||||
|
return new Set(ips.filter(Boolean)).size;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Shared pool — two or more unique IPs. One IP (even duplicated) is not a pool. */
|
||||||
|
export function isSharedPool(ips: readonly string[]): boolean {
|
||||||
|
return uniqueIpCount(ips) >= 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** LB / drain only when the service itself has a pool AND this FQDN is shared. */
|
||||||
|
export function canApplyLb(
|
||||||
|
serviceIps: readonly string[],
|
||||||
|
bindingIps: readonly string[],
|
||||||
|
): boolean {
|
||||||
|
return isSharedPool(serviceIps) && isSharedPool(bindingIps);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldRecordFailoverDnsDiff(input: {
|
||||||
|
configuredIps: readonly string[];
|
||||||
|
lbMode: LbMode;
|
||||||
|
added: readonly string[];
|
||||||
|
removed: readonly string[];
|
||||||
|
downIps: ReadonlySet<string>;
|
||||||
|
}): boolean {
|
||||||
|
if (!isSharedPool(input.configuredIps)) return false;
|
||||||
|
if (input.lbMode !== "weighted") return true;
|
||||||
|
return [...input.added, ...input.removed].some((ip) => input.downIps.has(ip));
|
||||||
|
}
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
import type { LbIpRow } from "./types.js";
|
import type { LbIpRow } from "./types.js";
|
||||||
import { isHealthy } from "./health.js";
|
import { isPoolMember } from "./health.js";
|
||||||
|
|
||||||
export function roundRobinDesired(rows: LbIpRow[]): string[] {
|
export function roundRobinDesired(rows: LbIpRow[]): string[] {
|
||||||
const healthy = rows.filter((r) => isHealthy(r.health));
|
const live = rows.filter((r) => isPoolMember(r.health));
|
||||||
const pool = healthy.length > 0 ? healthy : rows;
|
const pool = live.length > 0 ? live : rows;
|
||||||
return pool.map((r) => r.ip);
|
return pool.map((r) => r.ip);
|
||||||
}
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import type { LbIpRow } from "./types.js";
|
import type { LbIpRow } from "./types.js";
|
||||||
import { isHealthy } from "./health.js";
|
import { isPoolMember } from "./health.js";
|
||||||
|
|
||||||
/** Slot length for time-sliced weighted DNS (one A at a time). */
|
/** Slot length for time-sliced weighted DNS (one A at a time). */
|
||||||
export const WEIGHTED_SLOT_MS = 60_000;
|
export const WEIGHTED_SLOT_MS = 60_000;
|
||||||
@@ -9,8 +9,8 @@ export const WEIGHTED_DNS_TTL = 60;
|
|||||||
|
|
||||||
export function weightedDesired(rows: LbIpRow[], nowMs = Date.now()): string[] {
|
export function weightedDesired(rows: LbIpRow[], nowMs = Date.now()): string[] {
|
||||||
if (rows.length === 0) return [];
|
if (rows.length === 0) return [];
|
||||||
const healthy = rows.filter((r) => isHealthy(r.health));
|
const live = rows.filter((r) => isPoolMember(r.health));
|
||||||
const pool = healthy.length > 0 ? healthy : rows;
|
const pool = live.length > 0 ? live : rows;
|
||||||
if (pool.length === 1) return [pool[0]!.ip];
|
if (pool.length === 1) return [pool[0]!.ip];
|
||||||
|
|
||||||
const sorted = [...pool].sort((a, b) => a.ip.localeCompare(b.ip));
|
const sorted = [...pool].sort((a, b) => a.ip.localeCompare(b.ip));
|
||||||
|
|||||||
@@ -29,8 +29,12 @@ import * as domainService from "./domain-service.js";
|
|||||||
import { syncServiceToVpsTracker } from "./vps-tracker-sync.js";
|
import { syncServiceToVpsTracker } from "./vps-tracker-sync.js";
|
||||||
import { fireEnsureHealthWorker, DEFAULT_HEALTH_FALLBACKS } from "./health/health-worker-deploy.js";
|
import { fireEnsureHealthWorker, DEFAULT_HEALTH_FALLBACKS } from "./health/health-worker-deploy.js";
|
||||||
import {
|
import {
|
||||||
isHealthy,
|
canApplyLb,
|
||||||
|
isPoolMember,
|
||||||
|
isSharedPool,
|
||||||
|
resolveDesiredAIps,
|
||||||
selectActiveIpsByMode,
|
selectActiveIpsByMode,
|
||||||
|
shouldRecordFailoverDnsDiff,
|
||||||
withBindingLock,
|
withBindingLock,
|
||||||
WEIGHTED_DNS_TTL,
|
WEIGHTED_DNS_TTL,
|
||||||
type LbIpRow,
|
type LbIpRow,
|
||||||
@@ -38,12 +42,24 @@ import {
|
|||||||
} from "./routing/index.js";
|
} from "./routing/index.js";
|
||||||
|
|
||||||
export type { LbIpRow, LbTargetConfig };
|
export type { LbIpRow, LbTargetConfig };
|
||||||
export { selectActiveIpsByMode };
|
export {
|
||||||
|
canApplyLb,
|
||||||
|
resolveDesiredAIps,
|
||||||
|
selectActiveIpsByMode,
|
||||||
|
shouldRecordFailoverDnsDiff,
|
||||||
|
};
|
||||||
|
|
||||||
const AUTO_DNS_TTL = 1;
|
const AUTO_DNS_TTL = 1;
|
||||||
|
|
||||||
function ttlForLbMode(mode: LbMode): number {
|
function ttlForBinding(mode: LbMode, ips: readonly string[]): number {
|
||||||
return mode === "weighted" ? WEIGHTED_DNS_TTL : AUTO_DNS_TTL;
|
return mode === "weighted" && isSharedPool(ips) ? WEIGHTED_DNS_TTL : AUTO_DNS_TTL;
|
||||||
|
}
|
||||||
|
|
||||||
|
function enabledServiceIps(db: Db, serviceId: number): string[] {
|
||||||
|
return repos
|
||||||
|
.listServiceIpRows(db, serviceId)
|
||||||
|
.filter((row) => row.enabled)
|
||||||
|
.map((row) => row.ip);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function failoverARecordDiff(
|
export function failoverARecordDiff(
|
||||||
@@ -72,6 +88,22 @@ function recordFailoverDnsDiff(
|
|||||||
const { added, removed } = failoverARecordDiff(existingA, desiredIps);
|
const { added, removed } = failoverARecordDiff(existingA, desiredIps);
|
||||||
if (added.length === 0 && removed.length === 0) return;
|
if (added.length === 0 && removed.length === 0) return;
|
||||||
const binding = repos.getBinding(db, bindingId);
|
const binding = repos.getBinding(db, bindingId);
|
||||||
|
const configuredIps = repos.listBindingIps(db, bindingId);
|
||||||
|
const { config, rows } = getBindingLbState(db, bindingId);
|
||||||
|
const downIps = new Set(
|
||||||
|
rows.filter((row) => row.health === "down").map((row) => row.ip),
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
!shouldRecordFailoverDnsDiff({
|
||||||
|
configuredIps,
|
||||||
|
lbMode: config.lb_mode,
|
||||||
|
added,
|
||||||
|
removed,
|
||||||
|
downIps,
|
||||||
|
})
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
repos.insertFailoverLog(db, {
|
repos.insertFailoverLog(db, {
|
||||||
serviceId: binding.service_id,
|
serviceId: binding.service_id,
|
||||||
bindingId,
|
bindingId,
|
||||||
@@ -255,7 +287,7 @@ function getGroupLbState(
|
|||||||
} else {
|
} else {
|
||||||
existing.weight += weight;
|
existing.weight += weight;
|
||||||
existing.priority = Math.min(existing.priority, priority);
|
existing.priority = Math.min(existing.priority, priority);
|
||||||
if (isHealthy(existing.health) && status && !isHealthy(status.status as IpHealthState)) {
|
if (isPoolMember(existing.health) && status && !isPoolMember(status.status as IpHealthState)) {
|
||||||
existing.health = status.status as IpHealthState;
|
existing.health = status.status as IpHealthState;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -271,33 +303,27 @@ function getGroupLbState(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function computeActiveIps(
|
|
||||||
db: Db,
|
|
||||||
scope: HealthCheckScope,
|
|
||||||
refId: number,
|
|
||||||
): string[] {
|
|
||||||
const state =
|
|
||||||
scope === "binding"
|
|
||||||
? getBindingLbState(db, refId)
|
|
||||||
: getGroupLbState(db, refId);
|
|
||||||
return selectActiveIpsByMode(state.config, state.rows);
|
|
||||||
}
|
|
||||||
|
|
||||||
function desiredAIps(
|
function desiredAIps(
|
||||||
db: Db,
|
db: Db,
|
||||||
scope: HealthCheckScope,
|
scope: HealthCheckScope,
|
||||||
refId: number,
|
refId: number,
|
||||||
fallbackIps: string[],
|
fallbackIps: string[],
|
||||||
): string[] {
|
): string[] {
|
||||||
const config =
|
const state =
|
||||||
scope === "binding"
|
scope === "binding"
|
||||||
? getBindingLbState(db, refId).config
|
? getBindingLbState(db, refId)
|
||||||
: getGroupLbState(db, refId).config;
|
: getGroupLbState(db, refId);
|
||||||
if (config.lb_mode === "weighted" || config.health_check_enabled) {
|
const serviceIps =
|
||||||
const activeIps = computeActiveIps(db, scope, refId);
|
scope === "binding"
|
||||||
if (activeIps.length > 0) return activeIps;
|
? enabledServiceIps(db, repos.getBinding(db, refId).service_id)
|
||||||
}
|
: fallbackIps;
|
||||||
return fallbackIps;
|
return resolveDesiredAIps(
|
||||||
|
state.config,
|
||||||
|
state.rows,
|
||||||
|
fallbackIps,
|
||||||
|
Date.now(),
|
||||||
|
serviceIps,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function collectKnownZones(
|
async function collectKnownZones(
|
||||||
@@ -349,7 +375,7 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
|
|||||||
const { config, rows } = getBindingLbState(db, binding.id);
|
const { config, rows } = getBindingLbState(db, binding.id);
|
||||||
const bindingActiveIps = targetCname
|
const bindingActiveIps = targetCname
|
||||||
? []
|
? []
|
||||||
: selectActiveIpsByMode(config, rows);
|
: resolveDesiredAIps(config, rows, targetIps, Date.now(), ips);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
binding_id: binding.id,
|
binding_id: binding.id,
|
||||||
@@ -470,6 +496,21 @@ function fallbackCnameHealth(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function overlayLiveHealth(
|
||||||
|
stored: IpHealthState | undefined,
|
||||||
|
live: IpHealthState | undefined,
|
||||||
|
): IpHealthState {
|
||||||
|
if (live && live !== "unknown") return live;
|
||||||
|
return stored ?? live ?? "unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
function bestAliveDisplayStatus(statuses: readonly string[]): IpHealthState {
|
||||||
|
if (statuses.some((status) => status === "up")) return "up";
|
||||||
|
if (statuses.some((status) => status === "degraded")) return "degraded";
|
||||||
|
if (statuses.some((status) => status === "down")) return "down";
|
||||||
|
return "unknown";
|
||||||
|
}
|
||||||
|
|
||||||
function attachServiceHealth(
|
function attachServiceHealth(
|
||||||
db: Db,
|
db: Db,
|
||||||
views: ServiceView[],
|
views: ServiceView[],
|
||||||
@@ -477,10 +518,13 @@ function attachServiceHealth(
|
|||||||
const ids = views.map((v) => v.id);
|
const ids = views.map((v) => v.id);
|
||||||
const healthByService = repos.aggregateIpHealthByServiceIds(db, ids);
|
const healthByService = repos.aggregateIpHealthByServiceIds(db, ids);
|
||||||
const ipHealthByService = repos.listIpHealthByServiceIds(db, ids);
|
const ipHealthByService = repos.listIpHealthByServiceIds(db, ids);
|
||||||
|
const liveByService = repos.listLatestLiveHealthByServiceIds(db, ids);
|
||||||
return views.map((view) => {
|
return views.map((view) => {
|
||||||
const health = healthByService.get(view.id);
|
const health = healthByService.get(view.id);
|
||||||
const rows = ipHealthByService.get(view.id) ?? [];
|
const rows = ipHealthByService.get(view.id) ?? [];
|
||||||
|
const liveRows = liveByService.get(view.id) ?? [];
|
||||||
const byIp = new Map(rows.map((row) => [row.ip, row]));
|
const byIp = new Map(rows.map((row) => [row.ip, row]));
|
||||||
|
const liveByIp = new Map(liveRows.map((row) => [row.ip, row]));
|
||||||
const cnameFallback = fallbackCnameHealth(rows, view);
|
const cnameFallback = fallbackCnameHealth(rows, view);
|
||||||
const aRecordIps = new Set(
|
const aRecordIps = new Set(
|
||||||
(view.domains ?? []).flatMap((domain) =>
|
(view.domains ?? []).flatMap((domain) =>
|
||||||
@@ -489,20 +533,33 @@ function attachServiceHealth(
|
|||||||
);
|
);
|
||||||
const ip_health = (view.ips ?? []).map((ip) => {
|
const ip_health = (view.ips ?? []).map((ip) => {
|
||||||
const row = byIp.get(ip) ?? (aRecordIps.has(ip) ? undefined : cnameFallback);
|
const row = byIp.get(ip) ?? (aRecordIps.has(ip) ? undefined : cnameFallback);
|
||||||
|
const live = liveByIp.get(ip);
|
||||||
|
const status = overlayLiveHealth(row?.status, live?.status);
|
||||||
|
const extras = live && live.status !== "unknown" ? live : row;
|
||||||
return {
|
return {
|
||||||
ip,
|
ip,
|
||||||
status: row?.status ?? ("unknown" as const),
|
status,
|
||||||
latency_ms: row?.latency_ms ?? null,
|
latency_ms: extras?.latency_ms ?? null,
|
||||||
last_checked_at: row?.last_checked_at ?? null,
|
last_checked_at: extras?.last_checked_at ?? null,
|
||||||
last_error: row?.last_error ?? null,
|
last_error:
|
||||||
provider: row?.provider ?? "local",
|
live && live.status !== "unknown"
|
||||||
colo: row?.colo ?? null,
|
? live.last_error
|
||||||
|
: (row?.last_error ?? null),
|
||||||
|
provider: extras?.provider ?? "local",
|
||||||
|
colo: extras?.colo ?? null,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
const displayStatus = bestAliveDisplayStatus(ip_health.map((row) => row.status));
|
||||||
|
const latencyRow =
|
||||||
|
ip_health.find((row) => row.status === displayStatus && row.latency_ms != null) ??
|
||||||
|
ip_health.find((row) => row.latency_ms != null);
|
||||||
return {
|
return {
|
||||||
...view,
|
...view,
|
||||||
health_status: health?.health_status ?? "unknown",
|
health_status: overlayLiveHealth(health?.health_status, displayStatus),
|
||||||
health_latency_ms: health?.health_latency_ms ?? null,
|
health_latency_ms:
|
||||||
|
displayStatus !== "unknown"
|
||||||
|
? (latencyRow?.latency_ms ?? null)
|
||||||
|
: (health?.health_latency_ms ?? null),
|
||||||
ip_health,
|
ip_health,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
@@ -638,6 +695,7 @@ async function syncBindingDns(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const binding = repos.getBinding(db, bindingId);
|
const binding = repos.getBinding(db, bindingId);
|
||||||
|
const configuredIps = repos.listBindingIps(db, bindingId);
|
||||||
await syncBindingADns(
|
await syncBindingADns(
|
||||||
db,
|
db,
|
||||||
cf,
|
cf,
|
||||||
@@ -645,7 +703,7 @@ async function syncBindingDns(
|
|||||||
domainId,
|
domainId,
|
||||||
hostname,
|
hostname,
|
||||||
desiredIps,
|
desiredIps,
|
||||||
ttlForLbMode(binding.lb_mode),
|
ttlForBinding(binding.lb_mode, configuredIps),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1208,7 +1266,7 @@ async function syncGroupDomainDns(
|
|||||||
domainId,
|
domainId,
|
||||||
hostname,
|
hostname,
|
||||||
desiredIps,
|
desiredIps,
|
||||||
ttlForLbMode(group.lb_mode),
|
ttlForBinding(group.lb_mode, fallbackIps),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1701,9 +1759,12 @@ export async function reconcileDnsForTarget(
|
|||||||
const cnameTarget = binding.cname_target?.trim() || null;
|
const cnameTarget = binding.cname_target?.trim() || null;
|
||||||
if (cnameTarget) return;
|
if (cnameTarget) return;
|
||||||
const ips = repos.listServiceIps(db, service.id);
|
const ips = repos.listServiceIps(db, service.id);
|
||||||
|
const poolIps = enabledServiceIps(db, service.id);
|
||||||
const targetIps = repos.listBindingIps(db, binding.id);
|
const targetIps = repos.listBindingIps(db, binding.id);
|
||||||
validateTargetIpsInPool(targetIps, ips);
|
validateTargetIpsInPool(targetIps, ips);
|
||||||
const desiredIps = desiredAIps(db, "binding", refId, targetIps);
|
const desiredIps = canApplyLb(poolIps, targetIps)
|
||||||
|
? desiredAIps(db, "binding", refId, targetIps)
|
||||||
|
: targetIps;
|
||||||
await syncBindingDns(
|
await syncBindingDns(
|
||||||
db,
|
db,
|
||||||
cf,
|
cf,
|
||||||
@@ -1735,6 +1796,7 @@ export async function reconcileWeightedDns(
|
|||||||
for (const binding of repos.listAllBindings(db)) {
|
for (const binding of repos.listAllBindings(db)) {
|
||||||
if (binding.lb_mode !== "weighted") continue;
|
if (binding.lb_mode !== "weighted") continue;
|
||||||
if (binding.cname_target?.trim()) continue;
|
if (binding.cname_target?.trim()) continue;
|
||||||
|
if (!isSharedPool(binding.target_ips ?? [])) continue;
|
||||||
try {
|
try {
|
||||||
await withBindingLock(binding.id, async () => {
|
await withBindingLock(binding.id, async () => {
|
||||||
const latest = repos.getBinding(db, binding.id);
|
const latest = repos.getBinding(db, binding.id);
|
||||||
@@ -1743,7 +1805,8 @@ export async function reconcileWeightedDns(
|
|||||||
const service = repos.getService(db, latest.service_id);
|
const service = repos.getService(db, latest.service_id);
|
||||||
if (!shouldPushDns(db, service)) return;
|
if (!shouldPushDns(db, service)) return;
|
||||||
const targetIps = repos.listBindingIps(db, latest.id);
|
const targetIps = repos.listBindingIps(db, latest.id);
|
||||||
if (targetIps.length === 0) return;
|
const poolIps = enabledServiceIps(db, service.id);
|
||||||
|
if (!canApplyLb(poolIps, targetIps)) return;
|
||||||
const ips = repos.listServiceIps(db, service.id);
|
const ips = repos.listServiceIps(db, service.id);
|
||||||
validateTargetIpsInPool(targetIps, ips);
|
validateTargetIpsInPool(targetIps, ips);
|
||||||
const desiredIps = desiredAIps(db, "binding", latest.id, targetIps);
|
const desiredIps = desiredAIps(db, "binding", latest.id, targetIps);
|
||||||
|
|||||||
@@ -436,4 +436,46 @@ describe("CNAME health mapped onto service IPs", () => {
|
|||||||
const view = await getView(db, service.id);
|
const view = await getView(db, service.id);
|
||||||
expect(view.health_status).toBe("up");
|
expect(view.health_status).toBe("up");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("getView shows live OK over hysteresis unknown", async () => {
|
||||||
|
const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
|
||||||
|
const { getView } = await import("../src/services/service-config-service.js");
|
||||||
|
const { db, sqlite } = createMemoryDb();
|
||||||
|
runMigrations(sqlite);
|
||||||
|
|
||||||
|
const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
|
||||||
|
const service = repos.createService(db, "RW Panel", "rw-panel");
|
||||||
|
repos.replaceServiceIps(db, service.id, ["2.59.161.102"]);
|
||||||
|
const binding = repos.insertBinding(db, domain.id, service.id, "c", null);
|
||||||
|
repos.replaceBindingIpsWithMeta(db, binding.id, [
|
||||||
|
{ ip: "2.59.161.102", weight: 1, priority: 1 },
|
||||||
|
]);
|
||||||
|
repos.upsertIpHealthStatus(
|
||||||
|
db,
|
||||||
|
"binding",
|
||||||
|
binding.id,
|
||||||
|
"2.59.161.102",
|
||||||
|
"unknown",
|
||||||
|
63,
|
||||||
|
0,
|
||||||
|
null,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
repos.insertHealthProbeLog(db, {
|
||||||
|
scope: "binding",
|
||||||
|
refId: binding.id,
|
||||||
|
ip: "2.59.161.102",
|
||||||
|
provider: "local",
|
||||||
|
status: "up",
|
||||||
|
ok: true,
|
||||||
|
latencyMs: 63,
|
||||||
|
colo: null,
|
||||||
|
error: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
const view = await getView(db, service.id);
|
||||||
|
expect(view.health_status).toBe("up");
|
||||||
|
expect(view.ip_health[0]?.status).toBe("up");
|
||||||
|
expect(view.ip_health[0]?.latency_ms).toBe(63);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import {
|
import {
|
||||||
|
resolveDesiredAIps,
|
||||||
selectActiveIpsByMode,
|
selectActiveIpsByMode,
|
||||||
|
shouldRecordFailoverDnsDiff,
|
||||||
type LbIpRow,
|
type LbIpRow,
|
||||||
type LbTargetConfig,
|
type LbTargetConfig,
|
||||||
} from "../src/services/service-config-service.js";
|
} from "../src/services/service-config-service.js";
|
||||||
@@ -68,6 +70,18 @@ describe("selectActiveIpsByMode", () => {
|
|||||||
expect(selectActiveIpsByMode(config, rows)).toEqual(["1.1.1.1"]);
|
expect(selectActiveIpsByMode(config, rows)).toEqual(["1.1.1.1"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("failover returns recovering unknown primary immediately", () => {
|
||||||
|
const config: LbTargetConfig = {
|
||||||
|
lb_mode: "failover",
|
||||||
|
health_check_enabled: true,
|
||||||
|
};
|
||||||
|
const rows = [
|
||||||
|
row("1.1.1.1", { priority: 1, health: "unknown" }),
|
||||||
|
row("2.2.2.2", { priority: 2, health: "up" }),
|
||||||
|
];
|
||||||
|
expect(selectActiveIpsByMode(config, rows)).toEqual(["1.1.1.1"]);
|
||||||
|
});
|
||||||
|
|
||||||
it("failover falls back to min-priority ip among all when none healthy", () => {
|
it("failover falls back to min-priority ip among all when none healthy", () => {
|
||||||
const config: LbTargetConfig = {
|
const config: LbTargetConfig = {
|
||||||
lb_mode: "failover",
|
lb_mode: "failover",
|
||||||
@@ -103,6 +117,17 @@ describe("selectActiveIpsByMode", () => {
|
|||||||
).toEqual(["1.1.1.1"]);
|
).toEqual(["1.1.1.1"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("weighted includes recovering unknown in the cycle", () => {
|
||||||
|
const rows = [
|
||||||
|
row("1.1.1.1", { weight: 1, health: "up" }),
|
||||||
|
row("2.2.2.2", { weight: 3, health: "unknown" }),
|
||||||
|
];
|
||||||
|
expect(selectActiveIpsByMode(weightedConfig, rows, 0)).toEqual(["1.1.1.1"]);
|
||||||
|
expect(
|
||||||
|
selectActiveIpsByMode(weightedConfig, rows, WEIGHTED_SLOT_MS),
|
||||||
|
).toEqual(["2.2.2.2"]);
|
||||||
|
});
|
||||||
|
|
||||||
it("weighted with one ip always returns that ip", () => {
|
it("weighted with one ip always returns that ip", () => {
|
||||||
expect(
|
expect(
|
||||||
selectActiveIpsByMode(weightedConfig, [row("1.1.1.1", { weight: 5 })], 0),
|
selectActiveIpsByMode(weightedConfig, [row("1.1.1.1", { weight: 5 })], 0),
|
||||||
@@ -124,7 +149,7 @@ describe("selectActiveIpsByMode", () => {
|
|||||||
expect(selectActiveIpsByMode(weightedConfig, [], 0)).toEqual([]);
|
expect(selectActiveIpsByMode(weightedConfig, [], 0)).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("round_robin excludes unknown when another ip is up", () => {
|
it("round_robin puts recovering unknown back with live ips", () => {
|
||||||
const config: LbTargetConfig = {
|
const config: LbTargetConfig = {
|
||||||
lb_mode: "round_robin",
|
lb_mode: "round_robin",
|
||||||
health_check_enabled: true,
|
health_check_enabled: true,
|
||||||
@@ -133,7 +158,25 @@ describe("selectActiveIpsByMode", () => {
|
|||||||
row("1.1.1.1", { health: "up" }),
|
row("1.1.1.1", { health: "up" }),
|
||||||
row("2.2.2.2", { health: "unknown" }),
|
row("2.2.2.2", { health: "unknown" }),
|
||||||
];
|
];
|
||||||
expect(selectActiveIpsByMode(config, rows)).toEqual(["1.1.1.1"]);
|
expect(selectActiveIpsByMode(config, rows).sort()).toEqual([
|
||||||
|
"1.1.1.1",
|
||||||
|
"2.2.2.2",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("round_robin keeps degraded in the pool with live ips", () => {
|
||||||
|
const config: LbTargetConfig = {
|
||||||
|
lb_mode: "round_robin",
|
||||||
|
health_check_enabled: true,
|
||||||
|
};
|
||||||
|
const rows = [
|
||||||
|
row("1.1.1.1", { health: "up" }),
|
||||||
|
row("2.2.2.2", { health: "degraded" }),
|
||||||
|
];
|
||||||
|
expect(selectActiveIpsByMode(config, rows).sort()).toEqual([
|
||||||
|
"1.1.1.1",
|
||||||
|
"2.2.2.2",
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns empty array for no rows", () => {
|
it("returns empty array for no rows", () => {
|
||||||
@@ -144,3 +187,115 @@ describe("selectActiveIpsByMode", () => {
|
|||||||
expect(selectActiveIpsByMode(config, [])).toEqual([]);
|
expect(selectActiveIpsByMode(config, [])).toEqual([]);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("resolveDesiredAIps", () => {
|
||||||
|
it("keeps a dedicated single IP even when down", () => {
|
||||||
|
expect(
|
||||||
|
resolveDesiredAIps(
|
||||||
|
weightedConfig,
|
||||||
|
[row("1.1.1.1", { health: "down" })],
|
||||||
|
["1.1.1.1"],
|
||||||
|
),
|
||||||
|
).toEqual(["1.1.1.1"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not drain when the service has only one unique IP", () => {
|
||||||
|
expect(
|
||||||
|
resolveDesiredAIps(
|
||||||
|
weightedConfig,
|
||||||
|
[row("1.1.1.1", { health: "down" })],
|
||||||
|
["1.1.1.1", "1.1.1.1"],
|
||||||
|
0,
|
||||||
|
["1.1.1.1"],
|
||||||
|
),
|
||||||
|
).toEqual(["1.1.1.1", "1.1.1.1"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not apply overlay when service pool is a single IP", () => {
|
||||||
|
const rows = [
|
||||||
|
row("1.1.1.1", { health: "up" }),
|
||||||
|
row("2.2.2.2", { health: "down" }),
|
||||||
|
];
|
||||||
|
expect(
|
||||||
|
resolveDesiredAIps(
|
||||||
|
weightedConfig,
|
||||||
|
rows,
|
||||||
|
["1.1.1.1", "2.2.2.2"],
|
||||||
|
0,
|
||||||
|
["1.1.1.1"],
|
||||||
|
),
|
||||||
|
).toEqual(["1.1.1.1", "2.2.2.2"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("applies weighted overlay on a shared pool", () => {
|
||||||
|
const rows = [
|
||||||
|
row("1.1.1.1", { weight: 1, health: "up" }),
|
||||||
|
row("2.2.2.2", { weight: 3, health: "up" }),
|
||||||
|
];
|
||||||
|
expect(
|
||||||
|
resolveDesiredAIps(weightedConfig, rows, ["1.1.1.1", "2.2.2.2"], 0),
|
||||||
|
).toEqual(selectActiveIpsByMode(weightedConfig, rows, 0));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("shouldRecordFailoverDnsDiff", () => {
|
||||||
|
it("skips duplicate listings of the same IP", () => {
|
||||||
|
expect(
|
||||||
|
shouldRecordFailoverDnsDiff({
|
||||||
|
configuredIps: ["1.1.1.1", "1.1.1.1"],
|
||||||
|
lbMode: "failover",
|
||||||
|
added: [],
|
||||||
|
removed: ["1.1.1.1"],
|
||||||
|
downIps: new Set(["1.1.1.1"]),
|
||||||
|
}),
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips dedicated extra-FQDN", () => {
|
||||||
|
expect(
|
||||||
|
shouldRecordFailoverDnsDiff({
|
||||||
|
configuredIps: ["1.1.1.1"],
|
||||||
|
lbMode: "failover",
|
||||||
|
added: [],
|
||||||
|
removed: ["1.1.1.1"],
|
||||||
|
downIps: new Set(["1.1.1.1"]),
|
||||||
|
}),
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips weighted live-to-live slot swap", () => {
|
||||||
|
expect(
|
||||||
|
shouldRecordFailoverDnsDiff({
|
||||||
|
configuredIps: ["1.1.1.1", "2.2.2.2"],
|
||||||
|
lbMode: "weighted",
|
||||||
|
added: ["2.2.2.2"],
|
||||||
|
removed: ["1.1.1.1"],
|
||||||
|
downIps: new Set(),
|
||||||
|
}),
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("logs weighted swap when a down ip leaves the pool", () => {
|
||||||
|
expect(
|
||||||
|
shouldRecordFailoverDnsDiff({
|
||||||
|
configuredIps: ["1.1.1.1", "2.2.2.2"],
|
||||||
|
lbMode: "weighted",
|
||||||
|
added: ["2.2.2.2"],
|
||||||
|
removed: ["1.1.1.1"],
|
||||||
|
downIps: new Set(["1.1.1.1"]),
|
||||||
|
}),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("logs failover diffs on a shared pool", () => {
|
||||||
|
expect(
|
||||||
|
shouldRecordFailoverDnsDiff({
|
||||||
|
configuredIps: ["1.1.1.1", "2.2.2.2"],
|
||||||
|
lbMode: "failover",
|
||||||
|
added: ["2.2.2.2"],
|
||||||
|
removed: ["1.1.1.1"],
|
||||||
|
downIps: new Set(["1.1.1.1"]),
|
||||||
|
}),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -63,8 +63,8 @@ export function FailoverTimeline({
|
|||||||
return (
|
return (
|
||||||
<EmptyState
|
<EmptyState
|
||||||
icon={ShieldCheckIcon}
|
icon={ShieldCheckIcon}
|
||||||
title="Нет инцидентов Failover"
|
title="Нет инцидентов"
|
||||||
description="Нет Down и нет выходов из пула"
|
description="Нет Down и нет выходов из общего пула"
|
||||||
stackedIcon={false}
|
stackedIcon={false}
|
||||||
centered={false}
|
centered={false}
|
||||||
/>
|
/>
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import {
|
|||||||
type HealthLogProbe,
|
type HealthLogProbe,
|
||||||
type HealthLogStatus,
|
type HealthLogStatus,
|
||||||
} from '@/lib/health-log'
|
} from '@/lib/health-log'
|
||||||
import type { FailoverLogEntry } from '@/lib/schemas'
|
import type { FailoverLogEntry, ServiceView } from '@/lib/schemas'
|
||||||
import { Badge } from '@/components/reui/badge'
|
import { Badge } from '@/components/reui/badge'
|
||||||
import {
|
import {
|
||||||
Frame,
|
Frame,
|
||||||
@@ -28,6 +28,23 @@ import {
|
|||||||
AlertTitle,
|
AlertTitle,
|
||||||
} from '@/components/reui/alert'
|
} from '@/components/reui/alert'
|
||||||
|
|
||||||
|
type LbMode = ServiceView['lb_mode']
|
||||||
|
|
||||||
|
const PANEL_COPY: Record<LbMode, { title: string; description: string }> = {
|
||||||
|
failover: {
|
||||||
|
title: 'Failover (приоритет)',
|
||||||
|
description: 'Down снимается с общего FQDN. История: кто вышел из пула и кто вернулся',
|
||||||
|
},
|
||||||
|
weighted: {
|
||||||
|
title: 'Веса (подмена IP)',
|
||||||
|
description: 'На общем FQDN один IP по весам. Down выводится из цикла',
|
||||||
|
},
|
||||||
|
round_robin: {
|
||||||
|
title: 'Round Robin',
|
||||||
|
description: 'На общем FQDN все живые A. Down снимается с пула',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
function failoverCountLabel(count: number): string {
|
function failoverCountLabel(count: number): string {
|
||||||
const mod10 = count % 10
|
const mod10 = count % 10
|
||||||
const mod100 = count % 100
|
const mod100 = count % 100
|
||||||
@@ -38,8 +55,17 @@ function failoverCountLabel(count: number): string {
|
|||||||
return `${count} адресов Down`
|
return `${count} адресов Down`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function alertDescription(events: { kind: string; fqdns: string[] }[]): string {
|
||||||
|
const removedCount = events.filter((event) => event.kind === 'removed').length
|
||||||
|
if (removedCount > 0) return 'Сняты с общего FQDN'
|
||||||
|
if (events.some((event) => event.fqdns.length > 0)) {
|
||||||
|
return 'Остались в A-записях общего FQDN как last-resort'
|
||||||
|
}
|
||||||
|
return 'Down: персональные FQDN не меняются'
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Failover — текущие Down + кто вышел из пула и кто вернулся.
|
* Балансировка — текущие Down + кто вышел из общего пула.
|
||||||
* Preview: https://reui.io/preview/base/components/c-timeline-10
|
* Preview: https://reui.io/preview/base/components/c-timeline-10
|
||||||
* Preview: https://reui.io/preview/base/empty-state-12
|
* Preview: https://reui.io/preview/base/empty-state-12
|
||||||
* Docs: https://reui.io/docs/components/base/frame
|
* Docs: https://reui.io/docs/components/base/frame
|
||||||
@@ -48,16 +74,19 @@ function failoverCountLabel(count: number): string {
|
|||||||
* Docs: https://reui.io/docs/components/base/alert
|
* Docs: https://reui.io/docs/components/base/alert
|
||||||
*/
|
*/
|
||||||
export function ServiceFailoverPanel({
|
export function ServiceFailoverPanel({
|
||||||
|
lbMode = 'round_robin',
|
||||||
ipHealth,
|
ipHealth,
|
||||||
bindings,
|
bindings,
|
||||||
history,
|
history,
|
||||||
probes = [],
|
probes = [],
|
||||||
}: {
|
}: {
|
||||||
|
lbMode?: LbMode
|
||||||
ipHealth: readonly FailoverHealthInput[]
|
ipHealth: readonly FailoverHealthInput[]
|
||||||
bindings: readonly FailoverBindingPool[]
|
bindings: readonly FailoverBindingPool[]
|
||||||
history: readonly FailoverLogEntry[]
|
history: readonly FailoverLogEntry[]
|
||||||
probes?: readonly HealthLogProbe[]
|
probes?: readonly HealthLogProbe[]
|
||||||
}) {
|
}) {
|
||||||
|
const copy = PANEL_COPY[lbMode] ?? PANEL_COPY.round_robin
|
||||||
const liveByIp = latestHealthByIp(probes)
|
const liveByIp = latestHealthByIp(probes)
|
||||||
const overlayHealth = ipHealth.map((row) => {
|
const overlayHealth = ipHealth.map((row) => {
|
||||||
const live = liveByIp.get(row.ip)
|
const live = liveByIp.get(row.ip)
|
||||||
@@ -77,14 +106,13 @@ export function ServiceFailoverPanel({
|
|||||||
})
|
})
|
||||||
const events = toFailoverEvents(overlayHealth, bindings)
|
const events = toFailoverEvents(overlayHealth, bindings)
|
||||||
const mergedHistory = mergeFailoverHistory(history, probes, bindings)
|
const mergedHistory = mergeFailoverHistory(history, probes, bindings)
|
||||||
const removedCount = events.filter((event) => event.kind === 'removed').length
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Frame stacked spacing="sm" className="min-w-0 w-full">
|
<Frame stacked spacing="sm" className="min-w-0 w-full">
|
||||||
<FramePanel className="flex flex-col gap-3">
|
<FramePanel className="flex flex-col gap-3">
|
||||||
<FrameHeader className="gap-1 px-0 py-0">
|
<FrameHeader className="gap-1 px-0 py-0">
|
||||||
<FrameTitle className="flex flex-wrap items-center gap-2">
|
<FrameTitle className="flex flex-wrap items-center gap-2">
|
||||||
Failover
|
{copy.title}
|
||||||
{events.length > 0 ? (
|
{events.length > 0 ? (
|
||||||
<Badge variant="destructive-light" size="xs" radius="full">
|
<Badge variant="destructive-light" size="xs" radius="full">
|
||||||
{events.length}
|
{events.length}
|
||||||
@@ -95,20 +123,14 @@ export function ServiceFailoverPanel({
|
|||||||
</Badge>
|
</Badge>
|
||||||
)}
|
)}
|
||||||
</FrameTitle>
|
</FrameTitle>
|
||||||
<FrameDescription>
|
<FrameDescription>{copy.description}</FrameDescription>
|
||||||
Текущие Down и история: кто вышел из пула и кто вернулся
|
|
||||||
</FrameDescription>
|
|
||||||
</FrameHeader>
|
</FrameHeader>
|
||||||
|
|
||||||
{events.length > 0 ? (
|
{events.length > 0 ? (
|
||||||
<Alert variant="destructive">
|
<Alert variant="destructive">
|
||||||
<UnplugIcon aria-hidden="true" />
|
<UnplugIcon aria-hidden="true" />
|
||||||
<AlertTitle>{failoverCountLabel(events.length)}</AlertTitle>
|
<AlertTitle>{failoverCountLabel(events.length)}</AlertTitle>
|
||||||
<AlertDescription>
|
<AlertDescription>{alertDescription(events)}</AlertDescription>
|
||||||
{removedCount > 0
|
|
||||||
? 'Сняты с FQDN или остались last-resort'
|
|
||||||
: 'Остались в A-записях как last-resort'}
|
|
||||||
</AlertDescription>
|
|
||||||
</Alert>
|
</Alert>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
|
|||||||
@@ -51,7 +51,28 @@ describe('toFailoverEvents', () => {
|
|||||||
expect(isFailoverEventStatus('unhealthy')).toBe(false)
|
expect(isFailoverEventStatus('unhealthy')).toBe(false)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Down всегда виден, даже без A-записей', () => {
|
it('один IP у сервиса — не инцидент пула', () => {
|
||||||
|
const events = toFailoverEvents(
|
||||||
|
[
|
||||||
|
row({
|
||||||
|
ip: '10.0.0.1',
|
||||||
|
status: 'down',
|
||||||
|
consecutive_failures: 9,
|
||||||
|
last_error: 'fetch failed',
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
fqdn: 'solo.example.com',
|
||||||
|
configured: ['10.0.0.1'],
|
||||||
|
active: [],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
expect(events).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Down без shared pool не событие балансировки', () => {
|
||||||
const events = toFailoverEvents([
|
const events = toFailoverEvents([
|
||||||
row({
|
row({
|
||||||
ip: '130.49.213.153',
|
ip: '130.49.213.153',
|
||||||
@@ -60,19 +81,7 @@ describe('toFailoverEvents', () => {
|
|||||||
last_error: 'fetch failed',
|
last_error: 'fetch failed',
|
||||||
}),
|
}),
|
||||||
])
|
])
|
||||||
expect(events).toEqual([
|
expect(events).toEqual([])
|
||||||
{
|
|
||||||
id: '130.49.213.153',
|
|
||||||
address: '130.49.213.153',
|
|
||||||
status: 'down',
|
|
||||||
kind: 'last-resort',
|
|
||||||
fqdns: [],
|
|
||||||
consecutiveFailures: 9,
|
|
||||||
lastFailureReason: 'fetch failed',
|
|
||||||
lastCheckAt: null,
|
|
||||||
},
|
|
||||||
])
|
|
||||||
expect(failoverEventCopy(events[0]!)).toBe('Down, в A-записях last-resort')
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('OK вне пула не инцидент (standby)', () => {
|
it('OK вне пула не инцидент (standby)', () => {
|
||||||
@@ -137,6 +146,7 @@ describe('toFailoverEvents', () => {
|
|||||||
},
|
},
|
||||||
])
|
])
|
||||||
expect(failoverEventCopy(events[0]!)).toBe('Снята с gt.rkns.top')
|
expect(failoverEventCopy(events[0]!)).toBe('Снята с gt.rkns.top')
|
||||||
|
expect(events[0]?.fqdns).not.toContain('nsgt.rkns.top')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Down только last-resort на своём FQDN', () => {
|
it('Down только last-resort на своём FQDN', () => {
|
||||||
@@ -150,11 +160,7 @@ describe('toFailoverEvents', () => {
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
expect(events[0]?.kind).toBe('last-resort')
|
expect(events).toEqual([])
|
||||||
expect(events[0]?.fqdns).toEqual(['nsgt.rkns.top'])
|
|
||||||
expect(failoverEventCopy(events[0]!)).toBe(
|
|
||||||
'Down, в A-записях last-resort на nsgt.rkns.top',
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('down без A-записи на configured FQDN — снятие', () => {
|
it('down без A-записи на configured FQDN — снятие', () => {
|
||||||
@@ -279,7 +285,9 @@ describe('mergeFailoverHistory', () => {
|
|||||||
'removed:probe',
|
'removed:probe',
|
||||||
])
|
])
|
||||||
expect(history[0]?.copy).toBe('130.49.213.153 добавлена на gt.rkns.top')
|
expect(history[0]?.copy).toBe('130.49.213.153 добавлена на gt.rkns.top')
|
||||||
expect(history[1]?.copy).toContain('вышла из пула')
|
expect(history[1]?.copy).toBe(
|
||||||
|
'130.49.213.153 вышла из пула (gt.rkns.top)',
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('keeps DNS over a probe return in the same 2-minute window', () => {
|
it('keeps DNS over a probe return in the same 2-minute window', () => {
|
||||||
@@ -291,4 +299,38 @@ describe('mergeFailoverHistory', () => {
|
|||||||
expect(history).toHaveLength(1)
|
expect(history).toHaveLength(1)
|
||||||
expect(history[0]?.source).toBe('dns')
|
expect(history[0]?.source).toBe('dns')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('drops probe leave/return when the service has no shared pool', () => {
|
||||||
|
const history = mergeFailoverHistory(
|
||||||
|
[],
|
||||||
|
[
|
||||||
|
probe({ id: 1, status: 'up', checked_at: '2026-08-20T09:00:00Z' }),
|
||||||
|
probe({ id: 2, status: 'down', checked_at: '2026-08-20T09:10:00Z' }),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
fqdn: 'solo.example.com',
|
||||||
|
configured: ['130.49.213.153'],
|
||||||
|
active: ['130.49.213.153'],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
expect(history).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('drops dedicated extra-FQDN DNS rows', () => {
|
||||||
|
const history = mergeFailoverHistory(
|
||||||
|
[
|
||||||
|
dns({
|
||||||
|
id: 11,
|
||||||
|
fqdn: 'nsgt.rkns.top',
|
||||||
|
action: 'added',
|
||||||
|
created_at: '2026-08-20 09:26:00',
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
[],
|
||||||
|
mskHip,
|
||||||
|
)
|
||||||
|
expect(history).toEqual([])
|
||||||
|
})
|
||||||
})
|
})
|
||||||
@@ -44,6 +44,22 @@ export interface FailoverBindingPool {
|
|||||||
active: readonly string[]
|
active: readonly string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Unique A targets. Duplicates of the same IP are not a pool. */
|
||||||
|
export function uniqueIpCount(ips: readonly string[]): number {
|
||||||
|
return new Set(ips.filter(Boolean)).size
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Shared pool FQDN — two or more unique A targets. */
|
||||||
|
export function isSharedPoolBinding(binding: FailoverBindingPool): boolean {
|
||||||
|
return uniqueIpCount(binding.configured) >= 2
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasSharedPool(
|
||||||
|
bindings: readonly FailoverBindingPool[],
|
||||||
|
): boolean {
|
||||||
|
return bindings.some(isSharedPoolBinding)
|
||||||
|
}
|
||||||
|
|
||||||
/** Инцидент только при down. up / degraded / unknown — не вывод из пула. */
|
/** Инцидент только при down. up / degraded / unknown — не вывод из пула. */
|
||||||
export function isFailoverEventStatus(status: string): boolean {
|
export function isFailoverEventStatus(status: string): boolean {
|
||||||
return status === 'down'
|
return status === 'down'
|
||||||
@@ -61,17 +77,19 @@ export function failoverEventCopy(event: FailoverEvent): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Текущие Down всегда в панели. Per-FQDN: снята с hostname или last-resort.
|
* Инциденты пула только если у сервиса есть shared FQDN (2+ уникальных IP).
|
||||||
* Standby (up / degraded / unknown) — не инцидент.
|
* Один IP — не балансировка и не вывод из пула; Down смотрит health-монитор.
|
||||||
*/
|
*/
|
||||||
export function toFailoverEvents(
|
export function toFailoverEvents(
|
||||||
ipHealth: readonly FailoverHealthInput[],
|
ipHealth: readonly FailoverHealthInput[],
|
||||||
bindings: readonly FailoverBindingPool[] = [],
|
bindings: readonly FailoverBindingPool[] = [],
|
||||||
): FailoverEvent[] {
|
): FailoverEvent[] {
|
||||||
|
if (!hasSharedPool(bindings)) return []
|
||||||
return ipHealth.filter((row) => isFailoverEventStatus(row.status)).map((row) => {
|
return ipHealth.filter((row) => isFailoverEventStatus(row.status)).map((row) => {
|
||||||
const removedFqdns: string[] = []
|
const removedFqdns: string[] = []
|
||||||
const lastResortFqdns: string[] = []
|
const lastResortFqdns: string[] = []
|
||||||
for (const binding of bindings) {
|
for (const binding of bindings) {
|
||||||
|
if (!isSharedPoolBinding(binding)) continue
|
||||||
const configured = binding.configured.includes(row.ip)
|
const configured = binding.configured.includes(row.ip)
|
||||||
const active = binding.active.includes(row.ip)
|
const active = binding.active.includes(row.ip)
|
||||||
if (configured && !active) removedFqdns.push(binding.fqdn)
|
if (configured && !active) removedFqdns.push(binding.fqdn)
|
||||||
@@ -98,7 +116,18 @@ function fqdnsForIp(
|
|||||||
ip: string,
|
ip: string,
|
||||||
bindings: readonly FailoverBindingPool[],
|
bindings: readonly FailoverBindingPool[],
|
||||||
): string[] {
|
): string[] {
|
||||||
return bindings.filter((binding) => binding.configured.includes(ip)).map((binding) => binding.fqdn)
|
return bindings
|
||||||
|
.filter((binding) => isSharedPoolBinding(binding) && binding.configured.includes(ip))
|
||||||
|
.map((binding) => binding.fqdn)
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPoolFqdn(
|
||||||
|
fqdn: string,
|
||||||
|
bindings: readonly FailoverBindingPool[],
|
||||||
|
): boolean {
|
||||||
|
const binding = bindings.find((item) => item.fqdn === fqdn)
|
||||||
|
if (!binding) return true
|
||||||
|
return isSharedPoolBinding(binding)
|
||||||
}
|
}
|
||||||
|
|
||||||
function fqdnLabel(fqdns: string[]): string {
|
function fqdnLabel(fqdns: string[]): string {
|
||||||
@@ -201,10 +230,12 @@ export function mergeFailoverHistory(
|
|||||||
bindings: readonly FailoverBindingPool[] = [],
|
bindings: readonly FailoverBindingPool[] = [],
|
||||||
): FailoverHistoryItem[] {
|
): FailoverHistoryItem[] {
|
||||||
const merged = [
|
const merged = [
|
||||||
...dns.map(dnsHistoryItem),
|
...dns
|
||||||
...toIpAliveTransitions(probes).map((transition) =>
|
.filter((item) => isPoolFqdn(item.fqdn, bindings))
|
||||||
probeHistoryItem(transition, bindings),
|
.map(dnsHistoryItem),
|
||||||
),
|
...toIpAliveTransitions(probes)
|
||||||
|
.filter((transition) => fqdnsForIp(transition.ip, bindings).length > 0)
|
||||||
|
.map((transition) => probeHistoryItem(transition, bindings)),
|
||||||
]
|
]
|
||||||
merged.sort(
|
merged.sort(
|
||||||
(a, b) => eventTime(b.created_at) - eventTime(a.created_at) || a.id.localeCompare(b.id),
|
(a, b) => eventTime(b.created_at) - eventTime(a.created_at) || a.id.localeCompare(b.id),
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import {
|
|||||||
latestHealthByIp,
|
latestHealthByIp,
|
||||||
providerHealthStatuses,
|
providerHealthStatuses,
|
||||||
resolveIpDisplayHealth,
|
resolveIpDisplayHealth,
|
||||||
|
resolveServiceDisplayHealth,
|
||||||
worstHealthStatus,
|
worstHealthStatus,
|
||||||
type HealthLogProbe,
|
type HealthLogProbe,
|
||||||
} from '@/lib/health-log'
|
} from '@/lib/health-log'
|
||||||
@@ -146,3 +147,22 @@ describe('resolveIpDisplayHealth', () => {
|
|||||||
expect(resolveIpDisplayHealth('unknown', undefined)).toBe('unknown')
|
expect(resolveIpDisplayHealth('unknown', undefined)).toBe('unknown')
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('resolveServiceDisplayHealth', () => {
|
||||||
|
it('shows OK when live probes recovered and stored is still unknown', () => {
|
||||||
|
expect(
|
||||||
|
resolveServiceDisplayHealth(
|
||||||
|
'unknown',
|
||||||
|
[{ ip: '2.59.161.102', status: 'unknown' }],
|
||||||
|
[
|
||||||
|
probe({
|
||||||
|
id: 1,
|
||||||
|
ip: '2.59.161.102',
|
||||||
|
status: 'up',
|
||||||
|
checked_at: '2026-08-20T18:00:00Z',
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
).toBe('up')
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -194,3 +194,22 @@ export function resolveIpDisplayHealth(
|
|||||||
if (live && live !== 'unknown') return live
|
if (live && live !== 'unknown') return live
|
||||||
return stored ?? live ?? 'unknown'
|
return stored ?? live ?? 'unknown'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Service KPI / card: any-up of per-IP overlay (live probes beat hysteresis). */
|
||||||
|
export function resolveServiceDisplayHealth(
|
||||||
|
stored: HealthLogStatus | undefined,
|
||||||
|
ipHealth: readonly { ip: string; status: string }[],
|
||||||
|
probes: readonly HealthLogProbe[] = [],
|
||||||
|
): HealthLogStatus {
|
||||||
|
const liveByIp = latestHealthByIp(probes)
|
||||||
|
const statuses =
|
||||||
|
ipHealth.length > 0
|
||||||
|
? ipHealth.map((row) =>
|
||||||
|
resolveIpDisplayHealth(
|
||||||
|
row.status as HealthLogStatus,
|
||||||
|
liveByIp.get(row.ip)?.status,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
: [...liveByIp.values()].map((row) => row.status)
|
||||||
|
return resolveIpDisplayHealth(stored, bestAliveHealthStatus(statuses))
|
||||||
|
}
|
||||||
@@ -36,6 +36,8 @@ export const serviceGroupsQueryOptions = () =>
|
|||||||
}
|
}
|
||||||
return parsed.data
|
return parsed.data
|
||||||
},
|
},
|
||||||
|
refetchInterval: 10_000,
|
||||||
|
staleTime: 5_000,
|
||||||
})
|
})
|
||||||
|
|
||||||
export const servicesQueryOptions = () =>
|
export const servicesQueryOptions = () =>
|
||||||
@@ -101,6 +103,8 @@ export const serviceViewQueryOptions = (id: number) =>
|
|||||||
const data = await api.get<unknown>(`/api/v1/services/${id}`)
|
const data = await api.get<unknown>(`/api/v1/services/${id}`)
|
||||||
return serviceViewSchema.parse(data)
|
return serviceViewSchema.parse(data)
|
||||||
},
|
},
|
||||||
|
refetchInterval: 10_000,
|
||||||
|
staleTime: 5_000,
|
||||||
})
|
})
|
||||||
|
|
||||||
export const serviceHealthLogQueryOptions = (id: number) =>
|
export const serviceHealthLogQueryOptions = (id: number) =>
|
||||||
@@ -110,6 +114,8 @@ export const serviceHealthLogQueryOptions = (id: number) =>
|
|||||||
const data = await api.get<unknown>(`/api/v1/services/${id}/health-log`)
|
const data = await api.get<unknown>(`/api/v1/services/${id}/health-log`)
|
||||||
return z.object({ items: z.array(healthProbeLogSchema) }).parse(data)
|
return z.object({ items: z.array(healthProbeLogSchema) }).parse(data)
|
||||||
},
|
},
|
||||||
|
refetchInterval: 10_000,
|
||||||
|
staleTime: 5_000,
|
||||||
})
|
})
|
||||||
|
|
||||||
export const serviceFailoverLogQueryOptions = (id: number) =>
|
export const serviceFailoverLogQueryOptions = (id: number) =>
|
||||||
|
|||||||
@@ -32,9 +32,11 @@ import {
|
|||||||
ServiceHealthMonitor,
|
ServiceHealthMonitor,
|
||||||
} from '@/components/reui-kit'
|
} from '@/components/reui-kit'
|
||||||
import { api } from '@/lib/api-client'
|
import { api } from '@/lib/api-client'
|
||||||
|
import { hasSharedPool } from '@/lib/failover-events'
|
||||||
import {
|
import {
|
||||||
enabledHealthProviders,
|
enabledHealthProviders,
|
||||||
providerHealthStatuses,
|
providerHealthStatuses,
|
||||||
|
resolveServiceDisplayHealth,
|
||||||
} from '@/lib/health-log'
|
} from '@/lib/health-log'
|
||||||
import type { ServiceView, UpdateServiceConfigInput } from '@/lib/schemas'
|
import type { ServiceView, UpdateServiceConfigInput } from '@/lib/schemas'
|
||||||
import {
|
import {
|
||||||
@@ -115,6 +117,22 @@ function ServiceDetailPage() {
|
|||||||
})),
|
})),
|
||||||
[service?.domains],
|
[service?.domains],
|
||||||
)
|
)
|
||||||
|
const uniqueEnabledIps = useMemo(
|
||||||
|
() =>
|
||||||
|
(service?.ips ?? []).filter((ip) => service?.ip_enabled[ip] !== false),
|
||||||
|
[service?.ips, service?.ip_enabled],
|
||||||
|
)
|
||||||
|
const displayHealth = useMemo(
|
||||||
|
() =>
|
||||||
|
resolveServiceDisplayHealth(
|
||||||
|
service?.health_status,
|
||||||
|
service?.ip_health ?? [],
|
||||||
|
logItems,
|
||||||
|
),
|
||||||
|
[service?.health_status, service?.ip_health, logItems],
|
||||||
|
)
|
||||||
|
const showPoolPanel =
|
||||||
|
uniqueEnabledIps.length >= 2 && hasSharedPool(failoverBindings)
|
||||||
const nodes = (nodesQuery.data as OverviewPayload['nodes']) ?? []
|
const nodes = (nodesQuery.data as OverviewPayload['nodes']) ?? []
|
||||||
|
|
||||||
const [editOpen, setEditOpen] = useState(false)
|
const [editOpen, setEditOpen] = useState(false)
|
||||||
@@ -251,7 +269,7 @@ function ServiceDetailPage() {
|
|||||||
actions={
|
actions={
|
||||||
<>
|
<>
|
||||||
<LbModeTile mode={service.lb_mode} />
|
<LbModeTile mode={service.lb_mode} />
|
||||||
<HealthCheckBadge status={service.health_status} />
|
<HealthCheckBadge status={displayHealth} />
|
||||||
<Tooltip>
|
<Tooltip>
|
||||||
<TooltipTrigger
|
<TooltipTrigger
|
||||||
render={
|
render={
|
||||||
@@ -278,20 +296,20 @@ function ServiceDetailPage() {
|
|||||||
id: 'status',
|
id: 'status',
|
||||||
icon: <ActivityIcon />,
|
icon: <ActivityIcon />,
|
||||||
label: 'Статус',
|
label: 'Статус',
|
||||||
value: service.health_status === 'up' ? 'OK' : service.health_status,
|
value: displayHealth === 'up' ? 'OK' : displayHealth,
|
||||||
variant:
|
variant:
|
||||||
service.health_status === 'down'
|
displayHealth === 'down'
|
||||||
? 'destructive'
|
? 'destructive'
|
||||||
: service.health_status === 'degraded'
|
: displayHealth === 'degraded'
|
||||||
? 'warning'
|
? 'warning'
|
||||||
: 'default',
|
: 'default',
|
||||||
iconClassName:
|
iconClassName:
|
||||||
service.health_status === 'down'
|
displayHealth === 'down'
|
||||||
? 'text-destructive'
|
? 'text-destructive'
|
||||||
: service.health_status === 'degraded'
|
: displayHealth === 'degraded'
|
||||||
? 'text-warning'
|
? 'text-warning'
|
||||||
: 'text-success',
|
: 'text-success',
|
||||||
hint: <HealthCheckBadge status={service.health_status} size="xs" />,
|
hint: <HealthCheckBadge status={displayHealth} size="xs" />,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: 'fqdn',
|
id: 'fqdn',
|
||||||
@@ -305,21 +323,33 @@ function ServiceDetailPage() {
|
|||||||
icon: <NetworkIcon />,
|
icon: <NetworkIcon />,
|
||||||
label: 'IP',
|
label: 'IP',
|
||||||
value: String(service.ips.length),
|
value: String(service.ips.length),
|
||||||
hint: `${service.active_ips.length} в пуле`,
|
hint: showPoolPanel
|
||||||
|
? `${service.active_ips.length} в пуле`
|
||||||
|
: uniqueEnabledIps.length <= 1
|
||||||
|
? 'без балансировки'
|
||||||
|
: service.ips.join(', ') || 'нет',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: 'pool',
|
id: 'pool',
|
||||||
icon: <ServerIcon />,
|
icon: <ServerIcon />,
|
||||||
label: 'Активный пул',
|
label: showPoolPanel ? 'Активный пул' : 'Адреса',
|
||||||
value: String((overview?.active_addresses ?? service.active_ips).length),
|
value: String(
|
||||||
hint: (overview?.active_addresses ?? service.active_ips).join(', ') || 'нет',
|
(overview?.active_addresses ?? service.active_ips).length,
|
||||||
|
),
|
||||||
|
hint:
|
||||||
|
(overview?.active_addresses ?? service.active_ips).join(', ') ||
|
||||||
|
'нет',
|
||||||
},
|
},
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<section
|
<section
|
||||||
aria-label="Мониторинг"
|
aria-label="Мониторинг"
|
||||||
className="grid min-w-0 items-start gap-2 @3xl:grid-cols-2"
|
className={
|
||||||
|
showPoolPanel
|
||||||
|
? 'grid min-w-0 items-start gap-2 @3xl:grid-cols-2'
|
||||||
|
: 'grid min-w-0 items-start gap-2'
|
||||||
|
}
|
||||||
>
|
>
|
||||||
<ServiceHealthMonitor
|
<ServiceHealthMonitor
|
||||||
items={logItems}
|
items={logItems}
|
||||||
@@ -327,12 +357,15 @@ function ServiceDetailPage() {
|
|||||||
statuses={providerStatuses}
|
statuses={providerStatuses}
|
||||||
isLoading={logQuery.isLoading}
|
isLoading={logQuery.isLoading}
|
||||||
/>
|
/>
|
||||||
<ServiceFailoverPanel
|
{showPoolPanel ? (
|
||||||
ipHealth={service.ip_health}
|
<ServiceFailoverPanel
|
||||||
bindings={failoverBindings}
|
lbMode={service.lb_mode}
|
||||||
history={failoverHistory}
|
ipHealth={service.ip_health}
|
||||||
probes={logItems}
|
bindings={failoverBindings}
|
||||||
/>
|
history={failoverHistory}
|
||||||
|
probes={logItems}
|
||||||
|
/>
|
||||||
|
) : null}
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
{service.ips.length === 0 && service.domains.length === 0 ? (
|
{service.ips.length === 0 && service.domains.length === 0 ? (
|
||||||
|
|||||||
+4
-3
@@ -44,9 +44,10 @@ health-check работают на двух уровнях:
|
|||||||
- **Change Domain** — перенос привязок между зонами `POST /api/v1/services/:id/change-domain`.
|
- **Change Domain** — перенос привязок между зонами `POST /api/v1/services/:id/change-domain`.
|
||||||
|
|
||||||
Режимы LB: `round_robin`, `failover`, `weighted`. В Cloudflare free `weighted`
|
Режимы LB: `round_robin`, `failover`, `weighted`. В Cloudflare free `weighted`
|
||||||
работает как `round_robin` (одна A на IP). `unknown` **не** считается healthy и
|
работает как `round_robin` (одна A на IP). В A-пул попадает всё, кроме **Down**
|
||||||
не попадает в пул, пока нет успешных проб; восстановление — `UNHEALTHY → CHECKING → HEALTHY`
|
(`unknown` / checking / Slow возвращаются в DNS на первой успешной пробе).
|
||||||
после `HEALTH_SUCCESS_RECOVERIES` (default 2). Пороги и cron движка задаются в
|
Бейдж Healthy — `UNHEALTHY → CHECKING → HEALTHY` после `HEALTH_SUCCESS_RECOVERIES`
|
||||||
|
(default 2). Пороги и cron движка задаются в
|
||||||
**Настройки → Health-check** (env — fallback, пока значения не сохранены в UI).
|
**Настройки → Health-check** (env — fallback, пока значения не сохранены в UI).
|
||||||
|
|
||||||
### Источники проб: Local, Cloudflare Worker, Globalping
|
### Источники проб: Local, Cloudflare Worker, Globalping
|
||||||
|
|||||||
@@ -2341,6 +2341,113 @@ export function listIpHealthByServiceIds(
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseIpHealthState(status: string | null): IpHealthState | undefined {
|
||||||
|
if (
|
||||||
|
status === "up" ||
|
||||||
|
status === "down" ||
|
||||||
|
status === "degraded" ||
|
||||||
|
status === "unknown"
|
||||||
|
) {
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function bestAliveIpState(statuses: readonly IpHealthState[]): IpHealthState {
|
||||||
|
if (statuses.some((status) => status === "up")) return "up";
|
||||||
|
if (statuses.some((status) => status === "degraded")) return "degraded";
|
||||||
|
if (statuses.some((status) => status === "down")) return "down";
|
||||||
|
return "unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Latest probe per service+IP+provider from health_probe_log, then any-up per IP.
|
||||||
|
* Display overlay — hysteresis in ip_health_status is unchanged (DNS).
|
||||||
|
*/
|
||||||
|
export function listLatestLiveHealthByServiceIds(
|
||||||
|
db: Db,
|
||||||
|
serviceIds: number[],
|
||||||
|
): Map<number, ServiceIpHealthRow[]> {
|
||||||
|
const result = new Map<number, ServiceIpHealthRow[]>();
|
||||||
|
if (serviceIds.length === 0) return result;
|
||||||
|
const idList = sql.join(
|
||||||
|
serviceIds.map((id) => sql`${id}`),
|
||||||
|
sql`, `,
|
||||||
|
);
|
||||||
|
const rows = db.all<{
|
||||||
|
service_id: number;
|
||||||
|
ip: string;
|
||||||
|
provider: string | null;
|
||||||
|
status: string | null;
|
||||||
|
latency_ms: number | null;
|
||||||
|
last_error: string | null;
|
||||||
|
colo: string | null;
|
||||||
|
checked_at: string | null;
|
||||||
|
}>(sql`
|
||||||
|
SELECT sb.service_id AS service_id,
|
||||||
|
l.ip AS ip,
|
||||||
|
l.provider AS provider,
|
||||||
|
l.status AS status,
|
||||||
|
l.latency_ms AS latency_ms,
|
||||||
|
l.error AS last_error,
|
||||||
|
l.colo AS colo,
|
||||||
|
l.checked_at AS checked_at
|
||||||
|
FROM health_probe_log l
|
||||||
|
INNER JOIN service_bindings sb
|
||||||
|
ON l.scope = 'binding' AND l.ref_id = sb.id
|
||||||
|
INNER JOIN (
|
||||||
|
SELECT sb2.service_id AS service_id,
|
||||||
|
l2.ip AS ip,
|
||||||
|
l2.provider AS provider,
|
||||||
|
MAX(l2.id) AS max_id
|
||||||
|
FROM health_probe_log l2
|
||||||
|
INNER JOIN service_bindings sb2
|
||||||
|
ON l2.scope = 'binding' AND l2.ref_id = sb2.id
|
||||||
|
WHERE sb2.service_id IN (${idList})
|
||||||
|
GROUP BY sb2.service_id, l2.ip, l2.provider
|
||||||
|
) latest
|
||||||
|
ON latest.max_id = l.id
|
||||||
|
`);
|
||||||
|
|
||||||
|
const byServiceIp = new Map<
|
||||||
|
string,
|
||||||
|
{ serviceId: number; ip: string; probes: ServiceIpHealthRow[] }
|
||||||
|
>();
|
||||||
|
for (const row of rows) {
|
||||||
|
const status = parseIpHealthState(row.status);
|
||||||
|
if (!status) continue;
|
||||||
|
const key = `${row.service_id}\0${row.ip}`;
|
||||||
|
const probe: ServiceIpHealthRow = {
|
||||||
|
ip: row.ip,
|
||||||
|
status,
|
||||||
|
latency_ms: row.latency_ms,
|
||||||
|
last_checked_at: row.checked_at,
|
||||||
|
last_error: row.last_error,
|
||||||
|
provider: normalizeStatusProvider(row.provider),
|
||||||
|
colo: row.colo,
|
||||||
|
};
|
||||||
|
const bucket = byServiceIp.get(key);
|
||||||
|
if (bucket) bucket.probes.push(probe);
|
||||||
|
else {
|
||||||
|
byServiceIp.set(key, {
|
||||||
|
serviceId: row.service_id,
|
||||||
|
ip: row.ip,
|
||||||
|
probes: [probe],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const { serviceId, ip, probes } of byServiceIp.values()) {
|
||||||
|
const status = bestAliveIpState(probes.map((probe) => probe.status));
|
||||||
|
const preferred =
|
||||||
|
probes.find((probe) => probe.status === status) ?? probes[0]!;
|
||||||
|
const list = result.get(serviceId) ?? [];
|
||||||
|
list.push({ ...preferred, ip, status });
|
||||||
|
result.set(serviceId, list);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
export function mergeHealthAggregates(
|
export function mergeHealthAggregates(
|
||||||
parts: Array<HealthAggregate | undefined | null>,
|
parts: Array<HealthAggregate | undefined | null>,
|
||||||
): HealthAggregate {
|
): HealthAggregate {
|
||||||
|
|||||||
Reference in new issue
Block a user