Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3d0ea33baf | ||
|
|
8a13888db7 | ||
|
|
f5d97e463a | ||
|
|
be8f94143f |
@@ -38,6 +38,10 @@ import {
|
||||
healthEngineFallbacksFromConfig,
|
||||
scheduleHealthCheckJob,
|
||||
} from "./services/health-check-scheduler.js";
|
||||
import {
|
||||
createWeightedDnsTask,
|
||||
scheduleWeightedDnsJob,
|
||||
} from "./services/weighted-dns-scheduler.js";
|
||||
import { fireEnsureHealthWorker } from "./services/health/health-worker-deploy.js";
|
||||
import { AsyncTask, CronJob } from "toad-scheduler";
|
||||
|
||||
@@ -133,6 +137,7 @@ export async function buildApp(opts: BuildAppOptions = {}) {
|
||||
app.decorate("reloadHealthCheckJob", () => {
|
||||
scheduleHealthCheckJob(app, config, healthTask);
|
||||
});
|
||||
scheduleWeightedDnsJob(app, createWeightedDnsTask(app));
|
||||
if (config.cloudflareApiToken) {
|
||||
fireEnsureHealthWorker(
|
||||
app.db,
|
||||
|
||||
@@ -2,25 +2,30 @@ import type { LbMode } from "@cfdm/shared";
|
||||
import { failoverDesired } from "./failover.js";
|
||||
import { roundRobinDesired } from "./round-robin.js";
|
||||
import type { LbIpRow, LbTargetConfig } from "./types.js";
|
||||
import { weightedDesired } from "./weighted.js";
|
||||
|
||||
export type { LbIpRow, LbTargetConfig } from "./types.js";
|
||||
export { isHealthy } from "./health.js";
|
||||
export { withBindingLock } from "./binding-lock.js";
|
||||
export { WEIGHTED_DNS_TTL, WEIGHTED_SLOT_MS, weightedDesired } from "./weighted.js";
|
||||
|
||||
export function selectActiveIpsByMode(
|
||||
config: LbTargetConfig,
|
||||
rows: LbIpRow[],
|
||||
nowMs = Date.now(),
|
||||
): string[] {
|
||||
if (rows.length === 0) return [];
|
||||
if (config.lb_mode === "failover") {
|
||||
return failoverDesired(rows);
|
||||
}
|
||||
// weighted = round_robin on DNS (one A per IP)
|
||||
if (config.lb_mode === "weighted") {
|
||||
return weightedDesired(rows, nowMs);
|
||||
}
|
||||
return roundRobinDesired(rows);
|
||||
}
|
||||
|
||||
export function strategyLabel(mode: LbMode): string {
|
||||
if (mode === "failover") return "Failover";
|
||||
if (mode === "weighted") return "Round Robin (weighted alias)";
|
||||
if (mode === "weighted") return "Weighted";
|
||||
return "Round Robin";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import type { LbIpRow } from "./types.js";
|
||||
import { isHealthy } from "./health.js";
|
||||
|
||||
/** Slot length for time-sliced weighted DNS (one A at a time). */
|
||||
export const WEIGHTED_SLOT_MS = 60_000;
|
||||
|
||||
/** Cloudflare DNS-only minimum TTL; Auto (1) is ~300s and would smear ratios. */
|
||||
export const WEIGHTED_DNS_TTL = 60;
|
||||
|
||||
export function weightedDesired(rows: LbIpRow[], nowMs = Date.now()): string[] {
|
||||
if (rows.length === 0) return [];
|
||||
const healthy = rows.filter((r) => isHealthy(r.health));
|
||||
const pool = healthy.length > 0 ? healthy : rows;
|
||||
if (pool.length === 1) return [pool[0]!.ip];
|
||||
|
||||
const sorted = [...pool].sort((a, b) => a.ip.localeCompare(b.ip));
|
||||
const cycle: string[] = [];
|
||||
for (const row of sorted) {
|
||||
const weight = Math.max(1, Math.round(row.weight));
|
||||
for (let i = 0; i < weight; i++) cycle.push(row.ip);
|
||||
}
|
||||
const slot = Math.floor(nowMs / WEIGHTED_SLOT_MS) % cycle.length;
|
||||
return [cycle[slot]!];
|
||||
}
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
isHealthy,
|
||||
selectActiveIpsByMode,
|
||||
withBindingLock,
|
||||
WEIGHTED_DNS_TTL,
|
||||
type LbIpRow,
|
||||
type LbTargetConfig,
|
||||
} from "./routing/index.js";
|
||||
@@ -39,6 +40,12 @@ import {
|
||||
export type { LbIpRow, LbTargetConfig };
|
||||
export { selectActiveIpsByMode };
|
||||
|
||||
const AUTO_DNS_TTL = 1;
|
||||
|
||||
function ttlForLbMode(mode: LbMode): number {
|
||||
return mode === "weighted" ? WEIGHTED_DNS_TTL : AUTO_DNS_TTL;
|
||||
}
|
||||
|
||||
export function failoverARecordDiff(
|
||||
existingA: readonly string[],
|
||||
desiredIps: readonly string[],
|
||||
@@ -276,6 +283,23 @@ function computeActiveIps(
|
||||
return selectActiveIpsByMode(state.config, state.rows);
|
||||
}
|
||||
|
||||
function desiredAIps(
|
||||
db: Db,
|
||||
scope: HealthCheckScope,
|
||||
refId: number,
|
||||
fallbackIps: string[],
|
||||
): string[] {
|
||||
const config =
|
||||
scope === "binding"
|
||||
? getBindingLbState(db, refId).config
|
||||
: getGroupLbState(db, refId).config;
|
||||
if (config.lb_mode === "weighted" || config.health_check_enabled) {
|
||||
const activeIps = computeActiveIps(db, scope, refId);
|
||||
if (activeIps.length > 0) return activeIps;
|
||||
}
|
||||
return fallbackIps;
|
||||
}
|
||||
|
||||
async function collectKnownZones(
|
||||
db: Db,
|
||||
cf: CloudflareClient,
|
||||
@@ -613,7 +637,16 @@ async function syncBindingDns(
|
||||
return;
|
||||
}
|
||||
|
||||
await syncBindingADns(db, cf, bindingId, domainId, hostname, desiredIps);
|
||||
const binding = repos.getBinding(db, bindingId);
|
||||
await syncBindingADns(
|
||||
db,
|
||||
cf,
|
||||
bindingId,
|
||||
domainId,
|
||||
hostname,
|
||||
desiredIps,
|
||||
ttlForLbMode(binding.lb_mode),
|
||||
);
|
||||
}
|
||||
|
||||
async function syncBindingCnameDns(
|
||||
@@ -700,6 +733,7 @@ async function syncBindingADns(
|
||||
domainId: number,
|
||||
hostname: string,
|
||||
desiredIps: string[],
|
||||
ttl: number,
|
||||
): Promise<void> {
|
||||
const domain = repos.getDomain(db, domainId);
|
||||
const zoneName = domain.zone_name;
|
||||
@@ -735,13 +769,15 @@ async function syncBindingADns(
|
||||
|
||||
for (const ip of desiredIps) {
|
||||
const existing = refreshed.find((r) => r.content === ip);
|
||||
const recordName = dnsNameForBinding(hostname, zoneName);
|
||||
let recordId: number;
|
||||
if (existing) {
|
||||
if (!dnsRecordNamesMatch(existing.name, hostname, zoneName)) {
|
||||
if (!dnsRecordNamesMatch(existing.name, hostname, zoneName) || existing.ttl !== ttl) {
|
||||
await dnsService.update(db, cf, domainId, existing.id, {
|
||||
record_type: "A",
|
||||
name: dnsNameForBinding(hostname, zoneName),
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
}
|
||||
@@ -759,12 +795,24 @@ async function syncBindingADns(
|
||||
if (adopted) {
|
||||
repos.linkBindingRecord(db, bindingId, adopted.id);
|
||||
recordId = adopted.id;
|
||||
if (
|
||||
!dnsRecordNamesMatch(adopted.name, hostname, zoneName) ||
|
||||
adopted.ttl !== ttl
|
||||
) {
|
||||
await dnsService.update(db, cf, domainId, adopted.id, {
|
||||
record_type: "A",
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const record = await dnsService.create(db, cf, domainId, {
|
||||
record_type: "A",
|
||||
name: dnsNameForBinding(hostname, zoneName),
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl: 1,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
repos.linkBindingRecord(db, bindingId, record.id);
|
||||
@@ -1001,12 +1049,7 @@ async function syncServiceBindingsToDns(
|
||||
}
|
||||
validateTargetIpsInPool(targetIps, ips);
|
||||
|
||||
if (binding.health_check_enabled) {
|
||||
const activeIps = computeActiveIps(db, "binding", binding.id);
|
||||
if (activeIps.length > 0) {
|
||||
targetIps = activeIps;
|
||||
}
|
||||
}
|
||||
const desiredIps = desiredAIps(db, "binding", binding.id, targetIps);
|
||||
|
||||
await syncBindingDns(
|
||||
db,
|
||||
@@ -1014,7 +1057,7 @@ async function syncServiceBindingsToDns(
|
||||
binding.id,
|
||||
binding.domain_id,
|
||||
binding.hostname,
|
||||
targetIps,
|
||||
desiredIps,
|
||||
null,
|
||||
);
|
||||
}
|
||||
@@ -1046,6 +1089,7 @@ async function syncGroupDomainDnsRecords(
|
||||
domainId: number,
|
||||
hostname: string,
|
||||
desiredIps: string[],
|
||||
ttl: number = AUTO_DNS_TTL,
|
||||
): Promise<void> {
|
||||
const domain = repos.getDomain(db, domainId);
|
||||
const zoneName = domain.zone_name;
|
||||
@@ -1061,14 +1105,16 @@ async function syncGroupDomainDnsRecords(
|
||||
if (desiredIps.length === 0) return;
|
||||
|
||||
const refreshed = repos.listGroupDnsRecords(db, groupId);
|
||||
const recordName = dnsNameForBinding(hostname, zoneName);
|
||||
for (const ip of desiredIps) {
|
||||
const existing = refreshed.find((r) => r.content === ip);
|
||||
if (existing) {
|
||||
if (!dnsRecordNamesMatch(existing.name, hostname, zoneName)) {
|
||||
if (!dnsRecordNamesMatch(existing.name, hostname, zoneName) || existing.ttl !== ttl) {
|
||||
await dnsService.update(db, cf, domainId, existing.id, {
|
||||
record_type: "A",
|
||||
name: dnsNameForBinding(hostname, zoneName),
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
}
|
||||
@@ -1084,13 +1130,25 @@ async function syncGroupDomainDnsRecords(
|
||||
);
|
||||
if (adopted) {
|
||||
repos.linkGroupDnsRecord(db, groupId, adopted.id);
|
||||
if (
|
||||
!dnsRecordNamesMatch(adopted.name, hostname, zoneName) ||
|
||||
adopted.ttl !== ttl
|
||||
) {
|
||||
await dnsService.update(db, cf, domainId, adopted.id, {
|
||||
record_type: "A",
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const record = await dnsService.create(db, cf, domainId, {
|
||||
record_type: "A",
|
||||
name: dnsNameForBinding(hostname, zoneName),
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl: 1,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
repos.linkGroupDnsRecord(db, groupId, record.id);
|
||||
@@ -1141,9 +1199,8 @@ async function syncGroupDomainDns(
|
||||
const knownZones = await collectKnownZones(db, cf);
|
||||
const { zoneName, hostname } = parseFqdn(domainValue, knownZones);
|
||||
const domainId = await resolveDomainId(db, cf, zoneName);
|
||||
const desiredIps = group.health_check_enabled
|
||||
? computeActiveIps(db, "group", groupId)
|
||||
: await collectGroupDnsIps(db, groupId);
|
||||
const fallbackIps = await collectGroupDnsIps(db, groupId);
|
||||
const desiredIps = desiredAIps(db, "group", groupId, fallbackIps);
|
||||
await syncGroupDomainDnsRecords(
|
||||
db,
|
||||
cf,
|
||||
@@ -1151,6 +1208,7 @@ async function syncGroupDomainDns(
|
||||
domainId,
|
||||
hostname,
|
||||
desiredIps,
|
||||
ttlForLbMode(group.lb_mode),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1331,14 +1389,7 @@ export async function updateConfig(
|
||||
}
|
||||
|
||||
if (pushDns) {
|
||||
let effectiveIps = targetIps;
|
||||
const refreshedBinding = repos.getBinding(db, binding.id);
|
||||
if (refreshedBinding.health_check_enabled) {
|
||||
const activeIps = computeActiveIps(db, "binding", binding.id);
|
||||
if (activeIps.length > 0) {
|
||||
effectiveIps = activeIps;
|
||||
}
|
||||
}
|
||||
const effectiveIps = desiredAIps(db, "binding", binding.id, targetIps);
|
||||
await syncBindingDns(
|
||||
db,
|
||||
cf,
|
||||
@@ -1644,7 +1695,7 @@ export async function reconcileDnsForTarget(
|
||||
if (scope === "binding") {
|
||||
await withBindingLock(refId, async () => {
|
||||
const binding = repos.getBinding(db, refId);
|
||||
if (!binding.health_check_enabled) return;
|
||||
if (!binding.health_check_enabled && binding.lb_mode !== "weighted") return;
|
||||
const service = repos.getService(db, binding.service_id);
|
||||
if (!shouldPushDns(db, service)) return;
|
||||
const cnameTarget = binding.cname_target?.trim() || null;
|
||||
@@ -1652,8 +1703,7 @@ export async function reconcileDnsForTarget(
|
||||
const ips = repos.listServiceIps(db, service.id);
|
||||
const targetIps = repos.listBindingIps(db, binding.id);
|
||||
validateTargetIpsInPool(targetIps, ips);
|
||||
const activeIps = computeActiveIps(db, "binding", refId);
|
||||
const desiredIps = activeIps.length > 0 ? activeIps : targetIps;
|
||||
const desiredIps = desiredAIps(db, "binding", refId, targetIps);
|
||||
await syncBindingDns(
|
||||
db,
|
||||
cf,
|
||||
@@ -1668,8 +1718,59 @@ export async function reconcileDnsForTarget(
|
||||
}
|
||||
|
||||
const group = repos.getServiceGroup(db, refId);
|
||||
if (!group.enabled || !group.domain?.trim() || !group.health_check_enabled) {
|
||||
if (!group.enabled || !group.domain?.trim()) {
|
||||
return;
|
||||
}
|
||||
if (!group.health_check_enabled && group.lb_mode !== "weighted") {
|
||||
return;
|
||||
}
|
||||
await syncGroupDomainDns(db, cf, refId);
|
||||
}
|
||||
|
||||
export async function reconcileWeightedDns(
|
||||
db: Db,
|
||||
cf: CloudflareClient,
|
||||
): Promise<number> {
|
||||
let n = 0;
|
||||
for (const binding of repos.listAllBindings(db)) {
|
||||
if (binding.lb_mode !== "weighted") continue;
|
||||
if (binding.cname_target?.trim()) continue;
|
||||
try {
|
||||
await withBindingLock(binding.id, async () => {
|
||||
const latest = repos.getBinding(db, binding.id);
|
||||
if (latest.lb_mode !== "weighted") return;
|
||||
if (latest.cname_target?.trim()) return;
|
||||
const service = repos.getService(db, latest.service_id);
|
||||
if (!shouldPushDns(db, service)) return;
|
||||
const targetIps = repos.listBindingIps(db, latest.id);
|
||||
if (targetIps.length === 0) return;
|
||||
const ips = repos.listServiceIps(db, service.id);
|
||||
validateTargetIpsInPool(targetIps, ips);
|
||||
const desiredIps = desiredAIps(db, "binding", latest.id, targetIps);
|
||||
await syncBindingDns(
|
||||
db,
|
||||
cf,
|
||||
latest.id,
|
||||
latest.domain_id,
|
||||
latest.hostname,
|
||||
desiredIps,
|
||||
null,
|
||||
);
|
||||
n += 1;
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
for (const group of repos.listServiceGroups(db)) {
|
||||
if (group.lb_mode !== "weighted") continue;
|
||||
if (!group.enabled || !group.domain?.trim()) continue;
|
||||
try {
|
||||
await syncGroupDomainDns(db, cf, group.id);
|
||||
n += 1;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { AsyncTask, SimpleIntervalJob } from "toad-scheduler";
|
||||
import * as serviceConfigService from "./service-config-service.js";
|
||||
import { WEIGHTED_SLOT_MS } from "./routing/weighted.js";
|
||||
|
||||
export const WEIGHTED_DNS_JOB_ID = "weighted-dns";
|
||||
|
||||
export function createWeightedDnsTask(app: FastifyInstance): AsyncTask {
|
||||
return new AsyncTask(
|
||||
WEIGHTED_DNS_JOB_ID,
|
||||
async () => {
|
||||
const n = await serviceConfigService.reconcileWeightedDns(app.db, app.cf);
|
||||
if (n > 0) {
|
||||
app.log.info({ reconciled: n }, "weighted dns rotated");
|
||||
}
|
||||
},
|
||||
(err) => {
|
||||
app.log.warn({ err }, "weighted dns rotate failed");
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function scheduleWeightedDnsJob(
|
||||
app: FastifyInstance,
|
||||
task: AsyncTask,
|
||||
): void {
|
||||
const scheduler = app.scheduler;
|
||||
if (!scheduler) return;
|
||||
if (scheduler.existsById(WEIGHTED_DNS_JOB_ID)) {
|
||||
scheduler.removeById(WEIGHTED_DNS_JOB_ID);
|
||||
}
|
||||
scheduler.addSimpleIntervalJob(
|
||||
new SimpleIntervalJob(
|
||||
{ seconds: WEIGHTED_SLOT_MS / 1000, runImmediately: true },
|
||||
task,
|
||||
{ id: WEIGHTED_DNS_JOB_ID, preventOverrun: true },
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
type LbIpRow,
|
||||
type LbTargetConfig,
|
||||
} from "../src/services/service-config-service.js";
|
||||
import { WEIGHTED_SLOT_MS } from "../src/services/routing/weighted.js";
|
||||
|
||||
function row(
|
||||
ip: string,
|
||||
@@ -17,6 +18,11 @@ function row(
|
||||
};
|
||||
}
|
||||
|
||||
const weightedConfig: LbTargetConfig = {
|
||||
lb_mode: "weighted",
|
||||
health_check_enabled: true,
|
||||
};
|
||||
|
||||
describe("selectActiveIpsByMode", () => {
|
||||
it("round_robin returns all healthy ips, falls back to all if none healthy", () => {
|
||||
const config: LbTargetConfig = {
|
||||
@@ -75,22 +81,49 @@ describe("selectActiveIpsByMode", () => {
|
||||
expect(selectActiveIpsByMode(config, rows)).toEqual(["2.2.2.2"]);
|
||||
});
|
||||
|
||||
it("weighted returns all healthy ips (one A per ip; weights stored for display)", () => {
|
||||
const config: LbTargetConfig = {
|
||||
lb_mode: "weighted",
|
||||
health_check_enabled: true,
|
||||
};
|
||||
it("weighted 1:3 picks the lighter ip on slot 0 and the heavier on slot 1", () => {
|
||||
const rows = [
|
||||
row("1.1.1.1", { weight: 3, health: "up" }),
|
||||
row("2.2.2.2", { weight: 1, health: "up" }),
|
||||
row("3.3.3.3", { weight: 2, health: "down" }),
|
||||
row("1.1.1.1", { weight: 1, health: "up" }),
|
||||
row("2.2.2.2", { weight: 3, health: "up" }),
|
||||
];
|
||||
expect(selectActiveIpsByMode(config, rows).sort()).toEqual([
|
||||
"1.1.1.1",
|
||||
expect(selectActiveIpsByMode(weightedConfig, rows, 0)).toEqual(["1.1.1.1"]);
|
||||
expect(selectActiveIpsByMode(weightedConfig, rows, WEIGHTED_SLOT_MS)).toEqual([
|
||||
"2.2.2.2",
|
||||
]);
|
||||
});
|
||||
|
||||
it("weighted excludes down ips from the cycle", () => {
|
||||
const rows = [
|
||||
row("1.1.1.1", { weight: 1, health: "up" }),
|
||||
row("2.2.2.2", { weight: 3, health: "down" }),
|
||||
];
|
||||
expect(selectActiveIpsByMode(weightedConfig, rows, 0)).toEqual(["1.1.1.1"]);
|
||||
expect(
|
||||
selectActiveIpsByMode(weightedConfig, rows, WEIGHTED_SLOT_MS),
|
||||
).toEqual(["1.1.1.1"]);
|
||||
});
|
||||
|
||||
it("weighted with one ip always returns that ip", () => {
|
||||
expect(
|
||||
selectActiveIpsByMode(weightedConfig, [row("1.1.1.1", { weight: 5 })], 0),
|
||||
).toEqual(["1.1.1.1"]);
|
||||
});
|
||||
|
||||
it("weighted with all unknown rotates across every ip", () => {
|
||||
const rows = [
|
||||
row("1.1.1.1", { weight: 1, health: "unknown" }),
|
||||
row("2.2.2.2", { weight: 3, health: "unknown" }),
|
||||
];
|
||||
expect(selectActiveIpsByMode(weightedConfig, rows, 0)).toEqual(["1.1.1.1"]);
|
||||
expect(selectActiveIpsByMode(weightedConfig, rows, WEIGHTED_SLOT_MS)).toEqual([
|
||||
"2.2.2.2",
|
||||
]);
|
||||
});
|
||||
|
||||
it("weighted returns empty array for no rows", () => {
|
||||
expect(selectActiveIpsByMode(weightedConfig, [], 0)).toEqual([]);
|
||||
});
|
||||
|
||||
it("round_robin excludes unknown when another ip is up", () => {
|
||||
const config: LbTargetConfig = {
|
||||
lb_mode: "round_robin",
|
||||
|
||||
@@ -16,8 +16,8 @@ import { formatDate, formatRelative, sqliteUtcToIso } from '@/lib/format'
|
||||
import {
|
||||
failoverEventCopy,
|
||||
type FailoverEvent,
|
||||
type FailoverHistoryItem,
|
||||
} from '@/lib/failover-events'
|
||||
import type { FailoverLogEntry } from '@/lib/schemas'
|
||||
import { cn } from '@cfdm/ui/lib/utils'
|
||||
import type { ComponentProps } from 'react'
|
||||
|
||||
@@ -45,12 +45,6 @@ function separatorClass(tone: 'down' | 'added' | 'removed'): string {
|
||||
return 'bg-destructive/25'
|
||||
}
|
||||
|
||||
function failoverHistoryCopy(item: FailoverLogEntry): string {
|
||||
return item.action === 'removed'
|
||||
? `${item.ip} убрана с ${item.fqdn}`
|
||||
: `${item.ip} добавлена на ${item.fqdn}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Failover как sibling «Смены статуса»: ReUI Timeline + Badge.
|
||||
* Preview: https://reui.io/preview/base/components/c-timeline-10
|
||||
@@ -63,14 +57,14 @@ export function FailoverTimeline({
|
||||
history = [],
|
||||
}: {
|
||||
events: FailoverEvent[]
|
||||
history?: readonly FailoverLogEntry[]
|
||||
history?: readonly FailoverHistoryItem[]
|
||||
}) {
|
||||
if (events.length === 0 && history.length === 0) {
|
||||
return (
|
||||
<EmptyState
|
||||
icon={ShieldCheckIcon}
|
||||
title="Нет инцидентов Failover"
|
||||
description="Нет Down и нет смен A-записей по FQDN"
|
||||
description="Нет Down и нет выходов из пула"
|
||||
stackedIcon={false}
|
||||
centered={false}
|
||||
/>
|
||||
@@ -142,6 +136,10 @@ export function FailoverTimeline({
|
||||
<TimelineHeader>
|
||||
<TimelineTitle className="flex flex-wrap items-center gap-2">
|
||||
<span className="font-mono text-sm">{item.ip}</span>
|
||||
<HealthCheckBadge
|
||||
status={item.action === 'added' ? 'up' : 'down'}
|
||||
size="xs"
|
||||
/>
|
||||
<span className="text-muted-foreground text-xs">
|
||||
{item.fqdn}
|
||||
</span>
|
||||
@@ -151,9 +149,7 @@ export function FailoverTimeline({
|
||||
</TimelineDate>
|
||||
</TimelineHeader>
|
||||
<TimelineContent>
|
||||
<p className="text-foreground text-sm">
|
||||
{failoverHistoryCopy(item)}
|
||||
</p>
|
||||
<p className="text-foreground text-sm">{item.copy}</p>
|
||||
</TimelineContent>
|
||||
</TimelineItem>
|
||||
)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { FormFieldSimple } from '@/components/form-field'
|
||||
import { AppInput } from '@/components/app-input'
|
||||
import { SettingRow } from '@/components/setting-row'
|
||||
import { SelectField } from '@/components/select-field'
|
||||
import { Badge } from '@/components/reui/badge'
|
||||
import {
|
||||
NumberField,
|
||||
@@ -9,15 +10,8 @@ import {
|
||||
NumberFieldIncrement,
|
||||
NumberFieldInput,
|
||||
} from '@/components/reui/number-field'
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from '@cfdm/ui/components/select'
|
||||
import { Switch } from '@cfdm/ui/components/switch'
|
||||
import { FieldGroup } from '@cfdm/ui/components/field'
|
||||
import { Field, FieldGroup, FieldLabel } from '@cfdm/ui/components/field'
|
||||
import { Button } from '@cfdm/ui/components/button'
|
||||
import { ButtonGroup } from '@cfdm/ui/components/button-group'
|
||||
import { CableIcon, GlobeIcon } from 'lucide-react'
|
||||
@@ -59,9 +53,14 @@ export interface LbAndHealthConfig extends HealthCheckConfig {
|
||||
const defaultLbModeOptions = [
|
||||
{ value: 'round_robin', label: 'Round Robin' },
|
||||
{ value: 'failover', label: 'Failover (приоритет)' },
|
||||
{ value: 'weighted', label: 'Weighted (веса)' },
|
||||
{ value: 'weighted', label: 'Веса (подмена IP)' },
|
||||
]
|
||||
|
||||
export interface LbPoolMetaChange {
|
||||
weight?: number
|
||||
priority?: number
|
||||
}
|
||||
|
||||
function CompactNumberField({
|
||||
id,
|
||||
value,
|
||||
@@ -95,6 +94,89 @@ function CompactNumberField({
|
||||
)
|
||||
}
|
||||
|
||||
function PoolLbMetaFields({
|
||||
idPrefix,
|
||||
mode,
|
||||
ips,
|
||||
weights,
|
||||
priorities,
|
||||
onMetaChange,
|
||||
}: {
|
||||
idPrefix: string
|
||||
mode: Exclude<LbMode, 'round_robin'>
|
||||
ips: readonly string[]
|
||||
weights: Record<string, number>
|
||||
priorities: Record<string, number>
|
||||
onMetaChange?: (ip: string, meta: LbPoolMetaChange) => void
|
||||
}) {
|
||||
const isWeighted = mode === 'weighted'
|
||||
const minPriority =
|
||||
ips.length === 0
|
||||
? 1
|
||||
: Math.min(...ips.map((ip) => priorities[ip] ?? 1))
|
||||
|
||||
return (
|
||||
<SettingRow
|
||||
title={isWeighted ? 'Вес IP' : 'Приоритет IP'}
|
||||
description={
|
||||
isWeighted
|
||||
? 'Доля времени на общем FQDN: 1 и 3 = ¼ и ¾ цикла (слот 60 с)'
|
||||
: '1 — основной, больше — запасной'
|
||||
}
|
||||
compact
|
||||
stacked
|
||||
className="gap-3 px-0 py-3"
|
||||
contentClassName="min-w-0"
|
||||
>
|
||||
{ips.length === 0 ? (
|
||||
<p className="text-muted-foreground text-sm">Сначала добавьте IP выше</p>
|
||||
) : (
|
||||
<div className="flex w-full flex-col gap-2">
|
||||
{ips.map((ip) => {
|
||||
const isPrimary = (priorities[ip] ?? 1) === minPriority
|
||||
const fieldId = isWeighted
|
||||
? `${idPrefix}-weight-${ip}`
|
||||
: `${idPrefix}-priority-${ip}`
|
||||
return (
|
||||
<div key={ip} className="flex items-center gap-3">
|
||||
<span className="min-w-0 flex-1 truncate font-mono text-sm">{ip}</span>
|
||||
{!isWeighted ? (
|
||||
<Badge
|
||||
variant={isPrimary ? 'success-light' : 'outline'}
|
||||
size="xs"
|
||||
radius="full"
|
||||
>
|
||||
{isPrimary ? 'Основной' : 'Запасной'}
|
||||
</Badge>
|
||||
) : null}
|
||||
<Field className="w-28 gap-0">
|
||||
<FieldLabel htmlFor={fieldId} className="sr-only">
|
||||
{isWeighted ? `Вес ${ip}` : `Приоритет ${ip}`}
|
||||
</FieldLabel>
|
||||
<CompactNumberField
|
||||
id={fieldId}
|
||||
value={isWeighted ? (weights[ip] ?? 1) : (priorities[ip] ?? 1)}
|
||||
min={1}
|
||||
max={100}
|
||||
onValueChange={(next) =>
|
||||
onMetaChange?.(
|
||||
ip,
|
||||
isWeighted
|
||||
? { weight: next ?? 1 }
|
||||
: { priority: next ?? 1 },
|
||||
)
|
||||
}
|
||||
/>
|
||||
</Field>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</SettingRow>
|
||||
)
|
||||
}
|
||||
|
||||
export function HealthCheckConfigFields({
|
||||
value,
|
||||
onChange,
|
||||
@@ -102,6 +184,10 @@ export function HealthCheckConfigFields({
|
||||
lbModeOptions = defaultLbModeOptions,
|
||||
idPrefix = 'health',
|
||||
showLbMode = true,
|
||||
ips = [],
|
||||
weights = {},
|
||||
priorities = {},
|
||||
onMetaChange,
|
||||
className,
|
||||
}: {
|
||||
value: LbAndHealthConfig
|
||||
@@ -110,6 +196,10 @@ export function HealthCheckConfigFields({
|
||||
lbModeOptions?: { value: string; label: string }[]
|
||||
idPrefix?: string
|
||||
showLbMode?: boolean
|
||||
ips?: readonly string[]
|
||||
weights?: Record<string, number>
|
||||
priorities?: Record<string, number>
|
||||
onMetaChange?: (ip: string, meta: LbPoolMetaChange) => void
|
||||
className?: string
|
||||
}) {
|
||||
function patch(next: Partial<LbAndHealthConfig>) {
|
||||
@@ -134,25 +224,28 @@ export function HealthCheckConfigFields({
|
||||
compact
|
||||
className={rowClass}
|
||||
>
|
||||
<Select
|
||||
<SelectField
|
||||
modal={false}
|
||||
value={value.lb_mode}
|
||||
onValueChange={(v) => patch({ lb_mode: (v ?? 'round_robin') as LbMode })}
|
||||
>
|
||||
<SelectTrigger id={`${idPrefix}-lb-mode`} className="w-full">
|
||||
<SelectValue placeholder="Выберите режим" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{lbModeOptions.map((item) => (
|
||||
<SelectItem key={item.value} value={item.value}>
|
||||
{item.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
triggerId={`${idPrefix}-lb-mode`}
|
||||
placeholder="Выберите режим"
|
||||
options={lbModeOptions}
|
||||
/>
|
||||
</SettingRow>
|
||||
) : null}
|
||||
|
||||
{showLbMode && value.lb_mode !== 'round_robin' ? (
|
||||
<PoolLbMetaFields
|
||||
idPrefix={idPrefix}
|
||||
mode={value.lb_mode}
|
||||
ips={ips}
|
||||
weights={weights}
|
||||
priorities={priorities}
|
||||
onMetaChange={onMetaChange}
|
||||
/>
|
||||
) : null}
|
||||
|
||||
<SettingRow
|
||||
title="Провайдер health-check"
|
||||
description="Кто пробирует цель. Можно выбрать несколько источников."
|
||||
|
||||
@@ -2,10 +2,17 @@ import { UnplugIcon } from 'lucide-react'
|
||||
|
||||
import { FailoverTimeline } from '@/components/failover-timeline'
|
||||
import {
|
||||
mergeFailoverHistory,
|
||||
toFailoverEvents,
|
||||
type FailoverBindingPool,
|
||||
type FailoverHealthInput,
|
||||
} from '@/lib/failover-events'
|
||||
import {
|
||||
latestHealthByIp,
|
||||
resolveIpDisplayHealth,
|
||||
type HealthLogProbe,
|
||||
type HealthLogStatus,
|
||||
} from '@/lib/health-log'
|
||||
import type { FailoverLogEntry } from '@/lib/schemas'
|
||||
import { Badge } from '@/components/reui/badge'
|
||||
import {
|
||||
@@ -32,7 +39,7 @@ function failoverCountLabel(count: number): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Failover — текущие Down + журнал add/remove по FQDN.
|
||||
* Failover — текущие Down + кто вышел из пула и кто вернулся.
|
||||
* Preview: https://reui.io/preview/base/components/c-timeline-10
|
||||
* Preview: https://reui.io/preview/base/empty-state-12
|
||||
* Docs: https://reui.io/docs/components/base/frame
|
||||
@@ -44,12 +51,32 @@ export function ServiceFailoverPanel({
|
||||
ipHealth,
|
||||
bindings,
|
||||
history,
|
||||
probes = [],
|
||||
}: {
|
||||
ipHealth: readonly FailoverHealthInput[]
|
||||
bindings: readonly FailoverBindingPool[]
|
||||
history: readonly FailoverLogEntry[]
|
||||
probes?: readonly HealthLogProbe[]
|
||||
}) {
|
||||
const events = toFailoverEvents(ipHealth, bindings)
|
||||
const liveByIp = latestHealthByIp(probes)
|
||||
const overlayHealth = ipHealth.map((row) => {
|
||||
const live = liveByIp.get(row.ip)
|
||||
return {
|
||||
...row,
|
||||
status: resolveIpDisplayHealth(
|
||||
row.status as HealthLogStatus,
|
||||
live?.status,
|
||||
),
|
||||
last_error:
|
||||
live && live.status !== 'unknown' ? live.last_error : row.last_error,
|
||||
last_checked_at:
|
||||
live && live.status !== 'unknown'
|
||||
? live.last_checked_at
|
||||
: row.last_checked_at,
|
||||
}
|
||||
})
|
||||
const events = toFailoverEvents(overlayHealth, bindings)
|
||||
const mergedHistory = mergeFailoverHistory(history, probes, bindings)
|
||||
const removedCount = events.filter((event) => event.kind === 'removed').length
|
||||
|
||||
return (
|
||||
@@ -69,7 +96,7 @@ export function ServiceFailoverPanel({
|
||||
)}
|
||||
</FrameTitle>
|
||||
<FrameDescription>
|
||||
Текущие Down и история A-записей по FQDN
|
||||
Текущие Down и история: кто вышел из пула и кто вернулся
|
||||
</FrameDescription>
|
||||
</FrameHeader>
|
||||
|
||||
@@ -85,7 +112,7 @@ export function ServiceFailoverPanel({
|
||||
</Alert>
|
||||
) : null}
|
||||
|
||||
<FailoverTimeline events={events} history={history} />
|
||||
<FailoverTimeline events={events} history={mergedHistory} />
|
||||
</FramePanel>
|
||||
</Frame>
|
||||
)
|
||||
|
||||
@@ -45,6 +45,28 @@ describe('toAlignedSeries', () => {
|
||||
expect(keys).toEqual(['local', 'cloudflare', 'globalping'])
|
||||
})
|
||||
|
||||
it('keeps live latency when another IP in the same bucket is down', () => {
|
||||
const { points } = toAlignedSeries([
|
||||
probe({
|
||||
id: 1,
|
||||
latency_ms: 18,
|
||||
checked_at: '2026-01-01T00:00:10.000Z',
|
||||
}),
|
||||
probe({
|
||||
id: 2,
|
||||
status: 'down',
|
||||
ok: false,
|
||||
latency_ms: null,
|
||||
checked_at: '2026-01-01T00:00:12.000Z',
|
||||
}),
|
||||
])
|
||||
|
||||
expect(points).toHaveLength(1)
|
||||
expect(points[0]?.local).toBe(18)
|
||||
expect(points[0]?.localOk).toBe(true)
|
||||
expect(points[0]?.ok).toBe(true)
|
||||
})
|
||||
|
||||
it('does not plot down probes as latency 0', () => {
|
||||
const { points } = toAlignedSeries([
|
||||
probe({
|
||||
|
||||
@@ -124,8 +124,13 @@ export function toAlignedSeries(items: UptimeProbe[]): {
|
||||
}
|
||||
|
||||
const ok = probeOk(item)
|
||||
row[`${key}Ok`] = ok
|
||||
row[key] = ok ? item.latency_ms : null
|
||||
const prevOk = row[`${key}Ok`]
|
||||
row[`${key}Ok`] = prevOk === true || ok
|
||||
if (ok && item.latency_ms != null) {
|
||||
row[key] = item.latency_ms
|
||||
} else if (row[key] === undefined) {
|
||||
row[key] = null
|
||||
}
|
||||
}
|
||||
|
||||
const points = [...buckets.entries()]
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
DEFAULT_BINDING_HEALTH,
|
||||
emptyAddressBlock,
|
||||
hydrateAddressBlock,
|
||||
patchAddressIpMeta,
|
||||
toBindingDrafts,
|
||||
toDomainsPayload,
|
||||
type AddressBlockState,
|
||||
@@ -282,6 +283,12 @@ export function ServiceEditSheet({
|
||||
idPrefix="service-health"
|
||||
value={primaryHealthValue}
|
||||
onChange={handlePrimaryHealthChange}
|
||||
ips={address.nodes.map((node) => node.ip)}
|
||||
weights={address.target_ip_weights}
|
||||
priorities={address.target_ip_priorities}
|
||||
onMetaChange={(ip, meta) =>
|
||||
setAddress((current) => patchAddressIpMeta(current, ip, meta))
|
||||
}
|
||||
/>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
@@ -18,6 +18,11 @@ import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-colu
|
||||
import { IconTile } from '@/components/reui/icon-tile'
|
||||
import { createFilter, type Filter, type FilterFieldConfig } from '@/components/reui/filters'
|
||||
import { ResourcePage } from '@/components/reui-kit'
|
||||
import {
|
||||
latestHealthByIp,
|
||||
resolveIpDisplayHealth,
|
||||
type HealthLogProbe,
|
||||
} from '@/lib/health-log'
|
||||
import { certRelativeBadge } from '@/components/columns/certificates-columns'
|
||||
import { certMonitoringOptions } from '@/lib/cert-monitoring'
|
||||
import { formatDate } from '@/lib/format'
|
||||
@@ -99,8 +104,12 @@ function mapNodeHealth(status: string): HealthStatus {
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
function buildIpRows(service: ServiceView): ServiceIpRow[] {
|
||||
function buildIpRows(
|
||||
service: ServiceView,
|
||||
probes: readonly HealthLogProbe[] = [],
|
||||
): ServiceIpRow[] {
|
||||
const healthByIp = new Map(service.ip_health.map((row) => [row.ip, row]))
|
||||
const liveByIp = latestHealthByIp(probes)
|
||||
const weights = Object.assign(
|
||||
{},
|
||||
...service.domains.map((domain) => domain.target_ip_weights ?? {}),
|
||||
@@ -113,19 +122,25 @@ function buildIpRows(service: ServiceView): ServiceIpRow[] {
|
||||
|
||||
return service.ips.map((ip) => {
|
||||
const health = healthByIp.get(ip)
|
||||
const live = liveByIp.get(ip)
|
||||
const status = resolveIpDisplayHealth(health?.status, live?.status)
|
||||
const extras = live && live.status !== 'unknown' ? live : health
|
||||
return {
|
||||
id: ip,
|
||||
ip,
|
||||
status: health?.status ?? 'unknown',
|
||||
status,
|
||||
enabled: service.ip_enabled[ip] !== false,
|
||||
active: activeSet.has(ip),
|
||||
weight: weights[ip] ?? 1,
|
||||
priority: priorities[ip] ?? 1,
|
||||
latency_ms: health?.latency_ms ?? null,
|
||||
last_checked_at: health?.last_checked_at ?? null,
|
||||
last_error: health?.last_error ?? null,
|
||||
colo: health?.colo ?? null,
|
||||
provider: health?.provider ?? null,
|
||||
latency_ms: extras?.latency_ms ?? null,
|
||||
last_checked_at: extras?.last_checked_at ?? null,
|
||||
last_error:
|
||||
live && live.status !== 'unknown'
|
||||
? live.last_error
|
||||
: (health?.last_error ?? null),
|
||||
colo: extras?.colo ?? null,
|
||||
provider: extras?.provider ?? null,
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -151,17 +166,23 @@ function buildNodeRows(
|
||||
weight: number
|
||||
priority: number
|
||||
}>,
|
||||
probes: readonly HealthLogProbe[] = [],
|
||||
): ServiceNodeRow[] {
|
||||
return nodes.map((node) => ({
|
||||
id: String(node.id),
|
||||
nodeId: node.id,
|
||||
address: node.address,
|
||||
protocol: node.protocol,
|
||||
port: node.port,
|
||||
health_status: mapNodeHealth(node.health_status),
|
||||
weight: node.weight,
|
||||
priority: node.priority,
|
||||
}))
|
||||
const liveByIp = latestHealthByIp(probes)
|
||||
return nodes.map((node) => {
|
||||
const stored = mapNodeHealth(node.health_status)
|
||||
const live = liveByIp.get(node.address)
|
||||
return {
|
||||
id: String(node.id),
|
||||
nodeId: node.id,
|
||||
address: node.address,
|
||||
protocol: node.protocol,
|
||||
port: node.port,
|
||||
health_status: resolveIpDisplayHealth(stored, live?.status),
|
||||
weight: node.weight,
|
||||
priority: node.priority,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function NameCell({
|
||||
@@ -199,6 +220,7 @@ interface ServiceDetailGridProps {
|
||||
weight: number
|
||||
priority: number
|
||||
}>
|
||||
probes?: readonly HealthLogProbe[]
|
||||
togglingIp: string | null
|
||||
onToggleIp: (ip: string, enabled: boolean) => void
|
||||
onChangeIp: (row: ServiceFqdnRow) => void
|
||||
@@ -211,6 +233,7 @@ interface ServiceDetailGridProps {
|
||||
export function ServiceDetailGrid({
|
||||
service,
|
||||
nodes,
|
||||
probes = [],
|
||||
togglingIp,
|
||||
onToggleIp,
|
||||
onChangeIp,
|
||||
@@ -283,9 +306,9 @@ export function ServiceDetailGrid({
|
||||
},
|
||||
})
|
||||
|
||||
const ipRows = useMemo(() => buildIpRows(service), [service])
|
||||
const ipRows = useMemo(() => buildIpRows(service, probes), [service, probes])
|
||||
const fqdnRows = useMemo(() => buildFqdnRows(service), [service])
|
||||
const nodeRows = useMemo(() => buildNodeRows(nodes), [nodes])
|
||||
const nodeRows = useMemo(() => buildNodeRows(nodes, probes), [nodes, probes])
|
||||
const markActive = service.lb_mode === 'failover' || service.lb_mode === 'weighted'
|
||||
|
||||
const tabs = TABS.map((entry) => ({
|
||||
|
||||
@@ -71,7 +71,7 @@ const LB_MODE_META: Record<
|
||||
weighted: {
|
||||
icon: ScaleIcon,
|
||||
className: 'text-info',
|
||||
label: 'Weighted (веса)',
|
||||
label: 'Веса (подмена IP)',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -3,10 +3,14 @@ import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
failoverEventCopy,
|
||||
isFailoverEventStatus,
|
||||
mergeFailoverHistory,
|
||||
toFailoverEvents,
|
||||
toIpAliveTransitions,
|
||||
type FailoverBindingPool,
|
||||
type FailoverHealthInput,
|
||||
} from '@/lib/failover-events'
|
||||
import type { FailoverLogEntry } from '@/lib/schemas'
|
||||
import type { HealthLogProbe } from '@/lib/health-log'
|
||||
|
||||
function row(
|
||||
overrides: Partial<FailoverHealthInput> & Pick<FailoverHealthInput, 'ip'>,
|
||||
@@ -186,3 +190,105 @@ describe('toFailoverEvents', () => {
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
function probe(
|
||||
overrides: Partial<HealthLogProbe> & Pick<HealthLogProbe, 'id' | 'status' | 'checked_at'>,
|
||||
): HealthLogProbe {
|
||||
return {
|
||||
ip: '130.49.213.153',
|
||||
provider: 'local',
|
||||
ok: overrides.status === 'up',
|
||||
latency_ms: 12,
|
||||
colo: null,
|
||||
error: null,
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function dns(
|
||||
overrides: Partial<FailoverLogEntry> & Pick<FailoverLogEntry, 'id' | 'action' | 'created_at'>,
|
||||
): FailoverLogEntry {
|
||||
return {
|
||||
service_id: 13,
|
||||
binding_id: 1,
|
||||
fqdn: 'gt.rkns.top',
|
||||
ip: '130.49.213.153',
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
describe('toIpAliveTransitions', () => {
|
||||
it('emits leave then return, not the initial up', () => {
|
||||
const transitions = toIpAliveTransitions([
|
||||
probe({ id: 1, status: 'up', checked_at: '2026-08-20T09:00:00Z' }),
|
||||
probe({ id: 2, status: 'up', checked_at: '2026-08-20T09:01:00Z' }),
|
||||
probe({ id: 3, status: 'down', checked_at: '2026-08-20T09:02:00Z' }),
|
||||
probe({ id: 4, status: 'down', checked_at: '2026-08-20T09:03:00Z' }),
|
||||
probe({ id: 5, status: 'up', checked_at: '2026-08-20T09:04:00Z' }),
|
||||
])
|
||||
expect(transitions).toEqual([
|
||||
{ id: 'probe:3', ip: '130.49.213.153', alive: false, at: '2026-08-20T09:02:00Z' },
|
||||
{ id: 'probe:5', ip: '130.49.213.153', alive: true, at: '2026-08-20T09:04:00Z' },
|
||||
])
|
||||
})
|
||||
|
||||
it('any-up across providers: leave only when every source is down', () => {
|
||||
const transitions = toIpAliveTransitions([
|
||||
probe({
|
||||
id: 1,
|
||||
provider: 'local',
|
||||
status: 'up',
|
||||
checked_at: '2026-08-20T09:00:00Z',
|
||||
}),
|
||||
probe({
|
||||
id: 2,
|
||||
provider: 'cloudflare',
|
||||
status: 'down',
|
||||
checked_at: '2026-08-20T09:01:00Z',
|
||||
}),
|
||||
probe({
|
||||
id: 3,
|
||||
provider: 'local',
|
||||
status: 'down',
|
||||
checked_at: '2026-08-20T09:02:00Z',
|
||||
}),
|
||||
probe({
|
||||
id: 4,
|
||||
provider: 'local',
|
||||
status: 'up',
|
||||
checked_at: '2026-08-20T09:03:00Z',
|
||||
}),
|
||||
])
|
||||
expect(transitions.map((item) => item.id)).toEqual(['probe:3', 'probe:4'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('mergeFailoverHistory', () => {
|
||||
it('fills leave/return from probes when DNS has only the add', () => {
|
||||
const history = mergeFailoverHistory(
|
||||
[dns({ id: 10, action: 'added', created_at: '2026-08-20 09:26:00' })],
|
||||
[
|
||||
probe({ id: 1, status: 'up', checked_at: '2026-08-20T09:00:00Z' }),
|
||||
probe({ id: 2, status: 'down', checked_at: '2026-08-20T09:10:00Z' }),
|
||||
probe({ id: 3, status: 'up', checked_at: '2026-08-20T09:26:30Z' }),
|
||||
],
|
||||
mskHip,
|
||||
)
|
||||
expect(history.map((item) => `${item.action}:${item.source}`)).toEqual([
|
||||
'added:dns',
|
||||
'removed:probe',
|
||||
])
|
||||
expect(history[0]?.copy).toBe('130.49.213.153 добавлена на gt.rkns.top')
|
||||
expect(history[1]?.copy).toContain('вышла из пула')
|
||||
})
|
||||
|
||||
it('keeps DNS over a probe return in the same 2-minute window', () => {
|
||||
const history = mergeFailoverHistory(
|
||||
[dns({ id: 10, action: 'added', created_at: '2026-08-20 09:26:00' })],
|
||||
[probe({ id: 3, status: 'up', checked_at: '2026-08-20T09:26:30Z' })],
|
||||
mskHip,
|
||||
)
|
||||
expect(history).toHaveLength(1)
|
||||
expect(history[0]?.source).toBe('dns')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,5 +1,23 @@
|
||||
import {
|
||||
bestAliveHealthStatus,
|
||||
probeTime,
|
||||
type HealthLogProbe,
|
||||
type HealthLogStatus,
|
||||
} from '@/lib/health-log'
|
||||
import type { FailoverLogEntry } from '@/lib/schemas'
|
||||
|
||||
export type FailoverEventKind = 'removed' | 'last-resort'
|
||||
|
||||
export interface FailoverHistoryItem {
|
||||
id: string
|
||||
ip: string
|
||||
fqdn: string
|
||||
action: 'added' | 'removed'
|
||||
created_at: string
|
||||
copy: string
|
||||
source: 'dns' | 'probe'
|
||||
}
|
||||
|
||||
export interface FailoverEvent {
|
||||
id: string
|
||||
address: string
|
||||
@@ -73,3 +91,141 @@ export function toFailoverEvents(
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const DEDUPE_WINDOW_MS = 2 * 60 * 1000
|
||||
|
||||
function fqdnsForIp(
|
||||
ip: string,
|
||||
bindings: readonly FailoverBindingPool[],
|
||||
): string[] {
|
||||
return bindings.filter((binding) => binding.configured.includes(ip)).map((binding) => binding.fqdn)
|
||||
}
|
||||
|
||||
function fqdnLabel(fqdns: string[]): string {
|
||||
return fqdns.join(', ') || 'пул'
|
||||
}
|
||||
|
||||
function isAliveStatus(status: HealthLogStatus): boolean {
|
||||
return status === 'up' || status === 'degraded'
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-IP any-up flips: down → вышла из пула, up после down → вернулась.
|
||||
* Initial state is not an event.
|
||||
*/
|
||||
export function toIpAliveTransitions(
|
||||
probes: readonly HealthLogProbe[],
|
||||
): Array<{ id: string; ip: string; alive: boolean; at: string }> {
|
||||
const byIp = new Map<string, HealthLogProbe[]>()
|
||||
for (const item of probes) {
|
||||
const list = byIp.get(item.ip)
|
||||
if (list) list.push(item)
|
||||
else byIp.set(item.ip, [item])
|
||||
}
|
||||
|
||||
const out: Array<{ id: string; ip: string; alive: boolean; at: string }> = []
|
||||
for (const [ip, list] of byIp) {
|
||||
list.sort(
|
||||
(a, b) => probeTime(a.checked_at) - probeTime(b.checked_at) || a.id - b.id,
|
||||
)
|
||||
const latestByProvider = new Map<string, HealthLogProbe>()
|
||||
let prevAlive: boolean | undefined
|
||||
for (const probe of list) {
|
||||
latestByProvider.set(probe.provider, probe)
|
||||
const status = bestAliveHealthStatus(
|
||||
[...latestByProvider.values()].map((item) => item.status),
|
||||
)
|
||||
if (status === 'unknown') continue
|
||||
const alive = isAliveStatus(status)
|
||||
if (prevAlive !== undefined && alive !== prevAlive) {
|
||||
out.push({
|
||||
id: `probe:${probe.id}`,
|
||||
ip,
|
||||
alive,
|
||||
at: probe.checked_at,
|
||||
})
|
||||
}
|
||||
prevAlive = alive
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function dnsHistoryItem(item: FailoverLogEntry): FailoverHistoryItem {
|
||||
return {
|
||||
id: `dns:${item.id}`,
|
||||
ip: item.ip,
|
||||
fqdn: item.fqdn,
|
||||
action: item.action,
|
||||
created_at: item.created_at,
|
||||
copy:
|
||||
item.action === 'removed'
|
||||
? `${item.ip} убрана с ${item.fqdn}`
|
||||
: `${item.ip} добавлена на ${item.fqdn}`,
|
||||
source: 'dns',
|
||||
}
|
||||
}
|
||||
|
||||
function probeHistoryItem(
|
||||
transition: { id: string; ip: string; alive: boolean; at: string },
|
||||
bindings: readonly FailoverBindingPool[],
|
||||
): FailoverHistoryItem {
|
||||
const fqdns = fqdnsForIp(transition.ip, bindings)
|
||||
const fqdn = fqdnLabel(fqdns)
|
||||
const action = transition.alive ? 'added' : 'removed'
|
||||
return {
|
||||
id: transition.id,
|
||||
ip: transition.ip,
|
||||
fqdn,
|
||||
action,
|
||||
created_at: transition.at,
|
||||
copy:
|
||||
action === 'removed'
|
||||
? `${transition.ip} вышла из пула (${fqdn})`
|
||||
: `${transition.ip} вернулась в пул (${fqdn})`,
|
||||
source: 'probe',
|
||||
}
|
||||
}
|
||||
|
||||
function eventTime(value: string): number {
|
||||
return probeTime(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* DNS add/remove + health leave/return, newest first.
|
||||
* Same IP+action within 2 minutes: keep the DNS row (it has a concrete FQDN).
|
||||
*/
|
||||
export function mergeFailoverHistory(
|
||||
dns: readonly FailoverLogEntry[],
|
||||
probes: readonly HealthLogProbe[],
|
||||
bindings: readonly FailoverBindingPool[] = [],
|
||||
): FailoverHistoryItem[] {
|
||||
const merged = [
|
||||
...dns.map(dnsHistoryItem),
|
||||
...toIpAliveTransitions(probes).map((transition) =>
|
||||
probeHistoryItem(transition, bindings),
|
||||
),
|
||||
]
|
||||
merged.sort(
|
||||
(a, b) => eventTime(b.created_at) - eventTime(a.created_at) || a.id.localeCompare(b.id),
|
||||
)
|
||||
|
||||
const kept: FailoverHistoryItem[] = []
|
||||
for (const item of merged) {
|
||||
const duplicate = kept.find(
|
||||
(other) =>
|
||||
other.ip === item.ip &&
|
||||
other.action === item.action &&
|
||||
Math.abs(eventTime(other.created_at) - eventTime(item.created_at)) <=
|
||||
DEDUPE_WINDOW_MS,
|
||||
)
|
||||
if (!duplicate) {
|
||||
kept.push(item)
|
||||
continue
|
||||
}
|
||||
if (duplicate.source === 'probe' && item.source === 'dns') {
|
||||
kept[kept.indexOf(duplicate)] = item
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
@@ -4,7 +4,9 @@ import {
|
||||
bestAliveHealthStatus,
|
||||
collapseStatusChanges,
|
||||
enabledHealthProviders,
|
||||
latestHealthByIp,
|
||||
providerHealthStatuses,
|
||||
resolveIpDisplayHealth,
|
||||
worstHealthStatus,
|
||||
type HealthLogProbe,
|
||||
} from '@/lib/health-log'
|
||||
@@ -105,3 +107,42 @@ describe('bestAliveHealthStatus', () => {
|
||||
expect(bestAliveHealthStatus([])).toBe('unknown')
|
||||
})
|
||||
})
|
||||
|
||||
describe('latestHealthByIp', () => {
|
||||
it('any-up among latest-per-provider probes', () => {
|
||||
const items = [
|
||||
probe({
|
||||
id: 1,
|
||||
ip: '130.49.213.153',
|
||||
provider: 'local',
|
||||
status: 'up',
|
||||
checked_at: '2026-08-20T09:00:00Z',
|
||||
}),
|
||||
probe({
|
||||
id: 2,
|
||||
ip: '130.49.213.153',
|
||||
provider: 'cloudflare',
|
||||
status: 'down',
|
||||
checked_at: '2026-08-20T09:00:01Z',
|
||||
}),
|
||||
probe({
|
||||
id: 3,
|
||||
ip: '93.115.203.183',
|
||||
status: 'unknown',
|
||||
checked_at: '2026-08-20T08:59:00Z',
|
||||
}),
|
||||
]
|
||||
const byIp = latestHealthByIp(items)
|
||||
expect(byIp.get('130.49.213.153')?.status).toBe('up')
|
||||
expect(byIp.get('93.115.203.183')?.status).toBe('unknown')
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveIpDisplayHealth', () => {
|
||||
it('prefers a live probe over stored unknown', () => {
|
||||
expect(resolveIpDisplayHealth('unknown', 'up')).toBe('up')
|
||||
expect(resolveIpDisplayHealth('down', 'up')).toBe('up')
|
||||
expect(resolveIpDisplayHealth('up', 'unknown')).toBe('up')
|
||||
expect(resolveIpDisplayHealth('unknown', undefined)).toBe('unknown')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -136,3 +136,61 @@ export function providerHealthStatuses(
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export interface IpDisplayHealth {
|
||||
status: HealthLogStatus
|
||||
latency_ms: number | null
|
||||
last_checked_at: string
|
||||
last_error: string | null
|
||||
colo: string | null
|
||||
provider: HealthCheckProvider
|
||||
}
|
||||
|
||||
/**
|
||||
* Latest probe per provider+IP, then any-up among those providers.
|
||||
* Used by the IP table so hysteresis `unknown` in ip_health does not hide a live OK.
|
||||
*/
|
||||
export function latestHealthByIp(
|
||||
items: readonly HealthLogProbe[],
|
||||
): Map<string, IpDisplayHealth> {
|
||||
const latest = new Map<string, HealthLogProbe>()
|
||||
const sorted = [...items].sort(
|
||||
(a, b) => probeTime(b.checked_at) - probeTime(a.checked_at) || b.id - a.id,
|
||||
)
|
||||
for (const item of sorted) {
|
||||
const key = `${item.provider}\0${item.ip}`
|
||||
if (!latest.has(key)) latest.set(key, item)
|
||||
}
|
||||
|
||||
const byIp = new Map<string, HealthLogProbe[]>()
|
||||
for (const item of latest.values()) {
|
||||
const list = byIp.get(item.ip)
|
||||
if (list) list.push(item)
|
||||
else byIp.set(item.ip, [item])
|
||||
}
|
||||
|
||||
const result = new Map<string, IpDisplayHealth>()
|
||||
for (const [ip, probes] of byIp) {
|
||||
const status = bestAliveHealthStatus(probes.map((probe) => probe.status))
|
||||
const preferred =
|
||||
probes.find((probe) => probe.status === status) ?? probes[0]!
|
||||
result.set(ip, {
|
||||
status,
|
||||
latency_ms: preferred.latency_ms,
|
||||
last_checked_at: preferred.checked_at,
|
||||
last_error: preferred.error,
|
||||
colo: preferred.colo,
|
||||
provider: preferred.provider,
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/** Prefer a concrete live probe over stored hysteresis `unknown`. */
|
||||
export function resolveIpDisplayHealth(
|
||||
stored: HealthLogStatus | undefined,
|
||||
live: HealthLogStatus | undefined,
|
||||
): HealthLogStatus {
|
||||
if (live && live !== 'unknown') return live
|
||||
return stored ?? live ?? 'unknown'
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
emptyAddressBlock,
|
||||
emptyBindingDraft,
|
||||
hydrateAddressBlock,
|
||||
patchAddressIpMeta,
|
||||
removeAddressNode,
|
||||
toAddressBindings,
|
||||
toDomainsPayload,
|
||||
@@ -82,6 +83,26 @@ describe('hydrateAddressBlock', () => {
|
||||
expect(state.preservedBindings).toHaveLength(1)
|
||||
expect(state.preservedBindings[0]?.fqdn).toBe('edge.example.com')
|
||||
})
|
||||
|
||||
it('поднимает веса и приоритеты с общего FQDN', () => {
|
||||
const drafts = [
|
||||
aRecord('gt.rkns.top', ['130.49.213.153', '93.115.203.183'], {
|
||||
target_ip_weights: { '130.49.213.153': 3, '93.115.203.183': 1 },
|
||||
target_ip_priorities: { '130.49.213.153': 2, '93.115.203.183': 1 },
|
||||
}),
|
||||
aRecord('nsgt.rkns.top', ['130.49.213.153']),
|
||||
]
|
||||
const state = hydrateAddressBlock(drafts, ['130.49.213.153', '93.115.203.183'])
|
||||
expect(state.target_ip_weights).toEqual({
|
||||
'130.49.213.153': 3,
|
||||
'93.115.203.183': 1,
|
||||
})
|
||||
expect(state.target_ip_priorities).toEqual({
|
||||
'130.49.213.153': 2,
|
||||
'93.115.203.183': 1,
|
||||
})
|
||||
expect(state.nodes[0]?.extraFqdn).toBe('nsgt.rkns.top')
|
||||
})
|
||||
})
|
||||
|
||||
describe('toDomainsPayload', () => {
|
||||
@@ -161,6 +182,35 @@ describe('addAddressNode / addCommonFqdn', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('patchAddressIpMeta', () => {
|
||||
it('меняет вес одного IP и не трогает extraFqdn', () => {
|
||||
const state = {
|
||||
...addAddressNode(addAddressNode(emptyAddressBlock(), '1.1.1.1'), '2.2.2.2'),
|
||||
nodes: [
|
||||
{ ip: '1.1.1.1', extraFqdn: 'msk.example.com' },
|
||||
{ ip: '2.2.2.2', extraFqdn: '' },
|
||||
],
|
||||
}
|
||||
const next = patchAddressIpMeta(state, '1.1.1.1', { weight: 7 })
|
||||
expect(next.target_ip_weights).toEqual({ '1.1.1.1': 7, '2.2.2.2': 1 })
|
||||
expect(next.target_ip_priorities).toEqual(state.target_ip_priorities)
|
||||
expect(next.nodes).toEqual(state.nodes)
|
||||
})
|
||||
|
||||
it('clamp веса и приоритета в 1–100', () => {
|
||||
const state = addAddressNode(emptyAddressBlock(), '10.0.0.1')
|
||||
expect(patchAddressIpMeta(state, '10.0.0.1', { weight: 0 }).target_ip_weights['10.0.0.1']).toBe(1)
|
||||
expect(
|
||||
patchAddressIpMeta(state, '10.0.0.1', { priority: 999 }).target_ip_priorities['10.0.0.1'],
|
||||
).toBe(100)
|
||||
})
|
||||
|
||||
it('игнорирует IP вне пула', () => {
|
||||
const state = addAddressNode(emptyAddressBlock(), '10.0.0.1')
|
||||
expect(patchAddressIpMeta(state, '8.8.8.8', { weight: 5 })).toBe(state)
|
||||
})
|
||||
})
|
||||
|
||||
describe('CNAME / preservedBindings', () => {
|
||||
it('сохраняет CNAME в preserved при круге hydrate → payload', () => {
|
||||
const cname: ServiceBindingDraft = {
|
||||
|
||||
@@ -243,6 +243,33 @@ export function addAddressNode(state: AddressBlockState, ip: string): AddressBlo
|
||||
}
|
||||
}
|
||||
|
||||
const LB_META_MIN = 1
|
||||
const LB_META_MAX = 100
|
||||
|
||||
function clampLbMeta(value: number): number {
|
||||
if (!Number.isFinite(value)) return LB_META_MIN
|
||||
return Math.min(LB_META_MAX, Math.max(LB_META_MIN, Math.round(value)))
|
||||
}
|
||||
|
||||
export function patchAddressIpMeta(
|
||||
state: AddressBlockState,
|
||||
ip: string,
|
||||
meta: { weight?: number; priority?: number },
|
||||
): AddressBlockState {
|
||||
if (!state.nodes.some((node) => node.ip === ip)) return state
|
||||
return {
|
||||
...state,
|
||||
target_ip_weights:
|
||||
meta.weight === undefined
|
||||
? state.target_ip_weights
|
||||
: { ...state.target_ip_weights, [ip]: clampLbMeta(meta.weight) },
|
||||
target_ip_priorities:
|
||||
meta.priority === undefined
|
||||
? state.target_ip_priorities
|
||||
: { ...state.target_ip_priorities, [ip]: clampLbMeta(meta.priority) },
|
||||
}
|
||||
}
|
||||
|
||||
function fqdnKey(value: string): string {
|
||||
return value.trim().toLowerCase()
|
||||
}
|
||||
|
||||
@@ -331,6 +331,7 @@ function ServiceDetailPage() {
|
||||
ipHealth={service.ip_health}
|
||||
bindings={failoverBindings}
|
||||
history={failoverHistory}
|
||||
probes={logItems}
|
||||
/>
|
||||
</section>
|
||||
|
||||
@@ -344,6 +345,7 @@ function ServiceDetailPage() {
|
||||
<ServiceDetailGrid
|
||||
service={service}
|
||||
nodes={nodes}
|
||||
probes={logItems}
|
||||
togglingIp={togglingIp}
|
||||
onToggleIp={(ip, enabled) => {
|
||||
setTogglingIp(ip)
|
||||
|
||||
Reference in New Issue
Block a user