Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a9a84fabac | ||
|
|
2b150fa3a6 | ||
|
|
3d0ea33baf |
@@ -38,6 +38,10 @@ import {
|
||||
healthEngineFallbacksFromConfig,
|
||||
scheduleHealthCheckJob,
|
||||
} from "./services/health-check-scheduler.js";
|
||||
import {
|
||||
createWeightedDnsTask,
|
||||
scheduleWeightedDnsJob,
|
||||
} from "./services/weighted-dns-scheduler.js";
|
||||
import { fireEnsureHealthWorker } from "./services/health/health-worker-deploy.js";
|
||||
import { AsyncTask, CronJob } from "toad-scheduler";
|
||||
|
||||
@@ -133,6 +137,7 @@ export async function buildApp(opts: BuildAppOptions = {}) {
|
||||
app.decorate("reloadHealthCheckJob", () => {
|
||||
scheduleHealthCheckJob(app, config, healthTask);
|
||||
});
|
||||
scheduleWeightedDnsJob(app, createWeightedDnsTask(app));
|
||||
if (config.cloudflareApiToken) {
|
||||
fireEnsureHealthWorker(
|
||||
app.db,
|
||||
|
||||
@@ -1,26 +1,48 @@
|
||||
import type { LbMode } from "@cfdm/shared";
|
||||
import { failoverDesired } from "./failover.js";
|
||||
import { isSharedPool } from "./pool.js";
|
||||
import { roundRobinDesired } from "./round-robin.js";
|
||||
import type { LbIpRow, LbTargetConfig } from "./types.js";
|
||||
import { weightedDesired } from "./weighted.js";
|
||||
|
||||
export type { LbIpRow, LbTargetConfig } from "./types.js";
|
||||
export { isHealthy } from "./health.js";
|
||||
export { withBindingLock } from "./binding-lock.js";
|
||||
export { isSharedPool, shouldRecordFailoverDnsDiff } from "./pool.js";
|
||||
export { WEIGHTED_DNS_TTL, WEIGHTED_SLOT_MS, weightedDesired } from "./weighted.js";
|
||||
|
||||
export function selectActiveIpsByMode(
|
||||
config: LbTargetConfig,
|
||||
rows: LbIpRow[],
|
||||
nowMs = Date.now(),
|
||||
): string[] {
|
||||
if (rows.length === 0) return [];
|
||||
if (config.lb_mode === "failover") {
|
||||
return failoverDesired(rows);
|
||||
}
|
||||
// weighted = round_robin on DNS (one A per IP)
|
||||
if (config.lb_mode === "weighted") {
|
||||
return weightedDesired(rows, nowMs);
|
||||
}
|
||||
return roundRobinDesired(rows);
|
||||
}
|
||||
|
||||
export function resolveDesiredAIps(
|
||||
config: LbTargetConfig,
|
||||
rows: LbIpRow[],
|
||||
fallbackIps: readonly string[],
|
||||
nowMs = Date.now(),
|
||||
): string[] {
|
||||
const fallback = [...fallbackIps];
|
||||
if (!isSharedPool(fallback)) return fallback;
|
||||
if (config.lb_mode === "weighted" || config.health_check_enabled) {
|
||||
const activeIps = selectActiveIpsByMode(config, rows, nowMs);
|
||||
if (activeIps.length > 0) return activeIps;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
export function strategyLabel(mode: LbMode): string {
|
||||
if (mode === "failover") return "Failover";
|
||||
if (mode === "weighted") return "Round Robin (weighted alias)";
|
||||
if (mode === "weighted") return "Weighted";
|
||||
return "Round Robin";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { LbMode } from "@cfdm/shared";
|
||||
|
||||
/** Shared pool FQDN — two or more A targets. Dedicated extra-FQDN has one IP. */
|
||||
export function isSharedPool(ips: readonly string[]): boolean {
|
||||
return ips.length >= 2;
|
||||
}
|
||||
|
||||
export function shouldRecordFailoverDnsDiff(input: {
|
||||
configuredIps: readonly string[];
|
||||
lbMode: LbMode;
|
||||
added: readonly string[];
|
||||
removed: readonly string[];
|
||||
downIps: ReadonlySet<string>;
|
||||
}): boolean {
|
||||
if (!isSharedPool(input.configuredIps)) return false;
|
||||
if (input.lbMode !== "weighted") return true;
|
||||
return [...input.added, ...input.removed].some((ip) => input.downIps.has(ip));
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import type { LbIpRow } from "./types.js";
|
||||
import { isHealthy } from "./health.js";
|
||||
|
||||
/** Slot length for time-sliced weighted DNS (one A at a time). */
|
||||
export const WEIGHTED_SLOT_MS = 60_000;
|
||||
|
||||
/** Cloudflare DNS-only minimum TTL; Auto (1) is ~300s and would smear ratios. */
|
||||
export const WEIGHTED_DNS_TTL = 60;
|
||||
|
||||
export function weightedDesired(rows: LbIpRow[], nowMs = Date.now()): string[] {
|
||||
if (rows.length === 0) return [];
|
||||
const healthy = rows.filter((r) => isHealthy(r.health));
|
||||
const pool = healthy.length > 0 ? healthy : rows;
|
||||
if (pool.length === 1) return [pool[0]!.ip];
|
||||
|
||||
const sorted = [...pool].sort((a, b) => a.ip.localeCompare(b.ip));
|
||||
const cycle: string[] = [];
|
||||
for (const row of sorted) {
|
||||
const weight = Math.max(1, Math.round(row.weight));
|
||||
for (let i = 0; i < weight; i++) cycle.push(row.ip);
|
||||
}
|
||||
const slot = Math.floor(nowMs / WEIGHTED_SLOT_MS) % cycle.length;
|
||||
return [cycle[slot]!];
|
||||
}
|
||||
@@ -30,14 +30,24 @@ import { syncServiceToVpsTracker } from "./vps-tracker-sync.js";
|
||||
import { fireEnsureHealthWorker, DEFAULT_HEALTH_FALLBACKS } from "./health/health-worker-deploy.js";
|
||||
import {
|
||||
isHealthy,
|
||||
isSharedPool,
|
||||
resolveDesiredAIps,
|
||||
selectActiveIpsByMode,
|
||||
shouldRecordFailoverDnsDiff,
|
||||
withBindingLock,
|
||||
WEIGHTED_DNS_TTL,
|
||||
type LbIpRow,
|
||||
type LbTargetConfig,
|
||||
} from "./routing/index.js";
|
||||
|
||||
export type { LbIpRow, LbTargetConfig };
|
||||
export { selectActiveIpsByMode };
|
||||
export { resolveDesiredAIps, selectActiveIpsByMode, shouldRecordFailoverDnsDiff };
|
||||
|
||||
const AUTO_DNS_TTL = 1;
|
||||
|
||||
function ttlForBinding(mode: LbMode, ipCount: number): number {
|
||||
return mode === "weighted" && ipCount >= 2 ? WEIGHTED_DNS_TTL : AUTO_DNS_TTL;
|
||||
}
|
||||
|
||||
export function failoverARecordDiff(
|
||||
existingA: readonly string[],
|
||||
@@ -65,6 +75,22 @@ function recordFailoverDnsDiff(
|
||||
const { added, removed } = failoverARecordDiff(existingA, desiredIps);
|
||||
if (added.length === 0 && removed.length === 0) return;
|
||||
const binding = repos.getBinding(db, bindingId);
|
||||
const configuredIps = repos.listBindingIps(db, bindingId);
|
||||
const { config, rows } = getBindingLbState(db, bindingId);
|
||||
const downIps = new Set(
|
||||
rows.filter((row) => row.health === "down").map((row) => row.ip),
|
||||
);
|
||||
if (
|
||||
!shouldRecordFailoverDnsDiff({
|
||||
configuredIps,
|
||||
lbMode: config.lb_mode,
|
||||
added,
|
||||
removed,
|
||||
downIps,
|
||||
})
|
||||
) {
|
||||
return;
|
||||
}
|
||||
repos.insertFailoverLog(db, {
|
||||
serviceId: binding.service_id,
|
||||
bindingId,
|
||||
@@ -264,16 +290,17 @@ function getGroupLbState(
|
||||
};
|
||||
}
|
||||
|
||||
function computeActiveIps(
|
||||
function desiredAIps(
|
||||
db: Db,
|
||||
scope: HealthCheckScope,
|
||||
refId: number,
|
||||
fallbackIps: string[],
|
||||
): string[] {
|
||||
const state =
|
||||
scope === "binding"
|
||||
? getBindingLbState(db, refId)
|
||||
: getGroupLbState(db, refId);
|
||||
return selectActiveIpsByMode(state.config, state.rows);
|
||||
return resolveDesiredAIps(state.config, state.rows, fallbackIps);
|
||||
}
|
||||
|
||||
async function collectKnownZones(
|
||||
@@ -325,7 +352,7 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
|
||||
const { config, rows } = getBindingLbState(db, binding.id);
|
||||
const bindingActiveIps = targetCname
|
||||
? []
|
||||
: selectActiveIpsByMode(config, rows);
|
||||
: resolveDesiredAIps(config, rows, targetIps);
|
||||
|
||||
return {
|
||||
binding_id: binding.id,
|
||||
@@ -613,7 +640,17 @@ async function syncBindingDns(
|
||||
return;
|
||||
}
|
||||
|
||||
await syncBindingADns(db, cf, bindingId, domainId, hostname, desiredIps);
|
||||
const binding = repos.getBinding(db, bindingId);
|
||||
const configuredIps = repos.listBindingIps(db, bindingId);
|
||||
await syncBindingADns(
|
||||
db,
|
||||
cf,
|
||||
bindingId,
|
||||
domainId,
|
||||
hostname,
|
||||
desiredIps,
|
||||
ttlForBinding(binding.lb_mode, configuredIps.length),
|
||||
);
|
||||
}
|
||||
|
||||
async function syncBindingCnameDns(
|
||||
@@ -700,6 +737,7 @@ async function syncBindingADns(
|
||||
domainId: number,
|
||||
hostname: string,
|
||||
desiredIps: string[],
|
||||
ttl: number,
|
||||
): Promise<void> {
|
||||
const domain = repos.getDomain(db, domainId);
|
||||
const zoneName = domain.zone_name;
|
||||
@@ -735,13 +773,15 @@ async function syncBindingADns(
|
||||
|
||||
for (const ip of desiredIps) {
|
||||
const existing = refreshed.find((r) => r.content === ip);
|
||||
const recordName = dnsNameForBinding(hostname, zoneName);
|
||||
let recordId: number;
|
||||
if (existing) {
|
||||
if (!dnsRecordNamesMatch(existing.name, hostname, zoneName)) {
|
||||
if (!dnsRecordNamesMatch(existing.name, hostname, zoneName) || existing.ttl !== ttl) {
|
||||
await dnsService.update(db, cf, domainId, existing.id, {
|
||||
record_type: "A",
|
||||
name: dnsNameForBinding(hostname, zoneName),
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
}
|
||||
@@ -759,12 +799,24 @@ async function syncBindingADns(
|
||||
if (adopted) {
|
||||
repos.linkBindingRecord(db, bindingId, adopted.id);
|
||||
recordId = adopted.id;
|
||||
if (
|
||||
!dnsRecordNamesMatch(adopted.name, hostname, zoneName) ||
|
||||
adopted.ttl !== ttl
|
||||
) {
|
||||
await dnsService.update(db, cf, domainId, adopted.id, {
|
||||
record_type: "A",
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const record = await dnsService.create(db, cf, domainId, {
|
||||
record_type: "A",
|
||||
name: dnsNameForBinding(hostname, zoneName),
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl: 1,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
repos.linkBindingRecord(db, bindingId, record.id);
|
||||
@@ -1001,12 +1053,7 @@ async function syncServiceBindingsToDns(
|
||||
}
|
||||
validateTargetIpsInPool(targetIps, ips);
|
||||
|
||||
if (binding.health_check_enabled) {
|
||||
const activeIps = computeActiveIps(db, "binding", binding.id);
|
||||
if (activeIps.length > 0) {
|
||||
targetIps = activeIps;
|
||||
}
|
||||
}
|
||||
const desiredIps = desiredAIps(db, "binding", binding.id, targetIps);
|
||||
|
||||
await syncBindingDns(
|
||||
db,
|
||||
@@ -1014,7 +1061,7 @@ async function syncServiceBindingsToDns(
|
||||
binding.id,
|
||||
binding.domain_id,
|
||||
binding.hostname,
|
||||
targetIps,
|
||||
desiredIps,
|
||||
null,
|
||||
);
|
||||
}
|
||||
@@ -1046,6 +1093,7 @@ async function syncGroupDomainDnsRecords(
|
||||
domainId: number,
|
||||
hostname: string,
|
||||
desiredIps: string[],
|
||||
ttl: number = AUTO_DNS_TTL,
|
||||
): Promise<void> {
|
||||
const domain = repos.getDomain(db, domainId);
|
||||
const zoneName = domain.zone_name;
|
||||
@@ -1061,14 +1109,16 @@ async function syncGroupDomainDnsRecords(
|
||||
if (desiredIps.length === 0) return;
|
||||
|
||||
const refreshed = repos.listGroupDnsRecords(db, groupId);
|
||||
const recordName = dnsNameForBinding(hostname, zoneName);
|
||||
for (const ip of desiredIps) {
|
||||
const existing = refreshed.find((r) => r.content === ip);
|
||||
if (existing) {
|
||||
if (!dnsRecordNamesMatch(existing.name, hostname, zoneName)) {
|
||||
if (!dnsRecordNamesMatch(existing.name, hostname, zoneName) || existing.ttl !== ttl) {
|
||||
await dnsService.update(db, cf, domainId, existing.id, {
|
||||
record_type: "A",
|
||||
name: dnsNameForBinding(hostname, zoneName),
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
}
|
||||
@@ -1084,13 +1134,25 @@ async function syncGroupDomainDnsRecords(
|
||||
);
|
||||
if (adopted) {
|
||||
repos.linkGroupDnsRecord(db, groupId, adopted.id);
|
||||
if (
|
||||
!dnsRecordNamesMatch(adopted.name, hostname, zoneName) ||
|
||||
adopted.ttl !== ttl
|
||||
) {
|
||||
await dnsService.update(db, cf, domainId, adopted.id, {
|
||||
record_type: "A",
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const record = await dnsService.create(db, cf, domainId, {
|
||||
record_type: "A",
|
||||
name: dnsNameForBinding(hostname, zoneName),
|
||||
name: recordName,
|
||||
content: ip,
|
||||
ttl: 1,
|
||||
ttl,
|
||||
proxied: false,
|
||||
});
|
||||
repos.linkGroupDnsRecord(db, groupId, record.id);
|
||||
@@ -1141,9 +1203,8 @@ async function syncGroupDomainDns(
|
||||
const knownZones = await collectKnownZones(db, cf);
|
||||
const { zoneName, hostname } = parseFqdn(domainValue, knownZones);
|
||||
const domainId = await resolveDomainId(db, cf, zoneName);
|
||||
const desiredIps = group.health_check_enabled
|
||||
? computeActiveIps(db, "group", groupId)
|
||||
: await collectGroupDnsIps(db, groupId);
|
||||
const fallbackIps = await collectGroupDnsIps(db, groupId);
|
||||
const desiredIps = desiredAIps(db, "group", groupId, fallbackIps);
|
||||
await syncGroupDomainDnsRecords(
|
||||
db,
|
||||
cf,
|
||||
@@ -1151,6 +1212,7 @@ async function syncGroupDomainDns(
|
||||
domainId,
|
||||
hostname,
|
||||
desiredIps,
|
||||
ttlForBinding(group.lb_mode, fallbackIps.length),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1331,14 +1393,7 @@ export async function updateConfig(
|
||||
}
|
||||
|
||||
if (pushDns) {
|
||||
let effectiveIps = targetIps;
|
||||
const refreshedBinding = repos.getBinding(db, binding.id);
|
||||
if (refreshedBinding.health_check_enabled) {
|
||||
const activeIps = computeActiveIps(db, "binding", binding.id);
|
||||
if (activeIps.length > 0) {
|
||||
effectiveIps = activeIps;
|
||||
}
|
||||
}
|
||||
const effectiveIps = desiredAIps(db, "binding", binding.id, targetIps);
|
||||
await syncBindingDns(
|
||||
db,
|
||||
cf,
|
||||
@@ -1644,16 +1699,16 @@ export async function reconcileDnsForTarget(
|
||||
if (scope === "binding") {
|
||||
await withBindingLock(refId, async () => {
|
||||
const binding = repos.getBinding(db, refId);
|
||||
if (!binding.health_check_enabled) return;
|
||||
if (!binding.health_check_enabled && binding.lb_mode !== "weighted") return;
|
||||
const service = repos.getService(db, binding.service_id);
|
||||
if (!shouldPushDns(db, service)) return;
|
||||
const cnameTarget = binding.cname_target?.trim() || null;
|
||||
if (cnameTarget) return;
|
||||
const ips = repos.listServiceIps(db, service.id);
|
||||
const targetIps = repos.listBindingIps(db, binding.id);
|
||||
if (!isSharedPool(targetIps)) return;
|
||||
validateTargetIpsInPool(targetIps, ips);
|
||||
const activeIps = computeActiveIps(db, "binding", refId);
|
||||
const desiredIps = activeIps.length > 0 ? activeIps : targetIps;
|
||||
const desiredIps = desiredAIps(db, "binding", refId, targetIps);
|
||||
await syncBindingDns(
|
||||
db,
|
||||
cf,
|
||||
@@ -1668,8 +1723,60 @@ export async function reconcileDnsForTarget(
|
||||
}
|
||||
|
||||
const group = repos.getServiceGroup(db, refId);
|
||||
if (!group.enabled || !group.domain?.trim() || !group.health_check_enabled) {
|
||||
if (!group.enabled || !group.domain?.trim()) {
|
||||
return;
|
||||
}
|
||||
if (!group.health_check_enabled && group.lb_mode !== "weighted") {
|
||||
return;
|
||||
}
|
||||
await syncGroupDomainDns(db, cf, refId);
|
||||
}
|
||||
|
||||
export async function reconcileWeightedDns(
|
||||
db: Db,
|
||||
cf: CloudflareClient,
|
||||
): Promise<number> {
|
||||
let n = 0;
|
||||
for (const binding of repos.listAllBindings(db)) {
|
||||
if (binding.lb_mode !== "weighted") continue;
|
||||
if (binding.cname_target?.trim()) continue;
|
||||
if (!isSharedPool(binding.target_ips ?? [])) continue;
|
||||
try {
|
||||
await withBindingLock(binding.id, async () => {
|
||||
const latest = repos.getBinding(db, binding.id);
|
||||
if (latest.lb_mode !== "weighted") return;
|
||||
if (latest.cname_target?.trim()) return;
|
||||
const service = repos.getService(db, latest.service_id);
|
||||
if (!shouldPushDns(db, service)) return;
|
||||
const targetIps = repos.listBindingIps(db, latest.id);
|
||||
if (!isSharedPool(targetIps)) return;
|
||||
const ips = repos.listServiceIps(db, service.id);
|
||||
validateTargetIpsInPool(targetIps, ips);
|
||||
const desiredIps = desiredAIps(db, "binding", latest.id, targetIps);
|
||||
await syncBindingDns(
|
||||
db,
|
||||
cf,
|
||||
latest.id,
|
||||
latest.domain_id,
|
||||
latest.hostname,
|
||||
desiredIps,
|
||||
null,
|
||||
);
|
||||
n += 1;
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
for (const group of repos.listServiceGroups(db)) {
|
||||
if (group.lb_mode !== "weighted") continue;
|
||||
if (!group.enabled || !group.domain?.trim()) continue;
|
||||
try {
|
||||
await syncGroupDomainDns(db, cf, group.id);
|
||||
n += 1;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { AsyncTask, SimpleIntervalJob } from "toad-scheduler";
|
||||
import * as serviceConfigService from "./service-config-service.js";
|
||||
import { WEIGHTED_SLOT_MS } from "./routing/weighted.js";
|
||||
|
||||
export const WEIGHTED_DNS_JOB_ID = "weighted-dns";
|
||||
|
||||
export function createWeightedDnsTask(app: FastifyInstance): AsyncTask {
|
||||
return new AsyncTask(
|
||||
WEIGHTED_DNS_JOB_ID,
|
||||
async () => {
|
||||
const n = await serviceConfigService.reconcileWeightedDns(app.db, app.cf);
|
||||
if (n > 0) {
|
||||
app.log.info({ reconciled: n }, "weighted dns rotated");
|
||||
}
|
||||
},
|
||||
(err) => {
|
||||
app.log.warn({ err }, "weighted dns rotate failed");
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function scheduleWeightedDnsJob(
|
||||
app: FastifyInstance,
|
||||
task: AsyncTask,
|
||||
): void {
|
||||
const scheduler = app.scheduler;
|
||||
if (!scheduler) return;
|
||||
if (scheduler.existsById(WEIGHTED_DNS_JOB_ID)) {
|
||||
scheduler.removeById(WEIGHTED_DNS_JOB_ID);
|
||||
}
|
||||
scheduler.addSimpleIntervalJob(
|
||||
new SimpleIntervalJob(
|
||||
{ seconds: WEIGHTED_SLOT_MS / 1000, runImmediately: true },
|
||||
task,
|
||||
{ id: WEIGHTED_DNS_JOB_ID, preventOverrun: true },
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1,9 +1,12 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
resolveDesiredAIps,
|
||||
selectActiveIpsByMode,
|
||||
shouldRecordFailoverDnsDiff,
|
||||
type LbIpRow,
|
||||
type LbTargetConfig,
|
||||
} from "../src/services/service-config-service.js";
|
||||
import { WEIGHTED_SLOT_MS } from "../src/services/routing/weighted.js";
|
||||
|
||||
function row(
|
||||
ip: string,
|
||||
@@ -17,6 +20,11 @@ function row(
|
||||
};
|
||||
}
|
||||
|
||||
const weightedConfig: LbTargetConfig = {
|
||||
lb_mode: "weighted",
|
||||
health_check_enabled: true,
|
||||
};
|
||||
|
||||
describe("selectActiveIpsByMode", () => {
|
||||
it("round_robin returns all healthy ips, falls back to all if none healthy", () => {
|
||||
const config: LbTargetConfig = {
|
||||
@@ -75,22 +83,49 @@ describe("selectActiveIpsByMode", () => {
|
||||
expect(selectActiveIpsByMode(config, rows)).toEqual(["2.2.2.2"]);
|
||||
});
|
||||
|
||||
it("weighted returns all healthy ips (one A per ip; weights stored for display)", () => {
|
||||
const config: LbTargetConfig = {
|
||||
lb_mode: "weighted",
|
||||
health_check_enabled: true,
|
||||
};
|
||||
it("weighted 1:3 picks the lighter ip on slot 0 and the heavier on slot 1", () => {
|
||||
const rows = [
|
||||
row("1.1.1.1", { weight: 3, health: "up" }),
|
||||
row("2.2.2.2", { weight: 1, health: "up" }),
|
||||
row("3.3.3.3", { weight: 2, health: "down" }),
|
||||
row("1.1.1.1", { weight: 1, health: "up" }),
|
||||
row("2.2.2.2", { weight: 3, health: "up" }),
|
||||
];
|
||||
expect(selectActiveIpsByMode(config, rows).sort()).toEqual([
|
||||
"1.1.1.1",
|
||||
expect(selectActiveIpsByMode(weightedConfig, rows, 0)).toEqual(["1.1.1.1"]);
|
||||
expect(selectActiveIpsByMode(weightedConfig, rows, WEIGHTED_SLOT_MS)).toEqual([
|
||||
"2.2.2.2",
|
||||
]);
|
||||
});
|
||||
|
||||
it("weighted excludes down ips from the cycle", () => {
|
||||
const rows = [
|
||||
row("1.1.1.1", { weight: 1, health: "up" }),
|
||||
row("2.2.2.2", { weight: 3, health: "down" }),
|
||||
];
|
||||
expect(selectActiveIpsByMode(weightedConfig, rows, 0)).toEqual(["1.1.1.1"]);
|
||||
expect(
|
||||
selectActiveIpsByMode(weightedConfig, rows, WEIGHTED_SLOT_MS),
|
||||
).toEqual(["1.1.1.1"]);
|
||||
});
|
||||
|
||||
it("weighted with one ip always returns that ip", () => {
|
||||
expect(
|
||||
selectActiveIpsByMode(weightedConfig, [row("1.1.1.1", { weight: 5 })], 0),
|
||||
).toEqual(["1.1.1.1"]);
|
||||
});
|
||||
|
||||
it("weighted with all unknown rotates across every ip", () => {
|
||||
const rows = [
|
||||
row("1.1.1.1", { weight: 1, health: "unknown" }),
|
||||
row("2.2.2.2", { weight: 3, health: "unknown" }),
|
||||
];
|
||||
expect(selectActiveIpsByMode(weightedConfig, rows, 0)).toEqual(["1.1.1.1"]);
|
||||
expect(selectActiveIpsByMode(weightedConfig, rows, WEIGHTED_SLOT_MS)).toEqual([
|
||||
"2.2.2.2",
|
||||
]);
|
||||
});
|
||||
|
||||
it("weighted returns empty array for no rows", () => {
|
||||
expect(selectActiveIpsByMode(weightedConfig, [], 0)).toEqual([]);
|
||||
});
|
||||
|
||||
it("round_robin excludes unknown when another ip is up", () => {
|
||||
const config: LbTargetConfig = {
|
||||
lb_mode: "round_robin",
|
||||
@@ -111,3 +146,75 @@ describe("selectActiveIpsByMode", () => {
|
||||
expect(selectActiveIpsByMode(config, [])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveDesiredAIps", () => {
|
||||
it("keeps a dedicated single IP even when down", () => {
|
||||
expect(
|
||||
resolveDesiredAIps(
|
||||
weightedConfig,
|
||||
[row("1.1.1.1", { health: "down" })],
|
||||
["1.1.1.1"],
|
||||
),
|
||||
).toEqual(["1.1.1.1"]);
|
||||
});
|
||||
|
||||
it("applies weighted overlay on a shared pool", () => {
|
||||
const rows = [
|
||||
row("1.1.1.1", { weight: 1, health: "up" }),
|
||||
row("2.2.2.2", { weight: 3, health: "up" }),
|
||||
];
|
||||
expect(
|
||||
resolveDesiredAIps(weightedConfig, rows, ["1.1.1.1", "2.2.2.2"], 0),
|
||||
).toEqual(selectActiveIpsByMode(weightedConfig, rows, 0));
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldRecordFailoverDnsDiff", () => {
|
||||
it("skips dedicated extra-FQDN", () => {
|
||||
expect(
|
||||
shouldRecordFailoverDnsDiff({
|
||||
configuredIps: ["1.1.1.1"],
|
||||
lbMode: "failover",
|
||||
added: [],
|
||||
removed: ["1.1.1.1"],
|
||||
downIps: new Set(["1.1.1.1"]),
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("skips weighted live-to-live slot swap", () => {
|
||||
expect(
|
||||
shouldRecordFailoverDnsDiff({
|
||||
configuredIps: ["1.1.1.1", "2.2.2.2"],
|
||||
lbMode: "weighted",
|
||||
added: ["2.2.2.2"],
|
||||
removed: ["1.1.1.1"],
|
||||
downIps: new Set(),
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("logs weighted swap when a down ip leaves the pool", () => {
|
||||
expect(
|
||||
shouldRecordFailoverDnsDiff({
|
||||
configuredIps: ["1.1.1.1", "2.2.2.2"],
|
||||
lbMode: "weighted",
|
||||
added: ["2.2.2.2"],
|
||||
removed: ["1.1.1.1"],
|
||||
downIps: new Set(["1.1.1.1"]),
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("logs failover diffs on a shared pool", () => {
|
||||
expect(
|
||||
shouldRecordFailoverDnsDiff({
|
||||
configuredIps: ["1.1.1.1", "2.2.2.2"],
|
||||
lbMode: "failover",
|
||||
added: ["2.2.2.2"],
|
||||
removed: ["1.1.1.1"],
|
||||
downIps: new Set(["1.1.1.1"]),
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -63,8 +63,8 @@ export function FailoverTimeline({
|
||||
return (
|
||||
<EmptyState
|
||||
icon={ShieldCheckIcon}
|
||||
title="Нет инцидентов Failover"
|
||||
description="Нет Down и нет выходов из пула"
|
||||
title="Нет инцидентов"
|
||||
description="Нет Down и нет выходов из общего пула"
|
||||
stackedIcon={false}
|
||||
centered={false}
|
||||
/>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { FormFieldSimple } from '@/components/form-field'
|
||||
import { AppInput } from '@/components/app-input'
|
||||
import { SettingRow } from '@/components/setting-row'
|
||||
import { SelectField } from '@/components/select-field'
|
||||
import { Badge } from '@/components/reui/badge'
|
||||
import {
|
||||
NumberField,
|
||||
@@ -9,13 +10,6 @@ import {
|
||||
NumberFieldIncrement,
|
||||
NumberFieldInput,
|
||||
} from '@/components/reui/number-field'
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from '@cfdm/ui/components/select'
|
||||
import { Switch } from '@cfdm/ui/components/switch'
|
||||
import { Field, FieldGroup, FieldLabel } from '@cfdm/ui/components/field'
|
||||
import { Button } from '@cfdm/ui/components/button'
|
||||
@@ -59,7 +53,7 @@ export interface LbAndHealthConfig extends HealthCheckConfig {
|
||||
const defaultLbModeOptions = [
|
||||
{ value: 'round_robin', label: 'Round Robin' },
|
||||
{ value: 'failover', label: 'Failover (приоритет)' },
|
||||
{ value: 'weighted', label: 'Weighted (веса)' },
|
||||
{ value: 'weighted', label: 'Веса (подмена IP)' },
|
||||
]
|
||||
|
||||
export interface LbPoolMetaChange {
|
||||
@@ -126,7 +120,7 @@ function PoolLbMetaFields({
|
||||
title={isWeighted ? 'Вес IP' : 'Приоритет IP'}
|
||||
description={
|
||||
isWeighted
|
||||
? 'Больше — чаще в пуле'
|
||||
? 'Доля времени на общем FQDN: 1 и 3 = ¼ и ¾ цикла (слот 60 с)'
|
||||
: '1 — основной, больше — запасной'
|
||||
}
|
||||
compact
|
||||
@@ -230,22 +224,14 @@ export function HealthCheckConfigFields({
|
||||
compact
|
||||
className={rowClass}
|
||||
>
|
||||
<Select
|
||||
<SelectField
|
||||
modal={false}
|
||||
value={value.lb_mode}
|
||||
onValueChange={(v) => patch({ lb_mode: (v ?? 'round_robin') as LbMode })}
|
||||
>
|
||||
<SelectTrigger id={`${idPrefix}-lb-mode`} className="w-full">
|
||||
<SelectValue placeholder="Выберите режим" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{lbModeOptions.map((item) => (
|
||||
<SelectItem key={item.value} value={item.value}>
|
||||
{item.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
triggerId={`${idPrefix}-lb-mode`}
|
||||
placeholder="Выберите режим"
|
||||
options={lbModeOptions}
|
||||
/>
|
||||
</SettingRow>
|
||||
) : null}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
type HealthLogProbe,
|
||||
type HealthLogStatus,
|
||||
} from '@/lib/health-log'
|
||||
import type { FailoverLogEntry } from '@/lib/schemas'
|
||||
import type { FailoverLogEntry, ServiceView } from '@/lib/schemas'
|
||||
import { Badge } from '@/components/reui/badge'
|
||||
import {
|
||||
Frame,
|
||||
@@ -28,6 +28,23 @@ import {
|
||||
AlertTitle,
|
||||
} from '@/components/reui/alert'
|
||||
|
||||
type LbMode = ServiceView['lb_mode']
|
||||
|
||||
const PANEL_COPY: Record<LbMode, { title: string; description: string }> = {
|
||||
failover: {
|
||||
title: 'Failover (приоритет)',
|
||||
description: 'Down снимается с общего FQDN. История: кто вышел из пула и кто вернулся',
|
||||
},
|
||||
weighted: {
|
||||
title: 'Веса (подмена IP)',
|
||||
description: 'На общем FQDN один IP по весам. Down выводится из цикла',
|
||||
},
|
||||
round_robin: {
|
||||
title: 'Round Robin',
|
||||
description: 'На общем FQDN все живые A. Down снимается с пула',
|
||||
},
|
||||
}
|
||||
|
||||
function failoverCountLabel(count: number): string {
|
||||
const mod10 = count % 10
|
||||
const mod100 = count % 100
|
||||
@@ -38,8 +55,17 @@ function failoverCountLabel(count: number): string {
|
||||
return `${count} адресов Down`
|
||||
}
|
||||
|
||||
function alertDescription(events: { kind: string; fqdns: string[] }[]): string {
|
||||
const removedCount = events.filter((event) => event.kind === 'removed').length
|
||||
if (removedCount > 0) return 'Сняты с общего FQDN'
|
||||
if (events.some((event) => event.fqdns.length > 0)) {
|
||||
return 'Остались в A-записях общего FQDN как last-resort'
|
||||
}
|
||||
return 'Down: персональные FQDN не меняются'
|
||||
}
|
||||
|
||||
/**
|
||||
* Failover — текущие Down + кто вышел из пула и кто вернулся.
|
||||
* Балансировка — текущие Down + кто вышел из общего пула.
|
||||
* Preview: https://reui.io/preview/base/components/c-timeline-10
|
||||
* Preview: https://reui.io/preview/base/empty-state-12
|
||||
* Docs: https://reui.io/docs/components/base/frame
|
||||
@@ -48,16 +74,19 @@ function failoverCountLabel(count: number): string {
|
||||
* Docs: https://reui.io/docs/components/base/alert
|
||||
*/
|
||||
export function ServiceFailoverPanel({
|
||||
lbMode = 'round_robin',
|
||||
ipHealth,
|
||||
bindings,
|
||||
history,
|
||||
probes = [],
|
||||
}: {
|
||||
lbMode?: LbMode
|
||||
ipHealth: readonly FailoverHealthInput[]
|
||||
bindings: readonly FailoverBindingPool[]
|
||||
history: readonly FailoverLogEntry[]
|
||||
probes?: readonly HealthLogProbe[]
|
||||
}) {
|
||||
const copy = PANEL_COPY[lbMode] ?? PANEL_COPY.round_robin
|
||||
const liveByIp = latestHealthByIp(probes)
|
||||
const overlayHealth = ipHealth.map((row) => {
|
||||
const live = liveByIp.get(row.ip)
|
||||
@@ -77,14 +106,13 @@ export function ServiceFailoverPanel({
|
||||
})
|
||||
const events = toFailoverEvents(overlayHealth, bindings)
|
||||
const mergedHistory = mergeFailoverHistory(history, probes, bindings)
|
||||
const removedCount = events.filter((event) => event.kind === 'removed').length
|
||||
|
||||
return (
|
||||
<Frame stacked spacing="sm" className="min-w-0 w-full">
|
||||
<FramePanel className="flex flex-col gap-3">
|
||||
<FrameHeader className="gap-1 px-0 py-0">
|
||||
<FrameTitle className="flex flex-wrap items-center gap-2">
|
||||
Failover
|
||||
{copy.title}
|
||||
{events.length > 0 ? (
|
||||
<Badge variant="destructive-light" size="xs" radius="full">
|
||||
{events.length}
|
||||
@@ -95,20 +123,14 @@ export function ServiceFailoverPanel({
|
||||
</Badge>
|
||||
)}
|
||||
</FrameTitle>
|
||||
<FrameDescription>
|
||||
Текущие Down и история: кто вышел из пула и кто вернулся
|
||||
</FrameDescription>
|
||||
<FrameDescription>{copy.description}</FrameDescription>
|
||||
</FrameHeader>
|
||||
|
||||
{events.length > 0 ? (
|
||||
<Alert variant="destructive">
|
||||
<UnplugIcon aria-hidden="true" />
|
||||
<AlertTitle>{failoverCountLabel(events.length)}</AlertTitle>
|
||||
<AlertDescription>
|
||||
{removedCount > 0
|
||||
? 'Сняты с FQDN или остались last-resort'
|
||||
: 'Остались в A-записях как last-resort'}
|
||||
</AlertDescription>
|
||||
<AlertDescription>{alertDescription(events)}</AlertDescription>
|
||||
</Alert>
|
||||
) : null}
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ const LB_MODE_META: Record<
|
||||
weighted: {
|
||||
icon: ScaleIcon,
|
||||
className: 'text-info',
|
||||
label: 'Weighted (веса)',
|
||||
label: 'Веса (подмена IP)',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -137,6 +137,7 @@ describe('toFailoverEvents', () => {
|
||||
},
|
||||
])
|
||||
expect(failoverEventCopy(events[0]!)).toBe('Снята с gt.rkns.top')
|
||||
expect(events[0]?.fqdns).not.toContain('nsgt.rkns.top')
|
||||
})
|
||||
|
||||
it('Down только last-resort на своём FQDN', () => {
|
||||
@@ -151,10 +152,8 @@ describe('toFailoverEvents', () => {
|
||||
],
|
||||
)
|
||||
expect(events[0]?.kind).toBe('last-resort')
|
||||
expect(events[0]?.fqdns).toEqual(['nsgt.rkns.top'])
|
||||
expect(failoverEventCopy(events[0]!)).toBe(
|
||||
'Down, в A-записях last-resort на nsgt.rkns.top',
|
||||
)
|
||||
expect(events[0]?.fqdns).toEqual([])
|
||||
expect(failoverEventCopy(events[0]!)).toBe('Down, в A-записях last-resort')
|
||||
})
|
||||
|
||||
it('down без A-записи на configured FQDN — снятие', () => {
|
||||
@@ -279,7 +278,9 @@ describe('mergeFailoverHistory', () => {
|
||||
'removed:probe',
|
||||
])
|
||||
expect(history[0]?.copy).toBe('130.49.213.153 добавлена на gt.rkns.top')
|
||||
expect(history[1]?.copy).toContain('вышла из пула')
|
||||
expect(history[1]?.copy).toBe(
|
||||
'130.49.213.153 вышла из пула (gt.rkns.top)',
|
||||
)
|
||||
})
|
||||
|
||||
it('keeps DNS over a probe return in the same 2-minute window', () => {
|
||||
@@ -291,4 +292,20 @@ describe('mergeFailoverHistory', () => {
|
||||
expect(history).toHaveLength(1)
|
||||
expect(history[0]?.source).toBe('dns')
|
||||
})
|
||||
|
||||
it('drops dedicated extra-FQDN DNS rows', () => {
|
||||
const history = mergeFailoverHistory(
|
||||
[
|
||||
dns({
|
||||
id: 11,
|
||||
fqdn: 'nsgt.rkns.top',
|
||||
action: 'added',
|
||||
created_at: '2026-08-20 09:26:00',
|
||||
}),
|
||||
],
|
||||
[],
|
||||
mskHip,
|
||||
)
|
||||
expect(history).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -44,6 +44,11 @@ export interface FailoverBindingPool {
|
||||
active: readonly string[]
|
||||
}
|
||||
|
||||
/** Shared pool FQDN — two or more A targets. Dedicated extra-FQDN has one IP. */
|
||||
export function isSharedPoolBinding(binding: FailoverBindingPool): boolean {
|
||||
return binding.configured.length >= 2
|
||||
}
|
||||
|
||||
/** Инцидент только при down. up / degraded / unknown — не вывод из пула. */
|
||||
export function isFailoverEventStatus(status: string): boolean {
|
||||
return status === 'down'
|
||||
@@ -72,6 +77,7 @@ export function toFailoverEvents(
|
||||
const removedFqdns: string[] = []
|
||||
const lastResortFqdns: string[] = []
|
||||
for (const binding of bindings) {
|
||||
if (!isSharedPoolBinding(binding)) continue
|
||||
const configured = binding.configured.includes(row.ip)
|
||||
const active = binding.active.includes(row.ip)
|
||||
if (configured && !active) removedFqdns.push(binding.fqdn)
|
||||
@@ -98,7 +104,18 @@ function fqdnsForIp(
|
||||
ip: string,
|
||||
bindings: readonly FailoverBindingPool[],
|
||||
): string[] {
|
||||
return bindings.filter((binding) => binding.configured.includes(ip)).map((binding) => binding.fqdn)
|
||||
return bindings
|
||||
.filter((binding) => isSharedPoolBinding(binding) && binding.configured.includes(ip))
|
||||
.map((binding) => binding.fqdn)
|
||||
}
|
||||
|
||||
function isPoolFqdn(
|
||||
fqdn: string,
|
||||
bindings: readonly FailoverBindingPool[],
|
||||
): boolean {
|
||||
const binding = bindings.find((item) => item.fqdn === fqdn)
|
||||
if (!binding) return true
|
||||
return isSharedPoolBinding(binding)
|
||||
}
|
||||
|
||||
function fqdnLabel(fqdns: string[]): string {
|
||||
@@ -201,7 +218,9 @@ export function mergeFailoverHistory(
|
||||
bindings: readonly FailoverBindingPool[] = [],
|
||||
): FailoverHistoryItem[] {
|
||||
const merged = [
|
||||
...dns.map(dnsHistoryItem),
|
||||
...dns
|
||||
.filter((item) => isPoolFqdn(item.fqdn, bindings))
|
||||
.map(dnsHistoryItem),
|
||||
...toIpAliveTransitions(probes).map((transition) =>
|
||||
probeHistoryItem(transition, bindings),
|
||||
),
|
||||
|
||||
@@ -328,6 +328,7 @@ function ServiceDetailPage() {
|
||||
isLoading={logQuery.isLoading}
|
||||
/>
|
||||
<ServiceFailoverPanel
|
||||
lbMode={service.lb_mode}
|
||||
ipHealth={service.ip_health}
|
||||
bindings={failoverBindings}
|
||||
history={failoverHistory}
|
||||
|
||||
Reference in New Issue
Block a user