Denozordec
1d497d01c2
Introduced a new `verify_tls` boolean option in health check configurations across various services, allowing users to specify whether to validate TLS certificates during health checks. Updated related components and services to accommodate this new option, ensuring proper handling in both the backend and frontend. Enhanced tests to validate the new functionality and ensure correct behavior with different configurations.
449 lines
13 KiB
TypeScript
449 lines
13 KiB
TypeScript
import { connect, isIP } from "node:net";
|
|
import { resolve4, resolve6 } from "node:dns/promises";
|
|
import { Agent, buildConnector, fetch as undiciFetch } from "undici";
|
|
import type { Db } from "@cfdm/db";
|
|
import { repos } from "@cfdm/db";
|
|
import type { HealthCheckTarget, IpHealthState } from "@cfdm/shared";
|
|
import { AppError } from "../errors.js";
|
|
|
|
export interface HealthCheckThresholds {
|
|
degradedFailures: number;
|
|
downFailures: number;
|
|
latencyWarnMs: number;
|
|
}
|
|
|
|
export interface ProbeResult {
|
|
ok: boolean;
|
|
latencyMs: number;
|
|
error: string | null;
|
|
}
|
|
|
|
/** Bracket IPv6 for URL authority; leave IPv4/hostname as-is. */
|
|
export function hostForUrl(ipOrHost: string): string {
|
|
return isIP(ipOrHost) === 6 ? `[${ipOrHost}]` : ipOrHost;
|
|
}
|
|
|
|
/**
|
|
* Build http(s) URL authority for the probe.
|
|
* Prefer FQDN in the URL (correct Host/SNI); TCP dial goes to configured IP via custom connector.
|
|
*/
|
|
export function buildHttpProbeUrl(
|
|
urlHost: string,
|
|
port: number,
|
|
pathWithSlash: string,
|
|
useTls: boolean,
|
|
): string {
|
|
const defaultPort = useTls ? 443 : 80;
|
|
const portPart = port === defaultPort ? "" : `:${port}`;
|
|
return `${useTls ? "https" : "http"}://${hostForUrl(urlHost)}${portPart}${pathWithSlash}`;
|
|
}
|
|
|
|
function tcpProbe(
|
|
ip: string,
|
|
port: number,
|
|
timeoutMs: number,
|
|
): Promise<ProbeResult> {
|
|
return new Promise((resolve) => {
|
|
const started = Date.now();
|
|
const socket = connect({ host: ip, port, timeout: timeoutMs });
|
|
let settled = false;
|
|
|
|
const finish = (result: ProbeResult) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
socket.destroy();
|
|
resolve(result);
|
|
};
|
|
|
|
socket.on("connect", () =>
|
|
finish({
|
|
ok: true,
|
|
latencyMs: Date.now() - started,
|
|
error: null,
|
|
}),
|
|
);
|
|
socket.on("timeout", () =>
|
|
finish({
|
|
ok: false,
|
|
latencyMs: Date.now() - started,
|
|
error: "connection timeout",
|
|
}),
|
|
);
|
|
socket.on("error", (err) =>
|
|
finish({
|
|
ok: false,
|
|
latencyMs: Date.now() - started,
|
|
error: err.message,
|
|
}),
|
|
);
|
|
});
|
|
}
|
|
|
|
/** Dial `connectAddr` for TCP/TLS while URL Host/SNI stay on the FQDN. */
|
|
function createIpPinnedAgent(
|
|
connectAddr: string,
|
|
sniHost: string,
|
|
useTls: boolean,
|
|
timeoutMs: number,
|
|
verifyTls: boolean,
|
|
): Agent {
|
|
const connector = buildConnector({
|
|
rejectUnauthorized: verifyTls,
|
|
timeout: timeoutMs,
|
|
});
|
|
return new Agent({
|
|
connect(opts, callback) {
|
|
connector(
|
|
{
|
|
...opts,
|
|
// Force socket to configured IP (or CNAME target), not public DNS of FQDN.
|
|
hostname: connectAddr,
|
|
host: connectAddr,
|
|
servername:
|
|
useTls && isIP(sniHost) === 0
|
|
? sniHost
|
|
: (opts.servername as string | undefined),
|
|
},
|
|
callback,
|
|
);
|
|
},
|
|
});
|
|
}
|
|
|
|
/**
|
|
* HTTP(S) probe: URL/Host/SNI use hostname (vhost), TCP connects to configured IP when numeric.
|
|
* Avoids re-resolving FQDN via public DNS (which skewed group vs binding latency for the same IP).
|
|
*/
|
|
async function httpProbe(
|
|
ip: string,
|
|
target: HealthCheckTarget,
|
|
timeoutMs: number,
|
|
): Promise<ProbeResult> {
|
|
const started = Date.now();
|
|
const path = target.path?.trim() || "/";
|
|
const pathWithSlash = path.startsWith("/") ? path : `/${path}`;
|
|
const port = target.port ?? 80;
|
|
const useTls = port === 443;
|
|
const connectAddr = String(ip || "").trim();
|
|
const headerHost = (target.hostname || "").trim() || connectAddr;
|
|
const url = buildHttpProbeUrl(headerHost, port, pathWithSlash, useTls);
|
|
const verifyTls = target.verify_tls === true;
|
|
|
|
const family = isIP(connectAddr);
|
|
const pinToIp = family === 4 || family === 6;
|
|
|
|
const dispatcher = pinToIp
|
|
? createIpPinnedAgent(connectAddr, headerHost, useTls, timeoutMs, verifyTls)
|
|
: useTls
|
|
? createIpPinnedAgent(connectAddr, headerHost, useTls, timeoutMs, verifyTls)
|
|
: undefined;
|
|
|
|
try {
|
|
const response = await undiciFetch(url, {
|
|
method: "GET",
|
|
signal: AbortSignal.timeout(timeoutMs),
|
|
redirect: "manual",
|
|
dispatcher,
|
|
});
|
|
const latency = Date.now() - started;
|
|
if (target.expected_status != null) {
|
|
if (response.status !== target.expected_status) {
|
|
return {
|
|
ok: false,
|
|
latencyMs: latency,
|
|
error: `expected ${target.expected_status}, got ${response.status}`,
|
|
};
|
|
}
|
|
return { ok: true, latencyMs: latency, error: null };
|
|
}
|
|
if (response.status >= 200 && response.status < 400) {
|
|
return { ok: true, latencyMs: latency, error: null };
|
|
}
|
|
return {
|
|
ok: false,
|
|
latencyMs: latency,
|
|
error: `unexpected status ${response.status}`,
|
|
};
|
|
} catch (err) {
|
|
return {
|
|
ok: false,
|
|
latencyMs: Date.now() - started,
|
|
error: err instanceof Error ? err.message : String(err),
|
|
};
|
|
}
|
|
}
|
|
|
|
/** Host reachability via TCP :443 then :80 (ICMP often unavailable in Node). */
|
|
async function pingProbe(hostname: string, timeoutMs: number): Promise<ProbeResult> {
|
|
const ports = [443, 80];
|
|
let last: ProbeResult = {
|
|
ok: false,
|
|
latencyMs: 0,
|
|
error: "unreachable",
|
|
};
|
|
for (const port of ports) {
|
|
last = await tcpProbe(hostname, port, timeoutMs);
|
|
if (last.ok) return last;
|
|
}
|
|
return last;
|
|
}
|
|
|
|
async function dnsProbe(hostname: string): Promise<ProbeResult> {
|
|
const started = Date.now();
|
|
try {
|
|
const [v4, v6] = await Promise.allSettled([
|
|
resolve4(hostname),
|
|
resolve6(hostname),
|
|
]);
|
|
const hasV4 = v4.status === "fulfilled" && v4.value.length > 0;
|
|
const hasV6 = v6.status === "fulfilled" && v6.value.length > 0;
|
|
if (!hasV4 && !hasV6) {
|
|
return {
|
|
ok: false,
|
|
latencyMs: Date.now() - started,
|
|
error: "no A/AAAA records",
|
|
};
|
|
}
|
|
return { ok: true, latencyMs: Date.now() - started, error: null };
|
|
} catch (err) {
|
|
return {
|
|
ok: false,
|
|
latencyMs: Date.now() - started,
|
|
error: err instanceof Error ? err.message : String(err),
|
|
};
|
|
}
|
|
}
|
|
|
|
export async function probeTarget(
|
|
target: HealthCheckTarget,
|
|
): Promise<ProbeResult> {
|
|
const port = target.port ?? (target.type === "http" ? 80 : 80);
|
|
const timeoutMs = target.timeout_ms || 3000;
|
|
if (target.type === "http") {
|
|
return httpProbe(target.ip, target, timeoutMs);
|
|
}
|
|
if (target.type === "ping") {
|
|
return pingProbe(target.hostname || target.ip, timeoutMs);
|
|
}
|
|
if (target.type === "dns") {
|
|
return dnsProbe(target.hostname || target.ip);
|
|
}
|
|
return tcpProbe(target.ip, port, timeoutMs);
|
|
}
|
|
|
|
function deriveState(
|
|
ok: boolean,
|
|
latencyMs: number,
|
|
prev: { consecutive_failures: number; status: string } | null,
|
|
thresholds: HealthCheckThresholds,
|
|
): { state: IpHealthState; failures: number } {
|
|
if (!ok) {
|
|
const failures = (prev?.consecutive_failures ?? 0) + 1;
|
|
if (failures >= thresholds.downFailures) {
|
|
return { state: "down", failures };
|
|
}
|
|
if (failures >= thresholds.degradedFailures) {
|
|
return { state: "degraded", failures };
|
|
}
|
|
return { state: "degraded", failures };
|
|
}
|
|
if (latencyMs > thresholds.latencyWarnMs) {
|
|
return { state: "degraded", failures: 0 };
|
|
}
|
|
return { state: "up", failures: 0 };
|
|
}
|
|
|
|
export interface RunAllChecksOptions {
|
|
thresholds: HealthCheckThresholds;
|
|
/** Pause between unique physical probes (default 2000). Same IP is only probed once. */
|
|
probeGapMs?: number;
|
|
onStatusChange?: (
|
|
target: HealthCheckTarget,
|
|
prevState: IpHealthState | null,
|
|
nextState: IpHealthState,
|
|
) => void;
|
|
}
|
|
|
|
function sleep(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
/**
|
|
* One network hit per key. Group+binding on the same IP share a single TCP/HTTP probe
|
|
* so anti-bot / rate-limit on the origin is not tripped by back-to-back checks.
|
|
*/
|
|
export function physicalProbeKey(target: HealthCheckTarget): string {
|
|
const port = target.port ?? (target.type === "http" ? 80 : 80);
|
|
const ip = String(target.ip || "").trim().toLowerCase();
|
|
if (target.type === "http") {
|
|
const path = (target.path?.trim() || "/") || "/";
|
|
const expected = target.expected_status ?? "";
|
|
return `http|${ip}|${port}|${path}|${expected}`;
|
|
}
|
|
if (target.type === "tcp") return `tcp|${ip}|${port}`;
|
|
if (target.type === "ping") {
|
|
return `ping|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
|
|
}
|
|
if (target.type === "dns") {
|
|
return `dns|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
|
|
}
|
|
return `${target.type}|${ip}|${port}`;
|
|
}
|
|
|
|
export async function runAllChecks(
|
|
db: Db,
|
|
options: RunAllChecksOptions,
|
|
): Promise<number> {
|
|
const targets = repos.listHealthCheckTargets(db);
|
|
const gapMs = Math.max(0, options.probeGapMs ?? 2000);
|
|
|
|
const byPhysical = new Map<string, HealthCheckTarget[]>();
|
|
for (const target of targets) {
|
|
const key = physicalProbeKey(target);
|
|
const list = byPhysical.get(key);
|
|
if (list) list.push(target);
|
|
else byPhysical.set(key, [target]);
|
|
}
|
|
|
|
let probeIndex = 0;
|
|
for (const group of byPhysical.values()) {
|
|
if (probeIndex > 0 && gapMs > 0) {
|
|
await sleep(gapMs);
|
|
}
|
|
probeIndex += 1;
|
|
|
|
// Prefer binding hostname for SNI when several scopes share one IP.
|
|
const representative =
|
|
group.find((t) => t.scope === "binding") ?? group[0]!;
|
|
const result = await probeTarget(representative);
|
|
|
|
for (const target of group) {
|
|
const prev = repos.getIpHealthStatusRow(
|
|
db,
|
|
target.scope,
|
|
target.ref_id,
|
|
target.ip,
|
|
);
|
|
const { state, failures } = deriveState(
|
|
result.ok,
|
|
result.latencyMs,
|
|
prev
|
|
? {
|
|
consecutive_failures: prev.consecutive_failures,
|
|
status: prev.status,
|
|
}
|
|
: null,
|
|
options.thresholds,
|
|
);
|
|
const prevState: IpHealthState | null = prev
|
|
? (prev.status as IpHealthState)
|
|
: null;
|
|
repos.upsertIpHealthStatus(
|
|
db,
|
|
target.scope,
|
|
target.ref_id,
|
|
target.ip,
|
|
state,
|
|
result.latencyMs,
|
|
failures,
|
|
result.error,
|
|
);
|
|
if (prevState !== state) {
|
|
options.onStatusChange?.(target, prevState, state);
|
|
}
|
|
}
|
|
}
|
|
// Orphan rows (old IPs / hostname keys) still feed MAX latency on group badge.
|
|
repos.pruneStaleIpHealthStatus(db, targets);
|
|
return targets.length;
|
|
}
|
|
|
|
export async function runDomainMonitors(
|
|
db: Db,
|
|
thresholds: HealthCheckThresholds,
|
|
): Promise<number> {
|
|
const monitors = repos.listEnabledDomainMonitors(db);
|
|
let checked = 0;
|
|
for (const monitor of monitors) {
|
|
const target: HealthCheckTarget = {
|
|
scope: "binding",
|
|
ref_id: monitor.id,
|
|
ip: monitor.hostname,
|
|
hostname: monitor.hostname,
|
|
type: monitor.type as HealthCheckTarget["type"],
|
|
port: monitor.type === "http" ? (monitor.path?.includes("443") ? 443 : 80) : null,
|
|
path: monitor.path,
|
|
expected_status: monitor.expected_status,
|
|
timeout_ms: monitor.timeout_ms,
|
|
verify_tls: false,
|
|
};
|
|
let result: ProbeResult;
|
|
if (monitor.type === "http") {
|
|
result = await httpProbe(monitor.hostname, {
|
|
...target,
|
|
port: 443,
|
|
path: monitor.path ?? "/",
|
|
}, monitor.timeout_ms);
|
|
if (!result.ok) {
|
|
result = await httpProbe(monitor.hostname, {
|
|
...target,
|
|
port: 80,
|
|
path: monitor.path ?? "/",
|
|
}, monitor.timeout_ms);
|
|
}
|
|
} else if (monitor.type === "ping") {
|
|
result = await pingProbe(monitor.hostname, monitor.timeout_ms);
|
|
} else {
|
|
result = await dnsProbe(monitor.hostname);
|
|
}
|
|
const prevStatus = monitor.last_status as IpHealthState;
|
|
const { state } = deriveState(
|
|
result.ok,
|
|
result.latencyMs,
|
|
{
|
|
consecutive_failures: result.ok ? 0 : 1,
|
|
status: prevStatus,
|
|
},
|
|
thresholds,
|
|
);
|
|
repos.updateDomainMonitorResult(
|
|
db,
|
|
monitor.id,
|
|
state,
|
|
result.latencyMs,
|
|
result.error,
|
|
);
|
|
if (prevStatus !== state && prevStatus !== "unknown") {
|
|
const label =
|
|
state === "up" ? "OK" : state === "degraded" ? "Slow" : state === "down" ? "Down" : "—";
|
|
repos.insertNotificationLog(
|
|
db,
|
|
"domain_monitor",
|
|
"domain_monitor",
|
|
monitor.id,
|
|
`${monitor.hostname}: ${label}`,
|
|
result.error
|
|
? `${monitor.type.toUpperCase()} → ${label}. ${result.error}`
|
|
: `${monitor.type.toUpperCase()} → ${label}${result.latencyMs != null ? ` (${result.latencyMs} мс)` : ""}`,
|
|
);
|
|
}
|
|
checked += 1;
|
|
}
|
|
return checked;
|
|
}
|
|
|
|
export function listStatus(
|
|
db: Db,
|
|
scope: "binding" | "group",
|
|
refId: number,
|
|
) {
|
|
return repos.listIpHealthStatus(db, scope, refId);
|
|
}
|
|
|
|
export function requireValidScope(scope: string): "binding" | "group" {
|
|
if (scope !== "binding" && scope !== "group") {
|
|
throw AppError.validation(`invalid scope: ${scope}`);
|
|
}
|
|
return scope;
|
|
}
|