fix(health): разворачивать CNAME до IP в статусе таблицы

Пробы CNAME больше не ключуются hostname: цель разворачивается до origin/пула, а уже сохранённый статус по CNAME копируется на IP сервиса.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Denozordec
2026-08-20 02:13:09 +07:00
co-authored by Cursor
parent 3da6de9311
commit 1bf6cfa0d0
4 changed files with 298 additions and 25 deletions
@@ -18,6 +18,7 @@ import {
SYNC_PENDING_PUSH,
SYNC_SYNCED,
dnsRecordNamesMatch,
isIpLiteral,
normalizeDnsRecordName,
} from "@cfdm/shared";
import type { CloudflareClient } from "../lib/cf-client.js";
@@ -344,6 +345,64 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
};
}
const HEALTH_RANK: Record<string, number> = {
down: 3,
degraded: 2,
unknown: 1,
up: 0,
};
function cnameLookupKeys(value: string, zoneName?: string | null): string[] {
const trimmed = value.trim();
if (!trimmed) return [];
const noDot = trimmed.replace(/\.+$/, "");
const lower = noDot.toLowerCase();
const keys = new Set([trimmed, noDot, lower]);
if (zoneName && !lower.includes(".")) {
keys.add(`${lower}.${zoneName.trim().toLowerCase().replace(/\.+$/, "")}`);
}
return [...keys];
}
type ServiceHealthRow = {
ip: string;
status: IpHealthState;
latency_ms: number | null;
last_checked_at: string | null;
last_error: string | null;
provider: ServiceView["ip_health"][number]["provider"];
colo: string | null;
};
/** Health rows keyed by CNAME hostname (legacy probes) applied to service IPs. */
function fallbackCnameHealth(
rows: ServiceHealthRow[],
view: ServiceView,
): ServiceHealthRow | undefined {
const cnameKeys = new Set<string>();
for (const domain of view.domains ?? []) {
const cname = domain.target_cname?.trim();
if (!cname) continue;
for (const key of cnameLookupKeys(cname, domain.zone_name)) {
cnameKeys.add(key);
}
}
const hostnameRows = rows.filter((row) => !isIpLiteral(row.ip));
if (hostnameRows.length === 0) return undefined;
const matched =
cnameKeys.size === 0
? hostnameRows
: hostnameRows.filter((row) =>
cnameLookupKeys(row.ip).some((key) => cnameKeys.has(key)),
);
const candidates = matched.length > 0 ? matched : hostnameRows;
return candidates.reduce((worst, row) =>
(HEALTH_RANK[row.status] ?? 0) > (HEALTH_RANK[worst.status] ?? 0)
? row
: worst,
);
}
function attachServiceHealth(
db: Db,
views: ServiceView[],
@@ -353,11 +412,16 @@ function attachServiceHealth(
const ipHealthByService = repos.listIpHealthByServiceIds(db, ids);
return views.map((view) => {
const health = healthByService.get(view.id);
const byIp = new Map(
(ipHealthByService.get(view.id) ?? []).map((row) => [row.ip, row]),
const rows = ipHealthByService.get(view.id) ?? [];
const byIp = new Map(rows.map((row) => [row.ip, row]));
const cnameFallback = fallbackCnameHealth(rows, view);
const aRecordIps = new Set(
(view.domains ?? []).flatMap((domain) =>
domain.target_cname?.trim() ? [] : (domain.target_ips ?? []),
),
);
const ip_health = (view.ips ?? []).map((ip) => {
const row = byIp.get(ip);
const row = byIp.get(ip) ?? (aRecordIps.has(ip) ? undefined : cnameFallback);
return {
ip,
status: row?.status ?? ("unknown" as const),
+91
View File
@@ -235,4 +235,95 @@ describe("health-check state derivation via runAllChecks", () => {
expect(targets).toHaveLength(1);
expect(targets[0]?.verify_tls).toBe(true);
});
it("unwraps CNAME target to origin A record IPs", async () => {
const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
const { db, sqlite } = createMemoryDb();
runMigrations(sqlite);
const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
repos.insertDnsRecord(
db,
domain.id,
"A",
"ihome",
"2.59.161.102",
1,
false,
null,
"synced",
"cf",
null,
);
const service = repos.createService(db, "RW Sub", "rw-sub");
const binding = repos.insertBinding(db, domain.id, service.id, "s", null);
repos.setBindingCnameTarget(db, binding.id, "ihome.rkns.top");
repos.updateBindingLbConfig(db, binding.id, {
health_check_enabled: true,
health_check_type: "tcp",
health_check_port: 443,
});
const targets = repos.listHealthCheckTargets(db);
expect(targets).toHaveLength(1);
expect(targets[0]?.ip).toBe("2.59.161.102");
expect(targets[0]?.hostname).toBe("s.rkns.top");
});
it("unwraps CNAME target to service IP pool when origin DNS is empty", async () => {
const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
const { db, sqlite } = createMemoryDb();
runMigrations(sqlite);
const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
const service = repos.createService(db, "RW Sub", "rw-sub");
repos.replaceServiceIps(db, service.id, ["2.59.161.102"]);
const binding = repos.insertBinding(db, domain.id, service.id, "s", null);
repos.setBindingCnameTarget(db, binding.id, "ihome.rkns.top");
repos.updateBindingLbConfig(db, binding.id, {
health_check_enabled: true,
health_check_type: "tcp",
health_check_port: 443,
});
const targets = repos.listHealthCheckTargets(db);
expect(targets).toHaveLength(1);
expect(targets[0]?.ip).toBe("2.59.161.102");
expect(targets[0]?.hostname).toBe("s.rkns.top");
});
});
describe("CNAME health mapped onto service IPs", () => {
it("getView copies CNAME-keyed health onto the service IP row", async () => {
const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
const { getView } = await import("../src/services/service-config-service.js");
const { db, sqlite } = createMemoryDb();
runMigrations(sqlite);
const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
const service = repos.createService(db, "RW Sub", "rw-sub");
repos.replaceServiceIps(db, service.id, ["2.59.161.102"]);
const binding = repos.insertBinding(db, domain.id, service.id, "s", null);
repos.setBindingCnameTarget(db, binding.id, "ihome.rkns.top");
repos.upsertIpHealthStatus(
db,
"binding",
binding.id,
"ihome.rkns.top",
"up",
12,
0,
null,
);
const view = await getView(db, service.id);
expect(view.health_status).toBe("up");
expect(view.ip_health).toEqual([
expect.objectContaining({
ip: "2.59.161.102",
status: "up",
latency_ms: 12,
}),
]);
});
});
+53 -9
View File
@@ -2217,6 +2217,40 @@ function pruneStaleIpHealthStatus(db, activeTargets) {
}
return deleted;
}
function normalizeCnameHost(target, zoneName) {
const trimmed = target.trim().toLowerCase().replace(/\.+$/, "");
if (!trimmed) return "";
if (trimmed.includes(".")) return trimmed;
const zone = zoneName.trim().toLowerCase().replace(/\.+$/, "");
return zone ? `${trimmed}.${zone}` : trimmed;
}
function resolveCnameProbeIps(db, cnameTarget, zoneName, serviceId) {
const fqdn = normalizeCnameHost(cnameTarget, zoneName);
if (fqdn) {
const fromDns = listOriginIpsForFqdn(db, fqdn).filter(isIpLiteral);
if (fromDns.length > 0) return [...new Set(fromDns)];
}
if (serviceId > 0) {
return [...new Set(listServiceIps(db, serviceId).filter(isIpLiteral))];
}
return [];
}
function expandCnameHealthTargets(db, rows) {
const expanded = [];
for (const row of rows) {
const ips = resolveCnameProbeIps(
db,
row.ip,
row.zone_name ?? "",
row.service_id ?? 0
);
const resolved = ips.length > 0 ? ips : [row.ip];
for (const ip of resolved) {
expanded.push({ ...row, ip });
}
}
return expanded;
}
function listHealthCheckTargets(db) {
const fqdnExpr = sql2`CASE WHEN sb.hostname = '@' OR sb.hostname IS NULL THEN d.zone_name ELSE sb.hostname || '.' || d.zone_name END`;
const bindingTargets = db.all(sql2`
@@ -2235,7 +2269,7 @@ function listHealthCheckTargets(db) {
JOIN service_bindings sb ON sb.id = sbi.binding_id
JOIN domains d ON d.id = sb.domain_id
WHERE sb.health_check_enabled = 1
`);
`).filter((t) => isIpLiteral(t.ip));
const groupTargets = db.all(sql2`
SELECT DISTINCT 'group' AS scope, sg.id AS ref_id, sbi.ip,
sg.domain AS hostname,
@@ -2258,7 +2292,7 @@ function listHealthCheckTargets(db) {
AND s.enabled = 1
AND sg.enabled = 1
AND (sb.cname_target IS NULL OR sb.cname_target = '')
`);
`).filter((t) => isIpLiteral(t.ip));
const groupInheritedBindingTargets = db.all(sql2`
SELECT 'binding' AS scope, sb.id AS ref_id, sbi.ip,
${fqdnExpr} AS hostname,
@@ -2281,8 +2315,10 @@ function listHealthCheckTargets(db) {
AND s.enabled = 1
AND sg.enabled = 1
AND sb.health_check_enabled = 0
`);
const cnameBindingTargets = db.all(sql2`
`).filter((t) => isIpLiteral(t.ip));
const cnameBindingTargets = expandCnameHealthTargets(
db,
db.all(sql2`
SELECT 'binding' AS scope, sb.id AS ref_id, sb.cname_target AS ip,
${fqdnExpr} AS hostname,
sb.health_check_type AS type,
@@ -2293,7 +2329,9 @@ function listHealthCheckTargets(db) {
sb.health_check_verify_tls AS verify_tls,
sb.health_check_providers AS providers_json,
COALESCE(sb.health_check_aggregate, 'majority') AS aggregate,
COALESCE(sb.health_check_provider, 'local') AS provider
COALESCE(sb.health_check_provider, 'local') AS provider,
d.zone_name AS zone_name,
sb.service_id AS service_id
FROM service_bindings sb
JOIN domains d ON d.id = sb.domain_id
JOIN services s ON s.id = sb.service_id
@@ -2301,8 +2339,11 @@ function listHealthCheckTargets(db) {
AND sb.cname_target IS NOT NULL
AND sb.cname_target <> ''
AND s.enabled = 1
`);
const groupInheritedCnameBindingTargets = db.all(sql2`
`)
);
const groupInheritedCnameBindingTargets = expandCnameHealthTargets(
db,
db.all(sql2`
SELECT 'binding' AS scope, sb.id AS ref_id, sb.cname_target AS ip,
${fqdnExpr} AS hostname,
sg.health_check_type AS type,
@@ -2313,7 +2354,9 @@ function listHealthCheckTargets(db) {
sg.health_check_verify_tls AS verify_tls,
sg.health_check_providers AS providers_json,
COALESCE(sg.health_check_aggregate, 'majority') AS aggregate,
COALESCE(sg.health_check_provider, 'local') AS provider
COALESCE(sg.health_check_provider, 'local') AS provider,
d.zone_name AS zone_name,
sb.service_id AS service_id
FROM service_bindings sb
JOIN domains d ON d.id = sb.domain_id
JOIN services s ON s.id = sb.service_id
@@ -2325,7 +2368,8 @@ function listHealthCheckTargets(db) {
AND sb.health_check_enabled = 0
AND sb.cname_target IS NOT NULL
AND sb.cname_target <> ''
`);
`)
);
return [
...bindingTargets,
...groupTargets,
+87 -13
View File
@@ -2493,13 +2493,73 @@ export function pruneStaleIpHealthStatus(
// --- Health Check Targets ---
function normalizeCnameHost(target: string, zoneName: string): string {
const trimmed = target.trim().toLowerCase().replace(/\.+$/, "");
if (!trimmed) return "";
if (trimmed.includes(".")) return trimmed;
const zone = zoneName.trim().toLowerCase().replace(/\.+$/, "");
return zone ? `${trimmed}.${zone}` : trimmed;
}
/**
* Unwrap a CNAME health target to origin IPs:
* 1) A/AAAA from local dns_records (follows CNAME chain)
* 2) this service's IP pool
*
* Hostname is kept as fallback so probes still run when nothing resolves.
*/
function resolveCnameProbeIps(
db: Db,
cnameTarget: string,
zoneName: string,
serviceId: number,
): string[] {
const fqdn = normalizeCnameHost(cnameTarget, zoneName);
if (fqdn) {
const fromDns = listOriginIpsForFqdn(db, fqdn).filter(isIpLiteral);
if (fromDns.length > 0) return [...new Set(fromDns)];
}
if (serviceId > 0) {
return [...new Set(listServiceIps(db, serviceId).filter(isIpLiteral))];
}
return [];
}
type RawHealthTarget = HealthCheckTarget & {
providers_json?: string | null;
aggregate?: string | null;
zone_name?: string | null;
service_id?: number | null;
};
function expandCnameHealthTargets(
db: Db,
rows: RawHealthTarget[],
): RawHealthTarget[] {
const expanded: RawHealthTarget[] = [];
for (const row of rows) {
const ips = resolveCnameProbeIps(
db,
row.ip,
row.zone_name ?? "",
row.service_id ?? 0,
);
const resolved = ips.length > 0 ? ips : [row.ip];
for (const ip of resolved) {
expanded.push({ ...row, ip });
}
}
return expanded;
}
export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
// FQDN for a binding: "@" => zone_name, else "<hostname>.<zone_name>".
// Used as SNI / Host header for HTTP(S) probes — the raw `sb.hostname` is just the record name
// (e.g. "de" or "@"), which would break TLS SNI (ssl alert 112 "unrecognized name").
const fqdnExpr = sql`CASE WHEN sb.hostname = '@' OR sb.hostname IS NULL THEN d.zone_name ELSE sb.hostname || '.' || d.zone_name END`;
const bindingTargets = db.all<HealthCheckTarget>(sql`
const bindingTargets = db
.all<RawHealthTarget>(sql`
SELECT 'binding' AS scope, sb.id AS ref_id, sbi.ip,
${fqdnExpr} AS hostname,
sb.health_check_type AS type,
@@ -2515,12 +2575,14 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
JOIN service_bindings sb ON sb.id = sbi.binding_id
JOIN domains d ON d.id = sb.domain_id
WHERE sb.health_check_enabled = 1
`);
`)
.filter((t) => isIpLiteral(t.ip));
// Group FQDN probes the same A-record IPs that DNS publishes (binding IPs of
// enabled services) — NOT the full service_ips pool. Pool-only dead IPs must
// not mark the group Down while the published domain stays healthy.
const groupTargets = db.all<HealthCheckTarget>(sql`
const groupTargets = db
.all<RawHealthTarget>(sql`
SELECT DISTINCT 'group' AS scope, sg.id AS ref_id, sbi.ip,
sg.domain AS hostname,
sg.health_check_type AS type,
@@ -2542,13 +2604,14 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
AND s.enabled = 1
AND sg.enabled = 1
AND (sb.cname_target IS NULL OR sb.cname_target = '')
`);
`)
.filter((t) => isIpLiteral(t.ip));
// IPs of A-bindings of enabled services in a group whose group has health-check enabled.
// These inherit the group's health-check config (scope='binding', ref_id=binding_id),
// so per-domain badges reflect group rules. Skipped for bindings that already have
// their own health_check_enabled=1 (covered by bindingTargets above).
const groupInheritedBindingTargets = db.all<HealthCheckTarget>(sql`
const groupInheritedBindingTargets = db.all<RawHealthTarget>(sql`
SELECT 'binding' AS scope, sb.id AS ref_id, sbi.ip,
${fqdnExpr} AS hostname,
sg.health_check_type AS type,
@@ -2570,10 +2633,13 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
AND s.enabled = 1
AND sg.enabled = 1
AND sb.health_check_enabled = 0
`);
`)
.filter((t) => isIpLiteral(t.ip));
// CNAME-bindings with their own health_check_enabled: probe the CNAME target host.
const cnameBindingTargets = db.all<HealthCheckTarget>(sql`
// CNAME-bindings: unwrap to origin/service IPs so ip_health keys match the IP table.
const cnameBindingTargets = expandCnameHealthTargets(
db,
db.all<RawHealthTarget>(sql`
SELECT 'binding' AS scope, sb.id AS ref_id, sb.cname_target AS ip,
${fqdnExpr} AS hostname,
sb.health_check_type AS type,
@@ -2584,7 +2650,9 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
sb.health_check_verify_tls AS verify_tls,
sb.health_check_providers AS providers_json,
COALESCE(sb.health_check_aggregate, 'majority') AS aggregate,
COALESCE(sb.health_check_provider, 'local') AS provider
COALESCE(sb.health_check_provider, 'local') AS provider,
d.zone_name AS zone_name,
sb.service_id AS service_id
FROM service_bindings sb
JOIN domains d ON d.id = sb.domain_id
JOIN services s ON s.id = sb.service_id
@@ -2592,11 +2660,14 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
AND sb.cname_target IS NOT NULL
AND sb.cname_target <> ''
AND s.enabled = 1
`);
`),
);
// CNAME-bindings of services in a group with group health-check enabled
// (inherit group config). Only for bindings without their own health_check_enabled.
const groupInheritedCnameBindingTargets = db.all<HealthCheckTarget>(sql`
const groupInheritedCnameBindingTargets = expandCnameHealthTargets(
db,
db.all<RawHealthTarget>(sql`
SELECT 'binding' AS scope, sb.id AS ref_id, sb.cname_target AS ip,
${fqdnExpr} AS hostname,
sg.health_check_type AS type,
@@ -2607,7 +2678,9 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
sg.health_check_verify_tls AS verify_tls,
sg.health_check_providers AS providers_json,
COALESCE(sg.health_check_aggregate, 'majority') AS aggregate,
COALESCE(sg.health_check_provider, 'local') AS provider
COALESCE(sg.health_check_provider, 'local') AS provider,
d.zone_name AS zone_name,
sb.service_id AS service_id
FROM service_bindings sb
JOIN domains d ON d.id = sb.domain_id
JOIN services s ON s.id = sb.service_id
@@ -2619,7 +2692,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
AND sb.health_check_enabled = 0
AND sb.cname_target IS NOT NULL
AND sb.cname_target <> ''
`);
`),
);
return [
...bindingTargets,