cloudflare-domain-manager/apps/api/test/health-check.test.ts

import { describe, expect, it, beforeAll, afterAll } from "vitest";
import { createServer, type Server } from "node:net";
import { createServer as createHttpServer, type Server as HttpServer } from "node:http";
import * as healthCheckService from "../src/services/health-check-service.js";
import type { HealthCheckTarget } from "@cfdm/shared";

function startTcpServer(): Promise<{ server: Server; port: number }> {
  return new Promise((resolve) => {
    const server = createServer();
    server.listen(0, "127.0.0.1", () => {
      const address = server.address();
      const port =
        typeof address === "object" && address ? address.port : 0;
      resolve({ server, port });
    });
  });
}

describe("health-check URL helpers", () => {
  it("buildHttpProbeUrl uses FQDN in URL (IP pinned via connector)", () => {
    expect(
      healthCheckService.buildHttpProbeUrl("gt.rkns.top", 443, "/", true),
    ).toBe("https://gt.rkns.top/");
    expect(
      healthCheckService.buildHttpProbeUrl("gt.rkns.top", 8080, "/health", false),
    ).toBe("http://gt.rkns.top:8080/health");
    expect(
      healthCheckService.buildHttpProbeUrl("2001:db8::1", 443, "/", true),
    ).toBe("https://[2001:db8::1]/");
  });
});

describe("health-check probeTarget", () => {
  let server: Server;
  let port: number;

  beforeAll(async () => {
    const started = await startTcpServer();
    server = started.server;
    port = started.port;
  });

  afterAll(async () => {
    await new Promise<void>((resolve) => server.close(() => resolve()));
  });

  it("tcp probe succeeds for open port", async () => {
    const target: HealthCheckTarget = {
      scope: "binding",
      ref_id: 1,
      ip: "127.0.0.1",
      hostname: "test.local",
      type: "tcp",
      port,
      path: null,
      expected_status: null,
      timeout_ms: 1000,
      verify_tls: false,
      provider: "local",
    };
    const result = await healthCheckService.probeTarget(target);
    expect(result.ok).toBe(true);
    expect(result.error).toBeNull();
    expect(result.latencyMs).toBeGreaterThanOrEqual(0);
  });

  it("tcp probe fails for closed port", async () => {
    const target: HealthCheckTarget = {
      scope: "binding",
      ref_id: 1,
      ip: "127.0.0.1",
      hostname: "test.local",
      type: "tcp",
      port: 1,
      path: null,
      expected_status: null,
      timeout_ms: 500,
      verify_tls: false,
      provider: "local",
    };
    const result = await healthCheckService.probeTarget(target);
    expect(result.ok).toBe(false);
    expect(result.error).not.toBeNull();
  });

  it("http probe hits IP with Host=hostname (same IP, different FQDN)", async () => {
    let seenHost: string | undefined;
    const httpServer: HttpServer = createHttpServer((req, res) => {
      seenHost = req.headers.host;
      res.writeHead(200);
      res.end("ok");
    });
    const httpPort = await new Promise<number>((resolve) => {
      httpServer.listen(0, "127.0.0.1", () => {
        const address = httpServer.address();
        resolve(typeof address === "object" && address ? address.port : 0);
      });
    });

    try {
      const groupTarget: HealthCheckTarget = {
        scope: "group",
        ref_id: 1,
        ip: "127.0.0.1",
        hostname: "gt.rkns.top",
        type: "http",
        port: httpPort,
        path: "/",
        expected_status: 200,
        timeout_ms: 1000,
        verify_tls: false,
        provider: "local",
      };
      const bindingTarget: HealthCheckTarget = {
        ...groupTarget,
        scope: "binding",
        hostname: "rutg.rkns.top",
      };

      const groupResult = await healthCheckService.probeTarget(groupTarget);
      expect(groupResult.ok).toBe(true);
      expect(seenHost?.startsWith("gt.rkns.top")).toBe(true);

      const bindingResult = await healthCheckService.probeTarget(bindingTarget);
      expect(bindingResult.ok).toBe(true);
      expect(seenHost?.startsWith("rutg.rkns.top")).toBe(true);

      // Same loopback IP → latencies in the same ballpark (not ~1s DNS skew)
      expect(Math.abs(groupResult.latencyMs - bindingResult.latencyMs)).toBeLessThan(200);
    } finally {
      await new Promise<void>((resolve) => httpServer.close(() => resolve()));
    }
  });

  it("physicalProbeKey collapses group+binding on same IP for tcp", async () => {
    const { physicalProbeKey } = await import("../src/services/health-check-service.js");
    const group: HealthCheckTarget = {
      scope: "group",
      ref_id: 1,
      ip: "93.115.203.183",
      hostname: "gt.rkns.top",
      type: "tcp",
      port: 443,
      path: null,
      expected_status: null,
      timeout_ms: 3000,
      verify_tls: false,
      provider: "local",
    };
    const binding: HealthCheckTarget = {
      ...group,
      scope: "binding",
      ref_id: 2,
      hostname: "rutg.rkns.top",
    };
    expect(physicalProbeKey(group)).toBe(physicalProbeKey(binding));
  });
});

describe("health-check state derivation via runAllChecks", () => {
  it("marks ip down after threshold failures and up after recovery", async () => {
    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
    const { db, sqlite } = createMemoryDb();
    runMigrations(sqlite);

    const domain = repos.createDomain(db, null, "example.com", "zone-id");
    const service = repos.createService(db, "Svc", "svc");
    const binding = repos.insertBinding(
      db,
      domain.id,
      service.id,
      "@",
      null,
    );
    repos.updateBindingLbConfig(db, binding.id, {
      health_check_enabled: true,
      health_check_type: "tcp",
      health_check_port: 1,
      health_check_timeout_ms: 200,
    });
    repos.replaceBindingIpsWithMeta(db, binding.id, [
      { ip: "127.0.0.1", weight: 1, priority: 1 },
    ]);

    await healthCheckService.runAllChecks(db, {
      thresholds: {
        degradedFailures: 1,
        downFailures: 2,
        latencyWarnMs: 1000,
      },
    });
    let status = repos.getIpHealthStatusRow(
      db,
      "binding",
      binding.id,
      "127.0.0.1",
    );
    expect(status?.status).toBe("degraded");

    await healthCheckService.runAllChecks(db, {
      thresholds: {
        degradedFailures: 1,
        downFailures: 2,
        latencyWarnMs: 1000,
      },
    });
    status = repos.getIpHealthStatusRow(
      db,
      "binding",
      binding.id,
      "127.0.0.1",
    );
    expect(status?.status).toBe("down");
  });

  it("listHealthCheckTargets includes verify_tls from binding config", async () => {
    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
    const { db, sqlite } = createMemoryDb();
    runMigrations(sqlite);

    const domain = repos.createDomain(db, null, "example.com", "zone-id");
    const service = repos.createService(db, "Svc", "svc");
    const binding = repos.insertBinding(db, domain.id, service.id, "@", null);
    repos.updateBindingLbConfig(db, binding.id, {
      health_check_enabled: true,
      health_check_type: "http",
      health_check_port: 443,
      health_check_verify_tls: true,
    });
    repos.replaceBindingIpsWithMeta(db, binding.id, [
      { ip: "10.0.0.1", weight: 1, priority: 1 },
    ]);

    const targets = repos.listHealthCheckTargets(db);
    expect(targets).toHaveLength(1);
    expect(targets[0]?.verify_tls).toBe(true);
  });

  it("unwraps CNAME target to origin A record IPs", async () => {
    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
    const { db, sqlite } = createMemoryDb();
    runMigrations(sqlite);

    const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
    repos.insertDnsRecord(
      db,
      domain.id,
      "A",
      "ihome",
      "2.59.161.102",
      1,
      false,
      null,
      "synced",
      "cf",
      null,
    );
    const service = repos.createService(db, "RW Sub", "rw-sub");
    const binding = repos.insertBinding(db, domain.id, service.id, "s", null);
    repos.setBindingCnameTarget(db, binding.id, "ihome.rkns.top");
    repos.updateBindingLbConfig(db, binding.id, {
      health_check_enabled: true,
      health_check_type: "tcp",
      health_check_port: 443,
    });

    const targets = repos.listHealthCheckTargets(db);
    expect(targets).toHaveLength(1);
    expect(targets[0]?.ip).toBe("2.59.161.102");
    expect(targets[0]?.hostname).toBe("s.rkns.top");
  });

  it("unwraps CNAME target to service IP pool when origin DNS is empty", async () => {
    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
    const { db, sqlite } = createMemoryDb();
    runMigrations(sqlite);

    const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
    const service = repos.createService(db, "RW Sub", "rw-sub");
    repos.replaceServiceIps(db, service.id, ["2.59.161.102"]);
    const binding = repos.insertBinding(db, domain.id, service.id, "s", null);
    repos.setBindingCnameTarget(db, binding.id, "ihome.rkns.top");
    repos.updateBindingLbConfig(db, binding.id, {
      health_check_enabled: true,
      health_check_type: "tcp",
      health_check_port: 443,
    });

    const targets = repos.listHealthCheckTargets(db);
    expect(targets).toHaveLength(1);
    expect(targets[0]?.ip).toBe("2.59.161.102");
    expect(targets[0]?.hostname).toBe("s.rkns.top");
  });

  it("does not mark node unhealthy when binding majority is OK and group local fails", async () => {
    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
    const { db, sqlite } = createMemoryDb();
    runMigrations(sqlite);

    const tcp = await startTcpServer();
    try {
      const domain = repos.createDomain(db, null, "example.com", "zone-id");
      const group = repos.createServiceGroup(
        db,
        "VPN",
        "vpn",
        null,
        "vpn.example.com",
        {
          health_check_enabled: true,
          health_check_type: "http",
          health_check_port: 1,
          health_check_timeout_ms: 200,
          health_check_path: "/",
        },
      );
      const service = repos.createService(db, "Svc", "svc");
      repos.setServiceGroup(db, service.id, group.id);
      repos.setServiceEnabled(db, service.id, true);
      const binding = repos.insertBinding(db, domain.id, service.id, "@", null);
      repos.updateBindingLbConfig(db, binding.id, {
        health_check_enabled: true,
        health_check_type: "tcp",
        health_check_port: tcp.port,
        health_check_timeout_ms: 500,
      });
      repos.replaceBindingIpsWithMeta(db, binding.id, [
        { ip: "127.0.0.1", weight: 1, priority: 1 },
      ]);
      const node = repos.findNodeByIp(db, "127.0.0.1");
      expect(node).not.toBeNull();

      await healthCheckService.runAllChecks(db, {
        probeGapMs: 0,
        thresholds: {
          degradedFailures: 1,
          downFailures: 1,
          latencyWarnMs: 1000,
        },
      });

      const bindingHealth = repos.getIpHealthStatusRow(
        db,
        "binding",
        binding.id,
        "127.0.0.1",
      );
      const groupHealth = repos.getIpHealthStatusRow(
        db,
        "group",
        group.id,
        "127.0.0.1",
      );
      const after = repos.getNode(db, node!.id);

      expect(bindingHealth?.status).toBe("up");
      expect(groupHealth?.status).toBe("down");
      expect(after.health_status).toBe("healthy");
      expect(after.consecutive_failures).toBe(0);
      expect(after.last_failure_reason).toBeNull();
    } finally {
      await new Promise<void>((resolve) => tcp.server.close(() => resolve()));
    }
  });
});

describe("CNAME health mapped onto service IPs", () => {
  it("getView copies CNAME-keyed health onto the service IP row", async () => {
    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
    const { getView } = await import("../src/services/service-config-service.js");
    const { db, sqlite } = createMemoryDb();
    runMigrations(sqlite);

    const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
    const service = repos.createService(db, "RW Sub", "rw-sub");
    repos.replaceServiceIps(db, service.id, ["2.59.161.102"]);
    const binding = repos.insertBinding(db, domain.id, service.id, "s", null);
    repos.setBindingCnameTarget(db, binding.id, "ihome.rkns.top");
    repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true });
    repos.upsertIpHealthStatus(
      db,
      "binding",
      binding.id,
      "ihome.rkns.top",
      "up",
      12,
      0,
      null,
    );

    const view = await getView(db, service.id);
    expect(view.health_status).toBe("up");
    expect(view.ip_health).toEqual([
      expect.objectContaining({
        ip: "2.59.161.102",
        status: "up",
        latency_ms: 12,
      }),
    ]);
  });

  it("getView is up when any binding IP is up", async () => {
    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
    const { getView } = await import("../src/services/service-config-service.js");
    const { db, sqlite } = createMemoryDb();
    runMigrations(sqlite);

    const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
    const service = repos.createService(db, "MSK Hip", "msk-hip");
    repos.replaceServiceIps(db, service.id, ["10.0.0.1", "10.0.0.2"]);
    const binding = repos.insertBinding(db, domain.id, service.id, "gt", null);
    repos.replaceBindingIpsWithMeta(db, binding.id, [
      { ip: "10.0.0.1", weight: 1, priority: 1 },
      { ip: "10.0.0.2", weight: 1, priority: 1 },
    ]);
    repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true });
    repos.upsertIpHealthStatus(
      db,
      "binding",
      binding.id,
      "10.0.0.1",
      "up",
      12,
      0,
      null,
    );
    repos.upsertIpHealthStatus(
      db,
      "binding",
      binding.id,
      "10.0.0.2",
      "down",
      null,
      5,
      "timeout",
    );

    const view = await getView(db, service.id);
    expect(view.health_status).toBe("up");
  });

  it("getView shows live OK over hysteresis unknown", async () => {
    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
    const { getView } = await import("../src/services/service-config-service.js");
    const { db, sqlite } = createMemoryDb();
    runMigrations(sqlite);

    const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
    const service = repos.createService(db, "RW Panel", "rw-panel");
    repos.replaceServiceIps(db, service.id, ["2.59.161.102"]);
    const binding = repos.insertBinding(db, domain.id, service.id, "c", null);
    repos.replaceBindingIpsWithMeta(db, binding.id, [
      { ip: "2.59.161.102", weight: 1, priority: 1 },
    ]);
    repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true });
    repos.upsertIpHealthStatus(
      db,
      "binding",
      binding.id,
      "2.59.161.102",
      "unknown",
      63,
      0,
      null,
      1,
    );
    repos.insertHealthProbeLog(db, {
      scope: "binding",
      refId: binding.id,
      ip: "2.59.161.102",
      provider: "local",
      status: "up",
      ok: true,
      latencyMs: 63,
      colo: null,
      error: null,
    });

    const view = await getView(db, service.id);
    expect(view.health_status).toBe("up");
    expect(view.ip_health[0]?.status).toBe("up");
    expect(view.ip_health[0]?.latency_ms).toBe(63);
  });

  it("getView masks stale down when health-check is disabled", async () => {
    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
    const { getView } = await import("../src/services/service-config-service.js");
    const { db, sqlite } = createMemoryDb();
    runMigrations(sqlite);

    const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
    const service = repos.createService(db, "Main TG", "main-tg");
    repos.setServiceEnabled(db, service.id, true);
    repos.replaceServiceIps(db, service.id, ["130.49.213.176"]);
    const binding = repos.insertBinding(db, domain.id, service.id, "gt", null);
    repos.replaceBindingIpsWithMeta(db, binding.id, [
      { ip: "130.49.213.176", weight: 1, priority: 1 },
    ]);
    repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: false });
    repos.upsertIpHealthStatus(
      db,
      "binding",
      binding.id,
      "130.49.213.176",
      "down",
      null,
      5,
      "timeout",
    );

    const view = await getView(db, service.id);
    expect(view.health_status).toBe("unknown");
    expect(view.ip_health).toEqual([
      expect.objectContaining({
        ip: "130.49.213.176",
        status: "unknown",
        latency_ms: null,
        last_error: null,
      }),
    ]);
  });

  it("getView masks stale down when IP is disabled in pool", async () => {
    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
    const { getView } = await import("../src/services/service-config-service.js");
    const { db, sqlite } = createMemoryDb();
    runMigrations(sqlite);

    const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
    const service = repos.createService(db, "Main TG", "main-tg");
    repos.setServiceEnabled(db, service.id, true);
    repos.replaceServiceIps(db, service.id, ["130.49.213.176"]);
    repos.setServiceIpEnabled(db, service.id, "130.49.213.176", false);
    const binding = repos.insertBinding(db, domain.id, service.id, "gt", null);
    repos.replaceBindingIpsWithMeta(db, binding.id, [
      { ip: "130.49.213.176", weight: 1, priority: 1 },
    ]);
    repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true });
    repos.upsertIpHealthStatus(
      db,
      "binding",
      binding.id,
      "130.49.213.176",
      "down",
      null,
      5,
      "timeout",
    );

    const view = await getView(db, service.id);
    expect(view.health_status).toBe("unknown");
    expect(view.ip_health[0]?.status).toBe("unknown");
  });
});