feat(health-checks): enhance health check configuration and logging

- Added new health worker URL and token fields to the AppConfig interface, allowing for Cloudflare Worker integration.
- Updated health check routes to utilize the new worker configuration, enabling dynamic health checks via Cloudflare Workers.
- Introduced a health log endpoint for services, providing detailed logs of health probe results.
- Enhanced health check service logic to support both local and Cloudflare Worker providers, improving flexibility in health monitoring.
- Updated UI components to reflect changes in health check provider settings and display relevant health information.

This commit significantly improves the health check management capabilities, allowing for better integration with Cloudflare Workers and enhanced logging features.

packages/db/src/repos.ts

@@ -5,6 +5,7 @@ import type {
   DomainListItem,
   Group,
   GroupWithStats,
+  HealthCheckProvider,
   HealthCheckScope,
   HealthCheckTarget,
   HealthCheckType,
@@ -33,6 +34,7 @@ import {
   domains,
   groups,
   healthChecks,
+  healthProbeLog,
   ipHealthStatus,
   nodes,
   bindingNodes,
@@ -768,6 +770,10 @@ export function deleteService(db: Db, id: number): void {
 
 // --- Service Groups ---
 
+function normalizeHealthProvider(value: unknown): HealthCheckProvider {
+  return value === "cloudflare" ? "cloudflare" : "local";
+}
+
 function mapServiceGroup(row: typeof serviceGroups.$inferSelect): ServiceGroup {
   return {
     id: row.id,
@@ -785,6 +791,7 @@ function mapServiceGroup(row: typeof serviceGroups.$inferSelect): ServiceGroup {
     health_check_interval_sec: row.health_check_interval_sec,
     health_check_timeout_ms: row.health_check_timeout_ms,
     health_check_verify_tls: row.health_check_verify_tls,
+    health_check_provider: normalizeHealthProvider(row.health_check_provider),
     created_at: row.created_at,
     updated_at: row.updated_at,
   };
@@ -819,6 +826,7 @@ export interface ServiceGroupLbPatch {
   health_check_interval_sec?: number;
   health_check_timeout_ms?: number;
   health_check_verify_tls?: boolean;
+  health_check_provider?: HealthCheckProvider;
 }
 
 export function createServiceGroup(
@@ -845,6 +853,7 @@ export function createServiceGroup(
       health_check_interval_sec: lbPatch?.health_check_interval_sec ?? 30,
       health_check_timeout_ms: lbPatch?.health_check_timeout_ms ?? 3000,
       health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false,
+      health_check_provider: lbPatch?.health_check_provider ?? "local",
     })
     .returning({ id: serviceGroups.id })
     .get()!.id;
@@ -885,6 +894,8 @@ export function updateServiceGroup(
       update.health_check_timeout_ms = lbPatch.health_check_timeout_ms;
     if (lbPatch.health_check_verify_tls !== undefined)
       update.health_check_verify_tls = lbPatch.health_check_verify_tls;
+    if (lbPatch.health_check_provider !== undefined)
+      update.health_check_provider = lbPatch.health_check_provider;
   }
   const result = db
     .update(serviceGroups)
@@ -1427,6 +1438,7 @@ export interface BindingLbPatch {
   health_check_interval_sec?: number;
   health_check_timeout_ms?: number;
   health_check_verify_tls?: boolean;
+  health_check_provider?: HealthCheckProvider;
 }
 
 export function updateBindingLbConfig(
@@ -1457,6 +1469,8 @@ export function updateBindingLbConfig(
     update.health_check_timeout_ms = patch.health_check_timeout_ms;
   if (patch.health_check_verify_tls !== undefined)
     update.health_check_verify_tls = patch.health_check_verify_tls;
+  if (patch.health_check_provider !== undefined)
+    update.health_check_provider = patch.health_check_provider;
   db.update(serviceBindings)
     .set(update)
     .where(eq(serviceBindings.id, bindingId))
@@ -1564,7 +1578,7 @@ function dnsRecordMatchesHostname(
 const SERVICE_BINDING_SELECT_COLUMNS = `sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
   sb.lb_mode, sb.health_check_enabled, sb.health_check_type, sb.health_check_port,
   sb.health_check_path, sb.health_check_expected_status, sb.health_check_interval_sec,
-  sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.cname_target,
+  sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.health_check_provider, sb.cname_target,
   d.zone_name, d.group_id, g.name AS group_name,
   s.name AS service_name, s.slug AS service_slug,
   dr.content AS target_ip, dr.sync_status,
@@ -1943,7 +1957,7 @@ export function listIpHealthStatus(
   return db
     .all<IpHealthStatus>(sql`
     SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
-      last_checked_at, last_error
+      last_checked_at, last_error, colo, provider
     FROM ip_health_status
     WHERE scope = ${scope} AND ref_id = ${refId}
   `);
@@ -2108,6 +2122,10 @@ export type ServiceIpHealthRow = {
   ip: string;
   status: IpHealthState;
   latency_ms: number | null;
+  last_checked_at: string | null;
+  last_error: string | null;
+  provider: HealthCheckProvider;
+  colo: string | null;
 };
 
 /** Per-IP binding-scope health, worst status if the same IP is on several bindings. */
@@ -2126,11 +2144,19 @@ export function listIpHealthByServiceIds(
     ip: string;
     health_status: string | null;
     health_latency_ms: number | null;
+    last_checked_at: string | null;
+    last_error: string | null;
+    provider: string | null;
+    colo: string | null;
   }>(sql`
     SELECT sb.service_id AS service_id,
       ihs.ip AS ip,
       ${WORST_HEALTH_SQL} AS health_status,
-      MAX(ihs.latency_ms) AS health_latency_ms
+      MAX(ihs.latency_ms) AS health_latency_ms,
+      MAX(ihs.last_checked_at) AS last_checked_at,
+      MAX(ihs.last_error) AS last_error,
+      MAX(ihs.provider) AS provider,
+      MAX(ihs.colo) AS colo
     FROM ip_health_status ihs
     INNER JOIN service_bindings sb
       ON ihs.scope = 'binding' AND ihs.ref_id = sb.id
@@ -2147,6 +2173,10 @@ export function listIpHealthByServiceIds(
       ip: row.ip,
       status: parsed.health_status,
       latency_ms: parsed.health_latency_ms,
+      last_checked_at: row.last_checked_at,
+      last_error: row.last_error,
+      provider: normalizeHealthProvider(row.provider),
+      colo: row.colo,
     });
     result.set(row.service_id, list);
   }
@@ -2195,7 +2225,7 @@ export function getIpHealthStatusRow(
 ): IpHealthStatus | null {
   const rows = db.all<IpHealthStatus>(sql`
     SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
-      consecutive_successes, last_checked_at, last_error
+      consecutive_successes, last_checked_at, last_error, colo, provider
     FROM ip_health_status
     WHERE scope = ${scope} AND ref_id = ${refId} AND ip = ${ip}
     LIMIT 1
@@ -2213,13 +2243,18 @@ export function upsertIpHealthStatus(
   consecutiveFailures: number,
   lastError: string | null,
   consecutiveSuccesses = 0,
+  extras?: { colo?: string | null; provider?: HealthCheckProvider },
 ): void {
+  const colo = extras?.colo ?? null;
+  const provider = extras?.provider ?? "local";
   db.run(sql`
     INSERT INTO ip_health_status
       (scope, ref_id, ip, status, latency_ms, consecutive_failures,
-       consecutive_successes, last_checked_at, last_error, created_at, updated_at)
+       consecutive_successes, last_checked_at, last_error, colo, provider,
+       created_at, updated_at)
     VALUES (${scope}, ${refId}, ${ip}, ${status}, ${latencyMs}, ${consecutiveFailures},
-      ${consecutiveSuccesses}, datetime('now'), ${lastError}, datetime('now'), datetime('now'))
+      ${consecutiveSuccesses}, datetime('now'), ${lastError}, ${colo}, ${provider},
+      datetime('now'), datetime('now'))
     ON CONFLICT(scope, ref_id, ip) DO UPDATE SET
       status = excluded.status,
       latency_ms = excluded.latency_ms,
@@ -2227,6 +2262,8 @@ export function upsertIpHealthStatus(
       consecutive_successes = excluded.consecutive_successes,
       last_checked_at = excluded.last_checked_at,
       last_error = excluded.last_error,
+      colo = excluded.colo,
+      provider = excluded.provider,
       updated_at = datetime('now')
   `);
 }
@@ -2321,7 +2358,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sb.health_check_path AS path,
       sb.health_check_expected_status AS expected_status,
       sb.health_check_timeout_ms AS timeout_ms,
-      sb.health_check_verify_tls AS verify_tls
+      sb.health_check_verify_tls AS verify_tls,
+      COALESCE(sb.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN domains d ON d.id = sb.domain_id
@@ -2339,7 +2377,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN services s ON s.id = sb.service_id
@@ -2364,7 +2403,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN domains d ON d.id = sb.domain_id
@@ -2386,7 +2426,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sb.health_check_path AS path,
       sb.health_check_expected_status AS expected_status,
       sb.health_check_timeout_ms AS timeout_ms,
-      sb.health_check_verify_tls AS verify_tls
+      sb.health_check_verify_tls AS verify_tls,
+      COALESCE(sb.health_check_provider, 'local') AS provider
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     JOIN services s ON s.id = sb.service_id
@@ -2406,7 +2447,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     JOIN services s ON s.id = sb.service_id
@@ -2429,6 +2471,7 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
   ].map((t) => ({
     ...t,
     verify_tls: Boolean(t.verify_tls),
+    provider: normalizeHealthProvider(t.provider),
   }));
 }
 
@@ -2633,6 +2676,94 @@ export function listDomainMonitorResultsForDomain(
   `);
 }
 
+// --- Health probe journal ---
+
+const HEALTH_PROBE_LOG_KEEP = 50;
+
+export function insertHealthProbeLog(
+  db: Db,
+  entry: {
+    scope: HealthCheckScope;
+    refId: number;
+    ip: string;
+    provider: HealthCheckProvider;
+    status: string;
+    ok: boolean;
+    latencyMs: number | null;
+    colo: string | null;
+    error: string | null;
+  },
+): void {
+  db.insert(healthProbeLog)
+    .values({
+      scope: entry.scope,
+      ref_id: entry.refId,
+      ip: entry.ip,
+      provider: entry.provider,
+      status: entry.status,
+      ok: entry.ok,
+      latency_ms: entry.latencyMs,
+      colo: entry.colo,
+      error: entry.error,
+    })
+    .run();
+  db.run(sql`
+    DELETE FROM health_probe_log
+    WHERE id IN (
+      SELECT id FROM health_probe_log
+      WHERE scope = ${entry.scope} AND ref_id = ${entry.refId} AND ip = ${entry.ip}
+      ORDER BY checked_at DESC, id DESC
+      LIMIT -1 OFFSET ${HEALTH_PROBE_LOG_KEEP}
+    )
+  `);
+}
+
+export function listHealthProbeLogForService(
+  db: Db,
+  serviceId: number,
+  limit = 50,
+): {
+  id: number;
+  scope: string;
+  ref_id: number;
+  ip: string;
+  provider: HealthCheckProvider;
+  status: string;
+  ok: boolean;
+  latency_ms: number | null;
+  colo: string | null;
+  error: string | null;
+  checked_at: string;
+}[] {
+  const rows = db.all<{
+    id: number;
+    scope: string;
+    ref_id: number;
+    ip: string;
+    provider: string;
+    status: string;
+    ok: number;
+    latency_ms: number | null;
+    colo: string | null;
+    error: string | null;
+    checked_at: string;
+  }>(sql`
+    SELECT l.id, l.scope, l.ref_id, l.ip, l.provider, l.status, l.ok,
+      l.latency_ms, l.colo, l.error, l.checked_at
+    FROM health_probe_log l
+    INNER JOIN service_bindings sb
+      ON l.scope = 'binding' AND l.ref_id = sb.id
+    WHERE sb.service_id = ${serviceId}
+    ORDER BY l.checked_at DESC, l.id DESC
+    LIMIT ${limit}
+  `);
+  return rows.map((row) => ({
+    ...row,
+    provider: normalizeHealthProvider(row.provider),
+    ok: Boolean(row.ok),
+  }));
+}
+
 // --- Notification log ---
 
 export function insertNotificationLog(

packages/db/src/schema.ts

@@ -64,6 +64,7 @@ export const serviceGroups = sqliteTable("service_groups", {
   health_check_verify_tls: integer("health_check_verify_tls", { mode: "boolean" })
     .notNull()
     .default(false),
+  health_check_provider: text("health_check_provider").notNull().default("local"),
   created_at: text("created_at")
     .notNull()
     .default(sql`datetime('now')`),
@@ -165,6 +166,9 @@ export const serviceBindings = sqliteTable(
     })
       .notNull()
       .default(false),
+    health_check_provider: text("health_check_provider")
+      .notNull()
+      .default("local"),
     routing_strategy: text("routing_strategy").notNull().default("round_robin"),
     operation_version: integer("operation_version").notNull().default(0),
     created_at: text("created_at")
@@ -354,6 +358,8 @@ export const ipHealthStatus = sqliteTable(
       .default(0),
     last_checked_at: text("last_checked_at"),
     last_error: text("last_error"),
+    colo: text("colo"),
+    provider: text("provider").notNull().default("local"),
     created_at: text("created_at")
       .notNull()
       .default(sql`datetime('now')`),
@@ -385,6 +391,8 @@ export const appSettings = sqliteTable("app_settings", {
   health_down_failures: integer("health_down_failures"),
   health_latency_warn_ms: integer("health_latency_warn_ms"),
   health_success_recoveries: integer("health_success_recoveries"),
+  health_worker_url: text("health_worker_url"),
+  health_worker_token: text("health_worker_token"),
   created_at: text("created_at")
     .notNull()
     .default(sql`datetime('now')`),
@@ -441,6 +449,22 @@ export const domainMonitorResults = sqliteTable("domain_monitor_results", {
     .default(sql`datetime('now')`),
 });
 
+export const healthProbeLog = sqliteTable("health_probe_log", {
+  id: integer("id").primaryKey({ autoIncrement: true }),
+  scope: text("scope").notNull(),
+  ref_id: integer("ref_id").notNull(),
+  ip: text("ip").notNull(),
+  provider: text("provider").notNull(),
+  status: text("status").notNull(),
+  ok: integer("ok", { mode: "boolean" }).notNull(),
+  latency_ms: integer("latency_ms"),
+  colo: text("colo"),
+  error: text("error"),
+  checked_at: text("checked_at")
+    .notNull()
+    .default(sql`datetime('now')`),
+});
+
 export const notificationLog = sqliteTable("notification_log", {
   id: integer("id").primaryKey({ autoIncrement: true }),
   kind: text("kind").notNull(),
@@ -494,6 +518,7 @@ export const schema = {
   domainTags,
   domainMonitors,
   domainMonitorResults,
+  healthProbeLog,
   notificationLog,
   auditLog,
 };

packages/db/src/settings-repo.ts

@@ -19,6 +19,8 @@ export type AppSettingsDto = {
   vpsTrackerSyncEnabled: boolean;
   vpsTrackerLastSyncAt: string | null;
   showQuickActions: boolean;
+  healthWorkerUrl: string;
+  healthWorkerTokenSet: boolean;
 } & HealthEngineSettings;
 
 export type AppSettingsPatch = {
@@ -31,9 +33,14 @@ export type AppSettingsPatch = {
   healthDownFailures?: number;
   healthLatencyWarnMs?: number;
   healthSuccessRecoveries?: number;
+  healthWorkerUrl?: string;
+  healthWorkerToken?: string;
 };
 
-export type HealthEngineFallbacks = HealthEngineSettings;
+export type HealthEngineFallbacks = HealthEngineSettings & {
+  healthWorkerUrl: string;
+  healthWorkerTokenSet: boolean;
+};
 
 function coalesceInt(value: number | null | undefined, fallback: number): number {
   return value == null || Number.isNaN(value) || value < 1 ? fallback : value;
@@ -49,6 +56,8 @@ function toDto(
     healthDownFailures: 2,
     healthLatencyWarnMs: 1000,
     healthSuccessRecoveries: 2,
+    healthWorkerUrl: "",
+    healthWorkerTokenSet: false,
   };
   return {
     id: row.id,
@@ -77,6 +86,9 @@ function toDto(
       row.health_success_recoveries,
       env.healthSuccessRecoveries,
     ),
+    healthWorkerUrl: row.health_worker_url?.trim() || env.healthWorkerUrl,
+    healthWorkerTokenSet:
+      Boolean(row.health_worker_token?.trim()) || env.healthWorkerTokenSet,
   };
 }
 
@@ -103,6 +115,8 @@ export function getAppSettingsSecrets(db: Db): {
   vpsTrackerUrl: string;
   vpsTrackerIntegrationToken: string;
   vpsTrackerSyncEnabled: boolean;
+  healthWorkerUrl: string;
+  healthWorkerToken: string;
 } {
   const row = db
     .select()
@@ -114,6 +128,8 @@ export function getAppSettingsSecrets(db: Db): {
     vpsTrackerIntegrationToken:
       row?.vps_tracker_integration_token?.trim() ?? "",
     vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled),
+    healthWorkerUrl: row?.health_worker_url?.trim() ?? "",
+    healthWorkerToken: row?.health_worker_token?.trim() ?? "",
   };
 }
 
@@ -176,6 +192,15 @@ export function updateAppSettings(
         patch.healthSuccessRecoveries !== undefined
           ? patch.healthSuccessRecoveries
           : current.health_success_recoveries,
+      health_worker_url:
+        patch.healthWorkerUrl !== undefined
+          ? patch.healthWorkerUrl.trim() || null
+          : current.health_worker_url,
+      health_worker_token:
+        patch.healthWorkerToken !== undefined &&
+        patch.healthWorkerToken.trim() !== ""
+          ? patch.healthWorkerToken
+          : current.health_worker_token,
       updated_at: new Date().toISOString(),
     })
     .where(eq(appSettings.id, SETTINGS_ID))