feat(health-check): introduce health probe gap configuration and enhance health check logic

Added `healthProbeGapMs` configuration to control the minimum pause between probes to different physical targets. Updated health check service to utilize this configuration, ensuring efficient probing without overwhelming the targets. Enhanced the `runAllChecks` function to group probes by physical IP and implement the new gap logic. Updated related tests to validate the new functionality.

apps/api/src/app.ts

@@ -131,6 +131,7 @@ export async function buildApp(opts: BuildAppOptions = {}) {
         };
         const n = await healthCheckService.runAllChecks(app.db, {
           thresholds,
+          probeGapMs: config.healthProbeGapMs,
           onStatusChange: async (target, prev, next) => {
             try {
               const label =

apps/api/src/config.ts

@@ -14,6 +14,8 @@ export interface AppConfig {
   healthDegradedFailures: number;
   healthDownFailures: number;
   healthLatencyWarnMs: number;
+  /** Min pause between probes to different physical targets (same IP is probed once). */
+  healthProbeGapMs: number;
   logLevel: string;
   /** Portal SSO — when true, require portal JWT with apps includes cfdm */
   authRequired: boolean;
@@ -46,12 +48,14 @@ export function loadConfig(): AppConfig {
       ? resolve(process.env.STATIC_DIR)
       : null,
     certCheckCron: process.env.CERT_CHECK_CRON ?? "0 0 */6 * * *",
-    healthCheckCron: process.env.HEALTH_CHECK_CRON ?? "*/30 * * * * *",
+    // Default: every 2 minutes (was every 30s — hammered origins / anti-bot).
+    healthCheckCron: process.env.HEALTH_CHECK_CRON ?? "0 */2 * * * *",
     healthDegradedFailures:
       Number(process.env.HEALTH_DEGRADED_FAILURES ?? "1") || 1,
     healthDownFailures: Number(process.env.HEALTH_DOWN_FAILURES ?? "2") || 2,
     healthLatencyWarnMs:
       Number(process.env.HEALTH_LATENCY_WARN_MS ?? "1000") || 1000,
+    healthProbeGapMs: Number(process.env.HEALTH_PROBE_GAP_MS ?? "2000") || 2000,
     logLevel: process.env.LOG_LEVEL ?? "info",
     authRequired: boolEnv(process.env.AUTH_REQUIRED, false),
     authIssuer:

apps/api/src/routes/health-check.ts

@@ -23,6 +23,7 @@ export async function healthCheckRoutes(app: FastifyInstance) {
     };
     const checked = await healthCheckService.runAllChecks(request.server.db, {
       thresholds,
+      probeGapMs: config.healthProbeGapMs,
       onStatusChange: async (target, prev, next) => {
         try {
           const label =

apps/api/src/services/health-check-service.ts

@@ -1,6 +1,6 @@
 import { connect, isIP } from "node:net";
 import { resolve4, resolve6 } from "node:dns/promises";
-import { Agent, fetch as undiciFetch, interceptors } from "undici";
+import { Agent, buildConnector, fetch as undiciFetch } from "undici";
 import type { Db } from "@cfdm/db";
 import { repos } from "@cfdm/db";
 import type { HealthCheckTarget, IpHealthState } from "@cfdm/shared";
@@ -25,7 +25,7 @@ export function hostForUrl(ipOrHost: string): string {
 
 /**
  * Build http(s) URL authority for the probe.
- * Prefer FQDN in the URL (correct Host/SNI); IP is pinned via DNS interceptor.
+ * Prefer FQDN in the URL (correct Host/SNI); TCP dial goes to configured IP via custom connector.
  */
 export function buildHttpProbeUrl(
   urlHost: string,
@@ -79,6 +79,36 @@ function tcpProbe(
   });
 }
 
+/** Dial `connectAddr` for TCP/TLS while URL Host/SNI stay on the FQDN. */
+function createIpPinnedAgent(
+  connectAddr: string,
+  sniHost: string,
+  useTls: boolean,
+  timeoutMs: number,
+): Agent {
+  const connector = buildConnector({
+    rejectUnauthorized: false,
+    timeout: timeoutMs,
+  });
+  return new Agent({
+    connect(opts, callback) {
+      connector(
+        {
+          ...opts,
+          // Force socket to configured IP (or CNAME target), not public DNS of FQDN.
+          hostname: connectAddr,
+          host: connectAddr,
+          servername:
+            useTls && isIP(sniHost) === 0
+              ? sniHost
+              : (opts.servername as string | undefined),
+        },
+        callback,
+      );
+    },
+  });
+}
+
 /**
  * HTTP(S) probe: URL/Host/SNI use hostname (vhost), TCP connects to configured IP when numeric.
  * Avoids re-resolving FQDN via public DNS (which skewed group vs binding latency for the same IP).
@@ -95,37 +125,16 @@ async function httpProbe(
   const useTls = port === 443;
   const connectAddr = String(ip || "").trim();
   const headerHost = (target.hostname || "").trim() || connectAddr;
-  const urlHost = headerHost;
-  const url = buildHttpProbeUrl(urlHost, port, pathWithSlash, useTls);
+  const url = buildHttpProbeUrl(headerHost, port, pathWithSlash, useTls);
 
   const family = isIP(connectAddr);
   const pinToIp = family === 4 || family === 6;
 
-  let dispatcher: Agent | undefined;
-  if (pinToIp) {
-    // URL stays on FQDN (Host + SNI), lookup always returns the configured IP.
-    dispatcher = new Agent({
-      connect: {
-        ...(useTls && isIP(headerHost) === 0 ? { servername: headerHost } : {}),
-        rejectUnauthorized: false,
-      },
-    }).compose(
-      interceptors.dns({
-        dualStack: false,
-        affinity: family === 6 ? 6 : 4,
-        lookup: (_origin, _opts, cb) => {
-          cb(null, [{ address: connectAddr, family: family === 6 ? 6 : 4 }]);
-        },
-      }),
-    ) as Agent;
-  } else if (useTls) {
-    dispatcher = new Agent({
-      connect: {
-        ...(isIP(headerHost) === 0 ? { servername: headerHost } : {}),
-        rejectUnauthorized: false,
-      },
-    });
-  }
+  const dispatcher = pinToIp
+    ? createIpPinnedAgent(connectAddr, headerHost, useTls, timeoutMs)
+    : useTls
+      ? createIpPinnedAgent(connectAddr, headerHost, useTls, timeoutMs)
+      : undefined;
 
   try {
     const response = await undiciFetch(url, {
@@ -244,6 +253,8 @@ function deriveState(
 
 export interface RunAllChecksOptions {
   thresholds: HealthCheckThresholds;
+  /** Pause between unique physical probes (default 2000). Same IP is only probed once. */
+  probeGapMs?: number;
   onStatusChange?: (
     target: HealthCheckTarget,
     prevState: IpHealthState | null,
@@ -251,47 +262,97 @@ export interface RunAllChecksOptions {
   ) => void;
 }
 
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * One network hit per key. Group+binding on the same IP share a single TCP/HTTP probe
+ * so anti-bot / rate-limit on the origin is not tripped by back-to-back checks.
+ */
+export function physicalProbeKey(target: HealthCheckTarget): string {
+  const port = target.port ?? (target.type === "http" ? 80 : 80);
+  const ip = String(target.ip || "").trim().toLowerCase();
+  if (target.type === "http") {
+    const path = (target.path?.trim() || "/") || "/";
+    const expected = target.expected_status ?? "";
+    return `http|${ip}|${port}|${path}|${expected}`;
+  }
+  if (target.type === "tcp") return `tcp|${ip}|${port}`;
+  if (target.type === "ping") {
+    return `ping|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
+  }
+  if (target.type === "dns") {
+    return `dns|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
+  }
+  return `${target.type}|${ip}|${port}`;
+}
+
 export async function runAllChecks(
   db: Db,
   options: RunAllChecksOptions,
 ): Promise<number> {
   const targets = repos.listHealthCheckTargets(db);
+  const gapMs = Math.max(0, options.probeGapMs ?? 2000);
+
+  const byPhysical = new Map<string, HealthCheckTarget[]>();
   for (const target of targets) {
-    const prev = repos.getIpHealthStatusRow(
-      db,
-      target.scope,
-      target.ref_id,
-      target.ip,
-    );
-    const result = await probeTarget(target);
-    const { state, failures } = deriveState(
-      result.ok,
-      result.latencyMs,
-      prev
-        ? {
-            consecutive_failures: prev.consecutive_failures,
-            status: prev.status,
-          }
-        : null,
-      options.thresholds,
-    );
-    const prevState: IpHealthState | null = prev
-      ? (prev.status as IpHealthState)
-      : null;
-    repos.upsertIpHealthStatus(
-      db,
-      target.scope,
-      target.ref_id,
-      target.ip,
-      state,
-      result.latencyMs,
-      failures,
-      result.error,
-    );
-    if (prevState !== state) {
-      options.onStatusChange?.(target, prevState, state);
+    const key = physicalProbeKey(target);
+    const list = byPhysical.get(key);
+    if (list) list.push(target);
+    else byPhysical.set(key, [target]);
+  }
+
+  let probeIndex = 0;
+  for (const group of byPhysical.values()) {
+    if (probeIndex > 0 && gapMs > 0) {
+      await sleep(gapMs);
+    }
+    probeIndex += 1;
+
+    // Prefer binding hostname for SNI when several scopes share one IP.
+    const representative =
+      group.find((t) => t.scope === "binding") ?? group[0]!;
+    const result = await probeTarget(representative);
+
+    for (const target of group) {
+      const prev = repos.getIpHealthStatusRow(
+        db,
+        target.scope,
+        target.ref_id,
+        target.ip,
+      );
+      const { state, failures } = deriveState(
+        result.ok,
+        result.latencyMs,
+        prev
+          ? {
+              consecutive_failures: prev.consecutive_failures,
+              status: prev.status,
+            }
+          : null,
+        options.thresholds,
+      );
+      const prevState: IpHealthState | null = prev
+        ? (prev.status as IpHealthState)
+        : null;
+      repos.upsertIpHealthStatus(
+        db,
+        target.scope,
+        target.ref_id,
+        target.ip,
+        state,
+        result.latencyMs,
+        failures,
+        result.error,
+      );
+      if (prevState !== state) {
+        options.onStatusChange?.(target, prevState, state);
+      }
     }
   }
+  // Orphan rows (old IPs / hostname keys) still feed MAX latency on group badge.
+  repos.pruneStaleIpHealthStatus(db, targets);
   return targets.length;
 }
 

apps/api/test/health-check.test.ts

@@ -17,7 +17,7 @@ function startTcpServer(): Promise<{ server: Server; port: number }> {
 }
 
 describe("health-check URL helpers", () => {
-  it("buildHttpProbeUrl uses FQDN in URL (IP pinned via DNS interceptor)", () => {
+  it("buildHttpProbeUrl uses FQDN in URL (IP pinned via connector)", () => {
     expect(
       healthCheckService.buildHttpProbeUrl("gt.rkns.top", 443, "/", true),
     ).toBe("https://gt.rkns.top/");
@@ -125,6 +125,28 @@ describe("health-check probeTarget", () => {
       await new Promise<void>((resolve) => httpServer.close(() => resolve()));
     }
   });
+
+  it("physicalProbeKey collapses group+binding on same IP for tcp", async () => {
+    const { physicalProbeKey } = await import("../src/services/health-check-service.js");
+    const group: HealthCheckTarget = {
+      scope: "group",
+      ref_id: 1,
+      ip: "93.115.203.183",
+      hostname: "gt.rkns.top",
+      type: "tcp",
+      port: 443,
+      path: null,
+      expected_status: null,
+      timeout_ms: 3000,
+    };
+    const binding: HealthCheckTarget = {
+      ...group,
+      scope: "binding",
+      ref_id: 2,
+      hostname: "rutg.rkns.top",
+    };
+    expect(physicalProbeKey(group)).toBe(physicalProbeKey(binding));
+  });
 });
 
 describe("health-check state derivation via runAllChecks", () => {

packages/db/dist/index.js

@@ -194,6 +194,9 @@ var appSettings = sqliteTable("app_settings", {
     mode: "boolean"
   }).notNull().default(false),
   vps_tracker_last_sync_at: text("vps_tracker_last_sync_at"),
+  show_quick_actions: integer("show_quick_actions", {
+    mode: "boolean"
+  }).notNull().default(true),
   created_at: text("created_at").notNull().default(sql`datetime('now')`),
   updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
 });
@@ -345,6 +348,14 @@ var DEFAULT_APP_SWITCHER = {
       url: "http://192.168.100.67:6363",
       icon: "cloud",
       shortcut: "\u23182"
+    },
+    {
+      id: "evobgp",
+      name: "EvoBGP",
+      subtitle: "BGP \u043C\u0430\u0440\u0448\u0440\u0443\u0442\u0438\u0437\u0430\u0446\u0438\u044F",
+      url: "http://192.168.100.67:3000",
+      icon: "globe",
+      shortcut: "\u23183"
     }
   ]
 };
@@ -365,7 +376,8 @@ function toDto(row) {
       row.vps_tracker_integration_token?.trim()
     ),
     vpsTrackerSyncEnabled: Boolean(row.vps_tracker_sync_enabled),
-    vpsTrackerLastSyncAt: row.vps_tracker_last_sync_at
+    vpsTrackerLastSyncAt: row.vps_tracker_last_sync_at,
+    showQuickActions: row.show_quick_actions == null ? true : Boolean(row.show_quick_actions)
   };
 }
 function getAppSettings(db) {
@@ -397,6 +409,7 @@ function updateAppSettings(db, patch) {
     vps_tracker_url: patch.vpsTrackerUrl !== void 0 ? patch.vpsTrackerUrl : current.vps_tracker_url,
     vps_tracker_integration_token: patch.vpsTrackerIntegrationToken !== void 0 && patch.vpsTrackerIntegrationToken.trim() !== "" ? patch.vpsTrackerIntegrationToken : current.vps_tracker_integration_token,
     vps_tracker_sync_enabled: patch.vpsTrackerSyncEnabled !== void 0 ? patch.vpsTrackerSyncEnabled : current.vps_tracker_sync_enabled,
+    show_quick_actions: patch.showQuickActions !== void 0 ? patch.showQuickActions : current.show_quick_actions,
     updated_at: (/* @__PURE__ */ new Date()).toISOString()
   }).where(eq(appSettings.id, SETTINGS_ID)).run();
   return getAppSettings(db);
@@ -492,6 +505,7 @@ __export(repos_exports, {
   listUngroupedServices: () => listUngroupedServices,
   markDnsPendingDelete: () => markDnsPendingDelete,
   mergeHealthAggregates: () => mergeHealthAggregates,
+  pruneStaleIpHealthStatus: () => pruneStaleIpHealthStatus,
   reorderServices: () => reorderServices,
   replaceBindingIps: () => replaceBindingIps,
   replaceBindingIpsWithMeta: () => replaceBindingIpsWithMeta,
@@ -1443,6 +1457,32 @@ function deleteIpHealthStatusForIp(db, scope, refId, ip) {
     )
   ).run();
 }
+function pruneStaleIpHealthStatus(db, activeTargets) {
+  const byRef = /* @__PURE__ */ new Map();
+  for (const t of activeTargets) {
+    const key = `${t.scope}:${t.ref_id}`;
+    let ips = byRef.get(key);
+    if (!ips) {
+      ips = /* @__PURE__ */ new Set();
+      byRef.set(key, ips);
+    }
+    ips.add(t.ip);
+  }
+  let deleted = 0;
+  for (const [key, ips] of byRef) {
+    const sep = key.indexOf(":");
+    const scope = key.slice(0, sep);
+    const refId = Number(key.slice(sep + 1));
+    if (!Number.isFinite(refId)) continue;
+    for (const row of listIpHealthStatus(db, scope, refId)) {
+      if (!ips.has(row.ip)) {
+        deleteIpHealthStatusForIp(db, scope, refId, row.ip);
+        deleted += 1;
+      }
+    }
+  }
+  return deleted;
+}
 function listHealthCheckTargets(db) {
   const fqdnExpr = sql2`CASE WHEN sb.hostname = '@' OR sb.hostname IS NULL THEN d.zone_name ELSE sb.hostname || '.' || d.zone_name END`;
   const bindingTargets = db.all(sql2`

packages/db/src/repos.ts

@@ -1687,6 +1687,38 @@ export function deleteIpHealthStatusForIp(
     .run();
 }
 
+/** Drop health rows whose IP is no longer a live probe target for that scope/ref. */
+export function pruneStaleIpHealthStatus(
+  db: Db,
+  activeTargets: Array<{ scope: HealthCheckScope; ref_id: number; ip: string }>,
+): number {
+  const byRef = new Map<string, Set<string>>();
+  for (const t of activeTargets) {
+    const key = `${t.scope}:${t.ref_id}`;
+    let ips = byRef.get(key);
+    if (!ips) {
+      ips = new Set();
+      byRef.set(key, ips);
+    }
+    ips.add(t.ip);
+  }
+
+  let deleted = 0;
+  for (const [key, ips] of byRef) {
+    const sep = key.indexOf(":");
+    const scope = key.slice(0, sep) as HealthCheckScope;
+    const refId = Number(key.slice(sep + 1));
+    if (!Number.isFinite(refId)) continue;
+    for (const row of listIpHealthStatus(db, scope, refId)) {
+      if (!ips.has(row.ip)) {
+        deleteIpHealthStatusForIp(db, scope, refId, row.ip);
+        deleted += 1;
+      }
+    }
+  }
+  return deleted;
+}
+
 // --- Health Check Targets ---
 
 export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {