feat(health): добавить Globalping и мультивыбор источников проб

Несколько источников проб сразу и правило агрегации на сервисе вместо XOR Local/Cloudflare.

Co-authored-by: Cursor <cursoragent@cursor.com>

packages/db/src/repos.ts

@@ -5,10 +5,12 @@ import type {
   DomainListItem,
   Group,
   GroupWithStats,
+  HealthCheckAggregate,
   HealthCheckProvider,
   HealthCheckScope,
   HealthCheckTarget,
   HealthCheckType,
+  HealthStatusProvider,
   IpHealthState,
   IpHealthStatus,
   LbMode,
@@ -21,7 +23,15 @@ import type {
   Subdomain,
   SyncJob,
 } from "@cfdm/shared";
-import { dnsRecordNamesMatch, isIpLiteral } from "@cfdm/shared";
+import {
+  derivePrimaryProvider,
+  dnsRecordNamesMatch,
+  isIpLiteral,
+  parseHealthAggregate,
+  parseHealthProviders,
+  serializeHealthProviders,
+  normalizeStatusProvider,
+} from "@cfdm/shared";
 import { and, asc, count, eq, isNull, like, notInArray, or, sql } from "drizzle-orm";
 import type { Db } from "./client.js";
 import { ConflictError, NotFoundError } from "./errors.js";
@@ -770,8 +780,82 @@ export function deleteService(db: Db, id: number): void {
 
 // --- Service Groups ---
 
-function normalizeHealthProvider(value: unknown): HealthCheckProvider {
-  return value === "cloudflare" ? "cloudflare" : "local";
+function healthProviderColumns(patch: {
+  health_check_provider?: HealthCheckProvider;
+  health_check_providers?: HealthCheckProvider[];
+  health_check_aggregate?: HealthCheckAggregate;
+}): {
+  health_check_provider?: string;
+  health_check_providers?: string;
+  health_check_aggregate?: string;
+} {
+  const out: {
+    health_check_provider?: string;
+    health_check_providers?: string;
+    health_check_aggregate?: string;
+  } = {};
+  if (patch.health_check_providers !== undefined) {
+    const list = parseHealthProviders(patch.health_check_providers);
+    out.health_check_providers = serializeHealthProviders(list);
+    out.health_check_provider = derivePrimaryProvider(list);
+  } else if (patch.health_check_provider !== undefined) {
+    const list = parseHealthProviders(null, patch.health_check_provider);
+    out.health_check_providers = serializeHealthProviders(list);
+    out.health_check_provider = derivePrimaryProvider(list);
+  }
+  if (patch.health_check_aggregate !== undefined) {
+    out.health_check_aggregate = parseHealthAggregate(
+      patch.health_check_aggregate,
+    );
+  }
+  return out;
+}
+
+function mapHealthFields(row: {
+  health_check_provider?: unknown;
+  health_check_providers?: unknown;
+  health_check_aggregate?: unknown;
+}): {
+  health_check_provider: HealthCheckProvider;
+  health_check_providers: HealthCheckProvider[];
+  health_check_aggregate: HealthCheckAggregate;
+} {
+  const providers = parseHealthProviders(
+    row.health_check_providers,
+    row.health_check_provider,
+  );
+  return {
+    health_check_providers: providers,
+    health_check_provider: derivePrimaryProvider(providers),
+    health_check_aggregate: parseHealthAggregate(row.health_check_aggregate),
+  };
+}
+
+function mapServiceBinding(
+  row: typeof serviceBindings.$inferSelect,
+): ServiceBinding {
+  return {
+    id: row.id,
+    domain_id: row.domain_id,
+    service_id: row.service_id,
+    hostname: row.hostname,
+    cname_target: row.cname_target,
+    dns_record_id: row.dns_record_id,
+    lb_mode: row.lb_mode as LbMode,
+    health_check_enabled: row.health_check_enabled,
+    health_check_type: row.health_check_type as HealthCheckType,
+    health_check_port: row.health_check_port,
+    health_check_path: row.health_check_path,
+    health_check_expected_status: row.health_check_expected_status,
+    health_check_interval_sec: row.health_check_interval_sec,
+    health_check_timeout_ms: row.health_check_timeout_ms,
+    health_check_verify_tls: row.health_check_verify_tls,
+    ...mapHealthFields(row),
+    routing_strategy: row.routing_strategy as LbMode,
+    operation_version: row.operation_version,
+    created_at: row.created_at,
+    updated_at: row.updated_at,
+  };
 }
 
 function mapServiceGroup(row: typeof serviceGroups.$inferSelect): ServiceGroup {
@@ -791,7 +875,7 @@ function mapServiceGroup(row: typeof serviceGroups.$inferSelect): ServiceGroup {
     health_check_interval_sec: row.health_check_interval_sec,
     health_check_timeout_ms: row.health_check_timeout_ms,
     health_check_verify_tls: row.health_check_verify_tls,
-    health_check_provider: normalizeHealthProvider(row.health_check_provider),
+    ...mapHealthFields(row),
     created_at: row.created_at,
     updated_at: row.updated_at,
   };
@@ -827,6 +911,8 @@ export interface ServiceGroupLbPatch {
   health_check_timeout_ms?: number;
   health_check_verify_tls?: boolean;
   health_check_provider?: HealthCheckProvider;
+  health_check_providers?: HealthCheckProvider[];
+  health_check_aggregate?: HealthCheckAggregate;
 }
 
 export function createServiceGroup(
@@ -853,7 +939,11 @@ export function createServiceGroup(
       health_check_interval_sec: lbPatch?.health_check_interval_sec ?? 30,
       health_check_timeout_ms: lbPatch?.health_check_timeout_ms ?? 3000,
       health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false,
-      health_check_provider: lbPatch?.health_check_provider ?? "local",
+      ...healthProviderColumns({
+        health_check_provider: lbPatch?.health_check_provider ?? "local",
+        health_check_providers: lbPatch?.health_check_providers,
+        health_check_aggregate: lbPatch?.health_check_aggregate ?? "majority",
+      }),
     })
     .returning({ id: serviceGroups.id })
     .get()!.id;
@@ -894,8 +984,7 @@ export function updateServiceGroup(
       update.health_check_timeout_ms = lbPatch.health_check_timeout_ms;
     if (lbPatch.health_check_verify_tls !== undefined)
       update.health_check_verify_tls = lbPatch.health_check_verify_tls;
-    if (lbPatch.health_check_provider !== undefined)
-      update.health_check_provider = lbPatch.health_check_provider;
+    Object.assign(update, healthProviderColumns(lbPatch));
   }
   const result = db
     .update(serviceGroups)
@@ -1439,6 +1528,8 @@ export interface BindingLbPatch {
   health_check_timeout_ms?: number;
   health_check_verify_tls?: boolean;
   health_check_provider?: HealthCheckProvider;
+  health_check_providers?: HealthCheckProvider[];
+  health_check_aggregate?: HealthCheckAggregate;
 }
 
 export function updateBindingLbConfig(
@@ -1469,8 +1560,7 @@ export function updateBindingLbConfig(
     update.health_check_timeout_ms = patch.health_check_timeout_ms;
   if (patch.health_check_verify_tls !== undefined)
     update.health_check_verify_tls = patch.health_check_verify_tls;
-  if (patch.health_check_provider !== undefined)
-    update.health_check_provider = patch.health_check_provider;
+  Object.assign(update, healthProviderColumns(patch));
   db.update(serviceBindings)
     .set(update)
     .where(eq(serviceBindings.id, bindingId))
@@ -1578,7 +1668,8 @@ function dnsRecordMatchesHostname(
 const SERVICE_BINDING_SELECT_COLUMNS = `sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
   sb.lb_mode, sb.health_check_enabled, sb.health_check_type, sb.health_check_port,
   sb.health_check_path, sb.health_check_expected_status, sb.health_check_interval_sec,
-  sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.health_check_provider, sb.cname_target,
+  sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.health_check_provider,
+  sb.health_check_providers, sb.health_check_aggregate, sb.cname_target,
   d.zone_name, d.group_id, g.name AS group_name,
   s.name AS service_name, s.slug AS service_slug,
   dr.content AS target_ip, dr.sync_status,
@@ -1642,6 +1733,7 @@ function enrichServiceBindingView(
   return {
     ...row,
     cname_target: row.cname_target ?? null,
+    ...mapHealthFields(row),
     target_ips,
     target_ip: target_ips[0] ?? null,
     target_ip_weights,
@@ -1694,7 +1786,7 @@ export function getBinding(db: Db, id: number): ServiceBinding {
     .where(eq(serviceBindings.id, id))
     .get();
   if (!row) throw new NotFoundError(`service binding ${id}`);
-  return row as ServiceBinding;
+  return mapServiceBinding(row);
 }
 
 export function getBindingView(db: Db, id: number): ServiceBindingView {
@@ -1728,7 +1820,8 @@ export function findBinding(
       ),
     )
     .get();
-  return (row as ServiceBinding) ?? null;
+  if (!row) return null;
+  return mapServiceBinding(row);
 }
 
 export function insertBinding(
@@ -1792,7 +1885,8 @@ export function bindingsToRemove(
     .select()
     .from(serviceBindings)
     .where(eq(serviceBindings.service_id, serviceId))
-    .all() as ServiceBinding[];
+    .all()
+    .map(mapServiceBinding);
   return all.filter((b) => !keepIds.includes(b.id));
 }
 
@@ -2124,7 +2218,7 @@ export type ServiceIpHealthRow = {
   latency_ms: number | null;
   last_checked_at: string | null;
   last_error: string | null;
-  provider: HealthCheckProvider;
+  provider: HealthStatusProvider;
   colo: string | null;
 };
 
@@ -2175,7 +2269,7 @@ export function listIpHealthByServiceIds(
       latency_ms: parsed.health_latency_ms,
       last_checked_at: row.last_checked_at,
       last_error: row.last_error,
-      provider: normalizeHealthProvider(row.provider),
+      provider: normalizeStatusProvider(row.provider),
       colo: row.colo,
     });
     result.set(row.service_id, list);
@@ -2243,7 +2337,7 @@ export function upsertIpHealthStatus(
   consecutiveFailures: number,
   lastError: string | null,
   consecutiveSuccesses = 0,
-  extras?: { colo?: string | null; provider?: HealthCheckProvider },
+  extras?: { colo?: string | null; provider?: HealthStatusProvider },
 ): void {
   const colo = extras?.colo ?? null;
   const provider = extras?.provider ?? "local";
@@ -2359,6 +2453,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sb.health_check_expected_status AS expected_status,
       sb.health_check_timeout_ms AS timeout_ms,
       sb.health_check_verify_tls AS verify_tls,
+      sb.health_check_providers AS providers_json,
+      COALESCE(sb.health_check_aggregate, 'majority') AS aggregate,
       COALESCE(sb.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
@@ -2378,6 +2474,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
       sg.health_check_verify_tls AS verify_tls,
+      sg.health_check_providers AS providers_json,
+      COALESCE(sg.health_check_aggregate, 'majority') AS aggregate,
       COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
@@ -2404,6 +2502,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
       sg.health_check_verify_tls AS verify_tls,
+      sg.health_check_providers AS providers_json,
+      COALESCE(sg.health_check_aggregate, 'majority') AS aggregate,
       COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
@@ -2427,6 +2527,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sb.health_check_expected_status AS expected_status,
       sb.health_check_timeout_ms AS timeout_ms,
       sb.health_check_verify_tls AS verify_tls,
+      sb.health_check_providers AS providers_json,
+      COALESCE(sb.health_check_aggregate, 'majority') AS aggregate,
       COALESCE(sb.health_check_provider, 'local') AS provider
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
@@ -2448,6 +2550,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
       sg.health_check_verify_tls AS verify_tls,
+      sg.health_check_providers AS providers_json,
+      COALESCE(sg.health_check_aggregate, 'majority') AS aggregate,
       COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
@@ -2468,11 +2572,28 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
     ...groupInheritedBindingTargets,
     ...cnameBindingTargets,
     ...groupInheritedCnameBindingTargets,
-  ].map((t) => ({
-    ...t,
-    verify_tls: Boolean(t.verify_tls),
-    provider: normalizeHealthProvider(t.provider),
-  }));
+  ].map((t) => {
+    const row = t as HealthCheckTarget & {
+      providers_json?: string | null;
+      aggregate?: string | null;
+    };
+    const providers = parseHealthProviders(row.providers_json, row.provider);
+    return {
+      scope: row.scope,
+      ref_id: row.ref_id,
+      ip: row.ip,
+      hostname: row.hostname,
+      type: row.type,
+      port: row.port,
+      path: row.path,
+      expected_status: row.expected_status,
+      timeout_ms: row.timeout_ms,
+      verify_tls: Boolean(row.verify_tls),
+      providers,
+      aggregate: parseHealthAggregate(row.aggregate),
+      provider: derivePrimaryProvider(providers),
+    };
+  });
 }
 
 // --- Domain tags ---
@@ -2759,7 +2880,7 @@ export function listHealthProbeLogForService(
   `);
   return rows.map((row) => ({
     ...row,
-    provider: normalizeHealthProvider(row.provider),
+    provider: parseHealthProviders(null, row.provider)[0] ?? "local",
     ok: Boolean(row.ok),
   }));
 }

packages/db/src/schema.ts

@@ -65,6 +65,8 @@ export const serviceGroups = sqliteTable("service_groups", {
     .notNull()
     .default(false),
   health_check_provider: text("health_check_provider").notNull().default("local"),
+  health_check_providers: text("health_check_providers").notNull().default('["local"]'),
+  health_check_aggregate: text("health_check_aggregate").notNull().default("majority"),
   created_at: text("created_at")
     .notNull()
     .default(sql`datetime('now')`),
@@ -169,6 +171,12 @@ export const serviceBindings = sqliteTable(
     health_check_provider: text("health_check_provider")
       .notNull()
       .default("local"),
+    health_check_providers: text("health_check_providers")
+      .notNull()
+      .default('["local"]'),
+    health_check_aggregate: text("health_check_aggregate")
+      .notNull()
+      .default("majority"),
     routing_strategy: text("routing_strategy").notNull().default("round_robin"),
     operation_version: integer("operation_version").notNull().default(0),
     created_at: text("created_at")
@@ -398,6 +406,9 @@ export const appSettings = sqliteTable("app_settings", {
   health_worker_error: text("health_worker_error"),
   health_worker_deployed_at: text("health_worker_deployed_at"),
   health_worker_last_ingest_at: text("health_worker_last_ingest_at"),
+  globalping_token: text("globalping_token"),
+  globalping_locations: text("globalping_locations"),
+  globalping_limit: integer("globalping_limit"),
   created_at: text("created_at")
     .notNull()
     .default(sql`datetime('now')`),

packages/db/src/settings-repo.ts

@@ -28,6 +28,9 @@ export type AppSettingsDto = {
   healthWorkerError: string | null;
   healthWorkerDeployedAt: string | null;
   healthWorkerLastIngestAt: string | null;
+  globalpingTokenSet: boolean;
+  globalpingLocations: string;
+  globalpingLimit: number;
 } & HealthEngineSettings;
 
 export type AppSettingsPatch = {
@@ -47,6 +50,9 @@ export type AppSettingsPatch = {
   healthWorkerError?: string | null;
   healthWorkerDeployedAt?: string | null;
   healthWorkerLastIngestAt?: string | null;
+  globalpingToken?: string;
+  globalpingLocations?: string;
+  globalpingLimit?: number;
 };
 
 export type HealthEngineFallbacks = HealthEngineSettings & {
@@ -118,6 +124,14 @@ function toDto(
     healthWorkerDeployedAt: row.health_worker_deployed_at ?? null,
     healthWorkerLastIngestAt: row.health_worker_last_ingest_at ?? null,
     healthWorkerStatus: workerStatus(row, env.healthWorkerUrl),
+    globalpingTokenSet: Boolean(row.globalping_token?.trim()),
+    globalpingLocations: row.globalping_locations?.trim() || "World",
+    globalpingLimit:
+      row.globalping_limit == null ||
+      Number.isNaN(row.globalping_limit) ||
+      row.globalping_limit < 1
+        ? 3
+        : Math.min(10, row.globalping_limit),
   };
 }
 
@@ -146,12 +160,16 @@ export function getAppSettingsSecrets(db: Db): {
   vpsTrackerSyncEnabled: boolean;
   healthWorkerUrl: string;
   healthWorkerToken: string;
+  globalpingToken: string;
+  globalpingLocations: string;
+  globalpingLimit: number;
 } {
   const row = db
     .select()
     .from(appSettings)
     .where(eq(appSettings.id, SETTINGS_ID))
     .get();
+  const limit = row?.globalping_limit;
   return {
     vpsTrackerUrl: row?.vps_tracker_url?.trim() ?? "",
     vpsTrackerIntegrationToken:
@@ -159,6 +177,12 @@ export function getAppSettingsSecrets(db: Db): {
     vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled),
     healthWorkerUrl: row?.health_worker_url?.trim() ?? "",
     healthWorkerToken: row?.health_worker_token?.trim() ?? "",
+    globalpingToken: row?.globalping_token?.trim() ?? "",
+    globalpingLocations: row?.globalping_locations?.trim() || "World",
+    globalpingLimit:
+      limit == null || Number.isNaN(limit) || limit < 1
+        ? 3
+        : Math.min(10, limit),
   };
 }
 
@@ -249,6 +273,19 @@ export function updateAppSettings(
         patch.healthWorkerLastIngestAt !== undefined
           ? patch.healthWorkerLastIngestAt
           : current.health_worker_last_ingest_at,
+      globalping_token:
+        patch.globalpingToken !== undefined &&
+        patch.globalpingToken.trim() !== ""
+          ? patch.globalpingToken
+          : current.globalping_token,
+      globalping_locations:
+        patch.globalpingLocations !== undefined
+          ? patch.globalpingLocations.trim() || "World"
+          : current.globalping_locations,
+      globalping_limit:
+        patch.globalpingLimit !== undefined
+          ? Math.min(10, Math.max(1, patch.globalpingLimit))
+          : current.globalping_limit,
       updated_at: new Date().toISOString(),
     })
     .where(eq(appSettings.id, SETTINGS_ID))