feat: Implement load balancing and health check features for service groups, including DNS-based load balancing modes and health check configurations, enhancing service reliability and performance monitoring

packages/db/src/repos.ts

@@ -5,6 +5,11 @@ import type {
   DomainListItem,
   Group,
   GroupWithStats,
+  HealthCheckScope,
+  HealthCheckTarget,
+  HealthCheckType,
+  IpHealthStatus,
+  LbMode,
   Service,
   ServiceBinding,
   ServiceBindingView,
@@ -21,6 +26,7 @@ import {
   dnsRecords,
   domains,
   groups,
+  ipHealthStatus,
   serviceBindingIps,
   serviceBindingRecords,
   serviceBindings,
@@ -578,6 +584,22 @@ export function setServiceEnabled(
   return getService(db, id);
 }
 
+export function setServiceLb(
+  db: Db,
+  id: number,
+  weight: number,
+  priority: number,
+): void {
+  db.update(services)
+    .set({
+      lb_weight: weight,
+      lb_priority: priority,
+      updated_at: sql`datetime('now')`,
+    })
+    .where(eq(services.id, id))
+    .run();
+}
+
 export function setServiceGroup(
   db: Db,
   id: number,
@@ -648,6 +670,14 @@ function mapServiceGroup(row: typeof serviceGroups.$inferSelect): ServiceGroup {
     icon: row.icon,
     domain: row.domain,
     enabled: row.enabled,
+    lb_mode: row.lb_mode as LbMode,
+    health_check_enabled: row.health_check_enabled,
+    health_check_type: row.health_check_type as HealthCheckType,
+    health_check_port: row.health_check_port,
+    health_check_path: row.health_check_path,
+    health_check_expected_status: row.health_check_expected_status,
+    health_check_interval_sec: row.health_check_interval_sec,
+    health_check_timeout_ms: row.health_check_timeout_ms,
     created_at: row.created_at,
     updated_at: row.updated_at,
   };
@@ -672,16 +702,41 @@ export function getServiceGroup(db: Db, id: number): ServiceGroup {
   return mapServiceGroup(row);
 }
 
+export interface ServiceGroupLbPatch {
+  lb_mode?: LbMode;
+  health_check_enabled?: boolean;
+  health_check_type?: HealthCheckType;
+  health_check_port?: number | null;
+  health_check_path?: string | null;
+  health_check_expected_status?: number | null;
+  health_check_interval_sec?: number;
+  health_check_timeout_ms?: number;
+}
+
 export function createServiceGroup(
   db: Db,
   name: string,
   groupType: string,
   icon: string | null,
   domain: string | null,
+  lbPatch?: ServiceGroupLbPatch,
 ): ServiceGroup {
   const id = db
     .insert(serviceGroups)
-    .values({ name, type: groupType, icon, domain })
+    .values({
+      name,
+      type: groupType,
+      icon,
+      domain,
+      lb_mode: lbPatch?.lb_mode ?? "round_robin",
+      health_check_enabled: lbPatch?.health_check_enabled ?? false,
+      health_check_type: lbPatch?.health_check_type ?? "tcp",
+      health_check_port: lbPatch?.health_check_port ?? null,
+      health_check_path: lbPatch?.health_check_path ?? null,
+      health_check_expected_status: lbPatch?.health_check_expected_status ?? null,
+      health_check_interval_sec: lbPatch?.health_check_interval_sec ?? 30,
+      health_check_timeout_ms: lbPatch?.health_check_timeout_ms ?? 3000,
+    })
     .returning({ id: serviceGroups.id })
     .get()!.id;
   return getServiceGroup(db, id);
@@ -694,16 +749,35 @@ export function updateServiceGroup(
   groupType: string,
   icon: string | null,
   domain: string | null,
+  lbPatch?: ServiceGroupLbPatch,
 ): ServiceGroup {
+  const update: Record<string, unknown> = {
+    name,
+    type: groupType,
+    icon,
+    domain,
+    updated_at: sql`datetime('now')`,
+  };
+  if (lbPatch) {
+    if (lbPatch.lb_mode !== undefined) update.lb_mode = lbPatch.lb_mode;
+    if (lbPatch.health_check_enabled !== undefined)
+      update.health_check_enabled = lbPatch.health_check_enabled;
+    if (lbPatch.health_check_type !== undefined)
+      update.health_check_type = lbPatch.health_check_type;
+    if (lbPatch.health_check_port !== undefined)
+      update.health_check_port = lbPatch.health_check_port;
+    if (lbPatch.health_check_path !== undefined)
+      update.health_check_path = lbPatch.health_check_path;
+    if (lbPatch.health_check_expected_status !== undefined)
+      update.health_check_expected_status = lbPatch.health_check_expected_status;
+    if (lbPatch.health_check_interval_sec !== undefined)
+      update.health_check_interval_sec = lbPatch.health_check_interval_sec;
+    if (lbPatch.health_check_timeout_ms !== undefined)
+      update.health_check_timeout_ms = lbPatch.health_check_timeout_ms;
+  }
   const result = db
     .update(serviceGroups)
-    .set({
-      name,
-      type: groupType,
-      icon,
-      domain,
-      updated_at: sql`datetime('now')`,
-    })
+    .set(update)
     .where(eq(serviceGroups.id, id))
     .run();
   if (result.changes === 0) throw new NotFoundError(`service group ${id}`);
@@ -753,6 +827,12 @@ export function replaceServiceIps(
 
 // --- Service Binding IPs ---
 
+export interface BindingIpMeta {
+  ip: string;
+  weight: number;
+  priority: number;
+}
+
 export function listBindingIps(db: Db, bindingId: number): string[] {
   return db
     .select({ ip: serviceBindingIps.ip })
@@ -762,19 +842,93 @@ export function listBindingIps(db: Db, bindingId: number): string[] {
     .map((r) => r.ip);
 }
 
+export function listBindingIpsWithMeta(
+  db: Db,
+  bindingId: number,
+): BindingIpMeta[] {
+  return db
+    .select({
+      ip: serviceBindingIps.ip,
+      weight: serviceBindingIps.weight,
+      priority: serviceBindingIps.priority,
+    })
+    .from(serviceBindingIps)
+    .where(eq(serviceBindingIps.binding_id, bindingId))
+    .all();
+}
+
 export function replaceBindingIps(
   db: Db,
   bindingId: number,
   ips: string[],
+): void {
+  replaceBindingIpsWithMeta(
+    db,
+    bindingId,
+    ips.map((ip) => ({ ip, weight: 1, priority: 1 })),
+  );
+}
+
+export function replaceBindingIpsWithMeta(
+  db: Db,
+  bindingId: number,
+  entries: BindingIpMeta[],
 ): void {
   db.delete(serviceBindingIps)
     .where(eq(serviceBindingIps.binding_id, bindingId))
     .run();
-  for (const ip of ips) {
-    db.insert(serviceBindingIps).values({ binding_id: bindingId, ip }).run();
+  for (const entry of entries) {
+    db.insert(serviceBindingIps)
+      .values({
+        binding_id: bindingId,
+        ip: entry.ip,
+        weight: entry.weight,
+        priority: entry.priority,
+      })
+      .run();
   }
 }
 
+export interface BindingLbPatch {
+  lb_mode?: LbMode;
+  health_check_enabled?: boolean;
+  health_check_type?: HealthCheckType;
+  health_check_port?: number | null;
+  health_check_path?: string | null;
+  health_check_expected_status?: number | null;
+  health_check_interval_sec?: number;
+  health_check_timeout_ms?: number;
+}
+
+export function updateBindingLbConfig(
+  db: Db,
+  bindingId: number,
+  patch: BindingLbPatch,
+): void {
+  const update: Record<string, unknown> = {
+    updated_at: sql`datetime('now')`,
+  };
+  if (patch.lb_mode !== undefined) update.lb_mode = patch.lb_mode;
+  if (patch.health_check_enabled !== undefined)
+    update.health_check_enabled = patch.health_check_enabled;
+  if (patch.health_check_type !== undefined)
+    update.health_check_type = patch.health_check_type;
+  if (patch.health_check_port !== undefined)
+    update.health_check_port = patch.health_check_port;
+  if (patch.health_check_path !== undefined)
+    update.health_check_path = patch.health_check_path;
+  if (patch.health_check_expected_status !== undefined)
+    update.health_check_expected_status = patch.health_check_expected_status;
+  if (patch.health_check_interval_sec !== undefined)
+    update.health_check_interval_sec = patch.health_check_interval_sec;
+  if (patch.health_check_timeout_ms !== undefined)
+    update.health_check_timeout_ms = patch.health_check_timeout_ms;
+  db.update(serviceBindings)
+    .set(update)
+    .where(eq(serviceBindings.id, bindingId))
+    .run();
+}
+
 export function setBindingCnameTarget(
   db: Db,
   bindingId: number,
@@ -873,11 +1027,23 @@ function dnsRecordMatchesHostname(
   return dnsRecordNamesMatch(recordName, hostname, zoneName);
 }
 
+const SERVICE_BINDING_SELECT_COLUMNS = `sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
+  sb.lb_mode, sb.health_check_enabled, sb.health_check_type, sb.health_check_port,
+  sb.health_check_path, sb.health_check_expected_status, sb.health_check_interval_sec,
+  sb.health_check_timeout_ms, sb.cname_target,
+  d.zone_name, d.group_id, g.name AS group_name,
+  s.name AS service_name, s.slug AS service_slug,
+  dr.content AS target_ip, dr.sync_status,
+  sb.created_at, sb.updated_at`;
+
 function enrichServiceBindingView(
   db: Db,
-  row: Omit<ServiceBindingView, "target_ips"> & { target_ips?: string[] },
+  row: Omit<ServiceBindingView, "target_ips" | "target_ip_weights" | "target_ip_priorities"> & {
+    target_ips?: string[];
+  },
 ): ServiceBindingView {
-  const configured = listBindingIps(db, row.id);
+  const configured = listBindingIpsWithMeta(db, row.id);
+  const configuredIps = configured.map((c) => c.ip);
   const linkedRecords = listRecordsForBinding(db, row.id);
   const linkedIps = linkedRecords
     .filter((record) => record.record_type.toUpperCase() === "A")
@@ -885,7 +1051,7 @@ function enrichServiceBindingView(
 
   const target_ips = [
     ...new Set([
-      ...configured,
+      ...configuredIps,
       ...linkedIps,
       ...(row.target_ip ? [row.target_ip] : []),
     ]),
@@ -904,6 +1070,17 @@ function enrichServiceBindingView(
     target_ips.sort();
   }
 
+  const target_ip_weights: Record<string, number> = {};
+  const target_ip_priorities: Record<string, number> = {};
+  for (const entry of configured) {
+    target_ip_weights[entry.ip] = entry.weight;
+    target_ip_priorities[entry.ip] = entry.priority;
+  }
+  for (const ip of target_ips) {
+    if (target_ip_weights[ip] === undefined) target_ip_weights[ip] = 1;
+    if (target_ip_priorities[ip] === undefined) target_ip_priorities[ip] = 1;
+  }
+
   const sync_status =
     row.sync_status ??
     linkedRecords.find((record) => record.sync_status)?.sync_status ??
@@ -913,18 +1090,16 @@ function enrichServiceBindingView(
     ...row,
     target_ips,
     target_ip: target_ips[0] ?? null,
+    target_ip_weights,
+    target_ip_priorities,
     sync_status,
   };
 }
 
 export function listAllBindings(db: Db): ServiceBindingView[] {
   return db
-    .all<Omit<ServiceBindingView, "target_ips">>(sql`
-    SELECT sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
-      d.zone_name, d.group_id, g.name AS group_name,
-      s.name AS service_name, s.slug AS service_slug,
-      dr.content AS target_ip, dr.sync_status,
-      sb.created_at, sb.updated_at
+    .all<Omit<ServiceBindingView, "target_ips" | "target_ip_weights" | "target_ip_priorities">>(sql`
+    SELECT ${sql.raw(SERVICE_BINDING_SELECT_COLUMNS)}
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     LEFT JOIN groups g ON g.id = d.group_id
@@ -937,12 +1112,8 @@ export function listAllBindings(db: Db): ServiceBindingView[] {
 
 export function listBindingsByDomain(db: Db, domainId: number): ServiceBindingView[] {
   return db
-    .all<Omit<ServiceBindingView, "target_ips">>(sql`
-    SELECT sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
-      d.zone_name, d.group_id, g.name AS group_name,
-      s.name AS service_name, s.slug AS service_slug,
-      dr.content AS target_ip, dr.sync_status,
-      sb.created_at, sb.updated_at
+    .all<Omit<ServiceBindingView, "target_ips" | "target_ip_weights" | "target_ip_priorities">>(sql`
+    SELECT ${sql.raw(SERVICE_BINDING_SELECT_COLUMNS)}
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     LEFT JOIN groups g ON g.id = d.group_id
@@ -973,12 +1144,8 @@ export function getBinding(db: Db, id: number): ServiceBinding {
 }
 
 export function getBindingView(db: Db, id: number): ServiceBindingView {
-  const rows = db.all<Omit<ServiceBindingView, "target_ips">>(sql`
-    SELECT sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
-      d.zone_name, d.group_id, g.name AS group_name,
-      s.name AS service_name, s.slug AS service_slug,
-      dr.content AS target_ip, dr.sync_status,
-      sb.created_at, sb.updated_at
+  const rows = db.all<Omit<ServiceBindingView, "target_ips" | "target_ip_weights" | "target_ip_priorities">>(sql`
+    SELECT ${sql.raw(SERVICE_BINDING_SELECT_COLUMNS)}
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     LEFT JOIN groups g ON g.id = d.group_id
@@ -1225,3 +1392,129 @@ export function finishSyncJob(
     .where(eq(syncJobs.id, id))
     .run();
 }
+
+// --- IP Health Status ---
+
+export function listIpHealthStatus(
+  db: Db,
+  scope: HealthCheckScope,
+  refId: number,
+): IpHealthStatus[] {
+  return db
+    .all<IpHealthStatus>(sql`
+    SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
+      last_checked_at, last_error
+    FROM ip_health_status
+    WHERE scope = ${scope} AND ref_id = ${refId}
+  `);
+}
+
+export function getIpHealthStatusRow(
+  db: Db,
+  scope: HealthCheckScope,
+  refId: number,
+  ip: string,
+): IpHealthStatus | null {
+  const rows = db.all<IpHealthStatus>(sql`
+    SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
+      last_checked_at, last_error
+    FROM ip_health_status
+    WHERE scope = ${scope} AND ref_id = ${refId} AND ip = ${ip}
+    LIMIT 1
+  `);
+  return rows[0] ?? null;
+}
+
+export function upsertIpHealthStatus(
+  db: Db,
+  scope: HealthCheckScope,
+  refId: number,
+  ip: string,
+  status: string,
+  latencyMs: number | null,
+  consecutiveFailures: number,
+  lastError: string | null,
+): void {
+  db.run(sql`
+    INSERT INTO ip_health_status
+      (scope, ref_id, ip, status, latency_ms, consecutive_failures,
+       last_checked_at, last_error, created_at, updated_at)
+    VALUES (${scope}, ${refId}, ${ip}, ${status}, ${latencyMs}, ${consecutiveFailures},
+      datetime('now'), ${lastError}, datetime('now'), datetime('now'))
+    ON CONFLICT(scope, ref_id, ip) DO UPDATE SET
+      status = excluded.status,
+      latency_ms = excluded.latency_ms,
+      consecutive_failures = excluded.consecutive_failures,
+      last_checked_at = excluded.last_checked_at,
+      last_error = excluded.last_error,
+      updated_at = datetime('now')
+  `);
+}
+
+export function deleteIpHealthStatusForRef(
+  db: Db,
+  scope: HealthCheckScope,
+  refId: number,
+): void {
+  db.delete(ipHealthStatus)
+    .where(
+      and(
+        eq(ipHealthStatus.scope, scope),
+        eq(ipHealthStatus.ref_id, refId),
+      ),
+    )
+    .run();
+}
+
+export function deleteIpHealthStatusForIp(
+  db: Db,
+  scope: HealthCheckScope,
+  refId: number,
+  ip: string,
+): void {
+  db.delete(ipHealthStatus)
+    .where(
+      and(
+        eq(ipHealthStatus.scope, scope),
+        eq(ipHealthStatus.ref_id, refId),
+        eq(ipHealthStatus.ip, ip),
+      ),
+    )
+    .run();
+}
+
+// --- Health Check Targets ---
+
+export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
+  const bindingTargets = db.all<HealthCheckTarget>(sql`
+    SELECT 'binding' AS scope, sb.id AS ref_id, sbi.ip,
+      sb.hostname AS hostname,
+      sb.health_check_type AS type,
+      sb.health_check_port AS port,
+      sb.health_check_path AS path,
+      sb.health_check_expected_status AS expected_status,
+      sb.health_check_timeout_ms AS timeout_ms
+    FROM service_binding_ips sbi
+    JOIN service_bindings sb ON sb.id = sbi.binding_id
+    WHERE sb.health_check_enabled = 1
+  `);
+
+  const groupTargets = db.all<HealthCheckTarget>(sql`
+    SELECT 'group' AS scope, sg.id AS ref_id, sip.ip,
+      sg.domain AS hostname,
+      sg.health_check_type AS type,
+      sg.health_check_port AS port,
+      sg.health_check_path AS path,
+      sg.health_check_expected_status AS expected_status,
+      sg.health_check_timeout_ms AS timeout_ms
+    FROM services s
+    JOIN service_ips sip ON sip.service_id = s.id
+    JOIN service_groups sg ON sg.id = s.service_group_id
+    WHERE sg.health_check_enabled = 1
+      AND sg.domain IS NOT NULL
+      AND s.enabled = 1
+      AND (sg.enabled = 1)
+  `);
+
+  return [...bindingTargets, ...groupTargets];
+}

packages/db/src/schema.ts

@@ -29,6 +29,8 @@ export const services = sqliteTable("services", {
   subdomain: text("subdomain"),
   enabled: integer("enabled", { mode: "boolean" }).notNull().default(false),
   sort_order: integer("sort_order").notNull().default(0),
+  lb_weight: integer("lb_weight").notNull().default(1),
+  lb_priority: integer("lb_priority").notNull().default(1),
   created_at: text("created_at")
     .notNull()
     .default(sql`datetime('now')`),
@@ -44,6 +46,20 @@ export const serviceGroups = sqliteTable("service_groups", {
   icon: text("icon"),
   domain: text("domain"),
   enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
+  lb_mode: text("lb_mode").notNull().default("round_robin"),
+  health_check_enabled: integer("health_check_enabled", { mode: "boolean" })
+    .notNull()
+    .default(false),
+  health_check_type: text("health_check_type").notNull().default("tcp"),
+  health_check_port: integer("health_check_port"),
+  health_check_path: text("health_check_path"),
+  health_check_expected_status: integer("health_check_expected_status"),
+  health_check_interval_sec: integer("health_check_interval_sec")
+    .notNull()
+    .default(30),
+  health_check_timeout_ms: integer("health_check_timeout_ms")
+    .notNull()
+    .default(3000),
   created_at: text("created_at")
     .notNull()
     .default(sql`datetime('now')`),
@@ -123,6 +139,20 @@ export const serviceBindings = sqliteTable("service_bindings", {
   dns_record_id: integer("dns_record_id").references(() => dnsRecords.id, {
     onDelete: "set null",
   }),
+  lb_mode: text("lb_mode").notNull().default("round_robin"),
+  health_check_enabled: integer("health_check_enabled", { mode: "boolean" })
+    .notNull()
+    .default(false),
+  health_check_type: text("health_check_type").notNull().default("tcp"),
+  health_check_port: integer("health_check_port"),
+  health_check_path: text("health_check_path"),
+  health_check_expected_status: integer("health_check_expected_status"),
+  health_check_interval_sec: integer("health_check_interval_sec")
+    .notNull()
+    .default(30),
+  health_check_timeout_ms: integer("health_check_timeout_ms")
+    .notNull()
+    .default(3000),
   created_at: text("created_at")
     .notNull()
     .default(sql`datetime('now')`),
@@ -162,6 +192,8 @@ export const serviceBindingIps = sqliteTable(
       .notNull()
       .references(() => serviceBindings.id, { onDelete: "cascade" }),
     ip: text("ip").notNull(),
+    weight: integer("weight").notNull().default(1),
+    priority: integer("priority").notNull().default(1),
   },
   (t) => [primaryKey({ columns: [t.binding_id, t.ip] })],
 );
@@ -213,6 +245,29 @@ export const syncJobs = sqliteTable("sync_jobs", {
   finished_at: text("finished_at"),
 });
 
+export const ipHealthStatus = sqliteTable(
+  "ip_health_status",
+  {
+    scope: text("scope").notNull(),
+    ref_id: integer("ref_id").notNull(),
+    ip: text("ip").notNull(),
+    status: text("status").notNull().default("unknown"),
+    latency_ms: integer("latency_ms"),
+    consecutive_failures: integer("consecutive_failures")
+      .notNull()
+      .default(0),
+    last_checked_at: text("last_checked_at"),
+    last_error: text("last_error"),
+    created_at: text("created_at")
+      .notNull()
+      .default(sql`datetime('now')`),
+    updated_at: text("updated_at")
+      .notNull()
+      .default(sql`datetime('now')`),
+  },
+  (t) => [primaryKey({ columns: [t.scope, t.ref_id, t.ip] })],
+);
+
 export const schema = {
   groups,
   services,
@@ -227,4 +282,5 @@ export const schema = {
   serviceGroupDnsRecords,
   certificates,
   syncJobs,
+  ipHealthStatus,
 };