Compare commits

...
2 Commits
Author SHA1 Message Date
DenozordecandCursor 4c4908558b feat(health): деплоить probe-Worker из CFDM и опрашивать цели с edge
quality / changes (push) Successful in 9s
quality / commitlint (push) Skipped
quality / docker-check (push) Skipped
CD / update-wiki (push) Successful in 6s
quality / web (push) Successful in 1m4s
quality / api (push) Successful in 54s
CD / quality (push) Successful in 2m17s
CD / publish (push) Successful in 2m21s
Worker сам ходит на origin по Cron Trigger; CFDM кладёт цели в KV и забирает результаты без POST /probe.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-19 17:22:07 +07:00
Denozordec 2c92e78b24 feat(health-checks): enhance health check configuration and logging
quality / commitlint (push) Skipped
CD / update-wiki (push) Successful in 8s
quality / changes (push) Successful in 5s
quality / docker-check (push) Skipped
quality / web (push) Successful in 51s
quality / api (push) Successful in 44s
CD / quality (push) Successful in 1m44s
CD / publish (push) Successful in 1m35s
- Added new health worker URL and token fields to the AppConfig interface, allowing for Cloudflare Worker integration.
- Updated health check routes to utilize the new worker configuration, enabling dynamic health checks via Cloudflare Workers.
- Introduced a health log endpoint for services, providing detailed logs of health probe results.
- Enhanced health check service logic to support both local and Cloudflare Worker providers, improving flexibility in health monitoring.
- Updated UI components to reflect changes in health check provider settings and display relevant health information.

This commit significantly improves the health check management capabilities, allowing for better integration with Cloudflare Workers and enhanced logging features.
2026-08-19 16:27:34 +07:00
53 changed files with 3632 additions and 325 deletions
+1 -1
View File
@@ -5,7 +5,7 @@
"type": "module",
"scripts": {
"dev": "tsx watch src/server.ts",
"build": "tsup src/server.ts --format esm --dts",
"build": "tsup --config tsup.config.ts",
"start": "node dist/server.js",
"test": "vitest run"
},
+10
View File
@@ -35,8 +35,10 @@ import { auditRoutes } from "./routes/audit.js";
import * as certificateService from "./services/certificate-service.js";
import {
createHealthCheckTask,
healthEngineFallbacksFromConfig,
scheduleHealthCheckJob,
} from "./services/health-check-scheduler.js";
import { fireEnsureHealthWorker } from "./services/health/health-worker-deploy.js";
import { AsyncTask, CronJob } from "toad-scheduler";
export interface BuildAppOptions {
@@ -131,6 +133,14 @@ export async function buildApp(opts: BuildAppOptions = {}) {
app.decorate("reloadHealthCheckJob", () => {
scheduleHealthCheckJob(app, config, healthTask);
});
if (config.cloudflareApiToken) {
fireEnsureHealthWorker(
app.db,
app.cf,
healthEngineFallbacksFromConfig(config),
app.log,
);
}
}
return app;
+4
View File
@@ -17,6 +17,8 @@ export interface AppConfig {
healthLatencyWarnMs: number;
/** Min pause between probes to different physical targets (same IP is probed once). */
healthProbeGapMs: number;
healthWorkerUrl: string;
healthWorkerToken: string;
logLevel: string;
/** Portal SSO — when true, require portal JWT with apps includes cfdm */
authRequired: boolean;
@@ -61,6 +63,8 @@ export function loadConfig(): AppConfig {
healthLatencyWarnMs:
Number(process.env.HEALTH_LATENCY_WARN_MS ?? "1000") || 1000,
healthProbeGapMs: Number(process.env.HEALTH_PROBE_GAP_MS ?? "2000") || 2000,
healthWorkerUrl: (process.env.HEALTH_WORKER_URL ?? "").trim(),
healthWorkerToken: (process.env.HEALTH_WORKER_TOKEN ?? "").trim(),
logLevel: process.env.LOG_LEVEL ?? "info",
authRequired: boolEnv(process.env.AUTH_REQUIRED, false),
authIssuer:
+53
View File
@@ -7,6 +7,8 @@ import type {
} from "@cfdm/shared";
import { createDnsAdapter } from "./cloudflare/dns-service.js";
import { createHealthCheckAdapter, type CfHealthCheckPayload } from "./cloudflare/healthcheck-service.js";
import { createKvAdapter } from "./cloudflare/kv-service.js";
import { createWorkersAdapter } from "./cloudflare/workers-service.js";
import { createZoneAdapter } from "./cloudflare/zone-service.js";
export type { CfHealthCheckPayload };
@@ -15,11 +17,21 @@ export class CloudflareClient {
private readonly zones;
private readonly dns;
private readonly healthchecks;
private readonly kv;
private readonly workers;
private readonly token;
constructor(token: string) {
this.token = token.trim();
this.zones = createZoneAdapter(token);
this.dns = createDnsAdapter(token);
this.healthchecks = createHealthCheckAdapter(token);
this.kv = createKvAdapter(token);
this.workers = createWorkersAdapter(token);
}
get isConfigured(): boolean {
return this.token.length > 0;
}
listZones(): Promise<CfZone[]> {
@@ -81,4 +93,45 @@ export class CloudflareClient {
deleteHealthCheck(zoneId: string, id: string): Promise<void> {
return this.healthchecks.deleteHealthCheck(zoneId, id);
}
listAccounts() {
return this.workers.listAccounts();
}
listKvNamespaces(accountId: string) {
return this.kv.listNamespaces(accountId);
}
createKvNamespace(accountId: string, title: string) {
return this.kv.createNamespace(accountId, title);
}
kvGet(accountId: string, namespaceId: string, key: string) {
return this.kv.getValue(accountId, namespaceId, key);
}
kvPut(accountId: string, namespaceId: string, key: string, value: string) {
return this.kv.putValue(accountId, namespaceId, key, value);
}
putWorkerScript(opts: {
accountId: string;
scriptName: string;
source: string;
kvNamespaceId: string;
}) {
return this.workers.putScript(opts);
}
putWorkerSchedules(accountId: string, scriptName: string, crons: string[]) {
return this.workers.putSchedules(accountId, scriptName, crons);
}
enableWorkersDev(accountId: string, scriptName: string) {
return this.workers.enableWorkersDev(accountId, scriptName);
}
getWorkersSubdomain(accountId: string) {
return this.workers.getWorkersSubdomain(accountId);
}
}
+43
View File
@@ -17,6 +17,15 @@ export function mapCloudflareFailure(
): AppError {
const lower = message.toLowerCase();
if (status === 401 || status === 403 || lower.includes("authentication")) {
if (
operation.includes("workers") ||
operation.includes("kv_") ||
operation.includes("accounts")
) {
return AppError.cloudflareAuthFailed(
"Токену нужны права Account: Workers Scripts Write и Workers KV Storage Write. Zone DNS недостаточно.",
);
}
return AppError.cloudflareAuthFailed(
"Cloudflare отклонил токен. Проверьте CLOUDFLARE_API_TOKEN.",
);
@@ -67,6 +76,40 @@ export async function handleCfResponse<T>(
return body.result;
}
/** KV PUT / schedules often return `{ success: true }` without `result`. */
export async function handleCfSuccess(
response: Response,
operation: string,
): Promise<void> {
if (response.status === 429) {
const wait = parseRetryAfter(response.headers) ?? 5000;
throw AppError.rateLimited(
`Cloudflare временно ограничил запросы. Повторите через ${Math.ceil(wait / 1000)} с.`,
);
}
const text = await response.text();
if (!text) {
if (!response.ok) {
throw mapCloudflareFailure(operation, response.status, String(response.status));
}
return;
}
let body: CfResponse<unknown>;
try {
body = JSON.parse(text) as CfResponse<unknown>;
} catch {
if (!response.ok) {
throw mapCloudflareFailure(operation, response.status, text.slice(0, 180));
}
return;
}
if (!body.success) {
const msg =
body.errors?.map((e) => e.message).join("; ") ?? "unknown cloudflare error";
throw mapCloudflareFailure(operation, response.status, msg);
}
}
export async function cfRequest<T>(
token: string,
path: string,
+100
View File
@@ -0,0 +1,100 @@
import { CF_API_BASE, handleCfResponse, handleCfSuccess, mapCloudflareFailure } from "./http.js";
export interface CfKvNamespace {
id: string;
title: string;
}
export function createKvAdapter(token: string) {
return {
async listNamespaces(accountId: string): Promise<CfKvNamespace[]> {
const all: CfKvNamespace[] = [];
let page = 1;
while (true) {
const url = new URL(
`${CF_API_BASE}/accounts/${accountId}/storage/kv/namespaces`,
);
url.searchParams.set("per_page", "100");
url.searchParams.set("page", String(page));
const response = await fetch(url.toString(), {
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(30_000),
});
if (response.status >= 500 || response.status === 429) {
throw mapCloudflareFailure("kv_list", response.status, String(response.status));
}
const batch = await handleCfResponse<CfKvNamespace[]>(response, "kv_list");
all.push(...batch);
if (batch.length < 100) break;
page += 1;
}
return all;
},
async createNamespace(accountId: string, title: string): Promise<CfKvNamespace> {
const response = await fetch(
`${CF_API_BASE}/accounts/${accountId}/storage/kv/namespaces`,
{
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: JSON.stringify({ title }),
signal: AbortSignal.timeout(30_000),
},
);
if (response.status >= 500 || response.status === 429) {
throw mapCloudflareFailure("kv_create", response.status, String(response.status));
}
return handleCfResponse<CfKvNamespace>(response, "kv_create");
},
async getValue(
accountId: string,
namespaceId: string,
key: string,
): Promise<string | null> {
const response = await fetch(
`${CF_API_BASE}/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/values/${encodeURIComponent(key)}`,
{
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(30_000),
},
);
if (response.status === 404) return null;
if (response.status >= 500 || response.status === 429) {
throw mapCloudflareFailure("kv_get", response.status, String(response.status));
}
if (!response.ok) {
const text = await response.text().catch(() => "");
throw mapCloudflareFailure("kv_get", response.status, text.slice(0, 180));
}
return response.text();
},
async putValue(
accountId: string,
namespaceId: string,
key: string,
value: string,
): Promise<void> {
const response = await fetch(
`${CF_API_BASE}/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/values/${encodeURIComponent(key)}`,
{
method: "PUT",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "text/plain",
},
body: value,
signal: AbortSignal.timeout(30_000),
},
);
if (response.status >= 500 || response.status === 429) {
throw mapCloudflareFailure("kv_put", response.status, String(response.status));
}
await handleCfSuccess(response, "kv_put");
},
};
}
@@ -0,0 +1,142 @@
import {
CF_API_BASE,
handleCfResponse,
handleCfSuccess,
mapCloudflareFailure,
} from "./http.js";
export interface CfAccount {
id: string;
name?: string;
}
export interface CfWorkersSubdomain {
subdomain?: string;
enabled?: boolean;
}
export function createWorkersAdapter(token: string) {
return {
async listAccounts(): Promise<CfAccount[]> {
const response = await fetch(`${CF_API_BASE}/accounts?per_page=50`, {
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(30_000),
});
if (response.status >= 500 || response.status === 429) {
throw mapCloudflareFailure("list_accounts", response.status, String(response.status));
}
return handleCfResponse<CfAccount[]>(response, "list_accounts");
},
async putScript(opts: {
accountId: string;
scriptName: string;
source: string;
kvNamespaceId: string;
filename?: string;
}): Promise<void> {
const filename = opts.filename ?? "index.mjs";
const metadata = {
main_module: filename,
compatibility_date: "2025-04-01",
bindings: [
{
type: "kv_namespace",
name: "HEALTH_KV",
namespace_id: opts.kvNamespaceId,
},
],
};
const form = new FormData();
form.append(
"metadata",
new Blob([JSON.stringify(metadata)], { type: "application/json" }),
);
form.append(
filename,
new Blob([opts.source], { type: "application/javascript+module" }),
filename,
);
const response = await fetch(
`${CF_API_BASE}/accounts/${opts.accountId}/workers/scripts/${opts.scriptName}`,
{
method: "PUT",
headers: { Authorization: `Bearer ${token}` },
body: form,
signal: AbortSignal.timeout(60_000),
},
);
if (response.status >= 500 || response.status === 429) {
throw mapCloudflareFailure("workers_put_script", response.status, String(response.status));
}
await handleCfSuccess(response, "workers_put_script");
},
async putSchedules(
accountId: string,
scriptName: string,
crons: string[],
): Promise<void> {
const response = await fetch(
`${CF_API_BASE}/accounts/${accountId}/workers/scripts/${scriptName}/schedules`,
{
method: "PUT",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: JSON.stringify(crons.map((cron) => ({ cron }))),
signal: AbortSignal.timeout(30_000),
},
);
if (response.status >= 500 || response.status === 429) {
throw mapCloudflareFailure("workers_put_schedules", response.status, String(response.status));
}
await handleCfSuccess(response, "workers_put_schedules");
},
async enableWorkersDev(
accountId: string,
scriptName: string,
): Promise<void> {
const response = await fetch(
`${CF_API_BASE}/accounts/${accountId}/workers/scripts/${scriptName}/subdomain`,
{
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: JSON.stringify({ enabled: true }),
signal: AbortSignal.timeout(30_000),
},
);
if (response.status === 409) return;
if (response.status >= 500 || response.status === 429) {
throw mapCloudflareFailure("workers_subdomain", response.status, String(response.status));
}
if (!response.ok && response.status !== 200 && response.status !== 201) {
await handleCfSuccess(response, "workers_subdomain");
}
},
async getWorkersSubdomain(accountId: string): Promise<string | null> {
const response = await fetch(
`${CF_API_BASE}/accounts/${accountId}/workers/subdomain`,
{
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(30_000),
},
);
if (response.status === 404) return null;
if (response.status >= 500 || response.status === 429) {
throw mapCloudflareFailure("workers_get_subdomain", response.status, String(response.status));
}
const result = await handleCfResponse<CfWorkersSubdomain>(
response,
"workers_get_subdomain",
);
return result.subdomain?.trim() || null;
},
};
}
+17 -5
View File
@@ -1,8 +1,13 @@
import type { FastifyInstance } from "fastify";
import { healthStatusQuerySchema } from "@cfdm/shared";
import { repos } from "@cfdm/db";
import { getAppSettings, repos } from "@cfdm/db";
import * as healthCheckService from "../services/health-check-service.js";
import * as serviceConfigService from "../services/service-config-service.js";
import {
healthEngineFallbacksFromConfig,
} from "../services/health-check-scheduler.js";
import { mailboxFromSettings } from "../services/health/health-worker-deploy.js";
import { cronStaleAfterMs } from "../services/health/mailbox.js";
export async function healthCheckRoutes(app: FastifyInstance) {
app.get("/health-status", async (request) => {
@@ -16,15 +21,22 @@ export async function healthCheckRoutes(app: FastifyInstance) {
app.post("/health-check/run", async (request) => {
const config = request.server.config;
const fallbacks = healthEngineFallbacksFromConfig(config);
const settings = getAppSettings(
request.server.db,
fallbacks,
);
const thresholds = {
degradedFailures: config.healthDegradedFailures,
downFailures: config.healthDownFailures,
latencyWarnMs: config.healthLatencyWarnMs,
successRecoveries: config.healthSuccessRecoveries,
degradedFailures: settings.healthDegradedFailures,
downFailures: settings.healthDownFailures,
latencyWarnMs: settings.healthLatencyWarnMs,
successRecoveries: settings.healthSuccessRecoveries,
};
const checked = await healthCheckService.runAllChecks(request.server.db, {
thresholds,
probeGapMs: config.healthProbeGapMs,
mailbox: mailboxFromSettings(request.server.db, request.server.cf, fallbacks),
staleAfterMs: cronStaleAfterMs(settings.healthCheckCron),
onStatusChange: async (target, prev, next) => {
try {
const label =
+8
View File
@@ -65,6 +65,14 @@ export async function serviceRoutes(app: FastifyInstance) {
return serviceConfig.getView(request.server.db, Number(id));
});
app.get("/services/:id/health-log", async (request) => {
const { id } = request.params as { id: string };
repos.getService(request.server.db, Number(id));
return {
items: repos.listHealthProbeLogForService(request.server.db, Number(id)),
};
});
app.get("/services/:id/overview", async (request) => {
const { id } = request.params as { id: string };
return nodeService.getOverview(request.server.db, Number(id));
+21 -2
View File
@@ -10,6 +10,7 @@ import {
assertValidHealthCron,
healthEngineFallbacksFromConfig,
} from "../services/health-check-scheduler.js";
import { ensureHealthWorker } from "../services/health/health-worker-deploy.js";
export async function settingsRoutes(app: FastifyInstance) {
app.get("/settings", async (request) => {
@@ -40,11 +41,29 @@ export async function settingsRoutes(app: FastifyInstance) {
"ошибок до down не меньше, чем до degraded",
);
}
const next = updateAppSettings(request.server.db, body, fallbacks);
updateAppSettings(request.server.db, body, fallbacks);
if (body.healthCheckCron !== undefined) {
request.server.reloadHealthCheckJob?.();
const after = getAppSettings(request.server.db, fallbacks);
if (after.healthWorkerKvNamespaceId) {
try {
await ensureHealthWorker(
request.server.db,
request.server.cf,
fallbacks,
);
} catch {
// error stored in settings
}
}
}
return next;
return getAppSettings(request.server.db, fallbacks);
});
app.post("/settings/health/worker/ensure", async (request) => {
const fallbacks = healthEngineFallbacksFromConfig(request.server.config);
await ensureHealthWorker(request.server.db, request.server.cf, fallbacks);
return getAppSettings(request.server.db, fallbacks);
});
app.post("/settings/vps-tracker/test", async (request) => {
@@ -2,6 +2,7 @@ import type { FastifyInstance } from "fastify";
import { AsyncTask, CronJob } from "toad-scheduler";
import {
getAppSettings,
updateAppSettings,
type HealthEngineFallbacks,
} from "@cfdm/db";
import { repos } from "@cfdm/db";
@@ -9,6 +10,10 @@ import type { AppConfig } from "../config.js";
import { AppError } from "../errors.js";
import * as healthCheckService from "./health-check-service.js";
import * as serviceConfigService from "./service-config-service.js";
import {
mailboxFromSettings,
} from "./health/health-worker-deploy.js";
import { cronStaleAfterMs } from "./health/mailbox.js";
declare module "fastify" {
interface FastifyInstance {
@@ -27,6 +32,8 @@ export function healthEngineFallbacksFromConfig(
healthDownFailures: config.healthDownFailures,
healthLatencyWarnMs: config.healthLatencyWarnMs,
healthSuccessRecoveries: config.healthSuccessRecoveries,
healthWorkerUrl: config.healthWorkerUrl,
healthWorkerTokenSet: Boolean(config.healthWorkerToken),
};
}
@@ -63,9 +70,12 @@ export function createHealthCheckTask(
latencyWarnMs: settings.healthLatencyWarnMs,
successRecoveries: settings.healthSuccessRecoveries,
};
const mailbox = mailboxFromSettings(app.db, app.cf, fallbacks);
const n = await healthCheckService.runAllChecks(app.db, {
thresholds,
probeGapMs: config.healthProbeGapMs,
mailbox,
staleAfterMs: cronStaleAfterMs(settings.healthCheckCron),
onStatusChange: async (target, prev, next) => {
try {
const label =
@@ -98,6 +108,13 @@ export function createHealthCheckTask(
}
},
});
if (mailbox) {
updateAppSettings(
app.db,
{ healthWorkerLastIngestAt: new Date().toISOString() },
fallbacks,
);
}
const monitors = await healthCheckService.runDomainMonitors(
app.db,
thresholds,
+143 -62
View File
@@ -6,6 +6,15 @@ import { repos } from "@cfdm/db";
import type { HealthCheckTarget, IpHealthState } from "@cfdm/shared";
import { AppError } from "../errors.js";
import { nextHealthState } from "./health/state-machine.js";
import { LocalHealthCheckProvider } from "./health/local.js";
import { workerNotConfiguredResult } from "./health/worker.js";
import {
buildTargetsDoc,
indexResults,
isResultsStale,
originProbeKey,
type HealthMailbox,
} from "./health/mailbox.js";
export interface HealthCheckThresholds {
degradedFailures: number;
@@ -18,6 +27,7 @@ export interface ProbeResult {
ok: boolean;
latencyMs: number;
error: string | null;
colo?: string | null;
}
/** Bracket IPv6 for URL authority; leave IPv4/hostname as-is. */
@@ -261,6 +271,10 @@ export interface RunAllChecksOptions {
thresholds: HealthCheckThresholds;
/** Pause between unique physical probes (default 2000). Same IP is only probed once. */
probeGapMs?: number;
/** KV mailbox with Worker results. Missing → cloudflare targets fail, never Local fallback. */
mailbox?: HealthMailbox | null;
/** Results older than this are stale (default 10 min). */
staleAfterMs?: number;
onStatusChange?: (
target: HealthCheckTarget,
prevState: IpHealthState | null,
@@ -277,21 +291,83 @@ function sleep(ms: number): Promise<void> {
* so anti-bot / rate-limit on the origin is not tripped by back-to-back checks.
*/
export function physicalProbeKey(target: HealthCheckTarget): string {
const port = target.port ?? (target.type === "http" ? 80 : 80);
const ip = String(target.ip || "").trim().toLowerCase();
if (target.type === "http") {
const path = (target.path?.trim() || "/") || "/";
const expected = target.expected_status ?? "";
return `http|${ip}|${port}|${path}|${expected}`;
const kind = target.provider === "cloudflare" ? "cloudflare" : "local";
return `${kind}|${originProbeKey(target)}`;
}
function applyProbeResult(
db: Db,
target: HealthCheckTarget,
result: ProbeResult,
options: RunAllChecksOptions,
): void {
const prev = repos.getIpHealthStatusRow(
db,
target.scope,
target.ref_id,
target.ip,
);
const { state, failures, successes, node } = deriveState(
result.ok,
result.latencyMs,
prev
? {
consecutive_failures: prev.consecutive_failures,
consecutive_successes: prev.consecutive_successes,
status: prev.status,
}
: null,
options.thresholds,
);
const prevState: IpHealthState | null = prev
? (prev.status as IpHealthState)
: null;
const provider = target.provider === "cloudflare" ? "cloudflare" : "local";
repos.upsertIpHealthStatus(
db,
target.scope,
target.ref_id,
target.ip,
state,
result.latencyMs,
failures,
result.error,
successes,
{ colo: result.colo ?? null, provider },
);
repos.insertHealthProbeLog(db, {
scope: target.scope,
refId: target.ref_id,
ip: target.ip,
provider,
status: state,
ok: result.ok,
latencyMs: result.latencyMs,
colo: result.colo ?? null,
error: result.error,
});
const matchedNode = repos.findNodeByIp(db, target.ip);
if (matchedNode && matchedNode.enabled) {
repos.updateNode(db, matchedNode.id, {
health_status: node,
consecutive_failures: failures,
consecutive_successes: successes,
last_check_at: new Date().toISOString().replace("T", " ").slice(0, 19),
last_failure_reason: result.error,
});
}
if (target.type === "tcp") return `tcp|${ip}|${port}`;
if (target.type === "ping") {
return `ping|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
if (prevState !== state) {
options.onStatusChange?.(target, prevState, state);
}
if (target.type === "dns") {
return `dns|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
}
return `${target.type}|${ip}|${port}`;
}
function staleWorkerResult(colo: string | null): ProbeResult {
return {
ok: false,
latencyMs: 0,
error: "Cloudflare Worker: результаты устарели или KV пуст",
colo,
};
}
export async function runAllChecks(
@@ -300,6 +376,8 @@ export async function runAllChecks(
): Promise<number> {
const targets = repos.listHealthCheckTargets(db);
const gapMs = Math.max(0, options.probeGapMs ?? 2000);
const local = new LocalHealthCheckProvider();
const staleAfterMs = options.staleAfterMs ?? 10 * 60_000;
const byPhysical = new Map<string, HealthCheckTarget[]>();
for (const target of targets) {
@@ -309,67 +387,69 @@ export async function runAllChecks(
else byPhysical.set(key, [target]);
}
let probeIndex = 0;
const localGroups: HealthCheckTarget[][] = [];
const cloudflareGroups: HealthCheckTarget[][] = [];
for (const group of byPhysical.values()) {
if (group[0]?.provider === "cloudflare") cloudflareGroups.push(group);
else localGroups.push(group);
}
let probeIndex = 0;
for (const group of localGroups) {
if (probeIndex > 0 && gapMs > 0) {
await sleep(gapMs);
}
probeIndex += 1;
// Prefer binding hostname for SNI when several scopes share one IP.
const representative =
group.find((t) => t.scope === "binding") ?? group[0]!;
const result = await probeTarget(representative);
const result = await local.probe(representative);
for (const target of group) {
const prev = repos.getIpHealthStatusRow(
db,
target.scope,
target.ref_id,
target.ip,
);
const { state, failures, successes, node } = deriveState(
result.ok,
result.latencyMs,
prev
? {
consecutive_failures: prev.consecutive_failures,
consecutive_successes: prev.consecutive_successes,
status: prev.status,
}
: null,
options.thresholds,
);
const prevState: IpHealthState | null = prev
? (prev.status as IpHealthState)
: null;
repos.upsertIpHealthStatus(
db,
target.scope,
target.ref_id,
target.ip,
state,
result.latencyMs,
failures,
result.error,
successes,
);
const matchedNode = repos.findNodeByIp(db, target.ip);
if (matchedNode && matchedNode.enabled) {
repos.updateNode(db, matchedNode.id, {
health_status: node,
consecutive_failures: failures,
consecutive_successes: successes,
last_check_at: new Date().toISOString().replace("T", " ").slice(0, 19),
last_failure_reason: result.error,
});
applyProbeResult(db, target, result, options);
}
}
if (cloudflareGroups.length > 0) {
const mailbox = options.mailbox ?? null;
const resultsDoc = mailbox ? await mailbox.getResults() : null;
const byKey = indexResults(resultsDoc);
const stale = !mailbox || isResultsStale(resultsDoc, staleAfterMs);
const colo = resultsDoc?.colo ?? null;
if (mailbox) {
try {
const next = buildTargetsDoc(targets);
const current = await mailbox.getTargets();
if (current?.fingerprint !== next.fingerprint) {
await mailbox.putTargets(next);
}
} catch {
// ingest still proceeds
}
if (prevState !== state) {
options.onStatusChange?.(target, prevState, state);
}
for (const group of cloudflareGroups) {
const representative =
group.find((t) => t.scope === "binding") ?? group[0]!;
const item = byKey.get(originProbeKey(representative));
let result: ProbeResult;
if (!mailbox) {
result = workerNotConfiguredResult();
} else if (stale || !item) {
result = staleWorkerResult(colo);
} else {
result = {
ok: item.ok,
latencyMs: item.latencyMs,
error: item.error,
colo,
};
}
for (const target of group) {
applyProbeResult(db, target, result, options);
}
}
}
// Orphan rows (old IPs / hostname keys) still feed MAX latency on group badge.
repos.pruneStaleIpHealthStatus(db, targets);
return targets.length;
}
@@ -392,6 +472,7 @@ export async function runDomainMonitors(
expected_status: monitor.expected_status,
timeout_ms: monitor.timeout_ms,
verify_tls: false,
provider: "local",
};
let result: ProbeResult;
if (monitor.type === "http") {
@@ -0,0 +1,21 @@
import { existsSync, readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
export function loadHealthProbeWorkerSource(): string {
const dir = dirname(fileURLToPath(import.meta.url));
const candidates = [
join(dir, "health-probe-worker.mjs"),
join(process.cwd(), "dist/health-probe-worker.mjs"),
join(process.cwd(), "health-probe-worker.mjs"),
join(dir, "../../../../../workers/health-probe/src/index.mjs"),
join(process.cwd(), "../../workers/health-probe/src/index.mjs"),
join(process.cwd(), "workers/health-probe/src/index.mjs"),
];
for (const path of candidates) {
if (existsSync(path)) {
return readFileSync(path, "utf8");
}
}
throw new Error("не найден исходник Worker health-probe");
}
@@ -0,0 +1,185 @@
import { getAppSettings, repos, updateAppSettings, type HealthEngineFallbacks } from "@cfdm/db";
import type { Db } from "@cfdm/db";
import { HEALTH_PROBE_KV_TITLE, HEALTH_PROBE_SCRIPT_NAME } from "@cfdm/shared";
import type { CloudflareClient } from "../../lib/cf-client.js";
import { AppError } from "../../errors.js";
import { loadHealthProbeWorkerSource } from "./health-probe-script.js";
import {
buildTargetsDoc,
createCloudflareKvMailbox,
toCloudflareCron,
type HealthMailbox,
} from "./mailbox.js";
export const DEFAULT_HEALTH_FALLBACKS: HealthEngineFallbacks = {
healthCheckCron: "0 */2 * * * *",
healthDegradedFailures: 1,
healthDownFailures: 2,
healthLatencyWarnMs: 1000,
healthSuccessRecoveries: 2,
healthWorkerUrl: "",
healthWorkerTokenSet: false,
};
export async function resolveAccountId(
cf: CloudflareClient,
db: Db,
cached?: string | null,
): Promise<string> {
const trimmed = cached?.trim();
if (trimmed) return trimmed;
const domains = repos.listDomains(db);
for (const domain of domains) {
if (!domain.cf_zone_id) continue;
try {
const zone = await cf.getZone(domain.cf_zone_id);
const id = zone.account?.id?.trim();
if (id) return id;
} catch {
// try next zone / accounts list
}
}
const accounts = await cf.listAccounts();
const id = accounts[0]?.id?.trim();
if (!id) {
throw AppError.cloudflare(
"Не удалось определить Cloudflare account_id. Добавьте зону или расширьте права токена (Account Settings Read).",
);
}
return id;
}
export async function ensureKvNamespace(
cf: CloudflareClient,
accountId: string,
existingId?: string | null,
): Promise<string> {
if (existingId?.trim()) return existingId.trim();
const listed = await cf.listKvNamespaces(accountId);
const found = listed.find((ns) => ns.title === HEALTH_PROBE_KV_TITLE);
if (found?.id) return found.id;
const created = await cf.createKvNamespace(accountId, HEALTH_PROBE_KV_TITLE);
if (!created.id) {
throw AppError.cloudflare("Cloudflare не вернул id KV namespace");
}
return created.id;
}
export async function ensureHealthWorker(
db: Db,
cf: CloudflareClient,
fallbacks: HealthEngineFallbacks,
): Promise<{ url: string; kvNamespaceId: string; accountId: string }> {
const settings = getAppSettings(db, fallbacks);
try {
const accountId = await resolveAccountId(cf, db, settings.healthWorkerAccountId);
const kvNamespaceId = await ensureKvNamespace(
cf,
accountId,
settings.healthWorkerKvNamespaceId,
);
const source = loadHealthProbeWorkerSource();
await cf.putWorkerScript({
accountId,
scriptName: HEALTH_PROBE_SCRIPT_NAME,
source,
kvNamespaceId,
});
await cf.putWorkerSchedules(accountId, HEALTH_PROBE_SCRIPT_NAME, [
toCloudflareCron(settings.healthCheckCron),
]);
try {
await cf.enableWorkersDev(accountId, HEALTH_PROBE_SCRIPT_NAME);
} catch {
// workers.dev may already be on
}
const subdomain = await cf.getWorkersSubdomain(accountId);
const url = subdomain
? `https://${HEALTH_PROBE_SCRIPT_NAME}.${subdomain}.workers.dev`
: settings.healthWorkerUrl || `https://${HEALTH_PROBE_SCRIPT_NAME}.workers.dev`;
updateAppSettings(
db,
{
healthWorkerAccountId: accountId,
healthWorkerKvNamespaceId: kvNamespaceId,
healthWorkerUrl: url,
healthWorkerError: null,
healthWorkerDeployedAt: new Date().toISOString(),
},
fallbacks,
);
await syncCloudflareTargetsToKv(db, cf, fallbacks);
return { url, kvNamespaceId, accountId };
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
updateAppSettings(db, { healthWorkerError: message }, fallbacks);
throw err;
}
}
export async function maybeEnsureHealthWorker(
db: Db,
cf: CloudflareClient,
fallbacks: HealthEngineFallbacks,
): Promise<void> {
const hasCloudflare = repos
.listHealthCheckTargets(db)
.some((target) => target.provider === "cloudflare");
if (!hasCloudflare) return;
const settings = getAppSettings(db, fallbacks);
if (settings.healthWorkerKvNamespaceId.trim() && !settings.healthWorkerError) {
await syncCloudflareTargetsToKv(db, cf, fallbacks);
return;
}
await ensureHealthWorker(db, cf, fallbacks);
}
export function mailboxFromSettings(
db: Db,
cf: CloudflareClient,
fallbacks: HealthEngineFallbacks,
): HealthMailbox | null {
const settings = getAppSettings(db, fallbacks);
const accountId = settings.healthWorkerAccountId.trim();
const ns = settings.healthWorkerKvNamespaceId.trim();
if (!accountId || !ns) return null;
return createCloudflareKvMailbox(cf, accountId, ns);
}
export async function syncCloudflareTargetsToKv(
db: Db,
cf: CloudflareClient,
fallbacks: HealthEngineFallbacks,
mailbox?: HealthMailbox | null,
): Promise<void> {
const box = mailbox ?? mailboxFromSettings(db, cf, fallbacks);
if (!box) return;
const next = buildTargetsDoc(repos.listHealthCheckTargets(db));
const current = await box.getTargets();
if (current?.fingerprint === next.fingerprint) return;
await box.putTargets(next);
}
export function fireEnsureHealthWorker(
db: Db,
cf: CloudflareClient,
fallbacks: HealthEngineFallbacks,
log?: { warn: (obj: unknown, msg: string) => void },
): void {
if (process.env.VITEST) return;
if (!cf.isConfigured) return;
const hasCloudflare = repos
.listHealthCheckTargets(db)
.some((target) => target.provider === "cloudflare");
if (!hasCloudflare) {
void syncCloudflareTargetsToKv(db, cf, fallbacks).catch((err) => {
log?.warn({ err }, "health worker KV sync failed");
});
return;
}
void maybeEnsureHealthWorker(db, cf, fallbacks).catch((err) => {
log?.warn({ err }, "health worker ensure failed");
});
}
+145
View File
@@ -0,0 +1,145 @@
import type {
HealthCheckTarget,
HealthProbeResultItem,
HealthProbeResultsDoc,
HealthProbeTargetItem,
HealthProbeTargetsDoc,
} from "@cfdm/shared";
import { HEALTH_KV_RESULTS_KEY, HEALTH_KV_TARGETS_KEY } from "@cfdm/shared";
import type { CloudflareClient } from "../../lib/cf-client.js";
export interface HealthMailbox {
getTargets(): Promise<HealthProbeTargetsDoc | null>;
putTargets(doc: HealthProbeTargetsDoc): Promise<void>;
getResults(): Promise<HealthProbeResultsDoc | null>;
}
export function createCloudflareKvMailbox(
cf: CloudflareClient,
accountId: string,
namespaceId: string,
): HealthMailbox {
return {
async getTargets() {
return readJson<HealthProbeTargetsDoc>(cf, accountId, namespaceId, HEALTH_KV_TARGETS_KEY);
},
async putTargets(doc) {
await cf.kvPut(accountId, namespaceId, HEALTH_KV_TARGETS_KEY, JSON.stringify(doc));
},
async getResults() {
return readJson<HealthProbeResultsDoc>(cf, accountId, namespaceId, HEALTH_KV_RESULTS_KEY);
},
};
}
async function readJson<T>(
cf: CloudflareClient,
accountId: string,
namespaceId: string,
key: string,
): Promise<T | null> {
const raw = await cf.kvGet(accountId, namespaceId, key);
if (!raw) return null;
try {
return JSON.parse(raw) as T;
} catch {
return null;
}
}
export function originProbeKey(target: HealthCheckTarget): string {
const port = target.port ?? (target.type === "http" ? 80 : 80);
const ip = String(target.ip || "").trim().toLowerCase();
if (target.type === "http") {
const path = (target.path?.trim() || "/") || "/";
const expected = target.expected_status ?? "";
return `http|${ip}|${port}|${path}|${expected}`;
}
if (target.type === "tcp") return `tcp|${ip}|${port}`;
if (target.type === "ping") {
return `ping|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
}
if (target.type === "dns") {
return `dns|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
}
return `${target.type}|${ip}|${port}`;
}
export function cloudflareMailboxTargets(
targets: HealthCheckTarget[],
): HealthProbeTargetItem[] {
const unique = new Map<string, HealthProbeTargetItem>();
for (const target of targets) {
if (target.provider !== "cloudflare") continue;
if (target.type !== "tcp" && target.type !== "http") continue;
const key = originProbeKey(target);
if (unique.has(key)) continue;
unique.set(key, {
key,
ip: target.ip,
hostname: target.hostname || target.ip,
type: target.type,
port: target.port ?? (target.type === "http" ? 80 : 80),
path: target.path ?? "/",
expectedStatus: target.expected_status,
timeoutMs: target.timeout_ms ?? 3000,
verifyTls: Boolean(target.verify_tls),
});
}
return [...unique.values()].sort((a, b) => a.key.localeCompare(b.key));
}
export function fingerprintTargets(items: HealthProbeTargetItem[]): string {
return items
.map(
(item) =>
`${item.key}|${item.hostname}|${item.timeoutMs ?? ""}|${item.verifyTls ? "1" : "0"}`,
)
.join(";");
}
export function buildTargetsDoc(targets: HealthCheckTarget[]): HealthProbeTargetsDoc {
const items = cloudflareMailboxTargets(targets);
return {
fingerprint: fingerprintTargets(items),
updatedAt: new Date().toISOString(),
items,
};
}
export function indexResults(
doc: HealthProbeResultsDoc | null,
): Map<string, HealthProbeResultItem> {
const map = new Map<string, HealthProbeResultItem>();
if (!doc?.items) return map;
for (const item of doc.items) {
map.set(item.key, item);
}
return map;
}
export function isResultsStale(doc: HealthProbeResultsDoc | null, staleAfterMs: number): boolean {
if (!doc?.probedAt) return true;
const ts = Date.parse(doc.probedAt);
if (!Number.isFinite(ts)) return true;
return Date.now() - ts > staleAfterMs;
}
/** Drop seconds from toad 6-field cron for Cloudflare Workers (5-field). */
export function toCloudflareCron(expr: string): string {
const parts = expr.trim().split(/\s+/).filter(Boolean);
if (parts.length === 6) return parts.slice(1).join(" ");
if (parts.length === 5) return parts.join(" ");
throw new Error("некорректное cron-выражение");
}
export function cronStaleAfterMs(expr: string): number {
const cf = toCloudflareCron(expr);
const minute = cf.split(/\s+/)[0] ?? "*";
if (minute.startsWith("*/")) {
const n = Number(minute.slice(2));
if (Number.isFinite(n) && n > 0) return Math.max(n * 2, 5) * 60_000;
}
if (minute === "*") return 10 * 60_000;
return 10 * 60_000;
}
+13
View File
@@ -0,0 +1,13 @@
export function workerNotConfiguredResult(): {
ok: false;
latencyMs: number;
error: string;
colo: null;
} {
return {
ok: false,
latencyMs: 0,
error: "Cloudflare Worker не настроен (нет KV mailbox)",
colo: null,
};
}
@@ -24,6 +24,7 @@ import { isValidIpv4 } from "../lib/validators.js";
import * as dnsService from "./dns-service.js";
import * as domainService from "./domain-service.js";
import { syncServiceToVpsTracker } from "./vps-tracker-sync.js";
import { fireEnsureHealthWorker, DEFAULT_HEALTH_FALLBACKS } from "./health/health-worker-deploy.js";
import {
selectActiveIpsByMode,
withBindingLock,
@@ -50,6 +51,7 @@ export interface ServiceDomainInput {
health_check_interval_sec?: number;
health_check_timeout_ms?: number;
health_check_verify_tls?: boolean;
health_check_provider?: "local" | "cloudflare";
}
export interface ToggleRequest {
@@ -70,6 +72,7 @@ export interface ServiceGroupBody {
health_check_interval_sec?: number;
health_check_timeout_ms?: number;
health_check_verify_tls?: boolean;
health_check_provider?: "local" | "cloudflare";
}
export interface UpdateServiceGroupBody {
@@ -86,6 +89,7 @@ export interface UpdateServiceGroupBody {
health_check_interval_sec?: number;
health_check_timeout_ms?: number;
health_check_verify_tls?: boolean;
health_check_provider?: "local" | "cloudflare";
}
export interface UpdateServiceConfigRequest {
@@ -291,6 +295,7 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
health_check_interval_sec: binding.health_check_interval_sec,
health_check_timeout_ms: binding.health_check_timeout_ms,
health_check_verify_tls: binding.health_check_verify_tls,
health_check_provider: binding.health_check_provider ?? "local",
sync_status: aggregateSyncStatus(statuses),
};
});
@@ -334,6 +339,10 @@ function attachServiceHealth(
ip,
status: row?.status ?? ("unknown" as const),
latency_ms: row?.latency_ms ?? null,
last_checked_at: row?.last_checked_at ?? null,
last_error: row?.last_error ?? null,
provider: row?.provider ?? "local",
colo: row?.colo ?? null,
};
});
return {
@@ -1154,7 +1163,8 @@ export async function updateConfig(
input.health_check_expected_status !== undefined ||
input.health_check_interval_sec !== undefined ||
input.health_check_timeout_ms !== undefined ||
input.health_check_verify_tls !== undefined
input.health_check_verify_tls !== undefined ||
input.health_check_provider !== undefined
) {
repos.updateBindingLbConfig(db, binding.id, {
lb_mode: input.lb_mode,
@@ -1166,6 +1176,7 @@ export async function updateConfig(
health_check_interval_sec: input.health_check_interval_sec,
health_check_timeout_ms: input.health_check_timeout_ms,
health_check_verify_tls: input.health_check_verify_tls,
health_check_provider: input.health_check_provider,
});
}
@@ -1240,6 +1251,8 @@ export async function updateConfig(
void syncServiceToVpsTracker(db, id, removedBindingIds);
fireEnsureHealthWorker(db, cf, DEFAULT_HEALTH_FALLBACKS);
const [view] = attachServiceHealth(db, [await buildView(db, id)]);
return view!;
}
@@ -1251,7 +1264,7 @@ export async function createGroup(
): Promise<ServiceGroup> {
const groupType = body.type?.trim() || "custom";
const domain = await normalizeGroupDomain(db, cf, body.domain);
return repos.createServiceGroup(
const group = repos.createServiceGroup(
db,
body.name,
groupType,
@@ -1267,8 +1280,11 @@ export async function createGroup(
health_check_interval_sec: body.health_check_interval_sec,
health_check_timeout_ms: body.health_check_timeout_ms,
health_check_verify_tls: body.health_check_verify_tls,
health_check_provider: body.health_check_provider,
},
);
fireEnsureHealthWorker(db, cf, DEFAULT_HEALTH_FALLBACKS);
return group;
}
export async function updateGroup(
@@ -1303,6 +1319,7 @@ export async function updateGroup(
health_check_interval_sec: body.health_check_interval_sec,
health_check_timeout_ms: body.health_check_timeout_ms,
health_check_verify_tls: body.health_check_verify_tls,
health_check_provider: body.health_check_provider,
},
);
if (!domain && group.enabled) {
@@ -1310,6 +1327,7 @@ export async function updateGroup(
group = repos.getServiceGroup(db, id);
}
await syncEnabledServicesInGroup(db, cf, id);
fireEnsureHealthWorker(db, cf, DEFAULT_HEALTH_FALLBACKS);
return group;
}
+4
View File
@@ -56,6 +56,7 @@ describe("health-check probeTarget", () => {
expected_status: null,
timeout_ms: 1000,
verify_tls: false,
provider: "local",
};
const result = await healthCheckService.probeTarget(target);
expect(result.ok).toBe(true);
@@ -75,6 +76,7 @@ describe("health-check probeTarget", () => {
expected_status: null,
timeout_ms: 500,
verify_tls: false,
provider: "local",
};
const result = await healthCheckService.probeTarget(target);
expect(result.ok).toBe(false);
@@ -107,6 +109,7 @@ describe("health-check probeTarget", () => {
expected_status: 200,
timeout_ms: 1000,
verify_tls: false,
provider: "local",
};
const bindingTarget: HealthCheckTarget = {
...groupTarget,
@@ -142,6 +145,7 @@ describe("health-check probeTarget", () => {
expected_status: null,
timeout_ms: 3000,
verify_tls: false,
provider: "local",
};
const binding: HealthCheckTarget = {
...group,
+160
View File
@@ -0,0 +1,160 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { buildApp } from "../src/app.js";
import { loadConfig } from "../src/config.js";
import { toCloudflareCron } from "../src/services/health/mailbox.js";
import { HEALTH_PROBE_SCRIPT_NAME } from "@cfdm/shared";
async function authHeaders(app: Awaited<ReturnType<typeof buildApp>>) {
const res = await app.inject({
method: "POST",
url: "/api/v1/auth/login",
payload: { username: "admin", password: "admin" },
});
expect(res.statusCode).toBe(200);
const { token } = res.json() as { token: string };
return { authorization: `Bearer ${token}` };
}
function jsonOk(result: unknown, status = 200): Response {
return new Response(JSON.stringify({ success: true, result }), {
status,
headers: { "content-type": "application/json" },
});
}
describe("health worker deploy", () => {
afterEach(() => {
vi.unstubAllGlobals();
});
it("maps 6-field toad cron to 5-field Cloudflare cron", () => {
expect(toCloudflareCron("0 */2 * * * *")).toBe("*/2 * * * *");
expect(toCloudflareCron("*/5 * * * *")).toBe("*/5 * * * *");
});
it("POST ensure creates KV+script; 403 is not local fallback", async () => {
const calls: string[] = [];
vi.stubGlobal(
"fetch",
async (input: RequestInfo | URL, init?: RequestInit) => {
const url =
typeof input === "string" || input instanceof URL
? String(input)
: input.url;
const method = (
init?.method ??
(typeof Request !== "undefined" && input instanceof Request
? input.method
: "GET")
).toUpperCase();
calls.push(`${method} ${url}`);
if (
(url.includes("/accounts?") || /\/accounts$/.test(url.split("?")[0] ?? "")) &&
!url.includes("/storage/") &&
!url.includes("/workers/")
) {
return jsonOk([{ id: "acc-1", name: "Test" }]);
}
if (url.includes("/storage/kv/namespaces") && method === "GET" && !url.includes("/values/")) {
return jsonOk([]);
}
if (url.includes("/storage/kv/namespaces") && method === "POST") {
return jsonOk({ id: "kv-1", title: "cfdm-health-probe" });
}
if (
url.includes(`/workers/scripts/${HEALTH_PROBE_SCRIPT_NAME}`) &&
method === "PUT" &&
!url.includes("/schedules")
) {
return jsonOk({ id: "script-1" });
}
if (url.includes("/schedules") && method === "PUT") {
return jsonOk([{ cron: "*/2 * * * *" }]);
}
if (url.includes("/subdomain") && method === "POST") {
return jsonOk({ enabled: true });
}
if (url.includes("/workers/subdomain") && method === "GET") {
return jsonOk({ subdomain: "example" });
}
if (url.includes("/values/") && method === "GET") {
return new Response("null", { status: 404 });
}
if (url.includes("/values/") && method === "PUT") {
return jsonOk(null);
}
return jsonOk({});
},
);
const app = await buildApp({
config: { ...loadConfig(), staticDir: null, cloudflareApiToken: "cf-token" },
memory: true,
});
const headers = await authHeaders(app);
const res = await app.inject({
method: "POST",
url: "/api/v1/settings/health/worker/ensure",
headers,
});
expect(res.statusCode).toBe(200);
const body = res.json() as {
healthWorkerStatus: string;
healthWorkerUrl: string;
healthWorkerKvNamespaceId: string;
healthWorkerError: string | null;
};
expect(body.healthWorkerStatus).toBe("ready");
expect(body.healthWorkerKvNamespaceId).toBe("kv-1");
expect(body.healthWorkerUrl).toContain("cfdm-health-probe.example.workers.dev");
expect(body.healthWorkerError).toBeNull();
expect(calls.some((c) => c.includes("/workers/scripts/"))).toBe(true);
await app.close();
}, 20_000);
it("POST ensure 403 stores error, does not probe as local", async () => {
vi.stubGlobal(
"fetch",
async (input: RequestInfo | URL) => {
const url = String(input);
if (url.includes("/storage/kv/namespaces")) {
return new Response(
JSON.stringify({
success: false,
errors: [{ code: 10000, message: "Authentication error" }],
}),
{ status: 403, headers: { "content-type": "application/json" } },
);
}
if (url.includes("/accounts")) {
return jsonOk([{ id: "acc-1" }]);
}
return jsonOk({});
},
);
const app = await buildApp({
config: { ...loadConfig(), staticDir: null, cloudflareApiToken: "zone-only" },
memory: true,
});
const headers = await authHeaders(app);
const res = await app.inject({
method: "POST",
url: "/api/v1/settings/health/worker/ensure",
headers,
});
expect(res.statusCode).toBe(401);
const again = await app.inject({
method: "GET",
url: "/api/v1/settings",
headers,
});
const body = again.json() as {
healthWorkerStatus: string;
healthWorkerError: string | null;
};
expect(body.healthWorkerStatus).toBe("error");
expect(body.healthWorkerError).toMatch(/Workers Scripts Write|токен/i);
await app.close();
}, 20_000);
});
+245
View File
@@ -0,0 +1,245 @@
import { describe, expect, it } from "vitest";
import { buildApp } from "../src/app.js";
import { loadConfig } from "../src/config.js";
import { repos, type Db } from "@cfdm/db";
import * as healthCheckService from "../src/services/health-check-service.js";
import type { HealthMailbox } from "../src/services/health/mailbox.js";
import { originProbeKey } from "../src/services/health/mailbox.js";
import type { HealthCheckTarget } from "@cfdm/shared";
async function authHeaders(app: Awaited<ReturnType<typeof buildApp>>) {
const res = await app.inject({
method: "POST",
url: "/api/v1/auth/login",
payload: { username: "admin", password: "admin" },
});
expect(res.statusCode).toBe(200);
const { token } = res.json() as { token: string };
return { authorization: `Bearer ${token}` };
}
async function seedBinding(
db: Db,
opts: { provider: "local" | "cloudflare"; ip: string },
) {
const domain = repos.createDomain(db, null, "example.com", "zone-1");
const service = repos.createService(db, "Panel", "panel");
repos.setServiceEnabled(db, service.id, true);
repos.replaceServiceIps(db, service.id, [opts.ip]);
const binding = repos.insertBinding(db, domain.id, service.id, "panel", null);
repos.replaceBindingIpsWithMeta(db, binding.id, [
{ ip: opts.ip, weight: 1, priority: 1 },
]);
repos.updateBindingLbConfig(db, binding.id, {
health_check_enabled: true,
health_check_type: "tcp",
health_check_port: 1,
health_check_timeout_ms: 400,
health_check_provider: opts.provider,
});
return { service, binding, domain };
}
const thresholds = {
degradedFailures: 1,
downFailures: 2,
latencyWarnMs: 1000,
successRecoveries: 2,
};
function memoryMailbox(opts?: {
resultsOk?: boolean;
colo?: string;
probedAt?: string;
}): HealthMailbox {
let targets: unknown = null;
return {
async getTargets() {
return targets as never;
},
async putTargets(doc) {
targets = doc;
},
async getResults() {
if (!opts) return null;
const dummy: HealthCheckTarget = {
scope: "binding",
ref_id: 1,
ip: "203.0.113.10",
hostname: "panel.example.com",
type: "tcp",
port: 1,
path: null,
expected_status: null,
timeout_ms: 400,
verify_tls: false,
provider: "cloudflare",
};
return {
probedAt: opts.probedAt ?? new Date().toISOString(),
colo: opts.colo ?? "AMS",
items: [
{
key: originProbeKey(dummy),
ok: opts.resultsOk !== false,
latencyMs: 42,
error: opts.resultsOk === false ? "down" : null,
},
],
};
},
};
}
describe("health-check XOR worker mailbox", () => {
it("lists only local providers when no cloudflare bindings", async () => {
const app = await buildApp({
config: { ...loadConfig(), staticDir: null },
memory: true,
});
await seedBinding(app.db, {
provider: "local",
ip: "10.0.0.1",
});
const targets = repos.listHealthCheckTargets(app.db);
expect(targets.length).toBeGreaterThan(0);
expect(targets.every((t) => t.provider === "local")).toBe(true);
expect(targets.some((t) => t.provider === "cloudflare")).toBe(false);
await app.close();
});
it("cloudflare without mailbox does not fall back to local", async () => {
const app = await buildApp({
config: { ...loadConfig(), staticDir: null },
memory: true,
});
const { binding } = await seedBinding(app.db, {
provider: "cloudflare",
ip: "127.0.0.1",
});
await healthCheckService.runAllChecks(app.db, {
thresholds,
probeGapMs: 0,
mailbox: null,
});
const row = repos.getIpHealthStatusRow(
app.db,
"binding",
binding.id,
"127.0.0.1",
);
expect(row?.last_error).toMatch(/Worker не настроен/i);
expect(row?.provider).toBe("cloudflare");
await app.close();
});
it("KV results write colo and last_checked_at without HTTP /probe", async () => {
const app = await buildApp({
config: { ...loadConfig(), staticDir: null },
memory: true,
});
const headers = await authHeaders(app);
const { service, binding } = await seedBinding(app.db, {
provider: "cloudflare",
ip: "203.0.113.10",
});
const targets = repos.listHealthCheckTargets(app.db);
const cfTarget = targets.find((t) => t.ip === "203.0.113.10")!;
const mailbox: HealthMailbox = {
async getTargets() {
return null;
},
async putTargets() {
/* fingerprint sync */
},
async getResults() {
return {
probedAt: new Date().toISOString(),
colo: "AMS",
items: [
{
key: originProbeKey(cfTarget),
ok: true,
latencyMs: 42,
error: null,
},
],
};
},
};
await healthCheckService.runAllChecks(app.db, {
thresholds,
probeGapMs: 0,
mailbox,
});
const row = repos.getIpHealthStatusRow(
app.db,
"binding",
binding.id,
"203.0.113.10",
);
expect(row?.status).toBe("up");
expect(row?.colo).toBe("AMS");
expect(row?.last_checked_at).toBeTruthy();
expect(row?.provider).toBe("cloudflare");
const res = await app.inject({
method: "GET",
url: `/api/v1/services/${service.id}`,
headers,
});
expect(res.statusCode).toBe(200);
const body = res.json() as {
ip_health: Array<{
ip: string;
colo: string | null;
last_checked_at: string | null;
provider: string;
}>;
};
const ipRow = body.ip_health.find((item) => item.ip === "203.0.113.10");
expect(ipRow?.colo).toBe("AMS");
expect(ipRow?.provider).toBe("cloudflare");
const logRes = await app.inject({
method: "GET",
url: `/api/v1/services/${service.id}/health-log`,
headers,
});
expect(logRes.statusCode).toBe(200);
const logBody = logRes.json() as { items: Array<{ colo: string | null }> };
expect(logBody.items[0]?.colo).toBe("AMS");
await app.close();
});
it("stale KV results are recorded, not local probe", async () => {
const app = await buildApp({
config: { ...loadConfig(), staticDir: null },
memory: true,
});
const { binding } = await seedBinding(app.db, {
provider: "cloudflare",
ip: "203.0.113.20",
});
await healthCheckService.runAllChecks(app.db, {
thresholds,
probeGapMs: 0,
mailbox: memoryMailbox({
resultsOk: true,
colo: "SIN",
probedAt: new Date(Date.now() - 60 * 60_000).toISOString(),
}),
staleAfterMs: 60_000,
});
const row = repos.getIpHealthStatusRow(
app.db,
"binding",
binding.id,
"203.0.113.20",
);
expect(row?.last_error).toMatch(/устарели|KV/i);
expect(row?.provider).toBe("cloudflare");
await app.close();
});
});
+9 -1
View File
@@ -99,7 +99,15 @@ describe("service groups health enrichment", () => {
expect(groupView!.services[0]?.health_status).toBe("degraded");
expect(groupView!.services[0]?.health_latency_ms).toBe(120);
expect(groupView!.services[0]?.ip_health).toEqual([
{ ip: "1.2.3.4", status: "degraded", latency_ms: 120 },
{
ip: "1.2.3.4",
status: "degraded",
latency_ms: 120,
last_checked_at: expect.any(String),
last_error: null,
provider: "local",
colo: null,
},
]);
// group worst = degraded (from service) over up (group scope)
expect(groupView!.health_status).toBe("degraded");
+31
View File
@@ -40,12 +40,14 @@ describe("settings health engine", () => {
healthDownFailures: number;
healthLatencyWarnMs: number;
healthSuccessRecoveries: number;
healthWorkerStatus?: string;
};
expect(body.healthCheckCron).toBe("*/30 * * * * *");
expect(body.healthDegradedFailures).toBe(3);
expect(body.healthDownFailures).toBe(4);
expect(body.healthLatencyWarnMs).toBe(1500);
expect(body.healthSuccessRecoveries).toBe(5);
expect(body.healthWorkerStatus).toBe("missing");
await app.close();
});
@@ -133,4 +135,33 @@ describe("settings health engine", () => {
expect(res.statusCode).toBe(400);
await app.close();
});
it("PATCH worker URL; token is not returned in GET", async () => {
const app = await buildApp({
config: { ...loadConfig(), staticDir: null },
memory: true,
});
const headers = await authHeaders(app);
const res = await app.inject({
method: "PATCH",
url: "/api/v1/settings",
headers,
payload: {
healthWorkerUrl: "https://cfdm-health-probe.example.workers.dev",
healthWorkerToken: "super-secret",
},
});
expect(res.statusCode).toBe(200);
const body = res.json() as {
healthWorkerUrl: string;
healthWorkerTokenSet: boolean;
healthWorkerToken?: string;
};
expect(body.healthWorkerUrl).toBe(
"https://cfdm-health-probe.example.workers.dev",
);
expect(body.healthWorkerTokenSet).toBe(true);
expect(body.healthWorkerToken).toBeUndefined();
await app.close();
});
});
+17
View File
@@ -0,0 +1,17 @@
import { copyFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { defineConfig } from "tsup";
export default defineConfig({
entry: ["src/server.ts"],
format: ["esm"],
dts: true,
async onSuccess() {
const root = join(dirname(fileURLToPath(import.meta.url)), "../..");
copyFileSync(
join(root, "workers/health-probe/src/index.mjs"),
join(dirname(fileURLToPath(import.meta.url)), "dist/health-probe-worker.mjs"),
);
},
});
+1
View File
@@ -4,6 +4,7 @@ export default defineConfig({
test: {
environment: "node",
include: ["test/**/*.test.ts"],
testTimeout: 20_000,
typecheck: {
tsconfig: "./tsconfig.test.json",
},
@@ -54,6 +54,8 @@ interface HealthCheckBadgeProps {
latencyMs?: number | null
lastCheckedAt?: string | null
lastError?: string | null
colo?: string | null
provider?: 'local' | 'cloudflare' | string | null
title?: string
showLatency?: boolean
size?: 'xs' | 'sm'
@@ -65,6 +67,8 @@ export function HealthCheckBadge({
latencyMs,
lastCheckedAt,
lastError,
colo,
provider,
title,
showLatency = false,
size = 'sm',
@@ -79,6 +83,9 @@ export function HealthCheckBadge({
tooltipParts.push(`Статус: ${label}`)
if (latencyMs != null) tooltipParts.push(`Задержка: ${latencyMs} мс`)
if (lastCheckedAt) tooltipParts.push(`Проверка: ${formatDate(lastCheckedAt)}`)
if (colo) tooltipParts.push(`Colo: ${colo}`)
if (provider === 'cloudflare') tooltipParts.push('Провайдер: Cloudflare Worker')
if (provider === 'local') tooltipParts.push('Провайдер: Local')
if (lastError) tooltipParts.push(`Ошибка: ${lastError}`)
return (
@@ -190,7 +190,7 @@ export function HealthCheckConfigFields({
<SettingRow
title="Провайдер health-check"
description="Local TCP/HTTP или Cloudflare Health Checks API"
description="Откуда идёт проба: API CFDM или Cloudflare Worker (edge)"
labelFor={`${idPrefix}-provider`}
compact
className={rowClass}
@@ -208,21 +208,26 @@ export function HealthCheckConfigFields({
</SettingRow>
{value.provider === 'cloudflare' ? (
<Alert>
<AlertTitle>Cloudflare Health Checks</AlertTitle>
<AlertTitle>Cloudflare Worker</AlertTitle>
<AlertDescription>
Поля соответствуют официальному API зоны. Если план не позволяет Health
Checks, API вернёт ошибку останется Local. Workers не используются.
Проба с edge Cloudflare, не продукт Health Checks API (на Free его нет).
Worker создаётся автоматически и сам опрашивает IP (KV mailbox).
Статус деплоя в{' '}
<Link to="/settings/health" className="text-foreground underline">
Настройках Health-check
</Link>
. Если Worker не создан, цель не пробируется как Local.
</AlertDescription>
</Alert>
) : (
<Alert>
<AlertTitle>Local health-check</AlertTitle>
<AlertDescription>
Интервал и таймаут пробы ниже. Cron и пороги Slow/Down задаются в{' '}
Проба TCP/HTTP с сервера API. Cron и пороги Slow/Down в{' '}
<Link to="/settings/health" className="text-foreground underline">
Настройках Health-check
</Link>
, как параметры Cloudflare Health Checks в этой форме.
. Интервал в карточке не используется.
</AlertDescription>
</Alert>
)}
@@ -334,83 +339,18 @@ export function HealthCheckConfigFields({
</>
) : null}
<div className="grid grid-cols-2 gap-3">
<FormFieldSimple label="Интервал, сек" htmlFor={`${idPrefix}-interval`}>
<CompactNumberField
id={`${idPrefix}-interval`}
value={value.interval_sec}
min={5}
max={3600}
placeholder="30"
onValueChange={(next) =>
patch({ interval_sec: next ?? 30 })
}
/>
</FormFieldSimple>
<FormFieldSimple label="Таймаут, мс" htmlFor={`${idPrefix}-timeout`}>
<CompactNumberField
id={`${idPrefix}-timeout`}
value={value.timeout_ms}
min={100}
max={30000}
placeholder="3000"
onValueChange={(next) =>
patch({ timeout_ms: next ?? 3000 })
}
/>
</FormFieldSimple>
</div>
{value.provider === 'cloudflare' ? (
<div className="grid grid-cols-2 gap-3">
<FormFieldSimple
label="Retries"
htmlFor={`${idPrefix}-retries`}
hint="Cloudflare retries"
>
<CompactNumberField
id={`${idPrefix}-retries`}
value={value.retries ?? 2}
min={0}
max={10}
placeholder="2"
onValueChange={(retries) => patch({ retries: retries ?? 2 })}
/>
</FormFieldSimple>
<FormFieldSimple
label="Successes"
htmlFor={`${idPrefix}-successes`}
hint="consecutive_successes"
>
<CompactNumberField
id={`${idPrefix}-successes`}
value={value.consecutive_successes ?? 2}
min={1}
max={20}
placeholder="2"
onValueChange={(consecutive_successes) =>
patch({ consecutive_successes: consecutive_successes ?? 2 })
}
/>
</FormFieldSimple>
</div>
) : null}
{value.provider === 'cloudflare' && isHttp ? (
<FormFieldSimple label="HTTP method" htmlFor={`${idPrefix}-method`}>
<Select
modal={false}
value={value.method ?? 'GET'}
onValueChange={(v) => patch({ method: v ?? 'GET' })}
>
<SelectTrigger id={`${idPrefix}-method`} className="w-full">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="GET">GET</SelectItem>
<SelectItem value="HEAD">HEAD</SelectItem>
</SelectContent>
</Select>
</FormFieldSimple>
) : null}
<FormFieldSimple label="Таймаут, мс" htmlFor={`${idPrefix}-timeout`}>
<CompactNumberField
id={`${idPrefix}-timeout`}
value={value.timeout_ms}
min={100}
max={30000}
placeholder="3000"
onValueChange={(next) =>
patch({ timeout_ms: next ?? 3000 })
}
/>
</FormFieldSimple>
</div>
) : null}
</FieldGroup>
@@ -23,6 +23,8 @@ export interface HealthTimelineEvent {
latency_ms?: number | null
error?: string | null
checked_at: string
colo?: string | null
provider?: string | null
}
interface HealthTimelineProps {
@@ -63,6 +65,8 @@ export function HealthTimeline({ events }: HealthTimelineProps) {
<HealthCheckBadge
status={event.status}
latencyMs={event.latency_ms}
colo={event.colo}
provider={event.provider}
size="xs"
showLatency
/>
@@ -110,7 +110,7 @@ function toBindingDrafts(service: ServiceView): ServiceBindingDraft[] {
interval_sec: binding.health_check_interval_sec,
timeout_ms: binding.health_check_timeout_ms,
verify_tls: binding.health_check_verify_tls ?? false,
provider: 'local',
provider: binding.health_check_provider === 'cloudflare' ? 'cloudflare' : 'local',
},
target_ip_weights: binding.target_ip_weights ?? {},
target_ip_priorities: binding.target_ip_priorities ?? {},
@@ -138,6 +138,7 @@ function buildDomainsPayload(bindings: ServiceBindingDraft[]) {
health_check_interval_sec: binding.health.interval_sec,
health_check_timeout_ms: binding.health.timeout_ms,
health_check_verify_tls: binding.health.verify_tls,
health_check_provider: binding.health.provider,
}
: {
fqdn: binding.fqdn.trim(),
@@ -153,6 +154,7 @@ function buildDomainsPayload(bindings: ServiceBindingDraft[]) {
health_check_interval_sec: binding.health.interval_sec,
health_check_timeout_ms: binding.health.timeout_ms,
health_check_verify_tls: binding.health.verify_tls,
health_check_provider: binding.health.provider,
},
)
}
@@ -166,6 +166,10 @@ export function ServiceIpList({
<HealthCheckBadge
status={health?.status ?? 'unknown'}
latencyMs={health?.latency_ms}
lastCheckedAt={health?.last_checked_at}
lastError={health?.last_error}
colo={health?.colo}
provider={health?.provider}
size="xs"
/>
<TruncatedText
+22
View File
@@ -36,6 +36,7 @@ export const serviceGroupSchema = z.object({
health_check_interval_sec: z.number().default(30),
health_check_timeout_ms: z.number().default(3000),
health_check_verify_tls: z.coerce.boolean().default(false),
health_check_provider: z.enum(['local', 'cloudflare']).catch('local'),
created_at: z.string(),
updated_at: z.string(),
})
@@ -76,6 +77,7 @@ export const serviceDomainBindingSchema = z
health_check_interval_sec: z.number().default(30),
health_check_timeout_ms: z.number().default(3000),
health_check_verify_tls: z.coerce.boolean().default(false),
health_check_provider: z.enum(['local', 'cloudflare']).catch('local'),
sync_status: z.string().nullable().default(null),
})
.transform((binding) => ({
@@ -98,6 +100,24 @@ export const serviceIpHealthSchema = z.object({
ip: z.string(),
status: z.enum(['up', 'down', 'degraded', 'unknown']),
latency_ms: z.number().nullable(),
last_checked_at: z.string().nullable().optional(),
last_error: z.string().nullable().optional(),
provider: z.enum(['local', 'cloudflare']).optional(),
colo: z.string().nullable().optional(),
})
export const healthProbeLogSchema = z.object({
id: z.number(),
scope: z.string(),
ref_id: z.number(),
ip: z.string(),
provider: z.enum(['local', 'cloudflare']),
status: z.enum(['up', 'down', 'degraded', 'unknown']),
ok: z.coerce.boolean(),
latency_ms: z.number().nullable(),
colo: z.string().nullable(),
error: z.string().nullable(),
checked_at: z.string(),
})
export const serviceViewSchema = serviceSchema.extend({
@@ -168,6 +188,7 @@ export const serviceBindingSchema = z
health_check_interval_sec: z.number().default(30),
health_check_timeout_ms: z.number().default(3000),
health_check_verify_tls: z.coerce.boolean().default(false),
health_check_provider: z.enum(['local', 'cloudflare']).catch('local'),
sync_status: z.string().nullable().default(null),
created_at: z.string(),
updated_at: z.string(),
@@ -252,6 +273,7 @@ const healthCheckConfigFields = {
health_check_interval_sec: z.number().int().min(5).max(3600).optional(),
health_check_timeout_ms: z.number().int().min(100).max(30000).optional(),
health_check_verify_tls: z.boolean().optional(),
health_check_provider: z.enum(['local', 'cloudflare']).optional(),
}
const serviceDomainInputSchema = z
+21
View File
@@ -1,6 +1,7 @@
import { queryOptions } from '@tanstack/react-query'
import { api } from '@/lib/api-client'
import {
healthProbeLogSchema,
serviceBindingSchema,
serviceGroupsResponseSchema,
serviceViewSchema,
@@ -87,8 +88,28 @@ export async function deleteServiceBinding(id: number) {
export const serviceDetailKeys = {
overview: (id: number) => [...serviceKeys.all, id, 'overview'] as const,
nodes: (id: number) => [...serviceKeys.all, id, 'nodes'] as const,
healthLog: (id: number) => [...serviceKeys.all, id, 'health-log'] as const,
view: (id: number) => [...serviceKeys.all, id, 'view'] as const,
}
export const serviceViewQueryOptions = (id: number) =>
queryOptions({
queryKey: serviceDetailKeys.view(id),
queryFn: async () => {
const data = await api.get<unknown>(`/api/v1/services/${id}`)
return serviceViewSchema.parse(data)
},
})
export const serviceHealthLogQueryOptions = (id: number) =>
queryOptions({
queryKey: serviceDetailKeys.healthLog(id),
queryFn: async () => {
const data = await api.get<unknown>(`/api/v1/services/${id}/health-log`)
return z.object({ items: z.array(healthProbeLogSchema) }).parse(data)
},
})
export const serviceOverviewQueryOptions = (id: number) =>
queryOptions({
queryKey: serviceDetailKeys.overview(id),
@@ -1,22 +1,16 @@
import { createFileRoute } from '@tanstack/react-router'
import { useState } from 'react'
import { useForm } from 'react-hook-form'
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
import { toast } from 'sonner'
import { DetailPanel } from '@/components/reui-kit'
import { createFileRoute, Link } from '@tanstack/react-router'
import { useQuery } from '@tanstack/react-query'
import { ActivityIcon, GlobeIcon, ServerIcon } from 'lucide-react'
import { DetailPanel, KpiStatGrid } from '@/components/reui-kit'
import { EmptyState } from '@/components/empty-state'
import { FormSheet } from '@/components/form-sheet'
import { FormFieldSimple } from '@/components/form-field'
import { LoadingButton } from '@/components/loading-button'
import { Button } from '@cfdm/ui/components/button'
import { Input } from '@cfdm/ui/components/input'
import { Alert, AlertDescription, AlertTitle } from '@/components/reui/alert'
import { HealthProviderToggle } from '@/components/health-check-config-fields'
import { HealthTimeline } from '@/components/health/health-timeline'
import { HealthCheckBadge } from '@/components/health-check-badge'
import {
createOriginHealthCheck,
listOriginHealthChecks,
serviceOverviewQueryOptions,
serviceHealthLogQueryOptions,
serviceViewQueryOptions,
} from '@/queries'
import { formatDate } from '@/lib/format'
export const Route = createFileRoute('/_auth/services/$serviceId/health')({
component: ServiceHealthPage,
@@ -25,119 +19,86 @@ export const Route = createFileRoute('/_auth/services/$serviceId/health')({
export function ServiceHealthPage() {
const { serviceId } = Route.useParams()
const id = Number(serviceId)
const queryClient = useQueryClient()
const overview = useQuery(serviceOverviewQueryOptions(id))
const checksQuery = useQuery({
queryKey: ['health-checks'],
queryFn: listOriginHealthChecks,
})
const [open, setOpen] = useState(false)
const form = useForm<{
name: string
provider: 'local' | 'cloudflare'
protocol: string
}>({
defaultValues: { name: '', provider: 'local', protocol: 'tcp' },
})
const provider = form.watch('provider')
const checks = (checksQuery.data ?? []) as Array<{
id: number
name: string
provider: string
protocol: string
}>
const serviceQuery = useQuery(serviceViewQueryOptions(id))
const logQuery = useQuery(serviceHealthLogQueryOptions(id))
const service = serviceQuery.data
const items = logQuery.data?.items ?? []
const ipHealth = service?.ip_health ?? []
const createMut = useMutation({
mutationFn: (values: { name: string; provider: 'local' | 'cloudflare'; protocol: string }) =>
createOriginHealthCheck({
name: values.name,
provider: values.provider,
protocol: values.protocol,
}),
onSuccess: async () => {
toast.success('Health check сохранён')
await queryClient.invalidateQueries({ queryKey: ['health-checks'] })
setOpen(false)
},
onError: (e: unknown) =>
toast.error(
e instanceof Error
? e.message
: 'Cloudflare Health Checks недоступны для этой зоны',
const kpiCards = ipHealth.map((row) => {
const variant =
row.status === 'down'
? ('destructive' as const)
: row.status === 'degraded'
? ('warning' as const)
: ('default' as const)
return {
id: row.ip,
label: row.ip,
value: row.latency_ms != null ? `${row.latency_ms} мс` : '—',
hint: row.colo ? `colo ${row.colo}` : row.provider === 'cloudflare' ? 'Worker' : 'Local',
icon: row.provider === 'cloudflare' ? <GlobeIcon /> : <ServerIcon />,
variant,
footer: (
<HealthCheckBadge
status={row.status}
latencyMs={row.latency_ms}
lastCheckedAt={row.last_checked_at}
lastError={row.last_error}
colo={row.colo}
provider={row.provider}
size="xs"
/>
),
}
})
void overview
return (
<DetailPanel>
<DetailPanel.Header
title="Health checks"
description="Local TCP/HTTP или официальный Cloudflare Health Checks API."
actions={
<Button size="sm" onClick={() => setOpen(true)}>
Добавить проверку
</Button>
title="Health"
description="Снимок проб этого сервиса. Cloudflare = Worker с edge, не Health Checks API."
/>
<Alert>
<AlertTitle>XOR провайдеров</AlertTitle>
<AlertDescription>
Local ходит с API CFDM; Cloudflare через Worker. Cron и пороги Slow/Down общие, в{' '}
<Link to="/settings/health" className="text-foreground underline">
Настройках Health-check
</Link>
. Если Worker не задан, цель не пробируется как Local.
</AlertDescription>
</Alert>
{kpiCards.length > 0 ? (
<KpiStatGrid cards={kpiCards} />
) : (
<EmptyState
icon={ActivityIcon}
title="Нет проб"
description="Включите health-check на привязке — статус IP появится после cron."
/>
)}
<DetailPanel.Header
title="Журнал проб"
description={
items[0]?.checked_at
? `Последняя: ${formatDate(items[0].checked_at)}`
: 'Последние пробы по IP этого сервиса'
}
/>
{checks.length === 0 ? (
<EmptyState
title="Нет проверок"
description="Локальные пробы уже работают на привязках. Cloudflare Health Checks — опционально."
/>
) : (
<div className="flex flex-col gap-2">
{checks.map((check) => (
<div
key={check.id}
className="flex items-center justify-between gap-3 border-b py-3 last:border-0"
>
<div className="flex flex-col gap-1">
<span className="font-medium">{check.name}</span>
<span className="text-muted-foreground text-xs">
{check.provider} · {check.protocol}
</span>
</div>
</div>
))}
</div>
)}
<FormSheet
open={open}
onOpenChange={setOpen}
title="Health check"
description="Поля Cloudflare соответствуют официальному API (address, type, interval, timeout, retries)."
form={form}
onSubmit={(values) => createMut.mutate(values)}
footer={
<LoadingButton type="submit" isLoading={createMut.isPending}>
Сохранить
</LoadingButton>
}
>
<FormFieldSimple label="Имя" htmlFor="hc-name">
<Input id="hc-name" {...form.register('name')} />
</FormFieldSimple>
<FormFieldSimple label="Провайдер" htmlFor="hc-provider">
<HealthProviderToggle
id="hc-provider"
value={provider}
onChange={(next) => form.setValue('provider', next)}
/>
</FormFieldSimple>
{provider === 'cloudflare' ? (
<Alert>
<AlertTitle>Cloudflare Health Checks</AlertTitle>
<AlertDescription>
Если зона не поддерживает Health Checks, вернётся ошибка плана останется
Local. Workers не используются.
</AlertDescription>
</Alert>
) : null}
<FormFieldSimple label="Протокол" htmlFor="hc-protocol">
<Input id="hc-protocol" {...form.register('protocol')} placeholder="tcp" />
</FormFieldSimple>
</FormSheet>
<HealthTimeline
events={items.map((row) => ({
id: row.id,
hostname: row.ip,
type: row.provider,
status: row.status,
latency_ms: row.latency_ms,
error: row.error,
checked_at: row.checked_at,
colo: row.colo,
provider: row.provider,
}))}
/>
</DetailPanel>
)
}
+143 -4
View File
@@ -27,6 +27,9 @@ import {
} from '@/components/reui/number-field'
import { FieldGroup } from '@cfdm/ui/components/field'
import { Input } from '@cfdm/ui/components/input'
import { Alert, AlertDescription, AlertTitle } from '@/components/reui/alert'
import { Badge } from '@/components/reui/badge'
import { Button } from '@cfdm/ui/components/button'
const formSchema = z.object({
healthCheckCron: z.string().trim().min(1, 'Укажите cron').max(64),
@@ -45,9 +48,16 @@ const formSchema = z.object({
})
type FormValues = z.infer<typeof formSchema>
type HealthWorkerStatus = 'missing' | 'ready' | 'error'
type SettingsResponse = FormValues & {
id: string
healthWorkerUrl?: string
healthWorkerStatus?: HealthWorkerStatus
healthWorkerError?: string | null
healthWorkerDeployedAt?: string | null
healthWorkerLastIngestAt?: string | null
healthWorkerKvNamespaceId?: string
}
export const Route = createFileRoute('/_auth/settings/health')({
@@ -90,6 +100,28 @@ function CompactNumberInput({
)
}
function statusBadge(status: HealthWorkerStatus | undefined) {
if (status === 'ready') {
return (
<Badge variant="success-light" size="sm">
Готов
</Badge>
)
}
if (status === 'error') {
return (
<Badge variant="destructive-light" size="sm">
Ошибка
</Badge>
)
}
return (
<Badge variant="outline" size="sm">
Не создан
</Badge>
)
}
function HealthSettingsPage() {
const queryClient = useQueryClient()
const { data, isLoading } = useQuery({
@@ -121,7 +153,13 @@ function HealthSettingsPage() {
const saveMut = useMutation({
mutationFn: (values: FormValues) =>
api.patch<SettingsResponse>('/api/v1/settings', values),
api.patch<SettingsResponse>('/api/v1/settings', {
healthCheckCron: values.healthCheckCron,
healthDegradedFailures: values.healthDegradedFailures,
healthDownFailures: values.healthDownFailures,
healthLatencyWarnMs: values.healthLatencyWarnMs,
healthSuccessRecoveries: values.healthSuccessRecoveries,
}),
onSuccess: () => {
void queryClient.invalidateQueries({ queryKey: ['app-settings'] })
toast.success('Настройки health-check сохранены')
@@ -130,6 +168,17 @@ function HealthSettingsPage() {
toast.error(e instanceof Error ? e.message : 'Не удалось сохранить'),
})
const ensureMut = useMutation({
mutationFn: () =>
api.post<SettingsResponse>('/api/v1/settings/health/worker/ensure'),
onSuccess: () => {
void queryClient.invalidateQueries({ queryKey: ['app-settings'] })
toast.success('Worker создан или обновлён')
},
onError: (e: unknown) =>
toast.error(e instanceof Error ? e.message : 'Не удалось создать Worker'),
})
return (
<form
className="flex w-full flex-col gap-4"
@@ -144,15 +193,15 @@ function HealthSettingsPage() {
Local health-check
</FrameTitle>
<FrameDescription>
Расписание и пороги движка. Параметры Cloudflare Health Checks
задаются в карточке сервиса.
Расписание и пороги движка общие для Local и Cloudflare Worker.
Тип/порт/path задаются в карточке сервиса.
</FrameDescription>
</FrameHeader>
<FramePanel className="p-0">
<FieldGroup className="gap-0">
<SettingRow
title="Cron"
description="Расписание проб (6 полей: сек мин час день месяц день-недели). Env: HEALTH_CHECK_CRON."
description="Расписание проб CFDM (6 полей). Worker на edge получает 5-польное cron без секунд. Env: HEALTH_CHECK_CRON."
labelFor="health-cron"
stacked
>
@@ -276,6 +325,96 @@ function HealthSettingsPage() {
</FrameFooter>
</FramePanel>
</Frame>
<Frame dense spacing="sm" className="w-full">
<FrameHeader>
<FrameTitle className="flex items-center gap-2">
Cloudflare Worker
{statusBadge(data?.healthWorkerStatus)}
</FrameTitle>
<FrameDescription>
Worker сам опрашивает IP/порты с edge. CFDM создаёт скрипт через API
и забирает результаты из KV. Preview:{' '}
<a
href="https://reui.io/preview/base/settings-16"
className="underline"
target="_blank"
rel="noreferrer"
>
settings-16
</a>
.
</FrameDescription>
</FrameHeader>
<FramePanel className="flex flex-col gap-3 p-4">
<Alert variant={data?.healthWorkerStatus === 'error' ? 'destructive' : 'info'}>
<AlertTitle>Не Health Checks API</AlertTitle>
<AlertDescription>
На Free-плане продукта Health Checks нет. Нужен Account-токен с
Workers Scripts Write и Workers KV Storage Write Zone DNS
недостаточно. Лимиты Free: 5 cron на аккаунт, KV 1000 writes/сутки
(интервал 2 мин), до 48 целей за тик.
</AlertDescription>
</Alert>
{data?.healthWorkerError ? (
<Alert variant="destructive">
<AlertTitle>Ошибка деплоя</AlertTitle>
<AlertDescription>{data.healthWorkerError}</AlertDescription>
</Alert>
) : null}
<FieldGroup className="gap-0">
<SettingRow title="Скрипт" compact>
<span className="font-mono text-sm">cfdm-health-probe</span>
</SettingRow>
<SettingRow
title="KV namespace"
description="id mailbox targets/results"
compact
>
<span className="font-mono text-sm break-all">
{data?.healthWorkerKvNamespaceId || '—'}
</span>
</SettingRow>
<SettingRow
title="URL"
description="workers.dev после автодеплоя"
compact
>
<span className="font-mono text-sm break-all">
{data?.healthWorkerUrl || '—'}
</span>
</SettingRow>
<SettingRow
title="Последний деплой"
compact
>
<span className="text-sm text-muted-foreground">
{data?.healthWorkerDeployedAt || '—'}
</span>
</SettingRow>
<SettingRow
title="Последний ingest"
description="colo пишется в журнал проб"
compact
last
>
<span className="text-sm text-muted-foreground">
{data?.healthWorkerLastIngestAt || '—'}
</span>
</SettingRow>
</FieldGroup>
<div className="flex justify-end">
<Button
type="button"
variant="outline"
disabled={ensureMut.isPending}
onClick={() => ensureMut.mutate()}
>
{ensureMut.isPending ? 'Создаём…' : 'Создать / обновить Worker'}
</Button>
</div>
</FramePanel>
</Frame>
</form>
)
}
+1
View File
@@ -21,6 +21,7 @@ COPY apps/api apps/api
COPY packages/ui packages/ui
COPY packages/shared packages/shared
COPY packages/db packages/db
COPY workers/health-probe workers/health-probe
RUN --mount=type=cache,target=/root/.local/share/pnpm/store,sharing=locked \
pnpm turbo build --filter=web --filter=@cfdm/api \
&& pnpm --filter @cfdm/api deploy --prod /out \
+29 -3
View File
@@ -14,7 +14,7 @@ Manage Cloudflare zones, DNS records, domain groups, and TLS certificate expiry
| Variable | Description |
|----------|-------------|
| `CLOUDFLARE_API_TOKEN` | API token with Zone.DNS permissions |
| `CLOUDFLARE_API_TOKEN` | API token: Zone.DNS **и** для Worker — Account Workers Scripts Write + Workers KV Storage Write |
| `DATABASE_URL` | SQLite path (`sqlite:/data/app.db`) |
| `JWT_SECRET` | JWT signing secret |
| `ADMIN_USERNAME` | Admin username |
@@ -48,8 +48,34 @@ health-check работают на двух уровнях:
не попадает в пул, пока нет успешных проб; восстановление — `UNHEALTHY → CHECKING → HEALTHY`
после `HEALTH_SUCCESS_RECOVERIES` (default 2). Пороги и cron движка задаются в
**Настройки → Health-check** (env — fallback, пока значения не сохранены в UI).
Reconcile DNS запускается cron-задачей
`health-check`. Cloudflare Health Checks — официальный API зоны, Workers не используются.
### Local XOR Cloudflare Worker
Провайдер задаётся на привязке (`service_bindings.health_check_provider`): **local**
или **cloudflare**. Одновременно оба не работают.
| | Local | Cloudflare Worker |
|---|---|---|
| Кто пробирует | процесс API CFDM | Worker на edge (Cron Trigger) |
| Планировщик | глобальный cron CFDM | cron Worker + ingest KV в CFDM |
| Пороги Slow/Down | Настройки → Health-check | те же |
| Результат | SQLite `ip_health_status` | та же SQLite + `colo` из KV |
| Регионы Health Checks | нет | нет (на Free продукта нет) |
**Cloudflare в CFDM — это Worker**, не [Health Checks API](https://developers.cloudflare.com/api/resources/healthchecks).
Продукт Health Checks на Free-плане недоступен и **не используется**.
Worker **сам** опрашивает IP/порты/протоколы (TCP/HTTP, паттерн [UptimeFlare](https://github.com/lyc8503/UptimeFlare): `sockets.opened`, p-limit 5).
CFDM создаёт скрипт через Workers Scripts API, кладёт список целей в KV и читает результаты.
Публичный URL API не нужен. Если Worker/KV не готовы, cloudflare-цели **не** пробируются как Local.
Кнопка **Создать / обновить Worker****Настройки → Health-check**. Токен:
Account `Workers Scripts Write` + `Workers KV Storage Write`. Zone DNS недостаточно.
Free: 5 Cron Triggers на аккаунт; KV 1000 writes/сутки (интервал ≥ 2 мин);
≤ 48 целей за тик. Исходник: [`workers/health-probe/`](../workers/health-probe/).
Reconcile DNS запускается cron-задачей `health-check` после ingest KV.
## Docker
+894 -5
View File
File diff suppressed because one or more lines are too long
+141 -19
View File
@@ -52,6 +52,7 @@ var serviceGroups = sqliteTable("service_groups", {
health_check_interval_sec: integer("health_check_interval_sec").notNull().default(30),
health_check_timeout_ms: integer("health_check_timeout_ms").notNull().default(3e3),
health_check_verify_tls: integer("health_check_verify_tls", { mode: "boolean" }).notNull().default(false),
health_check_provider: text("health_check_provider").notNull().default("local"),
created_at: text("created_at").notNull().default(sql`datetime('now')`),
updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
});
@@ -117,6 +118,7 @@ var serviceBindings = sqliteTable(
health_check_verify_tls: integer("health_check_verify_tls", {
mode: "boolean"
}).notNull().default(false),
health_check_provider: text("health_check_provider").notNull().default("local"),
routing_strategy: text("routing_strategy").notNull().default("round_robin"),
operation_version: integer("operation_version").notNull().default(0),
created_at: text("created_at").notNull().default(sql`datetime('now')`),
@@ -252,6 +254,8 @@ var ipHealthStatus = sqliteTable(
consecutive_successes: integer("consecutive_successes").notNull().default(0),
last_checked_at: text("last_checked_at"),
last_error: text("last_error"),
colo: text("colo"),
provider: text("provider").notNull().default("local"),
created_at: text("created_at").notNull().default(sql`datetime('now')`),
updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
},
@@ -275,6 +279,13 @@ var appSettings = sqliteTable("app_settings", {
health_down_failures: integer("health_down_failures"),
health_latency_warn_ms: integer("health_latency_warn_ms"),
health_success_recoveries: integer("health_success_recoveries"),
health_worker_url: text("health_worker_url"),
health_worker_token: text("health_worker_token"),
health_worker_account_id: text("health_worker_account_id"),
health_worker_kv_namespace_id: text("health_worker_kv_namespace_id"),
health_worker_error: text("health_worker_error"),
health_worker_deployed_at: text("health_worker_deployed_at"),
health_worker_last_ingest_at: text("health_worker_last_ingest_at"),
created_at: text("created_at").notNull().default(sql`datetime('now')`),
updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
});
@@ -311,6 +322,19 @@ var domainMonitorResults = sqliteTable("domain_monitor_results", {
error: text("error"),
checked_at: text("checked_at").notNull().default(sql`datetime('now')`)
});
var healthProbeLog = sqliteTable("health_probe_log", {
id: integer("id").primaryKey({ autoIncrement: true }),
scope: text("scope").notNull(),
ref_id: integer("ref_id").notNull(),
ip: text("ip").notNull(),
provider: text("provider").notNull(),
status: text("status").notNull(),
ok: integer("ok", { mode: "boolean" }).notNull(),
latency_ms: integer("latency_ms"),
colo: text("colo"),
error: text("error"),
checked_at: text("checked_at").notNull().default(sql`datetime('now')`)
});
var notificationLog = sqliteTable("notification_log", {
id: integer("id").primaryKey({ autoIncrement: true }),
kind: text("kind").notNull(),
@@ -358,6 +382,7 @@ var schema = {
domainTags,
domainMonitors,
domainMonitorResults,
healthProbeLog,
notificationLog,
auditLog
};
@@ -502,13 +527,22 @@ var SETTINGS_ID = "settings-main";
function coalesceInt(value, fallback) {
return value == null || Number.isNaN(value) || value < 1 ? fallback : value;
}
function workerStatus(row, envUrl) {
if (row.health_worker_error?.trim()) return "error";
const url = row.health_worker_url?.trim() || envUrl;
const kv = row.health_worker_kv_namespace_id?.trim();
if (kv && url) return "ready";
return "missing";
}
function toDto(row, fallbacks) {
const env = fallbacks ?? {
healthCheckCron: "0 */2 * * * *",
healthDegradedFailures: 1,
healthDownFailures: 2,
healthLatencyWarnMs: 1e3,
healthSuccessRecoveries: 2
healthSuccessRecoveries: 2,
healthWorkerUrl: "",
healthWorkerTokenSet: false
};
return {
id: row.id,
@@ -535,7 +569,15 @@ function toDto(row, fallbacks) {
healthSuccessRecoveries: coalesceInt(
row.health_success_recoveries,
env.healthSuccessRecoveries
)
),
healthWorkerUrl: row.health_worker_url?.trim() || env.healthWorkerUrl,
healthWorkerTokenSet: Boolean(row.health_worker_token?.trim()) || env.healthWorkerTokenSet,
healthWorkerAccountId: row.health_worker_account_id?.trim() ?? "",
healthWorkerKvNamespaceId: row.health_worker_kv_namespace_id?.trim() ?? "",
healthWorkerError: row.health_worker_error?.trim() || null,
healthWorkerDeployedAt: row.health_worker_deployed_at ?? null,
healthWorkerLastIngestAt: row.health_worker_last_ingest_at ?? null,
healthWorkerStatus: workerStatus(row, env.healthWorkerUrl)
};
}
function getAppSettings(db, fallbacks) {
@@ -554,7 +596,9 @@ function getAppSettingsSecrets(db) {
return {
vpsTrackerUrl: row?.vps_tracker_url?.trim() ?? "",
vpsTrackerIntegrationToken: row?.vps_tracker_integration_token?.trim() ?? "",
vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled)
vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled),
healthWorkerUrl: row?.health_worker_url?.trim() ?? "",
healthWorkerToken: row?.health_worker_token?.trim() ?? ""
};
}
function updateAppSettings(db, patch, fallbacks) {
@@ -564,7 +608,6 @@ function updateAppSettings(db, patch, fallbacks) {
}
const current = db.select().from(appSettings).where(eq2(appSettings.id, SETTINGS_ID)).get();
db.update(appSettings).set({
// app_switcher_json: deprecated — source of truth is auth-portal
vps_tracker_url: patch.vpsTrackerUrl !== void 0 ? patch.vpsTrackerUrl : current.vps_tracker_url,
vps_tracker_integration_token: patch.vpsTrackerIntegrationToken !== void 0 && patch.vpsTrackerIntegrationToken.trim() !== "" ? patch.vpsTrackerIntegrationToken : current.vps_tracker_integration_token,
vps_tracker_sync_enabled: patch.vpsTrackerSyncEnabled !== void 0 ? patch.vpsTrackerSyncEnabled : current.vps_tracker_sync_enabled,
@@ -574,6 +617,13 @@ function updateAppSettings(db, patch, fallbacks) {
health_down_failures: patch.healthDownFailures !== void 0 ? patch.healthDownFailures : current.health_down_failures,
health_latency_warn_ms: patch.healthLatencyWarnMs !== void 0 ? patch.healthLatencyWarnMs : current.health_latency_warn_ms,
health_success_recoveries: patch.healthSuccessRecoveries !== void 0 ? patch.healthSuccessRecoveries : current.health_success_recoveries,
health_worker_url: patch.healthWorkerUrl !== void 0 ? patch.healthWorkerUrl.trim() || null : current.health_worker_url,
health_worker_token: patch.healthWorkerToken !== void 0 && patch.healthWorkerToken.trim() !== "" ? patch.healthWorkerToken : current.health_worker_token,
health_worker_account_id: patch.healthWorkerAccountId !== void 0 ? patch.healthWorkerAccountId?.trim() || null : current.health_worker_account_id,
health_worker_kv_namespace_id: patch.healthWorkerKvNamespaceId !== void 0 ? patch.healthWorkerKvNamespaceId?.trim() || null : current.health_worker_kv_namespace_id,
health_worker_error: patch.healthWorkerError !== void 0 ? patch.healthWorkerError?.trim() || null : current.health_worker_error,
health_worker_deployed_at: patch.healthWorkerDeployedAt !== void 0 ? patch.healthWorkerDeployedAt : current.health_worker_deployed_at,
health_worker_last_ingest_at: patch.healthWorkerLastIngestAt !== void 0 ? patch.healthWorkerLastIngestAt : current.health_worker_last_ingest_at,
updated_at: (/* @__PURE__ */ new Date()).toISOString()
}).where(eq2(appSettings.id, SETTINGS_ID)).run();
return getAppSettings(db, fallbacks);
@@ -644,6 +694,7 @@ __export(repos_exports, {
getSyncJob: () => getSyncJob,
insertBinding: () => insertBinding,
insertDnsRecord: () => insertDnsRecord,
insertHealthProbeLog: () => insertHealthProbeLog,
insertNotificationLog: () => insertNotificationLog,
linkBindingRecord: () => linkBindingRecord,
linkGroupDnsRecord: () => linkGroupDnsRecord,
@@ -670,6 +721,7 @@ __export(repos_exports, {
listGroups: () => listGroups,
listHealthCheckTargets: () => listHealthCheckTargets,
listHealthChecks: () => listHealthChecks,
listHealthProbeLogForService: () => listHealthProbeLogForService,
listIpHealthByServiceIds: () => listIpHealthByServiceIds,
listIpHealthStatus: () => listIpHealthStatus,
listNodes: () => listNodes,
@@ -1106,6 +1158,9 @@ function deleteService(db, id) {
const result = db.delete(services).where(eq3(services.id, id)).run();
if (result.changes === 0) throw new NotFoundError(`service ${id}`);
}
function normalizeHealthProvider(value) {
return value === "cloudflare" ? "cloudflare" : "local";
}
function mapServiceGroup(row) {
return {
id: row.id,
@@ -1123,6 +1178,7 @@ function mapServiceGroup(row) {
health_check_interval_sec: row.health_check_interval_sec,
health_check_timeout_ms: row.health_check_timeout_ms,
health_check_verify_tls: row.health_check_verify_tls,
health_check_provider: normalizeHealthProvider(row.health_check_provider),
created_at: row.created_at,
updated_at: row.updated_at
};
@@ -1149,7 +1205,8 @@ function createServiceGroup(db, name, groupType, icon, domain, lbPatch) {
health_check_expected_status: lbPatch?.health_check_expected_status ?? null,
health_check_interval_sec: lbPatch?.health_check_interval_sec ?? 30,
health_check_timeout_ms: lbPatch?.health_check_timeout_ms ?? 3e3,
health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false
health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false,
health_check_provider: lbPatch?.health_check_provider ?? "local"
}).returning({ id: serviceGroups.id }).get().id;
return getServiceGroup(db, id);
}
@@ -1179,6 +1236,8 @@ function updateServiceGroup(db, id, name, groupType, icon, domain, lbPatch) {
update.health_check_timeout_ms = lbPatch.health_check_timeout_ms;
if (lbPatch.health_check_verify_tls !== void 0)
update.health_check_verify_tls = lbPatch.health_check_verify_tls;
if (lbPatch.health_check_provider !== void 0)
update.health_check_provider = lbPatch.health_check_provider;
}
const result = db.update(serviceGroups).set(update).where(eq3(serviceGroups.id, id)).run();
if (result.changes === 0) throw new NotFoundError(`service group ${id}`);
@@ -1488,6 +1547,8 @@ function updateBindingLbConfig(db, bindingId, patch) {
update.health_check_timeout_ms = patch.health_check_timeout_ms;
if (patch.health_check_verify_tls !== void 0)
update.health_check_verify_tls = patch.health_check_verify_tls;
if (patch.health_check_provider !== void 0)
update.health_check_provider = patch.health_check_provider;
db.update(serviceBindings).set(update).where(eq3(serviceBindings.id, bindingId)).run();
}
function setBindingCnameTarget(db, bindingId, target) {
@@ -1546,7 +1607,7 @@ function dnsRecordMatchesHostname(recordName, hostname, zoneName) {
var SERVICE_BINDING_SELECT_COLUMNS = `sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
sb.lb_mode, sb.health_check_enabled, sb.health_check_type, sb.health_check_port,
sb.health_check_path, sb.health_check_expected_status, sb.health_check_interval_sec,
sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.cname_target,
sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.health_check_provider, sb.cname_target,
d.zone_name, d.group_id, g.name AS group_name,
s.name AS service_name, s.slug AS service_slug,
dr.content AS target_ip, dr.sync_status,
@@ -1767,7 +1828,7 @@ function finishSyncJob(db, id, status, message) {
function listIpHealthStatus(db, scope, refId) {
return db.all(sql2`
SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
last_checked_at, last_error
last_checked_at, last_error, colo, provider
FROM ip_health_status
WHERE scope = ${scope} AND ref_id = ${refId}
`);
@@ -1891,7 +1952,11 @@ function listIpHealthByServiceIds(db, serviceIds) {
SELECT sb.service_id AS service_id,
ihs.ip AS ip,
${WORST_HEALTH_SQL} AS health_status,
MAX(ihs.latency_ms) AS health_latency_ms
MAX(ihs.latency_ms) AS health_latency_ms,
MAX(ihs.last_checked_at) AS last_checked_at,
MAX(ihs.last_error) AS last_error,
MAX(ihs.provider) AS provider,
MAX(ihs.colo) AS colo
FROM ip_health_status ihs
INNER JOIN service_bindings sb
ON ihs.scope = 'binding' AND ihs.ref_id = sb.id
@@ -1907,7 +1972,11 @@ function listIpHealthByServiceIds(db, serviceIds) {
list.push({
ip: row.ip,
status: parsed.health_status,
latency_ms: parsed.health_latency_ms
latency_ms: parsed.health_latency_ms,
last_checked_at: row.last_checked_at,
last_error: row.last_error,
provider: normalizeHealthProvider(row.provider),
colo: row.colo
});
result.set(row.service_id, list);
}
@@ -1942,20 +2011,24 @@ function mergeHealthAggregates(parts) {
function getIpHealthStatusRow(db, scope, refId, ip) {
const rows = db.all(sql2`
SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
consecutive_successes, last_checked_at, last_error
consecutive_successes, last_checked_at, last_error, colo, provider
FROM ip_health_status
WHERE scope = ${scope} AND ref_id = ${refId} AND ip = ${ip}
LIMIT 1
`);
return rows[0] ?? null;
}
function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecutiveFailures, lastError, consecutiveSuccesses = 0) {
function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecutiveFailures, lastError, consecutiveSuccesses = 0, extras) {
const colo = extras?.colo ?? null;
const provider = extras?.provider ?? "local";
db.run(sql2`
INSERT INTO ip_health_status
(scope, ref_id, ip, status, latency_ms, consecutive_failures,
consecutive_successes, last_checked_at, last_error, created_at, updated_at)
consecutive_successes, last_checked_at, last_error, colo, provider,
created_at, updated_at)
VALUES (${scope}, ${refId}, ${ip}, ${status}, ${latencyMs}, ${consecutiveFailures},
${consecutiveSuccesses}, datetime('now'), ${lastError}, datetime('now'), datetime('now'))
${consecutiveSuccesses}, datetime('now'), ${lastError}, ${colo}, ${provider},
datetime('now'), datetime('now'))
ON CONFLICT(scope, ref_id, ip) DO UPDATE SET
status = excluded.status,
latency_ms = excluded.latency_ms,
@@ -1963,6 +2036,8 @@ function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecuti
consecutive_successes = excluded.consecutive_successes,
last_checked_at = excluded.last_checked_at,
last_error = excluded.last_error,
colo = excluded.colo,
provider = excluded.provider,
updated_at = datetime('now')
`);
}
@@ -2025,7 +2100,8 @@ function listHealthCheckTargets(db) {
sb.health_check_path AS path,
sb.health_check_expected_status AS expected_status,
sb.health_check_timeout_ms AS timeout_ms,
sb.health_check_verify_tls AS verify_tls
sb.health_check_verify_tls AS verify_tls,
COALESCE(sb.health_check_provider, 'local') AS provider
FROM service_binding_ips sbi
JOIN service_bindings sb ON sb.id = sbi.binding_id
JOIN domains d ON d.id = sb.domain_id
@@ -2039,7 +2115,8 @@ function listHealthCheckTargets(db) {
sg.health_check_path AS path,
sg.health_check_expected_status AS expected_status,
sg.health_check_timeout_ms AS timeout_ms,
sg.health_check_verify_tls AS verify_tls
sg.health_check_verify_tls AS verify_tls,
COALESCE(sg.health_check_provider, 'local') AS provider
FROM service_binding_ips sbi
JOIN service_bindings sb ON sb.id = sbi.binding_id
JOIN services s ON s.id = sb.service_id
@@ -2059,7 +2136,8 @@ function listHealthCheckTargets(db) {
sg.health_check_path AS path,
sg.health_check_expected_status AS expected_status,
sg.health_check_timeout_ms AS timeout_ms,
sg.health_check_verify_tls AS verify_tls
sg.health_check_verify_tls AS verify_tls,
COALESCE(sg.health_check_provider, 'local') AS provider
FROM service_binding_ips sbi
JOIN service_bindings sb ON sb.id = sbi.binding_id
JOIN domains d ON d.id = sb.domain_id
@@ -2079,7 +2157,8 @@ function listHealthCheckTargets(db) {
sb.health_check_path AS path,
sb.health_check_expected_status AS expected_status,
sb.health_check_timeout_ms AS timeout_ms,
sb.health_check_verify_tls AS verify_tls
sb.health_check_verify_tls AS verify_tls,
COALESCE(sb.health_check_provider, 'local') AS provider
FROM service_bindings sb
JOIN domains d ON d.id = sb.domain_id
JOIN services s ON s.id = sb.service_id
@@ -2096,7 +2175,8 @@ function listHealthCheckTargets(db) {
sg.health_check_path AS path,
sg.health_check_expected_status AS expected_status,
sg.health_check_timeout_ms AS timeout_ms,
sg.health_check_verify_tls AS verify_tls
sg.health_check_verify_tls AS verify_tls,
COALESCE(sg.health_check_provider, 'local') AS provider
FROM service_bindings sb
JOIN domains d ON d.id = sb.domain_id
JOIN services s ON s.id = sb.service_id
@@ -2117,7 +2197,8 @@ function listHealthCheckTargets(db) {
...groupInheritedCnameBindingTargets
].map((t) => ({
...t,
verify_tls: Boolean(t.verify_tls)
verify_tls: Boolean(t.verify_tls),
provider: normalizeHealthProvider(t.provider)
}));
}
function listDomainTags(db, domainId) {
@@ -2212,6 +2293,46 @@ function listDomainMonitorResultsForDomain(db, domainId, limit = 50) {
LIMIT ${limit}
`);
}
var HEALTH_PROBE_LOG_KEEP = 50;
function insertHealthProbeLog(db, entry) {
db.insert(healthProbeLog).values({
scope: entry.scope,
ref_id: entry.refId,
ip: entry.ip,
provider: entry.provider,
status: entry.status,
ok: entry.ok,
latency_ms: entry.latencyMs,
colo: entry.colo,
error: entry.error
}).run();
db.run(sql2`
DELETE FROM health_probe_log
WHERE id IN (
SELECT id FROM health_probe_log
WHERE scope = ${entry.scope} AND ref_id = ${entry.refId} AND ip = ${entry.ip}
ORDER BY checked_at DESC, id DESC
LIMIT -1 OFFSET ${HEALTH_PROBE_LOG_KEEP}
)
`);
}
function listHealthProbeLogForService(db, serviceId, limit = 50) {
const rows = db.all(sql2`
SELECT l.id, l.scope, l.ref_id, l.ip, l.provider, l.status, l.ok,
l.latency_ms, l.colo, l.error, l.checked_at
FROM health_probe_log l
INNER JOIN service_bindings sb
ON l.scope = 'binding' AND l.ref_id = sb.id
WHERE sb.service_id = ${serviceId}
ORDER BY l.checked_at DESC, l.id DESC
LIMIT ${limit}
`);
return rows.map((row) => ({
...row,
provider: normalizeHealthProvider(row.provider),
ok: Boolean(row.ok)
}));
}
function insertNotificationLog(db, kind, refType, refId, title, message) {
db.insert(notificationLog).values({
kind,
@@ -2249,6 +2370,7 @@ export {
groups,
healthCheck,
healthChecks,
healthProbeLog,
ipHealthStatus,
listAudit,
nodes,
@@ -0,0 +1,27 @@
-- XOR health-check: persist provider on bindings/groups; Worker URL/token;
-- colo + probe journal. Cloudflare = Worker edge probe, not Health Checks API.
ALTER TABLE service_bindings ADD COLUMN health_check_provider TEXT NOT NULL DEFAULT 'local';
ALTER TABLE service_groups ADD COLUMN health_check_provider TEXT NOT NULL DEFAULT 'local';
ALTER TABLE app_settings ADD COLUMN health_worker_url TEXT;
ALTER TABLE app_settings ADD COLUMN health_worker_token TEXT;
ALTER TABLE ip_health_status ADD COLUMN colo TEXT;
ALTER TABLE ip_health_status ADD COLUMN provider TEXT NOT NULL DEFAULT 'local';
CREATE TABLE IF NOT EXISTS health_probe_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
scope TEXT NOT NULL,
ref_id INTEGER NOT NULL,
ip TEXT NOT NULL,
provider TEXT NOT NULL,
status TEXT NOT NULL,
ok INTEGER NOT NULL,
latency_ms INTEGER,
colo TEXT,
error TEXT,
checked_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_health_probe_log_target
ON health_probe_log(scope, ref_id, ip, checked_at DESC);
@@ -0,0 +1,5 @@
ALTER TABLE app_settings ADD COLUMN health_worker_account_id TEXT;
ALTER TABLE app_settings ADD COLUMN health_worker_kv_namespace_id TEXT;
ALTER TABLE app_settings ADD COLUMN health_worker_error TEXT;
ALTER TABLE app_settings ADD COLUMN health_worker_deployed_at TEXT;
ALTER TABLE app_settings ADD COLUMN health_worker_last_ingest_at TEXT;
+142 -11
View File
@@ -5,6 +5,7 @@ import type {
DomainListItem,
Group,
GroupWithStats,
HealthCheckProvider,
HealthCheckScope,
HealthCheckTarget,
HealthCheckType,
@@ -33,6 +34,7 @@ import {
domains,
groups,
healthChecks,
healthProbeLog,
ipHealthStatus,
nodes,
bindingNodes,
@@ -768,6 +770,10 @@ export function deleteService(db: Db, id: number): void {
// --- Service Groups ---
function normalizeHealthProvider(value: unknown): HealthCheckProvider {
return value === "cloudflare" ? "cloudflare" : "local";
}
function mapServiceGroup(row: typeof serviceGroups.$inferSelect): ServiceGroup {
return {
id: row.id,
@@ -785,6 +791,7 @@ function mapServiceGroup(row: typeof serviceGroups.$inferSelect): ServiceGroup {
health_check_interval_sec: row.health_check_interval_sec,
health_check_timeout_ms: row.health_check_timeout_ms,
health_check_verify_tls: row.health_check_verify_tls,
health_check_provider: normalizeHealthProvider(row.health_check_provider),
created_at: row.created_at,
updated_at: row.updated_at,
};
@@ -819,6 +826,7 @@ export interface ServiceGroupLbPatch {
health_check_interval_sec?: number;
health_check_timeout_ms?: number;
health_check_verify_tls?: boolean;
health_check_provider?: HealthCheckProvider;
}
export function createServiceGroup(
@@ -845,6 +853,7 @@ export function createServiceGroup(
health_check_interval_sec: lbPatch?.health_check_interval_sec ?? 30,
health_check_timeout_ms: lbPatch?.health_check_timeout_ms ?? 3000,
health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false,
health_check_provider: lbPatch?.health_check_provider ?? "local",
})
.returning({ id: serviceGroups.id })
.get()!.id;
@@ -885,6 +894,8 @@ export function updateServiceGroup(
update.health_check_timeout_ms = lbPatch.health_check_timeout_ms;
if (lbPatch.health_check_verify_tls !== undefined)
update.health_check_verify_tls = lbPatch.health_check_verify_tls;
if (lbPatch.health_check_provider !== undefined)
update.health_check_provider = lbPatch.health_check_provider;
}
const result = db
.update(serviceGroups)
@@ -1427,6 +1438,7 @@ export interface BindingLbPatch {
health_check_interval_sec?: number;
health_check_timeout_ms?: number;
health_check_verify_tls?: boolean;
health_check_provider?: HealthCheckProvider;
}
export function updateBindingLbConfig(
@@ -1457,6 +1469,8 @@ export function updateBindingLbConfig(
update.health_check_timeout_ms = patch.health_check_timeout_ms;
if (patch.health_check_verify_tls !== undefined)
update.health_check_verify_tls = patch.health_check_verify_tls;
if (patch.health_check_provider !== undefined)
update.health_check_provider = patch.health_check_provider;
db.update(serviceBindings)
.set(update)
.where(eq(serviceBindings.id, bindingId))
@@ -1564,7 +1578,7 @@ function dnsRecordMatchesHostname(
const SERVICE_BINDING_SELECT_COLUMNS = `sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
sb.lb_mode, sb.health_check_enabled, sb.health_check_type, sb.health_check_port,
sb.health_check_path, sb.health_check_expected_status, sb.health_check_interval_sec,
sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.cname_target,
sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.health_check_provider, sb.cname_target,
d.zone_name, d.group_id, g.name AS group_name,
s.name AS service_name, s.slug AS service_slug,
dr.content AS target_ip, dr.sync_status,
@@ -1943,7 +1957,7 @@ export function listIpHealthStatus(
return db
.all<IpHealthStatus>(sql`
SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
last_checked_at, last_error
last_checked_at, last_error, colo, provider
FROM ip_health_status
WHERE scope = ${scope} AND ref_id = ${refId}
`);
@@ -2108,6 +2122,10 @@ export type ServiceIpHealthRow = {
ip: string;
status: IpHealthState;
latency_ms: number | null;
last_checked_at: string | null;
last_error: string | null;
provider: HealthCheckProvider;
colo: string | null;
};
/** Per-IP binding-scope health, worst status if the same IP is on several bindings. */
@@ -2126,11 +2144,19 @@ export function listIpHealthByServiceIds(
ip: string;
health_status: string | null;
health_latency_ms: number | null;
last_checked_at: string | null;
last_error: string | null;
provider: string | null;
colo: string | null;
}>(sql`
SELECT sb.service_id AS service_id,
ihs.ip AS ip,
${WORST_HEALTH_SQL} AS health_status,
MAX(ihs.latency_ms) AS health_latency_ms
MAX(ihs.latency_ms) AS health_latency_ms,
MAX(ihs.last_checked_at) AS last_checked_at,
MAX(ihs.last_error) AS last_error,
MAX(ihs.provider) AS provider,
MAX(ihs.colo) AS colo
FROM ip_health_status ihs
INNER JOIN service_bindings sb
ON ihs.scope = 'binding' AND ihs.ref_id = sb.id
@@ -2147,6 +2173,10 @@ export function listIpHealthByServiceIds(
ip: row.ip,
status: parsed.health_status,
latency_ms: parsed.health_latency_ms,
last_checked_at: row.last_checked_at,
last_error: row.last_error,
provider: normalizeHealthProvider(row.provider),
colo: row.colo,
});
result.set(row.service_id, list);
}
@@ -2195,7 +2225,7 @@ export function getIpHealthStatusRow(
): IpHealthStatus | null {
const rows = db.all<IpHealthStatus>(sql`
SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
consecutive_successes, last_checked_at, last_error
consecutive_successes, last_checked_at, last_error, colo, provider
FROM ip_health_status
WHERE scope = ${scope} AND ref_id = ${refId} AND ip = ${ip}
LIMIT 1
@@ -2213,13 +2243,18 @@ export function upsertIpHealthStatus(
consecutiveFailures: number,
lastError: string | null,
consecutiveSuccesses = 0,
extras?: { colo?: string | null; provider?: HealthCheckProvider },
): void {
const colo = extras?.colo ?? null;
const provider = extras?.provider ?? "local";
db.run(sql`
INSERT INTO ip_health_status
(scope, ref_id, ip, status, latency_ms, consecutive_failures,
consecutive_successes, last_checked_at, last_error, created_at, updated_at)
consecutive_successes, last_checked_at, last_error, colo, provider,
created_at, updated_at)
VALUES (${scope}, ${refId}, ${ip}, ${status}, ${latencyMs}, ${consecutiveFailures},
${consecutiveSuccesses}, datetime('now'), ${lastError}, datetime('now'), datetime('now'))
${consecutiveSuccesses}, datetime('now'), ${lastError}, ${colo}, ${provider},
datetime('now'), datetime('now'))
ON CONFLICT(scope, ref_id, ip) DO UPDATE SET
status = excluded.status,
latency_ms = excluded.latency_ms,
@@ -2227,6 +2262,8 @@ export function upsertIpHealthStatus(
consecutive_successes = excluded.consecutive_successes,
last_checked_at = excluded.last_checked_at,
last_error = excluded.last_error,
colo = excluded.colo,
provider = excluded.provider,
updated_at = datetime('now')
`);
}
@@ -2321,7 +2358,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
sb.health_check_path AS path,
sb.health_check_expected_status AS expected_status,
sb.health_check_timeout_ms AS timeout_ms,
sb.health_check_verify_tls AS verify_tls
sb.health_check_verify_tls AS verify_tls,
COALESCE(sb.health_check_provider, 'local') AS provider
FROM service_binding_ips sbi
JOIN service_bindings sb ON sb.id = sbi.binding_id
JOIN domains d ON d.id = sb.domain_id
@@ -2339,7 +2377,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
sg.health_check_path AS path,
sg.health_check_expected_status AS expected_status,
sg.health_check_timeout_ms AS timeout_ms,
sg.health_check_verify_tls AS verify_tls
sg.health_check_verify_tls AS verify_tls,
COALESCE(sg.health_check_provider, 'local') AS provider
FROM service_binding_ips sbi
JOIN service_bindings sb ON sb.id = sbi.binding_id
JOIN services s ON s.id = sb.service_id
@@ -2364,7 +2403,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
sg.health_check_path AS path,
sg.health_check_expected_status AS expected_status,
sg.health_check_timeout_ms AS timeout_ms,
sg.health_check_verify_tls AS verify_tls
sg.health_check_verify_tls AS verify_tls,
COALESCE(sg.health_check_provider, 'local') AS provider
FROM service_binding_ips sbi
JOIN service_bindings sb ON sb.id = sbi.binding_id
JOIN domains d ON d.id = sb.domain_id
@@ -2386,7 +2426,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
sb.health_check_path AS path,
sb.health_check_expected_status AS expected_status,
sb.health_check_timeout_ms AS timeout_ms,
sb.health_check_verify_tls AS verify_tls
sb.health_check_verify_tls AS verify_tls,
COALESCE(sb.health_check_provider, 'local') AS provider
FROM service_bindings sb
JOIN domains d ON d.id = sb.domain_id
JOIN services s ON s.id = sb.service_id
@@ -2406,7 +2447,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
sg.health_check_path AS path,
sg.health_check_expected_status AS expected_status,
sg.health_check_timeout_ms AS timeout_ms,
sg.health_check_verify_tls AS verify_tls
sg.health_check_verify_tls AS verify_tls,
COALESCE(sg.health_check_provider, 'local') AS provider
FROM service_bindings sb
JOIN domains d ON d.id = sb.domain_id
JOIN services s ON s.id = sb.service_id
@@ -2429,6 +2471,7 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
].map((t) => ({
...t,
verify_tls: Boolean(t.verify_tls),
provider: normalizeHealthProvider(t.provider),
}));
}
@@ -2633,6 +2676,94 @@ export function listDomainMonitorResultsForDomain(
`);
}
// --- Health probe journal ---
const HEALTH_PROBE_LOG_KEEP = 50;
export function insertHealthProbeLog(
db: Db,
entry: {
scope: HealthCheckScope;
refId: number;
ip: string;
provider: HealthCheckProvider;
status: string;
ok: boolean;
latencyMs: number | null;
colo: string | null;
error: string | null;
},
): void {
db.insert(healthProbeLog)
.values({
scope: entry.scope,
ref_id: entry.refId,
ip: entry.ip,
provider: entry.provider,
status: entry.status,
ok: entry.ok,
latency_ms: entry.latencyMs,
colo: entry.colo,
error: entry.error,
})
.run();
db.run(sql`
DELETE FROM health_probe_log
WHERE id IN (
SELECT id FROM health_probe_log
WHERE scope = ${entry.scope} AND ref_id = ${entry.refId} AND ip = ${entry.ip}
ORDER BY checked_at DESC, id DESC
LIMIT -1 OFFSET ${HEALTH_PROBE_LOG_KEEP}
)
`);
}
export function listHealthProbeLogForService(
db: Db,
serviceId: number,
limit = 50,
): {
id: number;
scope: string;
ref_id: number;
ip: string;
provider: HealthCheckProvider;
status: string;
ok: boolean;
latency_ms: number | null;
colo: string | null;
error: string | null;
checked_at: string;
}[] {
const rows = db.all<{
id: number;
scope: string;
ref_id: number;
ip: string;
provider: string;
status: string;
ok: number;
latency_ms: number | null;
colo: string | null;
error: string | null;
checked_at: string;
}>(sql`
SELECT l.id, l.scope, l.ref_id, l.ip, l.provider, l.status, l.ok,
l.latency_ms, l.colo, l.error, l.checked_at
FROM health_probe_log l
INNER JOIN service_bindings sb
ON l.scope = 'binding' AND l.ref_id = sb.id
WHERE sb.service_id = ${serviceId}
ORDER BY l.checked_at DESC, l.id DESC
LIMIT ${limit}
`);
return rows.map((row) => ({
...row,
provider: normalizeHealthProvider(row.provider),
ok: Boolean(row.ok),
}));
}
// --- Notification log ---
export function insertNotificationLog(
+30
View File
@@ -64,6 +64,7 @@ export const serviceGroups = sqliteTable("service_groups", {
health_check_verify_tls: integer("health_check_verify_tls", { mode: "boolean" })
.notNull()
.default(false),
health_check_provider: text("health_check_provider").notNull().default("local"),
created_at: text("created_at")
.notNull()
.default(sql`datetime('now')`),
@@ -165,6 +166,9 @@ export const serviceBindings = sqliteTable(
})
.notNull()
.default(false),
health_check_provider: text("health_check_provider")
.notNull()
.default("local"),
routing_strategy: text("routing_strategy").notNull().default("round_robin"),
operation_version: integer("operation_version").notNull().default(0),
created_at: text("created_at")
@@ -354,6 +358,8 @@ export const ipHealthStatus = sqliteTable(
.default(0),
last_checked_at: text("last_checked_at"),
last_error: text("last_error"),
colo: text("colo"),
provider: text("provider").notNull().default("local"),
created_at: text("created_at")
.notNull()
.default(sql`datetime('now')`),
@@ -385,6 +391,13 @@ export const appSettings = sqliteTable("app_settings", {
health_down_failures: integer("health_down_failures"),
health_latency_warn_ms: integer("health_latency_warn_ms"),
health_success_recoveries: integer("health_success_recoveries"),
health_worker_url: text("health_worker_url"),
health_worker_token: text("health_worker_token"),
health_worker_account_id: text("health_worker_account_id"),
health_worker_kv_namespace_id: text("health_worker_kv_namespace_id"),
health_worker_error: text("health_worker_error"),
health_worker_deployed_at: text("health_worker_deployed_at"),
health_worker_last_ingest_at: text("health_worker_last_ingest_at"),
created_at: text("created_at")
.notNull()
.default(sql`datetime('now')`),
@@ -441,6 +454,22 @@ export const domainMonitorResults = sqliteTable("domain_monitor_results", {
.default(sql`datetime('now')`),
});
export const healthProbeLog = sqliteTable("health_probe_log", {
id: integer("id").primaryKey({ autoIncrement: true }),
scope: text("scope").notNull(),
ref_id: integer("ref_id").notNull(),
ip: text("ip").notNull(),
provider: text("provider").notNull(),
status: text("status").notNull(),
ok: integer("ok", { mode: "boolean" }).notNull(),
latency_ms: integer("latency_ms"),
colo: text("colo"),
error: text("error"),
checked_at: text("checked_at")
.notNull()
.default(sql`datetime('now')`),
});
export const notificationLog = sqliteTable("notification_log", {
id: integer("id").primaryKey({ autoIncrement: true }),
kind: text("kind").notNull(),
@@ -494,6 +523,7 @@ export const schema = {
domainTags,
domainMonitors,
domainMonitorResults,
healthProbeLog,
notificationLog,
auditLog,
};
+75 -2
View File
@@ -1,4 +1,5 @@
import { eq } from "drizzle-orm";
import type { HealthWorkerStatus } from "@cfdm/shared";
import type { Db } from "./client.js";
import { appSettings } from "./schema.js";
@@ -19,6 +20,14 @@ export type AppSettingsDto = {
vpsTrackerSyncEnabled: boolean;
vpsTrackerLastSyncAt: string | null;
showQuickActions: boolean;
healthWorkerUrl: string;
healthWorkerTokenSet: boolean;
healthWorkerStatus: HealthWorkerStatus;
healthWorkerAccountId: string;
healthWorkerKvNamespaceId: string;
healthWorkerError: string | null;
healthWorkerDeployedAt: string | null;
healthWorkerLastIngestAt: string | null;
} & HealthEngineSettings;
export type AppSettingsPatch = {
@@ -31,14 +40,35 @@ export type AppSettingsPatch = {
healthDownFailures?: number;
healthLatencyWarnMs?: number;
healthSuccessRecoveries?: number;
healthWorkerUrl?: string;
healthWorkerToken?: string;
healthWorkerAccountId?: string | null;
healthWorkerKvNamespaceId?: string | null;
healthWorkerError?: string | null;
healthWorkerDeployedAt?: string | null;
healthWorkerLastIngestAt?: string | null;
};
export type HealthEngineFallbacks = HealthEngineSettings;
export type HealthEngineFallbacks = HealthEngineSettings & {
healthWorkerUrl: string;
healthWorkerTokenSet: boolean;
};
function coalesceInt(value: number | null | undefined, fallback: number): number {
return value == null || Number.isNaN(value) || value < 1 ? fallback : value;
}
function workerStatus(
row: typeof appSettings.$inferSelect,
envUrl: string,
): HealthWorkerStatus {
if (row.health_worker_error?.trim()) return "error";
const url = row.health_worker_url?.trim() || envUrl;
const kv = row.health_worker_kv_namespace_id?.trim();
if (kv && url) return "ready";
return "missing";
}
function toDto(
row: typeof appSettings.$inferSelect,
fallbacks?: HealthEngineFallbacks,
@@ -49,6 +79,8 @@ function toDto(
healthDownFailures: 2,
healthLatencyWarnMs: 1000,
healthSuccessRecoveries: 2,
healthWorkerUrl: "",
healthWorkerTokenSet: false,
};
return {
id: row.id,
@@ -77,6 +109,15 @@ function toDto(
row.health_success_recoveries,
env.healthSuccessRecoveries,
),
healthWorkerUrl: row.health_worker_url?.trim() || env.healthWorkerUrl,
healthWorkerTokenSet:
Boolean(row.health_worker_token?.trim()) || env.healthWorkerTokenSet,
healthWorkerAccountId: row.health_worker_account_id?.trim() ?? "",
healthWorkerKvNamespaceId: row.health_worker_kv_namespace_id?.trim() ?? "",
healthWorkerError: row.health_worker_error?.trim() || null,
healthWorkerDeployedAt: row.health_worker_deployed_at ?? null,
healthWorkerLastIngestAt: row.health_worker_last_ingest_at ?? null,
healthWorkerStatus: workerStatus(row, env.healthWorkerUrl),
};
}
@@ -103,6 +144,8 @@ export function getAppSettingsSecrets(db: Db): {
vpsTrackerUrl: string;
vpsTrackerIntegrationToken: string;
vpsTrackerSyncEnabled: boolean;
healthWorkerUrl: string;
healthWorkerToken: string;
} {
const row = db
.select()
@@ -114,6 +157,8 @@ export function getAppSettingsSecrets(db: Db): {
vpsTrackerIntegrationToken:
row?.vps_tracker_integration_token?.trim() ?? "",
vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled),
healthWorkerUrl: row?.health_worker_url?.trim() ?? "",
healthWorkerToken: row?.health_worker_token?.trim() ?? "",
};
}
@@ -138,7 +183,6 @@ export function updateAppSettings(
db.update(appSettings)
.set({
// app_switcher_json: deprecated — source of truth is auth-portal
vps_tracker_url:
patch.vpsTrackerUrl !== undefined
? patch.vpsTrackerUrl
@@ -176,6 +220,35 @@ export function updateAppSettings(
patch.healthSuccessRecoveries !== undefined
? patch.healthSuccessRecoveries
: current.health_success_recoveries,
health_worker_url:
patch.healthWorkerUrl !== undefined
? patch.healthWorkerUrl.trim() || null
: current.health_worker_url,
health_worker_token:
patch.healthWorkerToken !== undefined &&
patch.healthWorkerToken.trim() !== ""
? patch.healthWorkerToken
: current.health_worker_token,
health_worker_account_id:
patch.healthWorkerAccountId !== undefined
? patch.healthWorkerAccountId?.trim() || null
: current.health_worker_account_id,
health_worker_kv_namespace_id:
patch.healthWorkerKvNamespaceId !== undefined
? patch.healthWorkerKvNamespaceId?.trim() || null
: current.health_worker_kv_namespace_id,
health_worker_error:
patch.healthWorkerError !== undefined
? patch.healthWorkerError?.trim() || null
: current.health_worker_error,
health_worker_deployed_at:
patch.healthWorkerDeployedAt !== undefined
? patch.healthWorkerDeployedAt
: current.health_worker_deployed_at,
health_worker_last_ingest_at:
patch.healthWorkerLastIngestAt !== undefined
? patch.healthWorkerLastIngestAt
: current.health_worker_last_ingest_at,
updated_at: new Date().toISOString(),
})
.where(eq(appSettings.id, SETTINGS_ID))
+179 -1
View File
@@ -31,6 +31,7 @@ interface ServiceGroup$1 {
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: HealthCheckProvider;
created_at: string;
updated_at: string;
}
@@ -60,6 +61,7 @@ interface ServiceBinding {
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: HealthCheckProvider;
routing_strategy: LbMode;
operation_version: number;
created_at: string;
@@ -90,6 +92,7 @@ interface ServiceBindingView {
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: HealthCheckProvider;
sync_status: string | null;
created_at: string;
updated_at: string;
@@ -114,6 +117,7 @@ interface ServiceDomainBindingView {
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: HealthCheckProvider;
sync_status: string | null;
}
interface ServiceView$1 {
@@ -147,6 +151,10 @@ interface CfZone {
id: string;
name: string;
status: string;
account?: {
id: string;
name?: string;
};
}
interface CfDnsRecord {
id?: string;
@@ -193,11 +201,17 @@ interface IpHealthStatus {
consecutive_successes?: number;
last_checked_at: string | null;
last_error: string | null;
colo?: string | null;
provider?: HealthCheckProvider;
}
interface ServiceIpHealth$1 {
ip: string;
status: IpHealthState;
latency_ms: number | null;
last_checked_at?: string | null;
last_error?: string | null;
provider?: HealthCheckProvider;
colo?: string | null;
}
interface ServiceNode {
id: number;
@@ -275,6 +289,7 @@ interface HealthCheckTarget {
expected_status: number | null;
timeout_ms: number;
verify_tls: boolean;
provider: HealthCheckProvider;
}
declare class ValidationError extends Error {
@@ -378,6 +393,11 @@ declare const ipHealthStatusSchema: z.ZodObject<{
consecutive_successes: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
last_checked_at: z.ZodNullable<z.ZodString>;
last_error: z.ZodNullable<z.ZodString>;
colo: z.ZodOptional<z.ZodNullable<z.ZodString>>;
provider: z.ZodOptional<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
}, z.core.$strip>;
declare const serviceIpHealthSchema: z.ZodObject<{
ip: z.ZodString;
@@ -388,8 +408,37 @@ declare const serviceIpHealthSchema: z.ZodObject<{
degraded: "degraded";
}>;
latency_ms: z.ZodNullable<z.ZodNumber>;
last_checked_at: z.ZodOptional<z.ZodNullable<z.ZodString>>;
last_error: z.ZodOptional<z.ZodNullable<z.ZodString>>;
provider: z.ZodOptional<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
colo: z.ZodOptional<z.ZodNullable<z.ZodString>>;
}, z.core.$strip>;
type ServiceIpHealth = z.infer<typeof serviceIpHealthSchema>;
declare const healthProbeLogSchema: z.ZodObject<{
id: z.ZodNumber;
scope: z.ZodString;
ref_id: z.ZodNumber;
ip: z.ZodString;
provider: z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>;
status: z.ZodEnum<{
unknown: "unknown";
up: "up";
down: "down";
degraded: "degraded";
}>;
ok: z.ZodCoercedBoolean<unknown>;
latency_ms: z.ZodNullable<z.ZodNumber>;
colo: z.ZodNullable<z.ZodString>;
error: z.ZodNullable<z.ZodString>;
checked_at: z.ZodString;
}, z.core.$strip>;
type HealthProbeLog = z.infer<typeof healthProbeLogSchema>;
declare const groupSchema: z.ZodObject<{
id: z.ZodNumber;
name: z.ZodString;
@@ -443,6 +492,10 @@ declare const serviceGroupSchema: z.ZodObject<{
health_check_interval_sec: z.ZodDefault<z.ZodNumber>;
health_check_timeout_ms: z.ZodDefault<z.ZodNumber>;
health_check_verify_tls: z.ZodDefault<z.ZodCoercedBoolean<unknown>>;
health_check_provider: z.ZodCatch<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
created_at: z.ZodString;
updated_at: z.ZodString;
}, z.core.$strip>;
@@ -492,6 +545,10 @@ declare const serviceDomainBindingSchema: z.ZodPipe<z.ZodObject<{
health_check_interval_sec: z.ZodDefault<z.ZodNumber>;
health_check_timeout_ms: z.ZodDefault<z.ZodNumber>;
health_check_verify_tls: z.ZodDefault<z.ZodCoercedBoolean<unknown>>;
health_check_provider: z.ZodCatch<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
sync_status: z.ZodDefault<z.ZodNullable<z.ZodString>>;
}, z.core.$strip>, z.ZodTransform<{
target_ips: string[];
@@ -513,6 +570,7 @@ declare const serviceDomainBindingSchema: z.ZodPipe<z.ZodObject<{
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: "local" | "cloudflare";
sync_status: string | null;
target_ip?: string | null | undefined;
}, {
@@ -531,6 +589,7 @@ declare const serviceDomainBindingSchema: z.ZodPipe<z.ZodObject<{
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: "local" | "cloudflare";
sync_status: string | null;
target_ips?: string[] | undefined;
target_ip?: string | null | undefined;
@@ -584,6 +643,10 @@ declare const serviceViewSchema: z.ZodObject<{
health_check_interval_sec: z.ZodDefault<z.ZodNumber>;
health_check_timeout_ms: z.ZodDefault<z.ZodNumber>;
health_check_verify_tls: z.ZodDefault<z.ZodCoercedBoolean<unknown>>;
health_check_provider: z.ZodCatch<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
sync_status: z.ZodDefault<z.ZodNullable<z.ZodString>>;
}, z.core.$strip>, z.ZodTransform<{
target_ips: string[];
@@ -605,6 +668,7 @@ declare const serviceViewSchema: z.ZodObject<{
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: "local" | "cloudflare";
sync_status: string | null;
target_ip?: string | null | undefined;
}, {
@@ -623,6 +687,7 @@ declare const serviceViewSchema: z.ZodObject<{
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: "local" | "cloudflare";
sync_status: string | null;
target_ips?: string[] | undefined;
target_ip?: string | null | undefined;
@@ -646,6 +711,13 @@ declare const serviceViewSchema: z.ZodObject<{
degraded: "degraded";
}>;
latency_ms: z.ZodNullable<z.ZodNumber>;
last_checked_at: z.ZodOptional<z.ZodNullable<z.ZodString>>;
last_error: z.ZodOptional<z.ZodNullable<z.ZodString>>;
provider: z.ZodOptional<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
colo: z.ZodOptional<z.ZodNullable<z.ZodString>>;
}, z.core.$strip>>>;
ip_enabled: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
}, z.core.$strip>;
@@ -680,6 +752,10 @@ declare const serviceGroupViewSchema: z.ZodObject<{
health_check_interval_sec: z.ZodDefault<z.ZodNumber>;
health_check_timeout_ms: z.ZodDefault<z.ZodNumber>;
health_check_verify_tls: z.ZodDefault<z.ZodCoercedBoolean<unknown>>;
health_check_provider: z.ZodCatch<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
created_at: z.ZodString;
updated_at: z.ZodString;
services: z.ZodDefault<z.ZodArray<z.ZodObject<{
@@ -728,6 +804,10 @@ declare const serviceGroupViewSchema: z.ZodObject<{
health_check_interval_sec: z.ZodDefault<z.ZodNumber>;
health_check_timeout_ms: z.ZodDefault<z.ZodNumber>;
health_check_verify_tls: z.ZodDefault<z.ZodCoercedBoolean<unknown>>;
health_check_provider: z.ZodCatch<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
sync_status: z.ZodDefault<z.ZodNullable<z.ZodString>>;
}, z.core.$strip>, z.ZodTransform<{
target_ips: string[];
@@ -749,6 +829,7 @@ declare const serviceGroupViewSchema: z.ZodObject<{
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: "local" | "cloudflare";
sync_status: string | null;
target_ip?: string | null | undefined;
}, {
@@ -767,6 +848,7 @@ declare const serviceGroupViewSchema: z.ZodObject<{
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: "local" | "cloudflare";
sync_status: string | null;
target_ips?: string[] | undefined;
target_ip?: string | null | undefined;
@@ -790,6 +872,13 @@ declare const serviceGroupViewSchema: z.ZodObject<{
degraded: "degraded";
}>;
latency_ms: z.ZodNullable<z.ZodNumber>;
last_checked_at: z.ZodOptional<z.ZodNullable<z.ZodString>>;
last_error: z.ZodOptional<z.ZodNullable<z.ZodString>>;
provider: z.ZodOptional<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
colo: z.ZodOptional<z.ZodNullable<z.ZodString>>;
}, z.core.$strip>>>;
ip_enabled: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
}, z.core.$strip>>>;
@@ -833,6 +922,10 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{
health_check_interval_sec: z.ZodDefault<z.ZodNumber>;
health_check_timeout_ms: z.ZodDefault<z.ZodNumber>;
health_check_verify_tls: z.ZodDefault<z.ZodCoercedBoolean<unknown>>;
health_check_provider: z.ZodCatch<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
created_at: z.ZodString;
updated_at: z.ZodString;
services: z.ZodDefault<z.ZodArray<z.ZodObject<{
@@ -881,6 +974,10 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{
health_check_interval_sec: z.ZodDefault<z.ZodNumber>;
health_check_timeout_ms: z.ZodDefault<z.ZodNumber>;
health_check_verify_tls: z.ZodDefault<z.ZodCoercedBoolean<unknown>>;
health_check_provider: z.ZodCatch<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
sync_status: z.ZodDefault<z.ZodNullable<z.ZodString>>;
}, z.core.$strip>, z.ZodTransform<{
target_ips: string[];
@@ -902,6 +999,7 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: "local" | "cloudflare";
sync_status: string | null;
target_ip?: string | null | undefined;
}, {
@@ -920,6 +1018,7 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: "local" | "cloudflare";
sync_status: string | null;
target_ips?: string[] | undefined;
target_ip?: string | null | undefined;
@@ -943,6 +1042,13 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{
degraded: "degraded";
}>;
latency_ms: z.ZodNullable<z.ZodNumber>;
last_checked_at: z.ZodOptional<z.ZodNullable<z.ZodString>>;
last_error: z.ZodOptional<z.ZodNullable<z.ZodString>>;
provider: z.ZodOptional<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
colo: z.ZodOptional<z.ZodNullable<z.ZodString>>;
}, z.core.$strip>>>;
ip_enabled: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
}, z.core.$strip>>>;
@@ -1000,6 +1106,10 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{
health_check_interval_sec: z.ZodDefault<z.ZodNumber>;
health_check_timeout_ms: z.ZodDefault<z.ZodNumber>;
health_check_verify_tls: z.ZodDefault<z.ZodCoercedBoolean<unknown>>;
health_check_provider: z.ZodCatch<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
sync_status: z.ZodDefault<z.ZodNullable<z.ZodString>>;
}, z.core.$strip>, z.ZodTransform<{
target_ips: string[];
@@ -1021,6 +1131,7 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: "local" | "cloudflare";
sync_status: string | null;
target_ip?: string | null | undefined;
}, {
@@ -1039,6 +1150,7 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: "local" | "cloudflare";
sync_status: string | null;
target_ips?: string[] | undefined;
target_ip?: string | null | undefined;
@@ -1062,6 +1174,13 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{
degraded: "degraded";
}>;
latency_ms: z.ZodNullable<z.ZodNumber>;
last_checked_at: z.ZodOptional<z.ZodNullable<z.ZodString>>;
last_error: z.ZodOptional<z.ZodNullable<z.ZodString>>;
provider: z.ZodOptional<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
colo: z.ZodOptional<z.ZodNullable<z.ZodString>>;
}, z.core.$strip>>>;
ip_enabled: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
}, z.core.$strip>>>;
@@ -1265,6 +1384,10 @@ declare const healthCheckConfigSchema: z.ZodObject<{
health_check_interval_sec: z.ZodOptional<z.ZodNumber>;
health_check_timeout_ms: z.ZodOptional<z.ZodNumber>;
health_check_verify_tls: z.ZodOptional<z.ZodBoolean>;
health_check_provider: z.ZodOptional<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
}, z.core.$strip>;
type HealthCheckConfig = z.infer<typeof healthCheckConfigSchema>;
declare const createServiceSchema: z.ZodObject<{
@@ -1292,6 +1415,10 @@ declare const createServiceWithConfigSchema: z.ZodObject<{
health_check_interval_sec: z.ZodOptional<z.ZodNumber>;
health_check_timeout_ms: z.ZodOptional<z.ZodNumber>;
health_check_verify_tls: z.ZodOptional<z.ZodBoolean>;
health_check_provider: z.ZodOptional<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
fqdn: z.ZodString;
target_ips: z.ZodOptional<z.ZodArray<z.ZodString>>;
target_cname: z.ZodOptional<z.ZodString>;
@@ -1480,6 +1607,10 @@ declare const updateServiceConfigSchema: z.ZodObject<{
health_check_interval_sec: z.ZodOptional<z.ZodNumber>;
health_check_timeout_ms: z.ZodOptional<z.ZodNumber>;
health_check_verify_tls: z.ZodOptional<z.ZodBoolean>;
health_check_provider: z.ZodOptional<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
fqdn: z.ZodString;
target_ips: z.ZodOptional<z.ZodArray<z.ZodString>>;
target_cname: z.ZodOptional<z.ZodString>;
@@ -1507,6 +1638,10 @@ declare const createServiceGroupSchema: z.ZodObject<{
health_check_interval_sec: z.ZodOptional<z.ZodNumber>;
health_check_timeout_ms: z.ZodOptional<z.ZodNumber>;
health_check_verify_tls: z.ZodOptional<z.ZodBoolean>;
health_check_provider: z.ZodOptional<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
name: z.ZodString;
type: z.ZodDefault<z.ZodEnum<{
vpn: "vpn";
@@ -1537,6 +1672,10 @@ declare const updateServiceGroupSchema: z.ZodObject<{
health_check_interval_sec: z.ZodOptional<z.ZodNumber>;
health_check_timeout_ms: z.ZodOptional<z.ZodNumber>;
health_check_verify_tls: z.ZodOptional<z.ZodBoolean>;
health_check_provider: z.ZodOptional<z.ZodEnum<{
local: "local";
cloudflare: "cloudflare";
}>>;
name: z.ZodOptional<z.ZodString>;
type: z.ZodOptional<z.ZodEnum<{
vpn: "vpn";
@@ -1783,6 +1922,8 @@ declare const appSettingsPatchSchema: z.ZodObject<{
healthDownFailures: z.ZodOptional<z.ZodNumber>;
healthLatencyWarnMs: z.ZodOptional<z.ZodNumber>;
healthSuccessRecoveries: z.ZodOptional<z.ZodNumber>;
healthWorkerUrl: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodLiteral<"">]>>;
healthWorkerToken: z.ZodOptional<z.ZodString>;
}, z.core.$strip>;
type AppSettingsPatch = z.infer<typeof appSettingsPatchSchema>;
declare const vpsTrackerEventSchema: z.ZodObject<{
@@ -1799,6 +1940,43 @@ declare const vpsTrackerEventSchema: z.ZodObject<{
}, z.core.$strip>;
type VpsTrackerEvent = z.infer<typeof vpsTrackerEventSchema>;
declare const HEALTH_PROBE_SCRIPT_NAME = "cfdm-health-probe";
declare const HEALTH_PROBE_KV_TITLE = "cfdm-health-probe";
declare const HEALTH_KV_TARGETS_KEY = "targets";
declare const HEALTH_KV_RESULTS_KEY = "results";
declare const HEALTH_KV_CURSOR_KEY = "cursor";
declare const HEALTH_PROBE_BATCH = 48;
declare const HEALTH_PROBE_CONCURRENCY = 5;
type HealthWorkerStatus = "missing" | "ready" | "error";
interface HealthProbeTargetItem {
key: string;
ip: string;
hostname: string;
type: "tcp" | "http";
port: number;
path?: string;
expectedStatus?: number | null;
timeoutMs?: number;
verifyTls?: boolean;
}
interface HealthProbeTargetsDoc {
fingerprint: string;
updatedAt: string;
items: HealthProbeTargetItem[];
}
interface HealthProbeResultItem {
key: string;
ok: boolean;
latencyMs: number;
error: string | null;
}
interface HealthProbeResultsDoc {
probedAt: string;
colo: string | null;
fingerprint?: string;
items: HealthProbeResultItem[];
}
declare const AUDIT_SEVERITIES: readonly ["info", "warning", "critical"];
type AuditSeverity = (typeof AUDIT_SEVERITIES)[number];
declare const auditSeveritySchema: z.ZodEnum<{
@@ -1907,4 +2085,4 @@ declare const ingestAuditEventSchema: z.ZodObject<{
}, z.core.$strip>;
type IngestAuditEvent = z.infer<typeof ingestAuditEventSchema>;
export { AUDIT_SEVERITIES, AUDIT_SOURCE_APPS, AUDIT_TARGET_TYPES, type AppSettingsPatch, type AppSwitcherConfig, type AppSwitcherEntry, type AuditListQuery, type AuditLogEntry, type AuditSeverity, type AuditSourceApp, type AuditTargetType, type BulkUpdateDomainsInput, CERT_ERROR, CERT_EXPIRED, CERT_MONITORING_VALUES, CERT_MONITOR_AUTO, CERT_MONITOR_REQUIRED, CERT_MONITOR_SKIPPED, CERT_OK, CERT_UNKNOWN, CERT_WARNING, type CertMonitoring, type Certificate, type CfDnsRecord, type CfHealthCheck, type CfZone, type CfdmBindingSyncItem, type ChangeDomainInput, type ChangeIpInput, type CreateDnsRecordInput, type CreateDnsRecordPayload, type CreateDomainInput, type CreateDomainMonitorInput, type CreateGroupInput, type CreateOriginHealthCheckInput, type CreateServiceBindingInput, type CreateServiceGroupInput, type CreateServiceInput, type CreateServiceNodeInput, type CreateServiceWithConfigInput, type CreateSubdomainInput, type DnsRecord, type Domain, type DomainEnvironment, type DomainListItem, type DomainMonitor, type DomainMonitorResult, type DomainMonitorType, type Group, type GroupWithStats, type HealthCheckConfig, type HealthCheckProvider, type HealthCheckScope, type HealthCheckTarget, type HealthCheckType, type HealthStatusQuery, type IngestAuditEvent, type IpHealthState, type IpHealthStatus, type JwtClaims, type LbMode, type LoginInput, type LoginRequest, type LoginResponse, type NodeHealthState, type NotificationLog, type OriginHealthCheck, type OriginHealthCheckRecord, type ParsedFqdn, type PatchDnsRecordPayload, type ReorderServicesInput, SYNC_CONFLICT, SYNC_ERROR, SYNC_PENDING_DELETE, SYNC_PENDING_PUSH, SYNC_SYNCED, type Service, type ServiceBinding, type ServiceBindingView, type ServiceDomainBinding, type ServiceDomainBindingView, type ServiceGroup, type ServiceGroupView, type ServiceGroupsResponse, type ServiceIpHealth, type ServiceNode, type ServiceNodeRecord, type ServiceOverview, type ServiceView, type Subdomain, type SubdomainRecord, type SyncJob, type ToggleEnabledInput, type ToggleServiceIpInput, type UpdateDomainInput, type UpdateServiceConfigInput, type UpdateServiceGroupInput, type UpdateServiceNodeInput, type UpdateSubdomainInput, ValidationError, type VpsTrackerEvent, appSettingsPatchSchema, appSwitcherConfigSchema, appSwitcherEntrySchema, appSwitcherIconSchema, auditListQuerySchema, auditLogEntrySchema, auditSeveritySchema, auditSourceAppSchema, auditTargetTypeSchema, bindingToFqdn, bulkUpdateDomainsSchema, certMonitoringSchema, certStatusFromExpiry, certificateSchema, cfdmBindingSyncItemSchema, cfdmSyncBindingsBodySchema, changeDomainSchema, changeIpSchema, createDnsRecordSchema, createDomainMonitorSchema, createDomainSchema, createGroupSchema, createOriginHealthCheckSchema, createServiceBindingSchema, createServiceGroupSchema, createServiceNodeSchema, createServiceSchema, createServiceWithConfigSchema, createSubdomainSchema, dnsNameToSubdomainLabel, dnsRecordNamesMatch, dnsRecordSchema, domainEnvironmentSchema, domainListItemSchema, domainMonitorResultSchema, domainMonitorSchema, domainMonitorTypeSchema, domainSchema, fqdnToDisplay, groupSchema, groupWithStatsSchema, healthCheckConfigSchema, healthCheckProviderSchema, healthCheckScopeSchema, healthCheckTypeSchema, healthStatusQuerySchema, ingestAuditEventSchema, ipHealthStateSchema, ipHealthStatusSchema, isIpLiteral, isValidIpv4, lbModeSchema, loginSchema, nodeHealthStateSchema, normalizeDnsRecordName, notificationLogSchema, originHealthCheckSchema, parseFqdn, reorderServicesSchema, serviceBindingSchema, serviceDomainBindingSchema, serviceGroupSchema, serviceGroupTypeSchema, serviceGroupViewSchema, serviceGroupsResponseSchema, serviceIpHealthSchema, serviceNodeSchema, serviceSchema, serviceViewSchema, shouldMonitorService, subdomainLabelToFqdn, subdomainSchema, toggleEnabledSchema, toggleServiceIpSchema, updateDomainGroupSchema, updateDomainSchema, updateServiceConfigSchema, updateServiceGroupSchema, updateServiceNodeSchema, updateSubdomainSchema, validateDnsRecord, vpsTrackerEventSchema };
export { AUDIT_SEVERITIES, AUDIT_SOURCE_APPS, AUDIT_TARGET_TYPES, type AppSettingsPatch, type AppSwitcherConfig, type AppSwitcherEntry, type AuditListQuery, type AuditLogEntry, type AuditSeverity, type AuditSourceApp, type AuditTargetType, type BulkUpdateDomainsInput, CERT_ERROR, CERT_EXPIRED, CERT_MONITORING_VALUES, CERT_MONITOR_AUTO, CERT_MONITOR_REQUIRED, CERT_MONITOR_SKIPPED, CERT_OK, CERT_UNKNOWN, CERT_WARNING, type CertMonitoring, type Certificate, type CfDnsRecord, type CfHealthCheck, type CfZone, type CfdmBindingSyncItem, type ChangeDomainInput, type ChangeIpInput, type CreateDnsRecordInput, type CreateDnsRecordPayload, type CreateDomainInput, type CreateDomainMonitorInput, type CreateGroupInput, type CreateOriginHealthCheckInput, type CreateServiceBindingInput, type CreateServiceGroupInput, type CreateServiceInput, type CreateServiceNodeInput, type CreateServiceWithConfigInput, type CreateSubdomainInput, type DnsRecord, type Domain, type DomainEnvironment, type DomainListItem, type DomainMonitor, type DomainMonitorResult, type DomainMonitorType, type Group, type GroupWithStats, HEALTH_KV_CURSOR_KEY, HEALTH_KV_RESULTS_KEY, HEALTH_KV_TARGETS_KEY, HEALTH_PROBE_BATCH, HEALTH_PROBE_CONCURRENCY, HEALTH_PROBE_KV_TITLE, HEALTH_PROBE_SCRIPT_NAME, type HealthCheckConfig, type HealthCheckProvider, type HealthCheckScope, type HealthCheckTarget, type HealthCheckType, type HealthProbeLog, type HealthProbeResultItem, type HealthProbeResultsDoc, type HealthProbeTargetItem, type HealthProbeTargetsDoc, type HealthStatusQuery, type HealthWorkerStatus, type IngestAuditEvent, type IpHealthState, type IpHealthStatus, type JwtClaims, type LbMode, type LoginInput, type LoginRequest, type LoginResponse, type NodeHealthState, type NotificationLog, type OriginHealthCheck, type OriginHealthCheckRecord, type ParsedFqdn, type PatchDnsRecordPayload, type ReorderServicesInput, SYNC_CONFLICT, SYNC_ERROR, SYNC_PENDING_DELETE, SYNC_PENDING_PUSH, SYNC_SYNCED, type Service, type ServiceBinding, type ServiceBindingView, type ServiceDomainBinding, type ServiceDomainBindingView, type ServiceGroup, type ServiceGroupView, type ServiceGroupsResponse, type ServiceIpHealth, type ServiceNode, type ServiceNodeRecord, type ServiceOverview, type ServiceView, type Subdomain, type SubdomainRecord, type SyncJob, type ToggleEnabledInput, type ToggleServiceIpInput, type UpdateDomainInput, type UpdateServiceConfigInput, type UpdateServiceGroupInput, type UpdateServiceNodeInput, type UpdateSubdomainInput, ValidationError, type VpsTrackerEvent, appSettingsPatchSchema, appSwitcherConfigSchema, appSwitcherEntrySchema, appSwitcherIconSchema, auditListQuerySchema, auditLogEntrySchema, auditSeveritySchema, auditSourceAppSchema, auditTargetTypeSchema, bindingToFqdn, bulkUpdateDomainsSchema, certMonitoringSchema, certStatusFromExpiry, certificateSchema, cfdmBindingSyncItemSchema, cfdmSyncBindingsBodySchema, changeDomainSchema, changeIpSchema, createDnsRecordSchema, createDomainMonitorSchema, createDomainSchema, createGroupSchema, createOriginHealthCheckSchema, createServiceBindingSchema, createServiceGroupSchema, createServiceNodeSchema, createServiceSchema, createServiceWithConfigSchema, createSubdomainSchema, dnsNameToSubdomainLabel, dnsRecordNamesMatch, dnsRecordSchema, domainEnvironmentSchema, domainListItemSchema, domainMonitorResultSchema, domainMonitorSchema, domainMonitorTypeSchema, domainSchema, fqdnToDisplay, groupSchema, groupWithStatsSchema, healthCheckConfigSchema, healthCheckProviderSchema, healthCheckScopeSchema, healthCheckTypeSchema, healthProbeLogSchema, healthStatusQuerySchema, ingestAuditEventSchema, ipHealthStateSchema, ipHealthStatusSchema, isIpLiteral, isValidIpv4, lbModeSchema, loginSchema, nodeHealthStateSchema, normalizeDnsRecordName, notificationLogSchema, originHealthCheckSchema, parseFqdn, reorderServicesSchema, serviceBindingSchema, serviceDomainBindingSchema, serviceGroupSchema, serviceGroupTypeSchema, serviceGroupViewSchema, serviceGroupsResponseSchema, serviceIpHealthSchema, serviceNodeSchema, serviceSchema, serviceViewSchema, shouldMonitorService, subdomainLabelToFqdn, subdomainSchema, toggleEnabledSchema, toggleServiceIpSchema, updateDomainGroupSchema, updateDomainSchema, updateServiceConfigSchema, updateServiceGroupSchema, updateServiceNodeSchema, updateSubdomainSchema, validateDnsRecord, vpsTrackerEventSchema };
+45 -4
View File
@@ -190,12 +190,31 @@ var ipHealthStatusSchema = z.object({
consecutive_failures: z.number(),
consecutive_successes: z.number().optional().default(0),
last_checked_at: z.string().nullable(),
last_error: z.string().nullable()
last_error: z.string().nullable(),
colo: z.string().nullable().optional(),
provider: healthCheckProviderSchema.optional()
});
var serviceIpHealthSchema = z.object({
ip: z.string(),
status: ipHealthStateSchema,
latency_ms: z.number().nullable()
latency_ms: z.number().nullable(),
last_checked_at: z.string().nullable().optional(),
last_error: z.string().nullable().optional(),
provider: healthCheckProviderSchema.optional(),
colo: z.string().nullable().optional()
});
var healthProbeLogSchema = z.object({
id: z.number(),
scope: z.string(),
ref_id: z.number(),
ip: z.string(),
provider: healthCheckProviderSchema,
status: ipHealthStateSchema,
ok: z.coerce.boolean(),
latency_ms: z.number().nullable(),
colo: z.string().nullable(),
error: z.string().nullable(),
checked_at: z.string()
});
var groupSchema = z.object({
id: z.number(),
@@ -230,6 +249,7 @@ var serviceGroupSchema = z.object({
health_check_interval_sec: z.number().default(30),
health_check_timeout_ms: z.number().default(3e3),
health_check_verify_tls: z.coerce.boolean().default(false),
health_check_provider: healthCheckProviderSchema.catch("local"),
created_at: z.string(),
updated_at: z.string()
});
@@ -267,6 +287,7 @@ var serviceDomainBindingSchema = z.object({
health_check_interval_sec: z.number().default(30),
health_check_timeout_ms: z.number().default(3e3),
health_check_verify_tls: z.coerce.boolean().default(false),
health_check_provider: healthCheckProviderSchema.catch("local"),
sync_status: z.string().nullable().default(null)
}).transform((binding) => ({
...binding,
@@ -392,7 +413,8 @@ var healthCheckConfigFields = {
health_check_expected_status: z.number().int().min(100).max(599).nullable().optional(),
health_check_interval_sec: z.number().int().min(5).max(3600).optional(),
health_check_timeout_ms: z.number().int().min(100).max(3e4).optional(),
health_check_verify_tls: z.boolean().optional()
health_check_verify_tls: z.boolean().optional(),
health_check_provider: healthCheckProviderSchema.optional()
};
var healthCheckConfigSchema = z.object(healthCheckConfigFields);
var serviceDomainInputSchema = z.object({
@@ -703,7 +725,9 @@ var appSettingsPatchSchema = z3.object({
healthDegradedFailures: z3.number().int().min(1).max(20).optional(),
healthDownFailures: z3.number().int().min(1).max(50).optional(),
healthLatencyWarnMs: z3.number().int().min(50).max(6e4).optional(),
healthSuccessRecoveries: z3.number().int().min(1).max(20).optional()
healthSuccessRecoveries: z3.number().int().min(1).max(20).optional(),
healthWorkerUrl: z3.string().url().or(z3.literal("")).optional(),
healthWorkerToken: z3.string().optional()
}).superRefine((data, ctx) => {
if (data.healthDegradedFailures != null && data.healthDownFailures != null && data.healthDownFailures < data.healthDegradedFailures) {
ctx.addIssue({
@@ -725,6 +749,15 @@ var vpsTrackerEventSchema = z3.object({
timestamp: z3.string().datetime().optional()
});
// src/health-probe-mailbox.ts
var HEALTH_PROBE_SCRIPT_NAME = "cfdm-health-probe";
var HEALTH_PROBE_KV_TITLE = "cfdm-health-probe";
var HEALTH_KV_TARGETS_KEY = "targets";
var HEALTH_KV_RESULTS_KEY = "results";
var HEALTH_KV_CURSOR_KEY = "cursor";
var HEALTH_PROBE_BATCH = 48;
var HEALTH_PROBE_CONCURRENCY = 5;
// src/audit.ts
import { z as z4 } from "zod";
var AUDIT_SEVERITIES = ["info", "warning", "critical"];
@@ -796,6 +829,13 @@ export {
CERT_OK,
CERT_UNKNOWN,
CERT_WARNING,
HEALTH_KV_CURSOR_KEY,
HEALTH_KV_RESULTS_KEY,
HEALTH_KV_TARGETS_KEY,
HEALTH_PROBE_BATCH,
HEALTH_PROBE_CONCURRENCY,
HEALTH_PROBE_KV_TITLE,
HEALTH_PROBE_SCRIPT_NAME,
SYNC_CONFLICT,
SYNC_ERROR,
SYNC_PENDING_DELETE,
@@ -847,6 +887,7 @@ export {
healthCheckProviderSchema,
healthCheckScopeSchema,
healthCheckTypeSchema,
healthProbeLogSchema,
healthStatusQuerySchema,
ingestAuditEventSchema,
ipHealthStateSchema,
@@ -0,0 +1,41 @@
export const HEALTH_PROBE_SCRIPT_NAME = "cfdm-health-probe";
export const HEALTH_PROBE_KV_TITLE = "cfdm-health-probe";
export const HEALTH_KV_TARGETS_KEY = "targets";
export const HEALTH_KV_RESULTS_KEY = "results";
export const HEALTH_KV_CURSOR_KEY = "cursor";
export const HEALTH_PROBE_BATCH = 48;
export const HEALTH_PROBE_CONCURRENCY = 5;
export type HealthWorkerStatus = "missing" | "ready" | "error";
export interface HealthProbeTargetItem {
key: string;
ip: string;
hostname: string;
type: "tcp" | "http";
port: number;
path?: string;
expectedStatus?: number | null;
timeoutMs?: number;
verifyTls?: boolean;
}
export interface HealthProbeTargetsDoc {
fingerprint: string;
updatedAt: string;
items: HealthProbeTargetItem[];
}
export interface HealthProbeResultItem {
key: string;
ok: boolean;
latencyMs: number;
error: string | null;
}
export interface HealthProbeResultsDoc {
probedAt: string;
colo: string | null;
fingerprint?: string;
items: HealthProbeResultItem[];
}
+1
View File
@@ -5,6 +5,7 @@ export * from "./parse-fqdn.js";
export * from "./schemas.js";
export * from "./app-switcher.js";
export * from "./integration-vps-tracker.js";
export * from "./health-probe-mailbox.js";
export * from "./audit.js";
export type {
CfZone,
@@ -35,6 +35,8 @@ export const appSettingsPatchSchema = z.object({
healthDownFailures: z.number().int().min(1).max(50).optional(),
healthLatencyWarnMs: z.number().int().min(50).max(60_000).optional(),
healthSuccessRecoveries: z.number().int().min(1).max(20).optional(),
healthWorkerUrl: z.string().url().or(z.literal("")).optional(),
healthWorkerToken: z.string().optional(),
}).superRefine((data, ctx) => {
if (
data.healthDegradedFailures != null &&
+25
View File
@@ -45,6 +45,8 @@ export const ipHealthStatusSchema = z.object({
consecutive_successes: z.number().optional().default(0),
last_checked_at: z.string().nullable(),
last_error: z.string().nullable(),
colo: z.string().nullable().optional(),
provider: healthCheckProviderSchema.optional(),
})
export type IpHealthStatus = z.infer<typeof ipHealthStatusSchema>
@@ -53,10 +55,30 @@ export const serviceIpHealthSchema = z.object({
ip: z.string(),
status: ipHealthStateSchema,
latency_ms: z.number().nullable(),
last_checked_at: z.string().nullable().optional(),
last_error: z.string().nullable().optional(),
provider: healthCheckProviderSchema.optional(),
colo: z.string().nullable().optional(),
})
export type ServiceIpHealth = z.infer<typeof serviceIpHealthSchema>
export const healthProbeLogSchema = z.object({
id: z.number(),
scope: z.string(),
ref_id: z.number(),
ip: z.string(),
provider: healthCheckProviderSchema,
status: ipHealthStateSchema,
ok: z.coerce.boolean(),
latency_ms: z.number().nullable(),
colo: z.string().nullable(),
error: z.string().nullable(),
checked_at: z.string(),
})
export type HealthProbeLog = z.infer<typeof healthProbeLogSchema>
export const groupSchema = z.object({
id: z.number(),
name: z.string(),
@@ -93,6 +115,7 @@ export const serviceGroupSchema = z.object({
health_check_interval_sec: z.number().default(30),
health_check_timeout_ms: z.number().default(3000),
health_check_verify_tls: z.coerce.boolean().default(false),
health_check_provider: healthCheckProviderSchema.catch('local'),
created_at: z.string(),
updated_at: z.string(),
})
@@ -133,6 +156,7 @@ export const serviceDomainBindingSchema = z
health_check_interval_sec: z.number().default(30),
health_check_timeout_ms: z.number().default(3000),
health_check_verify_tls: z.coerce.boolean().default(false),
health_check_provider: healthCheckProviderSchema.catch('local'),
sync_status: z.string().nullable().default(null),
})
.transform((binding) => ({
@@ -304,6 +328,7 @@ const healthCheckConfigFields = {
health_check_interval_sec: z.number().int().min(5).max(3600).optional(),
health_check_timeout_ms: z.number().int().min(100).max(30000).optional(),
health_check_verify_tls: z.boolean().optional(),
health_check_provider: healthCheckProviderSchema.optional(),
}
export const healthCheckConfigSchema = z.object(healthCheckConfigFields)
+12
View File
@@ -22,6 +22,7 @@ export interface ServiceGroup {
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: HealthCheckProvider;
created_at: string;
updated_at: string;
}
@@ -121,6 +122,7 @@ export interface ServiceBinding {
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: HealthCheckProvider;
routing_strategy: LbMode;
operation_version: number;
created_at: string;
@@ -152,6 +154,7 @@ export interface ServiceBindingView {
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: HealthCheckProvider;
sync_status: string | null;
created_at: string;
updated_at: string;
@@ -177,6 +180,7 @@ export interface ServiceDomainBindingView {
health_check_interval_sec: number;
health_check_timeout_ms: number;
health_check_verify_tls: boolean;
health_check_provider: HealthCheckProvider;
sync_status: string | null;
}
@@ -225,6 +229,7 @@ export interface CfZone {
id: string;
name: string;
status: string;
account?: { id: string; name?: string };
}
export interface CfDnsRecord {
@@ -293,12 +298,18 @@ export interface IpHealthStatus {
consecutive_successes?: number;
last_checked_at: string | null;
last_error: string | null;
colo?: string | null;
provider?: HealthCheckProvider;
}
export interface ServiceIpHealth {
ip: string;
status: IpHealthState;
latency_ms: number | null;
last_checked_at?: string | null;
last_error?: string | null;
provider?: HealthCheckProvider;
colo?: string | null;
}
export interface ServiceNode {
@@ -382,4 +393,5 @@ export interface HealthCheckTarget {
expected_status: number | null;
timeout_ms: number;
verify_tls: boolean;
provider: HealthCheckProvider;
}
+17
View File
@@ -0,0 +1,17 @@
# CFDM health-probe Worker
Edge probe for CFDM. **Not** Cloudflare Health Checks API (unavailable on Free).
Production: CFDM creates this Worker via the Cloudflare API (KV mailbox + Cron Trigger).
You do not need `wrangler deploy`. Token needs **Account**: Workers Scripts Write and Workers KV Storage Write.
Local debug:
```powershell
cd workers/health-probe
npx wrangler dev
```
Worker reads KV `targets`, probes TCP (`cloudflare:sockets` + `opened`) or HTTP (`fetch` to IP + `Host`), writes KV `results`. Batch ≤ 48, concurrency 5.
`GET /` — liveness.
+11
View File
@@ -0,0 +1,11 @@
{
"name": "cfdm-health-probe",
"private": true,
"scripts": {
"dev": "wrangler dev",
"deploy": "wrangler deploy"
},
"devDependencies": {
"wrangler": "^4.20.0"
}
}
+228
View File
@@ -0,0 +1,228 @@
/**
* CFDM health-probe Worker. Cron Trigger reads KV `targets`, probes TCP/HTTP
* from the edge (UptimeFlare-style), writes KV `results`. CFDM is SoT in SQLite.
*/
const TARGETS_KEY = "targets";
const RESULTS_KEY = "results";
const CURSOR_KEY = "cursor";
const BATCH = 48;
const CONCURRENCY = 5;
const COOLDOWN_MS = 3 * 60 * 1000;
const UA = "CFDM-health-probe/1.0";
export default {
async fetch() {
return new Response(JSON.stringify({ ok: true, service: "cfdm-health-probe" }), {
headers: { "content-type": "application/json" },
});
},
async scheduled(_event, env) {
await probeBatch(env);
},
};
async function probeBatch(env) {
const raw = await env.HEALTH_KV.get(TARGETS_KEY);
if (!raw) return;
let doc;
try {
doc = JSON.parse(raw);
} catch {
return;
}
const items = Array.isArray(doc.items) ? doc.items : [];
if (items.length === 0) return;
let offset = 0;
const cursorRaw = await env.HEALTH_KV.get(CURSOR_KEY);
if (cursorRaw) {
try {
const cursor = JSON.parse(cursorRaw);
if (Number.isFinite(cursor.offset) && cursor.offset >= 0) {
offset = cursor.offset % items.length;
}
} catch {
offset = 0;
}
}
const slice = items.slice(offset, offset + BATCH);
const nextOffset = offset + slice.length >= items.length ? 0 : offset + slice.length;
const colo = await readColo();
const probed = await mapPool(slice, CONCURRENCY, async (target) => {
const type = target.type === "http" ? "http" : "tcp";
const port = Number(target.port) || (type === "http" ? 80 : 80);
const timeoutMs = Math.min(Math.max(Number(target.timeoutMs) || 3000, 100), 25_000);
const hostname = String(target.hostname ?? "").trim() || target.ip;
try {
const result =
type === "http"
? await httpProbe({
ip: target.ip,
hostname,
port,
path: target.path || "/",
expectedStatus: target.expectedStatus ?? 200,
timeoutMs,
verifyTls: Boolean(target.verifyTls),
})
: await tcpProbe(target.ip, port, timeoutMs);
return { key: target.key, ...result };
} catch (err) {
return {
key: target.key,
ok: false,
latencyMs: 0,
error: err instanceof Error ? err.message : "probe failed",
};
}
});
const fingerprint = resultFingerprint(probed);
const previousRaw = await env.HEALTH_KV.get(RESULTS_KEY);
let skipWrite = false;
if (previousRaw) {
try {
const prev = JSON.parse(previousRaw);
const age = Date.now() - Date.parse(prev.probedAt);
if (prev.fingerprint === fingerprint && Number.isFinite(age) && age < COOLDOWN_MS) {
skipWrite = true;
}
} catch {
skipWrite = false;
}
}
if (!skipWrite) {
const results = {
probedAt: new Date().toISOString(),
colo,
fingerprint,
items: probed,
};
await env.HEALTH_KV.put(RESULTS_KEY, JSON.stringify(results));
}
if (items.length > BATCH || offset !== 0) {
await env.HEALTH_KV.put(CURSOR_KEY, JSON.stringify({ offset: nextOffset }));
}
}
function resultFingerprint(items) {
return items
.map((item) => `${item.key}:${item.ok ? "1" : "0"}:${item.error ?? ""}`)
.sort()
.join("|");
}
async function mapPool(items, concurrency, fn) {
if (items.length === 0) return [];
const results = new Array(items.length);
let next = 0;
async function worker() {
while (next < items.length) {
const idx = next;
next += 1;
results[idx] = await fn(items[idx]);
}
}
const n = Math.min(concurrency, items.length);
await Promise.all(Array.from({ length: n }, () => worker()));
return results;
}
async function readColo() {
try {
const res = await fetch("https://www.cloudflare.com/cdn-cgi/trace", {
cf: { cacheTtlByStatus: { "100-599": -1 } },
});
const text = await res.text();
const line = text.split("\n").find((row) => row.startsWith("colo="));
return line ? line.slice(5).trim() || null : null;
} catch {
return null;
}
}
function withTimeout(promise, timeoutMs, label) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error(`${label} timeout`)), timeoutMs);
promise.then(
(value) => {
clearTimeout(timer);
resolve(value);
},
(err) => {
clearTimeout(timer);
reject(err);
},
);
});
}
async function tcpProbe(ip, port, timeoutMs) {
const started = Date.now();
const { connect } = await import("cloudflare:sockets");
const socket = connect({ hostname: ip, port });
try {
await withTimeout(socket.opened, timeoutMs, "tcp");
return { ok: true, latencyMs: Date.now() - started, error: null };
} catch (err) {
const message = err instanceof Error ? err.message : "tcp failed";
return { ok: false, latencyMs: Date.now() - started, error: message };
} finally {
try {
socket.close();
} catch {
// ignore
}
}
}
async function httpProbe(opts) {
const started = Date.now();
const useTls = opts.verifyTls || opts.port === 443;
const host = opts.ip.includes(":") ? `[${opts.ip}]` : opts.ip;
const path = opts.path.startsWith("/") ? opts.path : `/${opts.path}`;
const url = `${useTls ? "https" : "http"}://${host}:${opts.port}${path}`;
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), opts.timeoutMs);
try {
const res = await fetch(url, {
method: "GET",
headers: {
Host: opts.hostname,
"User-Agent": UA,
},
signal: controller.signal,
redirect: "manual",
cf: { cacheTtlByStatus: { "100-599": -1 } },
});
try {
await res.body?.cancel();
} catch {
// ignore
}
const latencyMs = Date.now() - started;
if (res.status !== opts.expectedStatus) {
return {
ok: false,
latencyMs,
error: `HTTP ${res.status} (ожидали ${opts.expectedStatus})`,
};
}
return { ok: true, latencyMs, error: null };
} catch (err) {
const message =
err instanceof Error
? err.name === "AbortError"
? "http timeout"
: err.message
: "http failed";
return { ok: false, latencyMs: Date.now() - started, error: message };
} finally {
clearTimeout(timer);
}
}
+10
View File
@@ -0,0 +1,10 @@
name = "cfdm-health-probe"
main = "src/index.mjs"
compatibility_date = "2025-04-01"
# Production Worker is created by CFDM (Workers Scripts API + KV + Cron Trigger).
# This file is for local `wrangler dev` only.
[[kv_namespaces]]
binding = "HEALTH_KV"
id = "00000000-0000-0000-0000-000000000000"