fix(api): дождаться заданий планировщика перед операцией с БД

backend/src/routes/scheduler.test.ts

@@ -3,12 +3,24 @@ import { test } from "node:test"
 import Fastify from "fastify"
 import { createSchedulerRoutes } from "./scheduler.js"
 import type { SchedulerRunReceipt } from "../services/scheduler-recorded-run.js"
+import { pauseSchedulerJobsAndWait, resumeSchedulerJobs } from "../services/scheduler-running.js"
+import { runSchedulerJobNow } from "../services/scheduler.js"
 const receipt: SchedulerRunReceipt = { status: "ok", error: null, result: { v: 1, job: "backups", sampledAt: "2026-10-01T00:00:00Z", due: true, created: 1, failures: 0, pruned: 0 } }
 async function setup(patch: Partial<Parameters<typeof createSchedulerRoutes>[0]> = {}) {
   const app = Fastify(); let calls = 0
   await app.register(createSchedulerRoutes({ status: async () => ({ jobs: [] }), runs: async () => ({ runs: [] }), refresh: async () => {}, run: async () => { calls++; return receipt }, ...patch }))
   return { app, calls: () => calls }
 }
+
+test("database pause denies the real manual runner with an accurate HTTP conflict", async () => {
+  await pauseSchedulerJobsAndWait()
+  const { app } = await setup({ run: runSchedulerJobNow })
+  try {
+    const response = await app.inject({ method: "POST", url: "/scheduler/jobs/backups/run-now" })
+    assert.equal(response.statusCode, 409)
+    assert.equal(response.json().error, "Планировщик приостановлен для операции с базой")
+  } finally { resumeSchedulerJobs(); await app.close() }
+})
 test("manual partial receipt is not ok and retains confirmed backup result", async () => {
   const { app } = await setup({ run: async () => ({ ...receipt, status: "error", error: "S3 unavailable" }) })
   try { const response = await app.inject({ method: "POST", url: "/scheduler/jobs/backups/run-now" }); assert.equal(response.statusCode, 200); assert.equal(response.json().ok, false); assert.equal(response.json().result.result.created, 1); assert.equal(response.json().result.error, "S3 unavailable") } finally { await app.close() }

backend/src/routes/scheduler.ts

@@ -52,7 +52,8 @@ return async (app) => {
     } catch (e) {
       const statusCode = (e as Error & { statusCode?: number }).statusCode
       if (statusCode === 409) {
-        return reply.status(409).send({ error: "Задача уже выполняется" })
+        const paused = (e as Error & { code?: string }).code === "SCHEDULER_PAUSED"
+        return reply.status(409).send({ error: paused ? "Планировщик приостановлен для операции с базой" : "Задача уже выполняется" })
       }
       const msg = e instanceof Error ? e.message : String(e)
       return reply.status(500).send({ error: msg })

backend/src/services/alert-collector-hooks.ts

@@ -5,7 +5,7 @@
  * Любой код, пишущий таблицы для `buildSignalSnapshot`, после успешного коммита должен вызывать
  * `scheduleAlertEngineAfterDataCollectors()`.
  */
-import { isSchedulerJobRunning } from "./scheduler-running.js"
+import { isSchedulerJobRunning, isSchedulerJobsPaused } from "./scheduler-running.js"
 
 const ALERT_AFTER_COLLECT_DEBOUNCE_MS = 250
 const ALERT_AFTER_COLLECT_WAIT_MS = 100
@@ -31,13 +31,16 @@ async function runAlertEngineWhenIdle(): Promise<void> {
   const run = alertEngineRunner
   if (!run) return
   while (isSchedulerJobRunning("alert_engine")) {
+    if (isSchedulerJobsPaused()) return
     await new Promise<void>((resolve) => setTimeout(resolve, ALERT_AFTER_COLLECT_WAIT_MS))
   }
+  if (isSchedulerJobsPaused()) return
   await run()
 }
 
 /** После успешной записи сигналов в БД коллекторами (не из `alert_engine`). */
 export function scheduleAlertEngineAfterDataCollectors(): void {
+  if (isSchedulerJobsPaused()) return
   if (alertAfterCollectTimer != null) clearTimeout(alertAfterCollectTimer)
   alertAfterCollectTimer = setTimeout(async () => {
     alertAfterCollectTimer = null

backend/src/services/scheduler-running.test.ts

@@ -0,0 +1,67 @@
+import assert from "node:assert/strict"
+import { test } from "node:test"
+import {
+  endSchedulerJob, isSchedulerJobsPaused, isSchedulerJobRunning,
+  pauseSchedulerJobsAndWait, resumeSchedulerJobs, tryBeginSchedulerJob,
+} from "./scheduler-running.js"
+import {
+  clearAlertCollectorHooksTimer, scheduleAlertEngineAfterDataCollectors, wireAlertEngineRunner,
+} from "./alert-collector-hooks.js"
+import { executeDatabaseOperation } from "./system-database-operation.js"
+
+test("database preparation waits for all jobs and denies new admission", async () => {
+  assert.equal(tryBeginSchedulerJob("traffic"), true)
+  assert.equal(tryBeginSchedulerJob("uptime_speed"), true)
+  let completed = false
+  const drained = pauseSchedulerJobsAndWait().then(() => { completed = true })
+  try {
+    assert.equal(isSchedulerJobsPaused(), true)
+    assert.equal(tryBeginSchedulerJob("manual"), false)
+    assert.throws(resumeSchedulerJobs, /ожидает/)
+    endSchedulerJob("traffic")
+    await Promise.resolve()
+    assert.equal(completed, false)
+    assert.equal(isSchedulerJobRunning("uptime_speed"), true)
+    endSchedulerJob("uptime_speed")
+    await drained
+    assert.equal(completed, true)
+    assert.equal(tryBeginSchedulerJob("manual"), false)
+  } finally {
+    endSchedulerJob("traffic"); endSchedulerJob("uptime_speed")
+    await drained; resumeSchedulerJobs()
+  }
+  assert.equal(tryBeginSchedulerJob("manual"), true)
+  endSchedulerJob("manual")
+})
+
+test("drain timeout prevents database command and recovery reopens admission", async () => {
+  assert.equal(tryBeginSchedulerJob("slow"), true)
+  let commands = 0, restarts = 0
+  try {
+    await assert.rejects(executeDatabaseOperation(async () => { commands++; return {} }, {
+      stopListener: () => {}, stopScheduler: () => pauseSchedulerJobsAndWait(10),
+      startListener: () => { restarts++ }, startScheduler: resumeSchedulerJobs,
+    }), /операция с базой отменена/)
+    assert.equal(commands, 0)
+    assert.equal(restarts, 1)
+    assert.equal(isSchedulerJobsPaused(), false)
+    assert.equal(isSchedulerJobRunning("slow"), true)
+    assert.equal(tryBeginSchedulerJob("slow"), false)
+  } finally { endSchedulerJob("slow"); resumeSchedulerJobs() }
+})
+
+test("a queued alert hook and collectors finishing during pause cannot start a job", async () => {
+  let runs = 0
+  wireAlertEngineRunner(async () => { runs++ })
+  scheduleAlertEngineAfterDataCollectors()
+  await pauseSchedulerJobsAndWait()
+  try {
+    scheduleAlertEngineAfterDataCollectors()
+    await new Promise(resolve => setTimeout(resolve, 300))
+    assert.equal(runs, 0)
+  } finally { clearAlertCollectorHooksTimer(); resumeSchedulerJobs() }
+  scheduleAlertEngineAfterDataCollectors()
+  await new Promise(resolve => setTimeout(resolve, 300))
+  assert.equal(runs, 1)
+  clearAlertCollectorHooksTimer()
+})

backend/src/services/scheduler-running.ts

@@ -3,6 +3,33 @@
  * Слот берётся до `await` в теле джобы — так `alert_engine` может дождаться REST и др. до чтения БД.
  */
 const runningSchedulerJobs = new Set<string>()
+let paused = false
+const idleWaiters = new Set<() => void>()
+
+export function isSchedulerJobsPaused(): boolean {
+  return paused
+}
+
+/** Close admission before waiting: completion includes the job's database journal. */
+export async function pauseSchedulerJobsAndWait(timeoutMs = 30_000): Promise<void> {
+  paused = true
+  if (!runningSchedulerJobs.size) return
+  await new Promise<void>((resolve, reject) => {
+    const complete = () => { clearTimeout(timer); idleWaiters.delete(complete); resolve() }
+    const timer = setTimeout(() => {
+      idleWaiters.delete(complete)
+      reject(new Error("Не удалось дождаться завершения заданий планировщика; операция с базой отменена"))
+    }, timeoutMs)
+    idleWaiters.add(complete)
+  })
+}
+
+export function resumeSchedulerJobs(): void {
+  if (runningSchedulerJobs.size && idleWaiters.size) {
+    throw new Error("Планировщик ещё ожидает завершения заданий")
+  }
+  paused = false
+}
 
 const ALERT_SNAPSHOT_SOURCE_JOB_KEYS = new Set<string>([
   "traffic",
@@ -15,13 +42,14 @@ const ALERT_SNAPSHOT_SOURCE_JOB_KEYS = new Set<string>([
 
 /** true, если слот занят (джоба уже выполняется). */
 export function tryBeginSchedulerJob(jobKey: string): boolean {
-  if (runningSchedulerJobs.has(jobKey)) return false
+  if (paused || runningSchedulerJobs.has(jobKey)) return false
   runningSchedulerJobs.add(jobKey)
   return true
 }
 
 export function endSchedulerJob(jobKey: string): void {
   runningSchedulerJobs.delete(jobKey)
+  if (!runningSchedulerJobs.size) for (const complete of [...idleWaiters]) complete()
 }
 
 export function isSchedulerJobRunning(jobKey: string): boolean {

backend/src/services/scheduler.ts

@@ -52,6 +52,9 @@ import {
   endSchedulerJob,
   isSchedulerJobRunning,
   tryBeginSchedulerJob,
+  isSchedulerJobsPaused,
+  pauseSchedulerJobsAndWait,
+  resumeSchedulerJobs,
 } from "./scheduler-running.js"
 import { appendEvent } from "../modules/events/service/events-service.js"
 import { runRecordedSchedulerJob, type SchedulerRunReceipt } from "./scheduler-recorded-run.js"
@@ -72,6 +75,7 @@ export const JOB_KEYS = [
 export type SchedulerJobKey = (typeof JOB_KEYS)[number]
 
 const timers = new Map<string, ReturnType<typeof setInterval>>()
+let schedulerGeneration = 0
 
 const RUN_LOG_RETENTION_MS = 30 * 24 * 60 * 60 * 1000
 
@@ -203,8 +207,9 @@ export async function runSchedulerJobNow(jobKey: string): Promise<SchedulerRunRe
     throw new Error(`Unknown job key: ${jobKey}`)
   }
   if (!tryBeginSchedulerJob(jobKey)) {
-    const err = new Error("Job already running")
+    const err = new Error(isSchedulerJobsPaused() ? "Планировщик приостановлен для операции с базой" : "Job already running")
     ;(err as Error & { statusCode?: number }).statusCode = 409
+    if (isSchedulerJobsPaused()) (err as Error & { code?: string }).code = "SCHEDULER_PAUSED"
     throw err
   }
   try {
@@ -222,9 +227,13 @@ function clearAllTimers() {
 
 /** Пересоздать интервалы после смены настроек. */
 export async function refreshScheduler(): Promise<void> {
+  if (isSchedulerJobsPaused()) return
+  const generation = ++schedulerGeneration
+  const superseded = () => generation !== schedulerGeneration || isSchedulerJobsPaused()
   clearAllTimers()
 
   const traffic = await getTrafficSettings()
+  if (superseded()) return
   if (traffic.enabled) {
     const ms = Math.max(5_000, traffic.intervalSec * 1000)
     void executeSchedulerJob("traffic").catch(() => {})
@@ -237,7 +246,9 @@ export async function refreshScheduler(): Promise<void> {
   }
 
   const apiPing = await getServersApiPingSettings()
+  if (superseded()) return
   const internetPath = await getInternetPathSettings()
+  if (superseded()) return
   if (apiPing.enabled) {
     const apiMs = Math.max(10_000, apiPing.intervalSec * 1000)
     void executeSchedulerJob("servers_rest_ping").catch(() => {})
@@ -250,6 +261,7 @@ export async function refreshScheduler(): Promise<void> {
   }
 
   const uptime = await getUptimeSettings()
+  if (superseded()) return
   const resOn = uptime.resourcesEnabled ?? uptime.enabled
   const pingOn = uptime.pingEnabled ?? uptime.enabled
   const spdOn = uptime.speedEnabled ?? uptime.enabled
@@ -308,6 +320,7 @@ export async function refreshScheduler(): Promise<void> {
   )
 
   const certRenew = await getCertificateRenewSettings()
+  if (superseded()) return
   if (certRenew.enabled) {
     const certRenewMs = Math.max(300_000, certRenew.intervalSec * 1000)
     void executeSchedulerJob("certificates_renew").catch(() => {})
@@ -320,6 +333,7 @@ export async function refreshScheduler(): Promise<void> {
   }
 
   const backupSchedule = await getBackupScheduleSettings()
+  if (superseded()) return
   if (backupSchedule.enabled) {
     const backupMs = 60_000
     void executeSchedulerJob("backups").catch(() => {})
@@ -332,6 +346,7 @@ export async function refreshScheduler(): Promise<void> {
   }
 
   const geoip = await getGeoipSettings()
+  if (superseded()) return
   if (geoip.enabled) {
     const geoipMs = Math.max(6 * 3600_000, geoip.updateIntervalSec * 1000)
     void executeSchedulerJob("geoip_update").catch(() => {})
@@ -354,9 +369,22 @@ export async function refreshScheduler(): Promise<void> {
 }
 
 export function stopScheduler(): void {
+  schedulerGeneration += 1
   clearAllTimers()
 }
 
+export async function stopSchedulerForDatabaseOperation(): Promise<void> {
+  const drained = pauseSchedulerJobsAndWait()
+  stopScheduler()
+  await drained
+  clearAlertCollectorHooksTimer()
+}
+
+export async function restartSchedulerAfterDatabaseOperation(): Promise<void> {
+  resumeSchedulerJobs()
+  await refreshScheduler()
+}
+
 export function isJobRunning(jobKey: string): boolean {
   return isSchedulerJobRunning(jobKey)
 }

backend/src/services/system-database-backup.ts

@@ -8,7 +8,7 @@ import { env } from "../config.js"
 import { pool } from "../db/index.js"
 import { withOperationLock } from "../db/operation-lock.js"
 import { executeDatabaseOperation, type DatabaseOperationWarnings } from "./system-database-operation.js"
-import { refreshScheduler, stopScheduler } from "./scheduler.js"
+import { restartSchedulerAfterDatabaseOperation, stopSchedulerForDatabaseOperation } from "./scheduler.js"
 import {
   startTrafficFlowListener,
   stopTrafficFlowListener,
@@ -49,9 +49,9 @@ function spawnCollected(command: string, args: string[], opts?: { stdin?: Buffer
 async function withDatabaseOperation<T extends object>(fn: () => Promise<T>): Promise<T & DatabaseOperationWarnings> {
   return withOperationLock(() => pool.connect(), BACKUP_LOCK_KEY, () => executeDatabaseOperation(fn, {
     stopListener: stopTrafficFlowListener,
-    stopScheduler,
+    stopScheduler: stopSchedulerForDatabaseOperation,
     startListener: startTrafficFlowListener,
-    startScheduler: refreshScheduler,
+    startScheduler: restartSchedulerAfterDatabaseOperation,
   }), "Операция с базой приложения уже выполняется")
 }