From 80af888cdb4e6b19b894bef81f73ff745dfe74ba Mon Sep 17 00:00:00 2001 From: Denozordec Date: Fri, 6 Nov 2026 22:36:53 +0700 Subject: [PATCH] =?UTF-8?q?perf(api):=20=D1=87=D0=B8=D1=81=D1=82=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D1=81=D1=82=D0=B0=D1=80=D1=8B=D0=B5=20=D0=B6?= =?UTF-8?q?=D1=83=D1=80=D0=BD=D0=B0=D0=BB=D1=8B=20=D0=B8=20=D0=BE=D0=B1?= =?UTF-8?q?=D0=BB=D0=B5=D0=B3=D1=87=D0=B8=D1=82=D1=8C=20=D1=81=D0=B8=D0=BD?= =?UTF-8?q?=D0=BA=20=D0=B8=20dashboard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - раз в сутки удаляются notification_log и sync_log старше 90 дней и vps_health_checks старше 30 дней; синки, зависшие в running дольше часа, помечаются ошибкой - dashboard stats не грузит тарифы, домены и проекты - запросы характеристик VPS в BILLmanager идут по 4 параллельно Co-Authored-By: Claude Opus 5.5 --- apps/api/src/routes/review-guards.test.ts | 52 +++++++++++++++++++ .../billmanager/map-concurrent.test.ts | 18 +++++++ apps/api/src/services/billmanager/sync.ts | 37 ++++++++----- apps/api/src/services/dashboard-stats.ts | 4 +- apps/api/src/services/scheduler.ts | 16 +++++- packages/db/src/index.ts | 1 + packages/db/src/maintenance/retention.test.ts | 29 +++++++++++ packages/db/src/maintenance/retention.ts | 42 +++++++++++++++ packages/db/src/repositories/snapshot.ts | 17 ++++++ 9 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 apps/api/src/services/billmanager/map-concurrent.test.ts create mode 100644 packages/db/src/maintenance/retention.test.ts create mode 100644 packages/db/src/maintenance/retention.ts diff --git a/apps/api/src/routes/review-guards.test.ts b/apps/api/src/routes/review-guards.test.ts index 910143e..f97b50d 100644 --- a/apps/api/src/routes/review-guards.test.ts +++ b/apps/api/src/routes/review-guards.test.ts @@ -135,3 +135,55 @@ describe('space viewer cannot write', () => { await app.close() }) }) + +describe('whole-DB operations need a global admin', () => { + beforeEach(() => { + process.env.AUTH_REQUIRED = 'true' + process.env.AUTH_JWT_SECRET = 'test-secret-at-least-8' + resetTestDb() + }) + afterEach(() => { + delete process.env.AUTH_REQUIRED + delete process.env.AUTH_JWT_SECRET + closeDb() + }) + + it('403 for settings admin, 200 for global admin', async () => { + const app = await buildApp() + const token = (sub: string, isAdmin: boolean) => + app.jwt.sign({ + sub, + email: `${sub}@t`, + name: sub, + apps: ['vps'], + permissions: ['vps:settings:admin'], + is_admin: isAdmin, + iss: app.authConfig.issuer, + }) + const get = (t: string) => + app.inject({ url: '/api/backup/json', headers: { authorization: `Bearer ${t}` } }) + expect((await get(token('space-admin', false))).statusCode).toBe(403) + const migrate = await app.inject({ + method: 'POST', + url: '/api/migrate', + headers: { authorization: `Bearer ${token('space-admin', false)}` }, + payload: {}, + }) + expect(migrate.statusCode).toBe(403) + expect((await get(token('root', true))).statusCode).toBe(200) + await app.close() + }) +}) + +describe('dashboard stats', () => { + beforeEach(() => resetTestDb()) + afterEach(() => closeDb()) + + it('computes from the slim snapshot', async () => { + const app = await buildApp() + const res = await app.inject({ url: '/api/dashboard/stats' }) + expect(res.statusCode).toBe(200) + expect(res.json()).toMatchObject({ totalVpsCount: 0, providerCount: 0 }) + await app.close() + }) +}) diff --git a/apps/api/src/services/billmanager/map-concurrent.test.ts b/apps/api/src/services/billmanager/map-concurrent.test.ts new file mode 100644 index 0000000..19a031e --- /dev/null +++ b/apps/api/src/services/billmanager/map-concurrent.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest' +import { mapConcurrent } from './sync.js' + +describe('mapConcurrent', () => { + it('keeps order and caps parallelism', async () => { + let active = 0 + let peak = 0 + const out = await mapConcurrent([5, 1, 4, 2, 3, 0], 2, async (n) => { + active++ + peak = Math.max(peak, active) + await new Promise((r) => setTimeout(r, n)) + active-- + return n * 10 + }) + expect(out).toEqual([50, 10, 40, 20, 30, 0]) + expect(peak).toBe(2) + }) +}) diff --git a/apps/api/src/services/billmanager/sync.ts b/apps/api/src/services/billmanager/sync.ts index 8e029fb..7cd6dde 100644 --- a/apps/api/src/services/billmanager/sync.ts +++ b/apps/api/src/services/billmanager/sync.ts @@ -88,6 +88,26 @@ async function mapVdsWithSpecs( return vps } +/** vds.edit lookups are one HTTP call per VPS — run a few at once, not all (hoster rate limits). */ +const VDS_SPECS_CONCURRENCY = 4 + +export async function mapConcurrent( + items: T[], + limit: number, + fn: (item: T) => Promise, +): Promise { + const out: R[] = new Array(items.length) + let next = 0 + const worker = async () => { + while (next < items.length) { + const i = next++ + out[i] = await fn(items[i]!) + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)) + return out +} + export async function syncFromBillmanager( account: BillmanagerSyncAccount, opts: SyncFromBillmanagerOptions = {}, @@ -138,20 +158,9 @@ export async function syncFromBillmanager( const syncSummary: SyncSummary = { added: [], updated: [], paymentsAdded: 0 } if (fetchVpsPayments) { - const mappedRaw: Awaited>[] = [] - for (const item of vdsItems) { - mappedRaw.push( - await mapVdsWithSpecs( - profile, - item, - providerId, - accountId, - tariffItems, - apiBaseUrl, - authinfo, - ), - ) - } + const mappedRaw = await mapConcurrent(vdsItems, VDS_SPECS_CONCURRENCY, (item) => + mapVdsWithSpecs(profile, item, providerId, accountId, tariffItems, apiBaseUrl, authinfo), + ) const mappedList = profile.map.enrichVdsBatch ? profile.map.enrichVdsBatch(mappedRaw, { balance: dashboardInfo?.balance ?? null, diff --git a/apps/api/src/services/dashboard-stats.ts b/apps/api/src/services/dashboard-stats.ts index 209873a..c896d2d 100644 --- a/apps/api/src/services/dashboard-stats.ts +++ b/apps/api/src/services/dashboard-stats.ts @@ -1,4 +1,4 @@ -import { getSnapshot } from '@cfdm/db/repositories/snapshot' +import { getDashboardSnapshot } from '@cfdm/db/repositories/snapshot' import { countExpiringWithin7Days, countInventoryIssues } from '@cfdm/shared/utils/inventory-health' import { accountBalanceApi } from '@cfdm/shared/utils/account-balance' import { isSyncApiType } from '@cfdm/shared/contracts/provider' @@ -76,7 +76,7 @@ export interface DashboardStats { } export function computeDashboardStats(): DashboardStats { - const snap = getSnapshot() + const snap = getDashboardSnapshot() const now = new Date() const activeVps = snap.vps.filter((v) => v.status === 'active') diff --git a/apps/api/src/services/scheduler.ts b/apps/api/src/services/scheduler.ts index 58f719a..dfa37bf 100644 --- a/apps/api/src/services/scheduler.ts +++ b/apps/api/src/services/scheduler.ts @@ -1,5 +1,5 @@ import { eq, sql } from 'drizzle-orm' -import { getDb, schema, runWithSpaceAsync, MAIN_SPACE_ID } from '@cfdm/db' +import { getDb, getSqlite, pruneOldLogs, schema, runWithSpaceAsync, MAIN_SPACE_ID } from '@cfdm/db' import { settingsRepository } from '@cfdm/db/repositories/settings' import { resolveSyncAccount, getProviderAdapter, type SyncReadyAccount } from './providers/index.js' @@ -25,6 +25,8 @@ let notifyIntervalId: ReturnType | null = null let uptimeIntervalId: ReturnType | null = null let cfdmSyncIntervalId: ReturnType | null = null let cfdmSyncInFlight = false +let retentionIntervalId: ReturnType | null = null +const RETENTION_INTERVAL_MS = 24 * 60 * 60 * 1000 type AccountRow = typeof schema.providerAccounts.$inferSelect type SettingsRow = typeof schema.settings.$inferSelect @@ -109,6 +111,12 @@ export const runNotificationTick = () => exclusive('notify', notificationTick) export const runScheduledSync = () => exclusive('sync', scheduledSync) export const runScheduledSyncTariffs = () => exclusive('tariffs', scheduledSyncTariffs) export const runScheduledUptimeChecks = () => exclusive('uptime', scheduledUptimeChecks) +export const runRetention = () => + exclusive('retention', async () => { + const r = pruneOldLogs(getSqlite()) + const total = r.notificationLog + r.syncLog + r.vpsHealthChecks + r.staleSyncs + if (total > 0) console.log('Retention:', JSON.stringify(r)) + }) async function notificationTick(): Promise { for (const settings of allSettings()) { @@ -334,6 +342,8 @@ export function startScheduler(): void { uptimeIntervalId = null if (cfdmSyncIntervalId) clearInterval(cfdmSyncIntervalId) cfdmSyncIntervalId = null + if (retentionIntervalId) clearInterval(retentionIntervalId) + retentionIntervalId = null try { const rows = allSettings() @@ -350,6 +360,8 @@ export function startScheduler(): void { ) void runNotificationTick() void runScheduledUptimeChecks() + retentionIntervalId = setInterval(() => void runRetention(), RETENTION_INTERVAL_MS) + void runRetention() const parts = [ `notify every ${notifyInterval} min`, @@ -393,6 +405,8 @@ export function stopScheduler(): void { uptimeIntervalId = null if (cfdmSyncIntervalId) clearInterval(cfdmSyncIntervalId) cfdmSyncIntervalId = null + if (retentionIntervalId) clearInterval(retentionIntervalId) + retentionIntervalId = null } export function restartScheduler(): void { diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 996c00e..c884ece 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -98,3 +98,4 @@ export { consolidateProviderApiFromAccounts, heuristicBillmanagerProviderApi, } from './maintenance/consolidate-api.js' +export { pruneOldLogs, RETENTION_DAYS, type PruneResult } from './maintenance/retention.js' diff --git a/packages/db/src/maintenance/retention.test.ts b/packages/db/src/maintenance/retention.test.ts new file mode 100644 index 0000000..b6f0c0b --- /dev/null +++ b/packages/db/src/maintenance/retention.test.ts @@ -0,0 +1,29 @@ +import Database from 'better-sqlite3' +import { describe, expect, it } from 'vitest' +import { ensureRuntimeSchema, resetRuntimeMigrate } from '../runtime-migrate.js' +import { pruneOldLogs } from './retention.js' + +describe('pruneOldLogs', () => { + it('drops old rows, keeps fresh ones, closes stale running syncs', () => { + const db = new Database(':memory:') + resetRuntimeMigrate() + ensureRuntimeSchema(db) + resetRuntimeMigrate() + const now = new Date('2026-06-01T00:00:00Z') + const old = '2026-01-01T00:00:00Z' + const fresh = '2026-05-31T23:30:00Z' + db.exec(`PRAGMA foreign_keys = OFF`) + for (const [id, at] of [['n-old', old], ['n-new', fresh]]) { + db.prepare(`INSERT INTO notification_log (id, event, channel, status, createdAt) VALUES (?, 'e', 'telegram', 'sent', ?)`).run(id, at) + db.prepare(`INSERT INTO vps_health_checks (id, vpsId, checkedAt, status) VALUES (?, 'v', ?, 'up')`).run(id, at) + } + db.prepare(`INSERT INTO sync_log (id, accountId, startedAt, status) VALUES ('s-old', 'a', ?, 'ok')`).run(old) + db.prepare(`INSERT INTO sync_log (id, accountId, startedAt, status) VALUES ('s-stuck', 'a', '2026-05-31T20:00:00Z', 'running')`).run() + db.prepare(`INSERT INTO sync_log (id, accountId, startedAt, status) VALUES ('s-live', 'a', ?, 'running')`).run(fresh) + + expect(pruneOldLogs(db, now)).toEqual({ notificationLog: 1, syncLog: 1, vpsHealthChecks: 1, staleSyncs: 1 }) + const status = (id: string) => (db.prepare(`SELECT status FROM sync_log WHERE id = ?`).get(id) as { status: string }).status + expect(status('s-stuck')).toBe('error') + expect(status('s-live')).toBe('running') + }) +}) diff --git a/packages/db/src/maintenance/retention.ts b/packages/db/src/maintenance/retention.ts new file mode 100644 index 0000000..713954e --- /dev/null +++ b/packages/db/src/maintenance/retention.ts @@ -0,0 +1,42 @@ +import type Database from 'better-sqlite3' + +const DAY_MS = 24 * 60 * 60 * 1000 + +export const RETENTION_DAYS = { + notificationLog: 90, + syncLog: 90, + vpsHealthChecks: 30, +} as const + +/** A sync that is still "running" after this long was cut off by a crash/restart. */ +const STALE_RUNNING_SYNC_MS = 60 * 60 * 1000 + +export interface PruneResult { + notificationLog: number + syncLog: number + vpsHealthChecks: number + staleSyncs: number +} + +/** Delete old journal rows (all spaces) and close sync_log rows left 'running' by a dead process. */ +export function pruneOldLogs(sqlite: Database.Database, now = new Date()): PruneResult { + const before = (days: number) => new Date(now.getTime() - days * DAY_MS).toISOString() + const nowIso = now.toISOString() + return sqlite.transaction(() => ({ + staleSyncs: sqlite + .prepare( + `UPDATE sync_log SET status = 'error', finishedAt = ?, error = 'Прервано: процесс перезапущен' + WHERE status = 'running' AND startedAt < ?`, + ) + .run(nowIso, new Date(now.getTime() - STALE_RUNNING_SYNC_MS).toISOString()).changes, + notificationLog: sqlite + .prepare(`DELETE FROM notification_log WHERE createdAt < ?`) + .run(before(RETENTION_DAYS.notificationLog)).changes, + syncLog: sqlite + .prepare(`DELETE FROM sync_log WHERE startedAt < ?`) + .run(before(RETENTION_DAYS.syncLog)).changes, + vpsHealthChecks: sqlite + .prepare(`DELETE FROM vps_health_checks WHERE checkedAt < ?`) + .run(before(RETENTION_DAYS.vpsHealthChecks)).changes, + }))() +} diff --git a/packages/db/src/repositories/snapshot.ts b/packages/db/src/repositories/snapshot.ts index 8ea2b31..fce8190 100644 --- a/packages/db/src/repositories/snapshot.ts +++ b/packages/db/src/repositories/snapshot.ts @@ -49,6 +49,23 @@ function listVpsWithShared(): VpsDto[] { return [...owned, ...shared] } +export type DashboardSnapshot = Pick< + Snapshot, + 'vps' | 'providers' | 'providerAccounts' | 'payments' | 'balanceLedger' | 'syncLog' +> + +/** What dashboard stats read — skips tariffs, domains, projects and settings of the full snapshot. */ +export function getDashboardSnapshot(): DashboardSnapshot { + return { + vps: listVpsWithShared(), + providers: providersRepository.list(), + providerAccounts: providerAccountsRepository.list(), + payments: paymentsRepository.list(), + balanceLedger: balanceLedgerRepository.list(), + syncLog: syncLogRepository.listRecent(50), + } +} + export function getSnapshot(): Snapshot { const spaceId = getCurrentSpaceId() return {