perf(api): чистить старые журналы и облегчить синк и dashboard
- раз в сутки удаляются notification_log и sync_log старше 90 дней и vps_health_checks старше 30 дней; синки, зависшие в running дольше часа, помечаются ошибкой - dashboard stats не грузит тарифы, домены и проекты - запросы характеристик VPS в BILLmanager идут по 4 параллельно Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0b9b1cad54
commit
80af888cdb
9 files changed
+199
-17
No files matched your search
@@ -135,3 +135,55 @@ describe('space viewer cannot write', () => {
|
||||
await app.close()
|
||||
})
|
||||
})
|
||||
|
||||
describe('whole-DB operations need a global admin', () => {
|
||||
beforeEach(() => {
|
||||
process.env.AUTH_REQUIRED = 'true'
|
||||
process.env.AUTH_JWT_SECRET = 'test-secret-at-least-8'
|
||||
resetTestDb()
|
||||
})
|
||||
afterEach(() => {
|
||||
delete process.env.AUTH_REQUIRED
|
||||
delete process.env.AUTH_JWT_SECRET
|
||||
closeDb()
|
||||
})
|
||||
|
||||
it('403 for settings admin, 200 for global admin', async () => {
|
||||
const app = await buildApp()
|
||||
const token = (sub: string, isAdmin: boolean) =>
|
||||
app.jwt.sign({
|
||||
sub,
|
||||
email: `${sub}@t`,
|
||||
name: sub,
|
||||
apps: ['vps'],
|
||||
permissions: ['vps:settings:admin'],
|
||||
is_admin: isAdmin,
|
||||
iss: app.authConfig.issuer,
|
||||
})
|
||||
const get = (t: string) =>
|
||||
app.inject({ url: '/api/backup/json', headers: { authorization: `Bearer ${t}` } })
|
||||
expect((await get(token('space-admin', false))).statusCode).toBe(403)
|
||||
const migrate = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/migrate',
|
||||
headers: { authorization: `Bearer ${token('space-admin', false)}` },
|
||||
payload: {},
|
||||
})
|
||||
expect(migrate.statusCode).toBe(403)
|
||||
expect((await get(token('root', true))).statusCode).toBe(200)
|
||||
await app.close()
|
||||
})
|
||||
})
|
||||
|
||||
describe('dashboard stats', () => {
|
||||
beforeEach(() => resetTestDb())
|
||||
afterEach(() => closeDb())
|
||||
|
||||
it('computes from the slim snapshot', async () => {
|
||||
const app = await buildApp()
|
||||
const res = await app.inject({ url: '/api/dashboard/stats' })
|
||||
expect(res.statusCode).toBe(200)
|
||||
expect(res.json()).toMatchObject({ totalVpsCount: 0, providerCount: 0 })
|
||||
await app.close()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,18 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { mapConcurrent } from './sync.js'
|
||||
|
||||
describe('mapConcurrent', () => {
|
||||
it('keeps order and caps parallelism', async () => {
|
||||
let active = 0
|
||||
let peak = 0
|
||||
const out = await mapConcurrent([5, 1, 4, 2, 3, 0], 2, async (n) => {
|
||||
active++
|
||||
peak = Math.max(peak, active)
|
||||
await new Promise((r) => setTimeout(r, n))
|
||||
active--
|
||||
return n * 10
|
||||
})
|
||||
expect(out).toEqual([50, 10, 40, 20, 30, 0])
|
||||
expect(peak).toBe(2)
|
||||
})
|
||||
})
|
||||
@@ -88,6 +88,26 @@ async function mapVdsWithSpecs(
|
||||
return vps
|
||||
}
|
||||
|
||||
/** vds.edit lookups are one HTTP call per VPS — run a few at once, not all (hoster rate limits). */
|
||||
const VDS_SPECS_CONCURRENCY = 4
|
||||
|
||||
export async function mapConcurrent<T, R>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const out: R[] = new Array(items.length)
|
||||
let next = 0
|
||||
const worker = async () => {
|
||||
while (next < items.length) {
|
||||
const i = next++
|
||||
out[i] = await fn(items[i]!)
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker))
|
||||
return out
|
||||
}
|
||||
|
||||
export async function syncFromBillmanager(
|
||||
account: BillmanagerSyncAccount,
|
||||
opts: SyncFromBillmanagerOptions = {},
|
||||
@@ -138,20 +158,9 @@ export async function syncFromBillmanager(
|
||||
const syncSummary: SyncSummary = { added: [], updated: [], paymentsAdded: 0 }
|
||||
|
||||
if (fetchVpsPayments) {
|
||||
const mappedRaw: Awaited<ReturnType<typeof mapVdsWithSpecs>>[] = []
|
||||
for (const item of vdsItems) {
|
||||
mappedRaw.push(
|
||||
await mapVdsWithSpecs(
|
||||
profile,
|
||||
item,
|
||||
providerId,
|
||||
accountId,
|
||||
tariffItems,
|
||||
apiBaseUrl,
|
||||
authinfo,
|
||||
),
|
||||
)
|
||||
}
|
||||
const mappedRaw = await mapConcurrent(vdsItems, VDS_SPECS_CONCURRENCY, (item) =>
|
||||
mapVdsWithSpecs(profile, item, providerId, accountId, tariffItems, apiBaseUrl, authinfo),
|
||||
)
|
||||
const mappedList = profile.map.enrichVdsBatch
|
||||
? profile.map.enrichVdsBatch(mappedRaw, {
|
||||
balance: dashboardInfo?.balance ?? null,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { getSnapshot } from '@cfdm/db/repositories/snapshot'
|
||||
import { getDashboardSnapshot } from '@cfdm/db/repositories/snapshot'
|
||||
import { countExpiringWithin7Days, countInventoryIssues } from '@cfdm/shared/utils/inventory-health'
|
||||
import { accountBalanceApi } from '@cfdm/shared/utils/account-balance'
|
||||
import { isSyncApiType } from '@cfdm/shared/contracts/provider'
|
||||
@@ -76,7 +76,7 @@ export interface DashboardStats {
|
||||
}
|
||||
|
||||
export function computeDashboardStats(): DashboardStats {
|
||||
const snap = getSnapshot()
|
||||
const snap = getDashboardSnapshot()
|
||||
const now = new Date()
|
||||
|
||||
const activeVps = snap.vps.filter((v) => v.status === 'active')
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { eq, sql } from 'drizzle-orm'
|
||||
import { getDb, schema, runWithSpaceAsync, MAIN_SPACE_ID } from '@cfdm/db'
|
||||
import { getDb, getSqlite, pruneOldLogs, schema, runWithSpaceAsync, MAIN_SPACE_ID } from '@cfdm/db'
|
||||
import { settingsRepository } from '@cfdm/db/repositories/settings'
|
||||
|
||||
import { resolveSyncAccount, getProviderAdapter, type SyncReadyAccount } from './providers/index.js'
|
||||
@@ -25,6 +25,8 @@ let notifyIntervalId: ReturnType<typeof setInterval> | null = null
|
||||
let uptimeIntervalId: ReturnType<typeof setInterval> | null = null
|
||||
let cfdmSyncIntervalId: ReturnType<typeof setInterval> | null = null
|
||||
let cfdmSyncInFlight = false
|
||||
let retentionIntervalId: ReturnType<typeof setInterval> | null = null
|
||||
const RETENTION_INTERVAL_MS = 24 * 60 * 60 * 1000
|
||||
|
||||
type AccountRow = typeof schema.providerAccounts.$inferSelect
|
||||
type SettingsRow = typeof schema.settings.$inferSelect
|
||||
@@ -109,6 +111,12 @@ export const runNotificationTick = () => exclusive('notify', notificationTick)
|
||||
export const runScheduledSync = () => exclusive('sync', scheduledSync)
|
||||
export const runScheduledSyncTariffs = () => exclusive('tariffs', scheduledSyncTariffs)
|
||||
export const runScheduledUptimeChecks = () => exclusive('uptime', scheduledUptimeChecks)
|
||||
export const runRetention = () =>
|
||||
exclusive('retention', async () => {
|
||||
const r = pruneOldLogs(getSqlite())
|
||||
const total = r.notificationLog + r.syncLog + r.vpsHealthChecks + r.staleSyncs
|
||||
if (total > 0) console.log('Retention:', JSON.stringify(r))
|
||||
})
|
||||
|
||||
async function notificationTick(): Promise<void> {
|
||||
for (const settings of allSettings()) {
|
||||
@@ -334,6 +342,8 @@ export function startScheduler(): void {
|
||||
uptimeIntervalId = null
|
||||
if (cfdmSyncIntervalId) clearInterval(cfdmSyncIntervalId)
|
||||
cfdmSyncIntervalId = null
|
||||
if (retentionIntervalId) clearInterval(retentionIntervalId)
|
||||
retentionIntervalId = null
|
||||
|
||||
try {
|
||||
const rows = allSettings()
|
||||
@@ -350,6 +360,8 @@ export function startScheduler(): void {
|
||||
)
|
||||
void runNotificationTick()
|
||||
void runScheduledUptimeChecks()
|
||||
retentionIntervalId = setInterval(() => void runRetention(), RETENTION_INTERVAL_MS)
|
||||
void runRetention()
|
||||
|
||||
const parts = [
|
||||
`notify every ${notifyInterval} min`,
|
||||
@@ -393,6 +405,8 @@ export function stopScheduler(): void {
|
||||
uptimeIntervalId = null
|
||||
if (cfdmSyncIntervalId) clearInterval(cfdmSyncIntervalId)
|
||||
cfdmSyncIntervalId = null
|
||||
if (retentionIntervalId) clearInterval(retentionIntervalId)
|
||||
retentionIntervalId = null
|
||||
}
|
||||
|
||||
export function restartScheduler(): void {
|
||||
|
||||
Reference in new issue
Block a user