fix(traffic): останавливать сбор при потере соединения блокировки

This commit is contained in:
Denozordec committed 2026-10-13 16:20:46 +07:00
1 parent 46de87e213
commit cbe44e8072
16 files changed
+1340 -18

No files matched your search

@@ -15,3 +15,11 @@ for(const state of ['busy','missing','ok','run-error','unlock-false','unlock-err
assert.equal(released,['missing','unlock-false','unlock-error','acquire-error'].includes(state))
}
console.log('traffic lock: busy/no work/acquisition uncertainty/run failure/unlock failure/destruction passed')
import {EventEmitter} from 'node:events'
for(const event of ['error','end']){
const events=new EventEmitter();let destroy=false,writes=0
const client={query:async()=>({rows:[{acquired:true,released:true}]}),release:(value?:boolean)=>{destroy=!!value},on:events.on.bind(events),removeListener:events.removeListener.bind(events)}
await assert.rejects(withTrafficCollectionLock(async()=>client,async guard=>{guard();events.emit(event,Error('own loss'));guard();writes++;return 42}),/Соединение блокировки/)
assert.equal(writes,0);assert.equal(destroy,true);assert.equal(events.listenerCount(event),0)
}
console.log('traffic lock: error/end handled, guarded work aborted, connection destroyed, listeners removed')
@@ -1,24 +1,35 @@
type Client = {
query: (sql:string,params:number[])=>Promise<{rows:Record<string,unknown>[]}>
release: (destroy?:boolean)=>void
on?: (event:"error"|"end",listener:()=>void)=>unknown
removeListener?: (event:"error"|"end",listener:()=>void)=>unknown
}
export const TRAFFIC_COLLECTION_LOCK_KEY=8_723_105
/** Coordinate collectors in the same database without queueing stale polling runs. */
export async function withTrafficCollectionLock<T>(connect:()=>Promise<Client>,run:()=>Promise<T>):Promise<{acquired:false}|{acquired:true;value:T}> {
export async function withTrafficCollectionLock<T>(connect:()=>Promise<Client>,run:(assertActive:()=>void)=>Promise<T>):Promise<{acquired:false}|{acquired:true;value:T}> {
const client=await connect()
let owned=false,destroy=true
let owned=false,destroy=true,lost=false
const lose=()=>{lost=true;destroy=true}
const assertActive=()=>{if(lost)throw Error('Соединение блокировки опроса трафика потеряно; опрос остановлен')}
client.on?.('error',lose)
client.on?.('end',lose)
try {
const result=await client.query('SELECT pg_try_advisory_lock($1) AS acquired',[TRAFFIC_COLLECTION_LOCK_KEY])
assertActive()
if(result.rows[0]?.acquired===false){destroy=false;return {acquired:false}}
if(result.rows[0]?.acquired!==true)throw Error('Блокировка опроса трафика не подтверждена')
owned=true;destroy=false
return {acquired:true,value:await run()}
const value=await run(assertActive)
assertActive()
return {acquired:true,value}
}finally{
if(owned){
if(owned&&!lost){
try{destroy=(await client.query('SELECT pg_advisory_unlock($1) AS released',[TRAFFIC_COLLECTION_LOCK_KEY])).rows[0]?.released!==true}
catch{destroy=true}
}
client.release(destroy)
client.removeListener?.('error',lose)
client.removeListener?.('end',lose)
}
}
+9 -2
View File
@@ -117,7 +117,7 @@ export async function collectTrafficOnce(): Promise<TrafficRunSnapshot> {
}
collecting = true
try {
const result = await withTrafficCollectionLock(() => pool.connect(), () => collectTrafficRun(sampledAt))
const result = await withTrafficCollectionLock(() => pool.connect(), assertActive => collectTrafficRun(sampledAt,assertActive))
return result.acquired ? result.value : {
v:SCHEDULER_RUN_SNAPSHOT_VERSION,job:"traffic",sampledAt,skipped:true,servers:[],
}
@@ -129,7 +129,7 @@ export async function collectTrafficOnce(): Promise<TrafficRunSnapshot> {
}
}
async function collectTrafficRun(sampledAt:string):Promise<TrafficRunSnapshot> {
async function collectTrafficRun(sampledAt:string,assertActive:()=>void=()=>{}):Promise<TrafficRunSnapshot> {
const startedAt = Date.now()
const now = sampledAt
const snapshot: TrafficRunSnapshot = {
@@ -140,14 +140,18 @@ async function collectTrafficRun(sampledAt:string):Promise<TrafficRunSnapshot> {
}
try {
assertActive()
const settings = await getSettingsRow()
const maxIntervalMs = Math.max(5, settings?.intervalSec ?? 30) * 3000
const enabledServers = await db.select().from(servers).where(eq(servers.enabled, true))
for (const srv of enabledServers) {
assertActive()
const name = srv.name || srv.host
const ensureSource = async () => {
assertActive()
const current = (await db.select({enabled:servers.enabled,updatedAt:servers.updatedAt}).from(servers).where(eq(servers.id,srv.id)).limit(1))[0]
assertActive()
if (!current?.enabled || Date.parse(current.updatedAt) !== Date.parse(srv.updatedAt))
throw new Error("Сервер выключен, удалён или его конфигурация изменилась во время опроса")
}
@@ -247,6 +251,7 @@ async function collectTrafficRun(sampledAt:string):Promise<TrafficRunSnapshot> {
} : {}),
})
} catch (err) {
assertActive()
await recordTrafficCollectionAttempt(srv, "error", now, err instanceof Error ? err.message : String(err))
snapshot.servers.push({
serverId: srv.id,
@@ -258,6 +263,7 @@ async function collectTrafficRun(sampledAt:string):Promise<TrafficRunSnapshot> {
}
}
assertActive()
await db.update(trafficSettings).set({
lastCollectedAt: now,
lastDurationMs: Date.now() - startedAt,
@@ -265,6 +271,7 @@ async function collectTrafficRun(sampledAt:string):Promise<TrafficRunSnapshot> {
updatedAt: now,
}).where(eq(trafficSettings.id, 1))
} catch (e) {
assertActive()
const msg = e instanceof Error ? e.message : String(e)
snapshot.fatalError = msg
await db.update(trafficSettings).set({