fix(traffic): останавливать сбор при потере соединения блокировки
This commit is contained in:
1 parent
46de87e213
commit
cbe44e8072
16 files changed
+1340
-18
No files matched your search
@@ -15,3 +15,11 @@ for(const state of ['busy','missing','ok','run-error','unlock-false','unlock-err
|
||||
assert.equal(released,['missing','unlock-false','unlock-error','acquire-error'].includes(state))
|
||||
}
|
||||
console.log('traffic lock: busy/no work/acquisition uncertainty/run failure/unlock failure/destruction passed')
|
||||
import {EventEmitter} from 'node:events'
|
||||
for(const event of ['error','end']){
|
||||
const events=new EventEmitter();let destroy=false,writes=0
|
||||
const client={query:async()=>({rows:[{acquired:true,released:true}]}),release:(value?:boolean)=>{destroy=!!value},on:events.on.bind(events),removeListener:events.removeListener.bind(events)}
|
||||
await assert.rejects(withTrafficCollectionLock(async()=>client,async guard=>{guard();events.emit(event,Error('own loss'));guard();writes++;return 42}),/Соединение блокировки/)
|
||||
assert.equal(writes,0);assert.equal(destroy,true);assert.equal(events.listenerCount(event),0)
|
||||
}
|
||||
console.log('traffic lock: error/end handled, guarded work aborted, connection destroyed, listeners removed')
|
||||
@@ -1,24 +1,35 @@
|
||||
type Client = {
|
||||
query: (sql:string,params:number[])=>Promise<{rows:Record<string,unknown>[]}>
|
||||
release: (destroy?:boolean)=>void
|
||||
on?: (event:"error"|"end",listener:()=>void)=>unknown
|
||||
removeListener?: (event:"error"|"end",listener:()=>void)=>unknown
|
||||
}
|
||||
export const TRAFFIC_COLLECTION_LOCK_KEY=8_723_105
|
||||
|
||||
/** Coordinate collectors in the same database without queueing stale polling runs. */
|
||||
export async function withTrafficCollectionLock<T>(connect:()=>Promise<Client>,run:()=>Promise<T>):Promise<{acquired:false}|{acquired:true;value:T}> {
|
||||
export async function withTrafficCollectionLock<T>(connect:()=>Promise<Client>,run:(assertActive:()=>void)=>Promise<T>):Promise<{acquired:false}|{acquired:true;value:T}> {
|
||||
const client=await connect()
|
||||
let owned=false,destroy=true
|
||||
let owned=false,destroy=true,lost=false
|
||||
const lose=()=>{lost=true;destroy=true}
|
||||
const assertActive=()=>{if(lost)throw Error('Соединение блокировки опроса трафика потеряно; опрос остановлен')}
|
||||
client.on?.('error',lose)
|
||||
client.on?.('end',lose)
|
||||
try {
|
||||
const result=await client.query('SELECT pg_try_advisory_lock($1) AS acquired',[TRAFFIC_COLLECTION_LOCK_KEY])
|
||||
assertActive()
|
||||
if(result.rows[0]?.acquired===false){destroy=false;return {acquired:false}}
|
||||
if(result.rows[0]?.acquired!==true)throw Error('Блокировка опроса трафика не подтверждена')
|
||||
owned=true;destroy=false
|
||||
return {acquired:true,value:await run()}
|
||||
const value=await run(assertActive)
|
||||
assertActive()
|
||||
return {acquired:true,value}
|
||||
}finally{
|
||||
if(owned){
|
||||
if(owned&&!lost){
|
||||
try{destroy=(await client.query('SELECT pg_advisory_unlock($1) AS released',[TRAFFIC_COLLECTION_LOCK_KEY])).rows[0]?.released!==true}
|
||||
catch{destroy=true}
|
||||
}
|
||||
client.release(destroy)
|
||||
client.removeListener?.('error',lose)
|
||||
client.removeListener?.('end',lose)
|
||||
}
|
||||
}
|
||||
@@ -117,7 +117,7 @@ export async function collectTrafficOnce(): Promise<TrafficRunSnapshot> {
|
||||
}
|
||||
collecting = true
|
||||
try {
|
||||
const result = await withTrafficCollectionLock(() => pool.connect(), () => collectTrafficRun(sampledAt))
|
||||
const result = await withTrafficCollectionLock(() => pool.connect(), assertActive => collectTrafficRun(sampledAt,assertActive))
|
||||
return result.acquired ? result.value : {
|
||||
v:SCHEDULER_RUN_SNAPSHOT_VERSION,job:"traffic",sampledAt,skipped:true,servers:[],
|
||||
}
|
||||
@@ -129,7 +129,7 @@ export async function collectTrafficOnce(): Promise<TrafficRunSnapshot> {
|
||||
}
|
||||
}
|
||||
|
||||
async function collectTrafficRun(sampledAt:string):Promise<TrafficRunSnapshot> {
|
||||
async function collectTrafficRun(sampledAt:string,assertActive:()=>void=()=>{}):Promise<TrafficRunSnapshot> {
|
||||
const startedAt = Date.now()
|
||||
const now = sampledAt
|
||||
const snapshot: TrafficRunSnapshot = {
|
||||
@@ -140,14 +140,18 @@ async function collectTrafficRun(sampledAt:string):Promise<TrafficRunSnapshot> {
|
||||
}
|
||||
|
||||
try {
|
||||
assertActive()
|
||||
const settings = await getSettingsRow()
|
||||
const maxIntervalMs = Math.max(5, settings?.intervalSec ?? 30) * 3000
|
||||
const enabledServers = await db.select().from(servers).where(eq(servers.enabled, true))
|
||||
|
||||
for (const srv of enabledServers) {
|
||||
assertActive()
|
||||
const name = srv.name || srv.host
|
||||
const ensureSource = async () => {
|
||||
assertActive()
|
||||
const current = (await db.select({enabled:servers.enabled,updatedAt:servers.updatedAt}).from(servers).where(eq(servers.id,srv.id)).limit(1))[0]
|
||||
assertActive()
|
||||
if (!current?.enabled || Date.parse(current.updatedAt) !== Date.parse(srv.updatedAt))
|
||||
throw new Error("Сервер выключен, удалён или его конфигурация изменилась во время опроса")
|
||||
}
|
||||
@@ -247,6 +251,7 @@ async function collectTrafficRun(sampledAt:string):Promise<TrafficRunSnapshot> {
|
||||
} : {}),
|
||||
})
|
||||
} catch (err) {
|
||||
assertActive()
|
||||
await recordTrafficCollectionAttempt(srv, "error", now, err instanceof Error ? err.message : String(err))
|
||||
snapshot.servers.push({
|
||||
serverId: srv.id,
|
||||
@@ -258,6 +263,7 @@ async function collectTrafficRun(sampledAt:string):Promise<TrafficRunSnapshot> {
|
||||
}
|
||||
}
|
||||
|
||||
assertActive()
|
||||
await db.update(trafficSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - startedAt,
|
||||
@@ -265,6 +271,7 @@ async function collectTrafficRun(sampledAt:string):Promise<TrafficRunSnapshot> {
|
||||
updatedAt: now,
|
||||
}).where(eq(trafficSettings.id, 1))
|
||||
} catch (e) {
|
||||
assertActive()
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
snapshot.fatalError = msg
|
||||
await db.update(trafficSettings).set({
|
||||
|
||||
Reference in new issue
Block a user