130 lines
18 KiB
TypeScript
130 lines
18 KiB
TypeScript
/** Explicit host-operator inspection/acceptance. No restore, service restart, role
|
|
* enrollment or source/READY issuer. Populated data is accepted, not certified. */
|
|
import assert from 'node:assert/strict'
|
|
import path from 'node:path'
|
|
import {fileURLToPath} from 'node:url'
|
|
import {mkdir,readdir} from 'node:fs/promises'
|
|
import {readProtected,protectedDirectory,writeFile,withJournalLease,sha} from './operator-files.mjs'
|
|
import {poolOptions,STARTUP_OPTIONS,assertJsDriverEnvironment} from './operator-contract.mjs'
|
|
import {readMaintenanceRecoveryFacts,readMaintenanceAcceptanceRecords} from '../../src/services/database-maintenance-startup.js'
|
|
import {parseMaintenanceJSON,runtimeDatabaseTargetSha256,evaluateMaintenanceAcceptances,validateMaintenanceKnownCurrent,validateMaintenanceAcceptance,MAINTENANCE_ACCEPTANCE_FORMAT,type MaintenanceKnownCurrent} from '../../src/services/database-maintenance-resolution.js'
|
|
import {captureMigrationSchema} from '../../src/db/migration-schema-inventory.js'
|
|
import {MIGRATION_MANIFEST,CURRENT_REQUIRED_TABLES,migrationManifestDigest,inventoryMigrationDirectory} from '../../src/db/migration-manifest.js'
|
|
import type {Pool,PoolClient} from 'pg'
|
|
const hex=(v:unknown):v is string=>typeof v==='string'&&/^[a-f0-9]{64}$/.test(v)
|
|
function object(v:unknown):asserts v is Record<string,unknown>{assert(v&&typeof v==='object'&&!Array.isArray(v))}
|
|
function exact(v:unknown,keys:string[]):asserts v is Record<string,unknown>{object(v);assert.equal(Object.keys(v).sort().join('\0'),keys.sort().join('\0'))}
|
|
const stamp=(v:unknown):v is string=>typeof v==='string'&&Number.isFinite(Date.parse(v))&&new Date(v).toISOString()===v
|
|
const stable=(v:unknown)=>JSON.stringify(v)
|
|
const flags=['state-dir','transport','transport-sha256','runtime-target','runtime-target-sha256']
|
|
const validityMs=900000
|
|
/** Explicit local-container transport never falls back from TLS or environment.
|
|
* Caller launch review supplies --network container:<exact reviewed PG id>;
|
|
* this module has no Docker socket/API and claims only loopback + physical SQL proof. */
|
|
export function maintenancePoolConfiguration(value:unknown,ca?:Buffer,caSha256?:string){
|
|
object(value);const mode=value.mode===undefined?'tls':value.mode;assert(mode==='tls'||mode==='local-loopback')
|
|
exact(value,mode==='local-loopback'?['format','mode','connection']:value.mode===undefined?['format','connection','tls']:['format','mode','connection','tls'])
|
|
assert.equal(value.format,'database-maintenance-transport-v1');exact(value.connection,['host','port','user','password','database']);const connection=value.connection
|
|
assert(typeof connection.host==='string'&&typeof connection.user==='string'&&/^[^\x00-\x1f\x7f]{1,63}$/.test(connection.user)&&typeof connection.password==='string'&&connection.password.length>0&&connection.password.length<=4096&&!connection.password.includes('\0')&&typeof connection.database==='string'&&/^[^\x00-\x1f\x7f]{1,63}$/.test(connection.database)&&typeof connection.port==='number'&&Number.isSafeInteger(connection.port)&&Number(connection.port)>0&&Number(connection.port)<=65535)
|
|
let options,identity:unknown
|
|
if(mode==='tls'){
|
|
exact(value.tls,['serverName','spkiSha256']);assert(typeof value.tls.serverName==='string'&&/^[a-zA-Z0-9][a-zA-Z0-9.-]{0,252}$/.test(value.tls.serverName)&&hex(value.tls.spkiSha256));assert(ca&&hex(caSha256))
|
|
options=poolOptions(connection,{roles:{publisher:{name:connection.user}},source:{databaseName:connection.database},tls:{...value.tls,caSha256}},ca)
|
|
identity={mode,host:connection.host,port:connection.port,database:connection.database,tls:{...value.tls,caSha256}}
|
|
}else{
|
|
assert(connection.host==='127.0.0.1'||connection.host==='::1','Plaintext maintenance requires exact loopback and reviewed PostgreSQL container namespace');assert(ca===undefined&&caSha256===undefined)
|
|
options={host:connection.host,port:connection.port,user:connection.user,password:connection.password,database:connection.database,max:1,connectionTimeoutMillis:5000,idleTimeoutMillis:5000,statement_timeout:10000,lock_timeout:5000,options:STARTUP_OPTIONS,application_name:'mmapp-maintenance-acceptance',client_encoding:'UTF8',ssl:false}
|
|
identity={mode,host:connection.host,port:connection.port,database:connection.database}
|
|
}
|
|
return {options,transportBinding:{mode,endpointSha256:sha(stable(identity))} as MaintenanceKnownCurrent['transport']}
|
|
}
|
|
export function maintenanceApproval(value:unknown,inspectionSha256:string,inspectionCreatedAt:string){
|
|
exact(value,['format','resolutionId','decision','actorId','inspectionSha256','issuedAt','expiresAt','managedBackendStopped','historicOutcomesRemainUnknown','automaticRestore'])
|
|
assert(value.format==='database-maintenance-current-state-approval-v1'&&value.decision==='accept-observed-current-state'&&typeof value.resolutionId==='string'&&/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(value.resolutionId))
|
|
assert(typeof value.actorId==='string'&&/^[^\x00-\x1f\x7f]{1,160}$/.test(value.actorId)&&value.inspectionSha256===inspectionSha256&&stamp(value.issuedAt)&&stamp(value.expiresAt))
|
|
const now=Date.now();assert(Date.parse(value.issuedAt)>=Date.parse(inspectionCreatedAt)&&Date.parse(value.issuedAt)<=now&&now<Date.parse(value.expiresAt)&&Date.parse(value.expiresAt)-Date.parse(value.issuedAt)<=validityMs)
|
|
assert(value.managedBackendStopped===true&&value.historicOutcomesRemainUnknown===true&&value.automaticRestore===false)
|
|
return value as unknown as {resolutionId:string;actorId:string;issuedAt:string;expiresAt:string}
|
|
}
|
|
/** Genuine observation, not a supplied snapshot factory. Owner/transaction ends
|
|
* must be confirmed before a caller can publish an acceptance. Exclusive key
|
|
* protects cooperative observations; stopped-backend assumption spans publication. */
|
|
export async function observeMaintenanceCurrent(pool:Pick<Pool,'connect'>,transport:MaintenanceKnownCurrent['transport']){
|
|
let client:PoolClient|undefined,locked=false,begun=false,bad=false,faulted=false
|
|
const fault=()=>{faulted=true},live=()=>{assert(!faulted)}
|
|
try{
|
|
client=await pool.connect();client.on('error',fault);client.on('end',fault)
|
|
const query=async(text:string,values:unknown[]=[])=>{live();const result=await client!.query(text,values);live();return result}
|
|
await query("SET statement_timeout='10s'");await query("SET lock_timeout='5s'")
|
|
const lease=await query('SELECT pg_catalog.pg_try_advisory_lock(8723110::bigint) AS locked');assert(lease.rows.length===1&&lease.rows[0].locked===true);locked=true
|
|
await query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY');begun=true
|
|
for(const sql of ["SET LOCAL search_path=pg_catalog","SET LOCAL TimeZone='UTC'","SET LOCAL DateStyle='ISO, YMD'","SET LOCAL standard_conforming_strings=on","SET LOCAL statement_timeout='10s'"])await query(sql)
|
|
const identity=await query(`SELECT c.system_identifier::text AS "systemIdentifier",d.oid::text AS "databaseOid",d.datname AS "databaseName",pg_catalog.current_setting('server_version_num')::int AS "serverVersionNum",pg_catalog.pg_is_in_recovery() AS "inRecovery" FROM pg_catalog.pg_database d CROSS JOIN pg_catalog.pg_control_system() c WHERE d.datname=pg_catalog.current_database()`)
|
|
assert.equal(identity.rows.length,1)
|
|
// The optional writer-profile control schema is not installed by ordinary
|
|
// application migrations. Observe its presence in this same transaction;
|
|
// detailed capture still refuses every other unexpected namespace.
|
|
const control=await query("SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname='mmapp_control'")
|
|
assert(control.rows.length<=1&&control.rows.every(row=>row.nspname==='mmapp_control'))
|
|
const schema=await captureMigrationSchema({query:async(text,values)=>query(text,values)},{scope:{applicationNamespaces:['public'],excludedControlNamespaces:control.rows.length?['mmapp_control']:[]},deadlineAt:Date.now()+60000})
|
|
const marker=schema.sections.relations.find(row=>row.namespace==='public'&&row.name==='schema_migrations'),column=schema.sections.columns.find(row=>row.namespace==='public'&&row.relation==='schema_migrations'&&row.name==='id'&&row.dropped===false)
|
|
assert(marker?.kind==='r'&&marker.persistence==='p'&&marker.row_security===false&&marker.force_row_security===false&&column?.type==='text')
|
|
for(const table of CURRENT_REQUIRED_TABLES)assert(schema.sections.relations.some(row=>row.namespace==='public'&&row.name===table&&['r','p'].includes(String(row.kind))&&row.row_security===false&&row.force_row_security===false))
|
|
const markers=(await query('SELECT id FROM public.schema_migrations ORDER BY id LIMIT $1',[MIGRATION_MANIFEST.length+1])).rows
|
|
assert.equal(stable(markers),stable(MIGRATION_MANIFEST.map(entry=>({id:entry.file.slice(0,-4)}))))
|
|
const knownCurrent={transport,source:identity.rows[0],schemaSha256:schema.sha256,rawSchemaSha256:schema.observedRaw.sha256,physicalSchemaSha256:schema.observedPhysical.sha256,manifestSha256:migrationManifestDigest(),markersSha256:sha(stable(markers))}
|
|
validateMaintenanceKnownCurrent(knownCurrent)
|
|
const end=await query('ROLLBACK');assert.equal(end.command,'ROLLBACK');begun=false
|
|
const unlock=await query('SELECT pg_catalog.pg_advisory_unlock(8723110::bigint) AS unlocked');assert(unlock.rows.length===1&&unlock.rows[0].unlocked===true);locked=false
|
|
return {knownCurrent,schema}
|
|
}catch(error){bad=true;throw error}finally{
|
|
if(client){let cleanupFailed=false
|
|
try{if(begun&&!faulted){const end=await client.query('ROLLBACK');assert.equal(end.command,'ROLLBACK')}if(locked&&!faulted){const end=await client.query('SELECT pg_catalog.pg_advisory_unlock(8723110::bigint) AS unlocked');assert(end.rows[0]?.unlocked===true)}}catch{cleanupFailed=true}
|
|
try{client.off('error',fault);client.off('end',fault)}catch{cleanupFailed=true}
|
|
try{client.release(bad||faulted||cleanupFailed)}catch{cleanupFailed=true}
|
|
assert(!cleanupFailed&&!faulted,'Read-only maintenance owner cleanup unconfirmed')
|
|
}
|
|
}
|
|
}
|
|
async function snapshot(stateDirectory:string,target:string,records:readonly Buffer[]){
|
|
const state=await readMaintenanceRecoveryFacts(stateDirectory);assert(state.kind==='journal'&&state.generation)
|
|
const accepted=evaluateMaintenanceAcceptances(records,state.journal,state.vault.evidence,target)
|
|
const unresolvedOperations=state.operations.filter(op=>op.state!=='applied'),vaultCoverage=state.vault.evidence.filter(marker=>state.vault.unresolvedFiles.includes(marker.file))
|
|
const newUnresolved=unresolvedOperations.some(op=>!accepted.acceptedCoreIds.has(op.operationId))||state.vault.unresolvedFiles.some(file=>!accepted.acceptedVaultFiles.has(file))
|
|
return {newUnresolved,external:{journalCoverage:{prefixBytes:state.journal.length,sha256:sha(state.journal),unresolvedOperations},vaultCoverage,generationObservedSha256:sha(state.generation),previousResolutionSha256:accepted.headSha256}}
|
|
}
|
|
export async function maintenanceMain(args:string[]){
|
|
assertJsDriverEnvironment(process.env);const resolve=args[0]==='maintenance-resolve';assert(resolve||args[0]==='maintenance-inspect')
|
|
const required=[...flags,...(resolve?['inspection','inspection-sha256','approval','approval-sha256']:['output'])],names=[...required,'ca','ca-sha256'],values=new Map<string,string>()
|
|
assert(args.length===1+required.length*2||args.length===1+(required.length+2)*2)
|
|
for(let i=1;i<args.length;i+=2){const name=args[i].slice(2);assert(args[i].startsWith('--')&&names.includes(name)&&!values.has(name)&&args[i+1]);values.set(name,args[i+1])}
|
|
for(const name of required)assert(values.has(name));assert(values.has('ca')===values.has('ca-sha256'));const get=(name:string)=>values.get(name)!
|
|
for(const name of [...values.keys()].filter(name=>name.endsWith('sha256')))assert(hex(get(name)))
|
|
const stateDirectory=get('state-dir');await protectedDirectory(stateDirectory)
|
|
const transport=parseMaintenanceJSON(await readProtected(get('transport'),16384,get('transport-sha256')));object(transport);const mode=transport.mode===undefined?'tls':transport.mode;assert(mode==='tls'||mode==='local-loopback');assert(values.has('ca')===(mode==='tls'))
|
|
const ca=mode==='tls'?await readProtected(get('ca'),65536,get('ca-sha256')):undefined
|
|
const runtime=parseMaintenanceJSON(await readProtected(get('runtime-target'),16384,get('runtime-target-sha256')));exact(runtime,['format','databaseUrl']);assert(runtime.format==='database-maintenance-runtime-target-v1'&&typeof runtime.databaseUrl==='string');const runtimeUrl=new URL(runtime.databaseUrl);assert(!runtimeUrl.username&&!runtimeUrl.password)
|
|
const target=runtimeDatabaseTargetSha256(runtime.databaseUrl),{options,transportBinding}=maintenancePoolConfiguration(transport,ca,mode==='tls'?get('ca-sha256'):undefined)
|
|
exact(transport.connection,['host','port','user','password','database']);assert.equal(decodeURIComponent(runtimeUrl.pathname.slice(1)),transport.connection.database)
|
|
const records=await readMaintenanceAcceptanceRecords(stateDirectory);if(resolve)assert(records.length<64,'Maintenance acceptance history bound reached');const currentState=await snapshot(stateDirectory,target,records),before=currentState.external;if(resolve)assert(currentState.newUnresolved,'No new unresolved history to accept')
|
|
await inventoryMigrationDirectory(path.join(path.dirname(path.dirname(path.dirname(fileURLToPath(import.meta.url)))),'drizzle'))
|
|
let inspection:Record<string,unknown>|undefined,approved:ReturnType<typeof maintenanceApproval>|undefined
|
|
if(resolve){inspection=parseMaintenanceJSON(await readProtected(get('inspection'),67108864,get('inspection-sha256'))) as Record<string,unknown>;exact(inspection,['format','createdAt','expiresAt','runtimeTargetSha256','acceptanceRequired','external','knownCurrent','schema']);assert(inspection.format==='database-maintenance-current-state-inspection-v1'&&stamp(inspection.createdAt)&&stamp(inspection.expiresAt)&&Date.parse(inspection.createdAt)<=Date.now()&&Date.now()<Date.parse(inspection.expiresAt)&&Date.parse(inspection.expiresAt)-Date.parse(inspection.createdAt)<=validityMs&&inspection.runtimeTargetSha256===target&&inspection.acceptanceRequired===true);validateMaintenanceKnownCurrent(inspection.knownCurrent);assert.equal(stable(inspection.external),stable(before));approved=maintenanceApproval(parseMaintenanceJSON(await readProtected(get('approval'),16384,get('approval-sha256'))),get('inspection-sha256'),inspection.createdAt)}
|
|
const {default:pg}=await import('pg');assertJsDriverEnvironment(process.env)
|
|
const runProof=async()=>{const pool=new pg.Pool(options);let failed=false;pool.on('error',()=>{failed=true});try{const result=await observeMaintenanceCurrent(pool,transportBinding);await pool.end();assert(!failed);return result}catch(error){try{await pool.end()}catch{/* preserve refusal */}throw error}}
|
|
if(!resolve){const result=await runProof();assert.equal(stable((await snapshot(stateDirectory,target,await readMaintenanceAcceptanceRecords(stateDirectory))).external),stable(before));const nowMs=Date.now(),now=new Date(nowMs).toISOString(),out={format:'database-maintenance-current-state-inspection-v1',createdAt:now,expiresAt:new Date(nowMs+validityMs).toISOString(),runtimeTargetSha256:target,acceptanceRequired:currentState.newUnresolved,external:before,...result};const raw=Buffer.from(stable(out));assert(raw.length<=67108864);await writeFile(get('output'),raw,{flag:'wx',mode:0o600});return {event:'database-maintenance-inspection',inspectionSha256:sha(raw),unresolvedCoreOperations:before.journalCoverage.unresolvedOperations.length,unresolvedVaultFiles:before.vaultCoverage.length,state:currentState.newUnresolved?'inspection-required':before.previousResolutionSha256?'historically-accepted':'no-unresolved-history',historicalAcceptanceHeadSha256:before.previousResolutionSha256,humanApprovalRequired:currentState.newUnresolved,knownCurrentObserved:true,archiveDataEquivalenceProved:false,historicOutcomesChanged:false,automaticRestore:false}}
|
|
const resolutionDirectory=path.join(stateDirectory,'maintenance-acceptances');await mkdir(resolutionDirectory,{recursive:true,mode:0o700});await protectedDirectory(resolutionDirectory)
|
|
return withJournalLease(resolutionDirectory,async()=>{
|
|
// Existing immutable records are reread while our proven O_EXCL lease is held.
|
|
// Runtime reader refuses this lease; never infer stale ownership from PID/time.
|
|
const expected=records.map(raw=>{const v=parseMaintenanceJSON(raw) as {resolutionId:string};return v.resolutionId+'.json'}).sort()
|
|
assert.equal((await readdir(resolutionDirectory)).sort().join('\0'),[...expected,'publisher-lease.lock'].sort().join('\0'))
|
|
for(let i=0;i<records.length;i++){const value=parseMaintenanceJSON(records[i]) as {resolutionId:string};assert.equal(sha(await readProtected(path.join(resolutionDirectory,value.resolutionId+'.json'),1048576)),sha(records[i]))}
|
|
const observed=await runProof();assert.equal(stable(observed.knownCurrent),stable(inspection!.knownCurrent));assert.equal(stable(observed.schema),stable(inspection!.schema))
|
|
assert.equal(stable((await snapshot(stateDirectory,target,records)).external),stable(before));assert(Date.now()<Date.parse(approved!.expiresAt)&&Date.now()<Date.parse(inspection!.expiresAt as string))
|
|
const record={format:MAINTENANCE_ACCEPTANCE_FORMAT,resolutionId:approved!.resolutionId,decision:'accept-observed-current-state',historicOutcomeChanged:false,automaticRestore:false,actor:{kind:'host-operator',id:approved!.actorId},approvalSha256:get('approval-sha256'),inspectionSha256:get('inspection-sha256'),approvedAt:approved!.issuedAt,approvalExpiresAt:approved!.expiresAt,acceptedAt:new Date().toISOString(),runtimeTargetSha256:target,...before,knownCurrent:observed.knownCurrent}
|
|
validateMaintenanceAcceptance(record);const raw=Buffer.from(stable(record));assert(raw.length<=1048576,'Maintenance acceptance byte bound exceeded');await writeFile(path.join(resolutionDirectory,record.resolutionId+'.json'),raw,{flag:'wx',mode:0o600})
|
|
return {event:'database-maintenance-current-state-accepted',outcome:'committed',resolutionId:record.resolutionId,acceptanceSha256:sha(raw),archiveDataEquivalenceProved:false,historicOutcomesChanged:false,automaticRestore:false}
|
|
})
|
|
}
|