chore(test): проверить хранение последних копий и частичное удаление

backend/scripts/acceptance/run-s3-backup-retention.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -euo pipefail
+umask 077
+cd /opt/mmapp-test
+mm240_db="mm_accept_166_$(date +%s%3N)"
+mm240_pass="$(openssl rand -hex 24)"
+mm240_role_created=false
+mm240_db_created=false
+cleanup240() {
+  docker rm mmapp-s3app240-owned >/dev/null 2>&1 || true
+  if $mm240_db_created; then docker exec mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1 -c "DROP DATABASE \"$mm240_db\" WITH (FORCE)"; fi
+  if $mm240_role_created; then docker exec mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1 -c "DROP ROLE \"$mm240_db\""; fi
+  remaining=$(docker exec mmapp-test-postgres psql -U mmapp -d mmapp -Atc "SELECT (SELECT count(*) FROM pg_database WHERE datname='$mm240_db')+(SELECT count(*) FROM pg_roles WHERE rolname='$mm240_db')")
+  [ "$remaining" = 0 ]
+  python3 - <<'PY'
+import json,pathlib
+p=pathlib.Path('/opt/mmapp-test/artifacts/s3-retention240/s3-retention240.json')
+if p.exists():
+ r=json.loads(p.read_text());r['ownedDatabaseAndRoleRemoved']=True;p.write_text(json.dumps(r,indent=2)+'\n')
+PY
+  printf 'OWNED_DATABASE_AND_ROLE_REMOVED\n'
+  unset mm240_pass
+}
+trap cleanup240 EXIT
+[[ "$mm240_db" =~ ^mm_accept_166_[0-9]+$ ]]
+existing=$(docker exec mmapp-test-postgres psql -U mmapp -d mmapp -Atc "SELECT (SELECT count(*) FROM pg_database WHERE datname='$mm240_db')+(SELECT count(*) FROM pg_roles WHERE rolname='$mm240_db')")
+[ "$existing" = 0 ]
+printf "CREATE ROLE \"%s\" LOGIN PASSWORD '%s' NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS;\n" "$mm240_db" "$mm240_pass" | docker exec -i mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1
+mm240_role_created=true
+docker exec mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1 -c "CREATE DATABASE \"$mm240_db\" OWNER \"$mm240_db\""
+mm240_db_created=true
+docker run --name mmapp-s3app240-owned --network mmapp-test_default --cap-drop ALL --security-opt no-new-privileges -e "MMAPP_ORIGIN_BASELINE=${2:-false}" --env-file /opt/mmapp-test/artifacts/s3-retention240/private.env -e NODE_ENV=test -e "DATABASE_URL=postgres://$mm240_db:$mm240_pass@postgres:5432/$mm240_db" -e "MMAPP_OWN_SQL_DB=$mm240_db" -e "DATABASE_PATH=/tmp/$mm240_db/legacy.db" -e "GEOIP_DIR=/tmp/$mm240_db/geoip" -v /opt/mmapp-test/artifacts/s3-retention240:/checks --entrypoint node "${1:-mmapp-test/backend:ux-backups237}" /checks/s3-backup-retention.mjs

backend/scripts/acceptance/s3-backup-retention.mjs

@@ -0,0 +1,84 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import { randomUUID, createHash } from 'node:crypto';
+import { createRequire } from 'node:module';
+const database = process.env.MMAPP_OWN_SQL_DB;
+assert(/^mm_accept_166_\d+$/.test(database));
+const url = new URL(process.env.DATABASE_URL); assert.equal(url.pathname, '/' + database); assert.equal(url.username, database);
+fs.mkdirSync('/tmp/' + database, { recursive: true }); process.chdir('/tmp/' + database);
+const require = createRequire('/app/backend/package.json'), { ListObjectVersionsCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3');
+const { pool, db, closePool } = await import('/app/backend/dist/db/index.js');
+const { initDatabase } = await import('/app/backend/dist/db/bootstrap.js');
+const { servers } = await import('/app/backend/dist/db/schema.js');
+const { databaseBackgroundWork } = await import('/app/backend/dist/services/database-background-work.js');
+const { createS3ClientFromConfig, s3PutObject, s3GetObject } = await import('/app/backend/dist/services/s3-backup-client.js');
+const { updateBackupStorageSettings, pruneBackupsForServer, readBackupContent, listBackups } = await import('/app/backend/dist/services/backup-service.js');
+const role = (await pool.query('SELECT rolsuper,rolcreatedb,rolcreaterole,rolbypassrls FROM pg_roles WHERE rolname=current_user')).rows[0];
+assert(Object.values(role).every(v => v === false)); await initDatabase();
+const prefix = 'mmapp-acceptance/' + randomUUID(), secret = process.env.MMAPP_TEST_S3_SECRET_KEY;
+const cfg = { endpoint: process.env.MMAPP_TEST_S3_ENDPOINT, region: process.env.MMAPP_TEST_S3_REGION, bucket: process.env.MMAPP_TEST_S3_BUCKET, accessKeyId: process.env.MMAPP_TEST_S3_ACCESS_KEY, secretAccessKey: secret, forcePathStyle: false };
+assert.equal(cfg.endpoint, 'https://s3.ru-1.storage.selcloud.ru'); assert.equal(cfg.bucket, 'mikrotik-test-s3'); assert(secret);
+const s3 = createS3ClientFromConfig(cfg), plannedKeys = [], seededIds = [], payloads = new Map();
+const report = { date: '2026-10-14', block: '12.5.240', scope: 'Owned PostgreSQL + compiled retention service + real Selectel S3; no CHR contacts', prefix, passed: false, checks: [], cleanup: {}, chrContacts: 0, bucketSettingsWrites: 0 };
+const save = () => fs.writeFileSync('/checks/s3-retention240.json', JSON.stringify(report, null, 2) + '\n');
+const check = (name, details = {}) => { report.checks.push({ name, ...details }); save(); };
+const catalog = async () => (await pool.query('SELECT id,storage,s3_key FROM backup_entries ORDER BY id')).rows;
+async function versions() {
+ let keyMarker, versionMarker; const all = [];
+ for (let n = 0; n < 10; n++) {
+  const r = await s3.send(new ListObjectVersionsCommand({ Bucket: cfg.bucket, Prefix: prefix + '/', KeyMarker: keyMarker, VersionIdMarker: versionMarker }));
+  all.push(...(r.Versions ?? []).map(v => ({ ...v, kind: 'version' })), ...(r.DeleteMarkers ?? []).map(v => ({ ...v, kind: 'delete-marker' })));
+  if (!r.IsTruncated) return all;
+  assert(r.NextKeyMarker || r.NextVersionIdMarker); keyMarker = r.NextKeyMarker; versionMarker = r.NextVersionIdMarker;
+ } throw Error('Own pagination limit');
+}
+try {
+ assert.equal((await versions()).length, 0);
+ const hosts = await db.insert(servers).values(['a', 'b', 'c'].map(s => ({ name: 'owned-retention240-' + s, host: '127.0.0.1', port: 9, useSsl: false, username: 'fixture', password: 'fixture', enabled: true, type: 'exit-node' }))).returning();
+ await updateBackupStorageSettings({ provider: 's3', s3Endpoint: cfg.endpoint, s3Region: cfg.region, s3Bucket: cfg.bucket, s3Prefix: prefix, s3AccessKeyId: cfg.accessKeyId, s3SecretAccessKey: secret, s3ForcePathStyle: false, keepLocalCopy: true });
+ fs.mkdirSync('storage/backups', { recursive: true });
+ const fixtures = [['a4',0,4,'both'],['a3',0,3,'s3'],['a2',0,2,'local'],['a1',0,1,'s3'],['b2',1,2,'both'],['b1',1,1,'local'],['b0',1,0,'s3'],['c0',2,0,'local']];
+ for (const [id, hostIndex, seconds, storage] of fixtures) {
+  const filename = id + '-retention240.rsc', bytes = Buffer.from('# Own retention240 ' + id + '\n'), key = storage === 'local' ? null : prefix + '/' + filename;
+  seededIds.push(id); payloads.set(id, bytes);
+  if (key) { plannedKeys.push(key); await s3PutObject(s3, cfg.bucket, key, bytes); assert.deepEqual(await s3GetObject(s3, cfg.bucket, key), bytes); }
+  if (storage !== 's3') fs.writeFileSync('storage/backups/' + filename, bytes);
+  await pool.query('INSERT INTO backup_entries(id,server_id,server_name,filename,size_bytes,kind,storage,s3_key,created_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)', [id, hosts[hostIndex].id, hosts[hostIndex].name, filename, bytes.length, id.endsWith('0') ? 'manual' : 'auto', storage, key, new Date(Date.UTC(2026,9,14,0,0,seconds)).toISOString()]);
+ }
+ check('eight-fixtures-three-servers', { remoteObjects: plannedKeys.length, localFiles: fixtures.filter(f => f[3] !== 's3').length });
+ const untouched = (await catalog()).filter(r => !r.id.startsWith('a'));
+ await updateBackupStorageSettings({ s3SecretAccessKey: secret + '-invalid' });
+ let failure;
+ try { await pruneBackupsForServer(String(hosts[0].id), 2); } catch (e) { failure = e; }
+ assert.equal(failure?.name, 'BackupPruneError'); assert.equal(failure.pruned, 1);
+ let remaining = await catalog(); assert.deepEqual(remaining.filter(r => !r.id.startsWith('a')), untouched);
+ assert.deepEqual(remaining.filter(r => r.id.startsWith('a')).map(r => r.id), ['a1','a3','a4']);
+ assert(!fs.existsSync('storage/backups/a2-retention240.rsc'));
+ assert.deepEqual(await s3GetObject(s3, cfg.bucket, prefix + '/a1-retention240.rsc'), payloads.get('a1'));
+ check('partial-failure-acknowledges-one-local-deletion', { pruned: failure.pruned, failedRowRetained: true, remoteObjectRetained: true, otherServersUnchanged: true });
+ await updateBackupStorageSettings({ s3SecretAccessKey: secret });
+ assert.equal(await pruneBackupsForServer(String(hosts[0].id), 2), 1);
+ assert.equal(await pruneBackupsForServer(String(hosts[0].id), 2), 0);
+ assert.deepEqual((await catalog()).filter(r => r.id.startsWith('a')).map(r => r.id), ['a3','a4']);
+ check('retry-retains-latest-two-and-repeat-noop', { retryPruned: 1, repeatPruned: 0 });
+ assert.equal(await pruneBackupsForServer(String(hosts[1].id), 1), 2);
+ assert.deepEqual((await catalog()).map(r => r.id), ['a3','a4','b2','c0']);
+ check('second-server-retains-latest-and-unselected-manual-copy', { pruned: 2, retained: ['a3','a4','b2','c0'] });
+ const copies = await listBackups();
+ for (const copy of copies) assert.deepEqual(await readBackupContent(copy), payloads.get(copy.id));
+ assert.equal(copies.find(r => r.id === 'a3').localFileStatus, 'not-stored');
+ assert(copies.filter(r => r.id !== 'a3').every(r => r.localFileStatus === 'available'));
+ check('retained-content-hashes-and-local-states', { copies: copies.map(r => ({ id: r.id, localFileStatus: r.localFileStatus, sha256: createHash('sha256').update(payloads.get(r.id)).digest('hex') })) });
+ report.passed = true;
+} catch (e) { report.failure = { name: e.name }; process.exitCode = 1; }
+finally {
+ try {
+  const existing = await versions(); assert(existing.every(v => plannedKeys.includes(v.Key) && v.VersionId));
+  report.cleanup = { plannedKeys, versionsBefore: existing.filter(v => v.kind === 'version').length, deleteMarkersBefore: existing.filter(v => v.kind === 'delete-marker').length, removed: 0 }; save();
+  for (const v of existing) { await s3.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: v.Key, VersionId: v.VersionId })); report.cleanup.removed++; save(); }
+  assert.equal((await versions()).length, 0); report.cleanup.versionsAndMarkersRemaining = 0;
+  if (seededIds.length) await pool.query('DELETE FROM backup_entries WHERE id=ANY($1::text[])', [seededIds]);
+  report.cleanup.ownCatalogRemaining = Number((await pool.query('SELECT count(*) AS n FROM backup_entries')).rows[0].n); assert.equal(report.cleanup.ownCatalogRemaining, 0);
+ } catch (e) { report.cleanup.failure = { name: e.name }; report.passed = false; process.exitCode = 1; }
+ await databaseBackgroundWork.drain(); await closePool(); s3.destroy(); save(); console.log(JSON.stringify(report));
+}