529 lines
18 KiB
JavaScript
529 lines
18 KiB
JavaScript
/**
|
|
* Разные утилитарные роуты (auto-urls, history, s3 meta, availability, etc.)
|
|
*/
|
|
|
|
const { s3, BUCKET_NAME, writeS3TextObject, headMeta, streamToString } = require('../services/s3Service');
|
|
const { sendError, sendOk } = require('../middleware/errorHandler');
|
|
const { resourceToKey, toIso } = require('../utils/helpers');
|
|
const { GetObjectCommand, PutObjectCommand, ListObjectVersionsCommand, CopyObjectCommand } = require('@aws-sdk/client-s3');
|
|
const validators = require('../lib/validators');
|
|
const net = require('net');
|
|
const http = require('http');
|
|
const https = require('https');
|
|
const { URL } = require('url');
|
|
|
|
// GET /api/s3/last-modified
|
|
async function getS3LastModified(req, res) {
|
|
try {
|
|
const keys = [
|
|
{ name: 'domainsNew', key: 'bgp_data/domains_community.txt' },
|
|
{ name: 'asns', key: 'bgp_data/asns.txt' },
|
|
{ name: 'servers', key: 'servers.json' },
|
|
{ name: 'filters', key: 'filters.json' },
|
|
{ name: 'ipRanges', key: 'bgp_data/ips.txt' },
|
|
{ name: 'uiSettings', key: 'bgp_data/rt_ui_settings.json' }
|
|
];
|
|
|
|
const results = await Promise.allSettled(
|
|
keys.map(k => headMeta(k.key))
|
|
);
|
|
|
|
const out = {};
|
|
results.forEach((r, idx) => {
|
|
const name = keys[idx].name;
|
|
if (r.status === 'fulfilled' && r.value) {
|
|
out[name] = r.value;
|
|
} else {
|
|
out[name] = null;
|
|
}
|
|
});
|
|
|
|
res.json(out);
|
|
} catch (error) {
|
|
console.error('Error fetching last modified dates from S3:', error);
|
|
return sendError(res, 500, 'Error fetching last modified dates from S3', 'E_S3');
|
|
}
|
|
}
|
|
|
|
// GET /api/history/:resource
|
|
async function getHistory(req, res) {
|
|
const { resource } = req.params;
|
|
const { countOnly, format } = req.query || {};
|
|
const key = resourceToKey(resource);
|
|
|
|
if (!key) return sendError(res, 400, 'Unknown resource', 'E_RESOURCE');
|
|
|
|
try {
|
|
const out = await s3.send(new ListObjectVersionsCommand({
|
|
Bucket: BUCKET_NAME,
|
|
Prefix: key,
|
|
MaxKeys: 50
|
|
}));
|
|
|
|
const versionsAll = (out.Versions || []).filter(v => v.Key === key);
|
|
|
|
if (countOnly === 'true') {
|
|
const total = versionsAll.length;
|
|
return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
|
|
}
|
|
|
|
const versions = versionsAll.slice(0, 10).map(v => ({
|
|
versionId: v.VersionId,
|
|
isLatest: v.IsLatest,
|
|
lastModified: toIso(v.LastModified),
|
|
size: v.Size,
|
|
etag: v.ETag
|
|
}));
|
|
|
|
return res.json(format === 'std' ? { items: versions, total: versionsAll.length, meta: {} } : { items: versions });
|
|
} catch (e) {
|
|
console.error('history error', e);
|
|
return sendError(res, 500, 'Error reading history', 'E_S3', { error: String(e?.message || e) });
|
|
}
|
|
}
|
|
|
|
// POST /api/history/:resource/rollback
|
|
async function postRollback(req, res) {
|
|
const { resource } = req.params;
|
|
const { versionId } = req.body || {};
|
|
const key = resourceToKey(resource);
|
|
|
|
if (!key || !versionId) {
|
|
return sendError(res, 400, 'Bad request', 'E_BAD_REQUEST');
|
|
}
|
|
|
|
try {
|
|
await s3.send(new CopyObjectCommand({
|
|
Bucket: BUCKET_NAME,
|
|
CopySource: `/${BUCKET_NAME}/${encodeURIComponent(key)}?versionId=${encodeURIComponent(versionId)}`,
|
|
Key: key
|
|
}));
|
|
|
|
const meta = await headMeta(key);
|
|
return sendOk(res, meta);
|
|
} catch (e) {
|
|
console.error('rollback error', e);
|
|
return sendError(res, 500, 'Error rollback', 'E_S3', { error: String(e?.message || e) });
|
|
}
|
|
}
|
|
|
|
// GET /api/auto-urls
|
|
async function getAutoUrls(req, res) {
|
|
const { checkIfNoneMatch } = require('../middleware/errorHandler');
|
|
|
|
try {
|
|
const { HeadObjectCommand } = require('@aws-sdk/client-s3');
|
|
const head = await s3.send(new HeadObjectCommand({
|
|
Bucket: BUCKET_NAME,
|
|
Key: 'bgp_data/auto_url/urls.txt'
|
|
})).catch(() => null);
|
|
|
|
const etag = head?.ETag || null;
|
|
if (etag) res.set('ETag', String(etag));
|
|
if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
|
|
if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
|
|
if (checkIfNoneMatch(req, res, etag)) return;
|
|
|
|
const data = await s3.send(new GetObjectCommand({
|
|
Bucket: BUCKET_NAME,
|
|
Key: 'bgp_data/auto_url/urls.txt'
|
|
}));
|
|
const fileContent = await streamToString(data.Body);
|
|
const urls = fileContent.split('\n').filter(line => line).map(line => {
|
|
const parts = line.trim().split(/\s+/);
|
|
return { url: parts[0] || '', community: parts[1] || '' };
|
|
});
|
|
res.json(urls);
|
|
} catch (error) {
|
|
if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
|
|
res.json([]);
|
|
} else {
|
|
console.error(error);
|
|
return sendError(res, 500, 'Error reading auto URLs from S3', 'E_S3');
|
|
}
|
|
}
|
|
}
|
|
|
|
// POST /api/auto-urls
|
|
async function postAutoUrls(req, res) {
|
|
const { urls } = req.body;
|
|
const fileContent = urls.map(u => `${u.url} ${u.community}`).join('\n');
|
|
|
|
try {
|
|
const meta = await writeS3TextObject('bgp_data/auto_url/urls.txt', fileContent);
|
|
return sendOk(res, meta);
|
|
} catch (error) {
|
|
console.error(error);
|
|
return sendError(res, 500, 'Error writing auto URLs to S3', 'E_S3');
|
|
}
|
|
}
|
|
|
|
// POST /api/auto-urls/process
|
|
async function processAutoUrls(req, res) {
|
|
try {
|
|
// Загрузка URL списков
|
|
let urls = [];
|
|
try {
|
|
const urlsData = await s3.send(new GetObjectCommand({
|
|
Bucket: BUCKET_NAME,
|
|
Key: 'bgp_data/auto_url/urls.txt'
|
|
}));
|
|
const urlsContent = await streamToString(urlsData.Body);
|
|
urls = urlsContent.split('\n').filter(line => line).map(line => {
|
|
const parts = line.trim().split(/\s+/);
|
|
return { url: parts[0] || '', community: parts[1] || '' };
|
|
});
|
|
} catch (error) {
|
|
if (error.code !== 'NoSuchKey') throw error;
|
|
}
|
|
|
|
if (urls.length === 0) {
|
|
return sendError(res, 400, 'No URLs to process', 'E_BAD_REQUEST');
|
|
}
|
|
|
|
// Загрузка текущих IP и доменов
|
|
let currentIps = [];
|
|
try {
|
|
const ipsData = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' }));
|
|
const ipsContent = await streamToString(ipsData.Body);
|
|
currentIps = ipsContent.split('\n').filter(line => line).map(line => {
|
|
const parts = line.trim().split(/\s+/);
|
|
return { ipRange: parts[0] || '', community: parts[1] || '' };
|
|
});
|
|
} catch (error) {
|
|
if (error.code !== 'NoSuchKey') throw error;
|
|
}
|
|
|
|
let currentDomains = [];
|
|
try {
|
|
const dData = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' }));
|
|
const dContent = await streamToString(dData.Body);
|
|
currentDomains = dContent.split('\n').filter(line => line).map(line => {
|
|
const parts = line.trim().split(/\s+/);
|
|
return { domain: (parts[0] || '').toLowerCase(), community: parts[1] || '' };
|
|
});
|
|
} catch (error) {
|
|
if (error.code !== 'NoSuchKey') throw error;
|
|
}
|
|
|
|
// Обработка URL
|
|
const newIps = [];
|
|
const newDomains = [];
|
|
|
|
for (const urlData of urls) {
|
|
try {
|
|
const url = String(urlData.url || '').trim();
|
|
const community = String(urlData.community || '').trim();
|
|
if (!url || !community) continue;
|
|
|
|
const content = await new Promise((resolve, reject) => {
|
|
const protocol = url.startsWith('https:') ? https : http;
|
|
const req = protocol.get(url, (r) => {
|
|
let data = '';
|
|
r.on('data', (chunk) => { data += chunk; });
|
|
r.on('end', () => resolve(data));
|
|
});
|
|
req.on('error', reject);
|
|
req.setTimeout(15000, () => req.destroy());
|
|
});
|
|
|
|
const lines = content.split('\n');
|
|
for (const raw of lines) {
|
|
const line = String(raw || '').trim();
|
|
if (!line || line.startsWith('#') || line.startsWith('//')) continue;
|
|
const token = line.split(/\s+/)[0]?.trim();
|
|
if (!token) continue;
|
|
|
|
if (validators.isValidCIDRv4(token) || validators.isValidCIDRv6(token)) {
|
|
newIps.push({ ipRange: token, community });
|
|
} else if (validators.isValidIPv4(token)) {
|
|
newIps.push({ ipRange: `${token}/32`, community });
|
|
} else if (validators.isValidDomain(token)) {
|
|
newDomains.push({ domain: token.toLowerCase(), community });
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error(`Error processing URL ${urlData.url}:`, error);
|
|
}
|
|
}
|
|
|
|
// Объединение и дедупликация
|
|
const existingIpRanges = new Set(currentIps.map(i => i.ipRange));
|
|
const uniqueNewIps = newIps.filter(i => !existingIpRanges.has(i.ipRange));
|
|
const allIps = [...currentIps, ...uniqueNewIps];
|
|
|
|
const existingDomains = new Set(currentDomains.map(d => d.domain));
|
|
const uniqueNewDomains = newDomains.filter(d => !existingDomains.has(d.domain));
|
|
const allDomains = [...currentDomains, ...uniqueNewDomains];
|
|
|
|
// Сохранение
|
|
const updatedIpsContent = allIps.map(i => `${i.ipRange} ${i.community}`).join('\n');
|
|
await s3.send(new PutObjectCommand({
|
|
Bucket: BUCKET_NAME,
|
|
Key: 'bgp_data/ips.txt',
|
|
Body: updatedIpsContent,
|
|
ContentType: 'text/plain'
|
|
}));
|
|
|
|
const updatedDomainsContent = allDomains.map(d => `${d.domain} ${d.community}`).join('\n');
|
|
await s3.send(new PutObjectCommand({
|
|
Bucket: BUCKET_NAME,
|
|
Key: 'bgp_data/domains_community.txt',
|
|
Body: updatedDomainsContent,
|
|
ContentType: 'text/plain'
|
|
}));
|
|
|
|
const msg = `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} IP и ${uniqueNewDomains.length} доменов`;
|
|
return res.json({
|
|
success: true,
|
|
message: msg,
|
|
processedUrls: urls.length,
|
|
newIpsCount: uniqueNewIps.length,
|
|
newDomainsCount: uniqueNewDomains.length,
|
|
totalIpsCount: allIps.length,
|
|
totalDomainsCount: allDomains.length
|
|
});
|
|
} catch (error) {
|
|
console.error('Error processing auto URLs:', error);
|
|
return sendError(res, 500, 'Error processing auto URLs', 'E_S3');
|
|
}
|
|
}
|
|
|
|
// GET /api/servers/availability
|
|
const availabilityCache = { at: 0, data: null };
|
|
|
|
function tcpCheck(host, port, timeoutMs) {
|
|
return new Promise((resolve) => {
|
|
const socket = new net.Socket();
|
|
let settled = false;
|
|
const settle = (ok) => {
|
|
if (!settled) {
|
|
settled = true;
|
|
try { socket.destroy(); } catch {}
|
|
resolve(ok);
|
|
}
|
|
};
|
|
socket.setTimeout(timeoutMs, () => settle(false));
|
|
socket.once('error', () => settle(false));
|
|
socket.connect(port, host, () => settle(true));
|
|
});
|
|
}
|
|
|
|
function anyTrue(promises) {
|
|
return new Promise((resolve) => {
|
|
if (!Array.isArray(promises) || promises.length === 0) return resolve(false);
|
|
let remaining = promises.length;
|
|
let resolved = false;
|
|
for (const p of promises) {
|
|
Promise.resolve(p).then((v) => {
|
|
if (v && !resolved) { resolved = true; resolve(true); }
|
|
}).finally(() => {
|
|
remaining -= 1;
|
|
if (remaining === 0 && !resolved) resolve(false);
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
async function checkOneServerFast(srv, perSocketTimeoutMs = 800, perServerBudgetMs = 1000) {
|
|
const hosts = [];
|
|
if (srv.ip) hosts.push(String(srv.ip));
|
|
if (srv.dns) hosts.push(String(srv.dns));
|
|
|
|
const tryOneHost = (host) => anyTrue([
|
|
tcpCheck(host, 443, perSocketTimeoutMs),
|
|
tcpCheck(host, 80, perSocketTimeoutMs),
|
|
]);
|
|
|
|
const run = anyTrue(hosts.map((h) => tryOneHost(h)));
|
|
const timeout = new Promise((resolve) => setTimeout(() => resolve(false), perServerBudgetMs));
|
|
return Promise.race([run, timeout]);
|
|
}
|
|
|
|
async function getServersAvailability(req, res) {
|
|
try {
|
|
const ttlSeconds = Math.max(0, Math.min(300, Number(req.query.ttlSeconds) || 30));
|
|
const now = Date.now();
|
|
|
|
if (availabilityCache.data && (now - availabilityCache.at) < ttlSeconds * 1000) {
|
|
return res.json({ ...availabilityCache.data, cached: true });
|
|
}
|
|
|
|
const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'servers.json' }));
|
|
let servers = [];
|
|
try {
|
|
servers = JSON.parse(await streamToString(data.Body));
|
|
if (!Array.isArray(servers)) servers = [];
|
|
} catch {
|
|
servers = [];
|
|
}
|
|
|
|
const checks = await Promise.allSettled(servers.map((s) => checkOneServerFast(s)));
|
|
const statuses = servers.map((s, i) => ({
|
|
ip: s.ip,
|
|
dns: s.dns,
|
|
online: checks[i].status === 'fulfilled' ? Boolean(checks[i].value) : false
|
|
}));
|
|
|
|
const online = statuses.filter((x) => x.online).length;
|
|
const payload = { online, total: servers.length, statuses };
|
|
availabilityCache.at = Date.now();
|
|
availabilityCache.data = payload;
|
|
res.json(payload);
|
|
} catch (e) {
|
|
console.error('availability error', e);
|
|
res.status(500).json({ online: 0, total: 0, statuses: [] });
|
|
}
|
|
}
|
|
|
|
// POST /api/update-bgp/background
|
|
async function updateBgpBackground(req, res) {
|
|
try {
|
|
const targetUrl = process.env.BGP_BACKGROUND_URL;
|
|
if (!targetUrl) {
|
|
return sendError(res, 500, 'BGP_BACKGROUND_URL is not configured', 'E_CONFIG');
|
|
}
|
|
|
|
const u = new URL(targetUrl);
|
|
const client = u.protocol === 'https:' ? https : http;
|
|
const options = {
|
|
method: 'POST',
|
|
hostname: u.hostname,
|
|
port: u.port || (u.protocol === 'https:' ? 443 : 80),
|
|
path: `${u.pathname}${u.search || ''}`,
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
'Accept': 'application/json',
|
|
},
|
|
timeout: 15000,
|
|
};
|
|
|
|
const body = req.body && Object.keys(req.body).length ? JSON.stringify(req.body) : '';
|
|
|
|
const upstream = client.request(options, (r) => {
|
|
let data = '';
|
|
r.setEncoding('utf8');
|
|
r.on('data', (chunk) => { data += chunk; });
|
|
r.on('end', () => {
|
|
const status = r.statusCode || 502;
|
|
try {
|
|
const json = data ? JSON.parse(data) : {};
|
|
return res.status(status).json(json);
|
|
} catch (_) {
|
|
return res.status(status).json({ ok: status >= 200 && status < 300, data });
|
|
}
|
|
});
|
|
});
|
|
|
|
upstream.on('timeout', () => {
|
|
try { upstream.destroy(); } catch {}
|
|
return sendError(res, 504, 'Upstream timeout', 'E_UPSTREAM_TIMEOUT');
|
|
});
|
|
|
|
upstream.on('error', (e) => {
|
|
return sendError(res, 502, 'Upstream error', 'E_UPSTREAM', { error: String(e?.message || e) });
|
|
});
|
|
|
|
if (body) upstream.write(body);
|
|
upstream.end();
|
|
} catch (e) {
|
|
return sendError(res, 500, 'Proxy error', 'E_PROXY', { error: String(e?.message || e) });
|
|
}
|
|
}
|
|
|
|
// GET /api/ws/url
|
|
async function getWsUrl(req, res) {
|
|
try {
|
|
const settings = await s3.send(new GetObjectCommand({
|
|
Bucket: BUCKET_NAME,
|
|
Key: 'bgp_data/rt_ui_settings.json'
|
|
})).then(async (d) => {
|
|
try { return JSON.parse(await streamToString(d.Body)); } catch { return {}; }
|
|
}).catch(() => ({}));
|
|
|
|
const url = settings?.wsUpdateUrl || process.env.WS_UPDATE_URL || '';
|
|
return res.json({ url });
|
|
} catch (e) {
|
|
return res.json({ url: '' });
|
|
}
|
|
}
|
|
|
|
// GET/POST /api/ui-settings
|
|
async function getUiSettings(req, res) {
|
|
const { checkIfNoneMatch } = require('../middleware/errorHandler');
|
|
|
|
try {
|
|
const { HeadObjectCommand } = require('@aws-sdk/client-s3');
|
|
const head = await s3.send(new HeadObjectCommand({
|
|
Bucket: BUCKET_NAME,
|
|
Key: 'bgp_data/rt_ui_settings.json'
|
|
})).catch(() => null);
|
|
|
|
const etag = head?.ETag || null;
|
|
if (etag) res.set('ETag', String(etag));
|
|
if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
|
|
if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
|
|
if (checkIfNoneMatch(req, res, etag)) return;
|
|
|
|
const data = await s3.send(new GetObjectCommand({
|
|
Bucket: BUCKET_NAME,
|
|
Key: 'bgp_data/rt_ui_settings.json'
|
|
}));
|
|
const jsonText = await streamToString(data.Body);
|
|
let settings = {};
|
|
try {
|
|
const parsed = JSON.parse(jsonText);
|
|
if (parsed && typeof parsed === 'object') settings = parsed;
|
|
} catch (parseError) {
|
|
settings = {};
|
|
}
|
|
return res.json(settings);
|
|
} catch (error) {
|
|
if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
|
|
return res.json({});
|
|
}
|
|
console.error('Error reading ui settings from S3:', error);
|
|
return sendError(res, 500, 'Error reading UI settings from S3', 'E_S3');
|
|
}
|
|
}
|
|
|
|
async function postUiSettings(req, res) {
|
|
const { settings, etag } = req.body || {};
|
|
const payload = (settings && typeof settings === 'object') ? settings : {};
|
|
|
|
try {
|
|
const { headS3ObjectEtag } = require('../services/s3Service');
|
|
let current = null;
|
|
const ifMatch = req.headers['if-match'] ? String(req.headers['if-match']).replace(/\"/g,'"') : null;
|
|
try { current = await headS3ObjectEtag('bgp_data/rt_ui_settings.json'); } catch {}
|
|
if (current && (ifMatch || etag) && current !== String(ifMatch || etag)) {
|
|
const meta = await headMeta('bgp_data/rt_ui_settings.json');
|
|
return sendError(res, 412, 'Precondition Failed: ETag mismatch', 'E_ETAG_MISMATCH', { currentEtag: current, meta });
|
|
}
|
|
} catch {}
|
|
|
|
try {
|
|
const { writeS3JsonObject } = require('../services/s3Service');
|
|
const meta = await writeS3JsonObject('bgp_data/rt_ui_settings.json', payload);
|
|
return sendOk(res, meta);
|
|
} catch (error) {
|
|
console.error('Error writing UI settings to S3:', error);
|
|
return sendError(res, 500, 'Error writing UI settings to S3', 'E_S3', { error: String(error?.message || error) });
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
getS3LastModified,
|
|
getHistory,
|
|
postRollback,
|
|
getAutoUrls,
|
|
postAutoUrls,
|
|
processAutoUrls,
|
|
getServersAvailability,
|
|
updateBgpBackground,
|
|
getWsUrl,
|
|
getUiSettings,
|
|
postUiSettings,
|
|
};
|
|
|