From d25e8fc8104d09a61b572f71901ecf66e6800d2f Mon Sep 17 00:00:00 2001 From: Denis Shatskiy Date: Sun, 10 Aug 2025 23:55:44 +0700 Subject: [PATCH] feat: Enhance backend API with pagination and validation for ASNs, Domains, and IP Ranges, and update frontend managers to support new API structure for improved data handling and user experience --- backend/package.json | 4 +- backend/server.js | 275 +++++++++++++++++++++++++---- frontend/src/ASNsNewManager.jsx | 5 +- frontend/src/DomainsNewManager.jsx | 7 +- frontend/src/IPRangesManager.jsx | 7 +- 5 files changed, 253 insertions(+), 45 deletions(-) diff --git a/backend/package.json b/backend/package.json index 7ac239f..1207abf 100644 --- a/backend/package.json +++ b/backend/package.json @@ -19,7 +19,9 @@ "aws-sdk": "^2.1692.0", "cors": "^2.8.5", "dotenv": "^17.0.1", - "express": "^4.19.2" + "express": "^4.19.2", + "compression": "^1.7.4", + "ajv": "^8.17.1" }, "devDependencies": { "nodemon": "^3.1.10" diff --git a/backend/server.js b/backend/server.js index 6f9e257..e19ddb0 100644 --- a/backend/server.js +++ b/backend/server.js @@ -3,12 +3,15 @@ const express = require('express'); const AWS = require('aws-sdk'); const cors = require('cors'); const path = require('path'); +const compression = require('compression'); +const Ajv = require('ajv'); const app = express(); const port = 3001; app.use(cors()); app.use(express.json()); +app.use(compression()); // Serve static files from the React app app.use(express.static(path.join(__dirname, 'public'))); @@ -26,6 +29,95 @@ const s3 = new AWS.S3({ const BUCKET_NAME = process.env.S3_BUCKET_NAME; const FILE_KEY = 'bgp_data/domains.txt'; +// AJV setup and schemas +const ajv = new Ajv({ allErrors: true, removeAdditional: 'failing' }); + +const schemaDomainsNew = { + type: 'array', + items: { + type: 'object', + required: ['domain', 'community'], + additionalProperties: false, + properties: { + domain: { type: 'string' }, + community: { type: 'string' } + } + } +}; +const schemaAsns = { + type: 'array', + items: { + type: 'object', + required: ['domain', 'type'], + additionalProperties: false, + properties: { + domain: { type: 'string' }, + type: { type: 'string' } + } + } +}; +const schemaIpRanges = { + type: 'array', + items: { + type: 'object', + required: ['ipRange', 'community'], + additionalProperties: false, + properties: { + ipRange: { type: 'string' }, + community: { type: 'string' } + } + } +}; +const schemaFilters = { + type: 'array', + items: { + type: 'object', + required: ['community', 'gateway'], + additionalProperties: true, + properties: { + community: { type: 'string' }, + gateway: { type: 'string' }, + description: { type: 'string' } + } + } +}; +const schemaServers = { + type: 'array', + items: { + type: 'object', + required: ['ip', 'dns', 'country', 'provider', 'tunnel'], + additionalProperties: true, + properties: { + ip: { type: 'string' }, + dns: { type: 'string' }, + country: { type: 'string' }, + provider: { type: 'string' }, + tunnel: { type: 'string' }, + gateway: { type: 'string' } + } + } +}; +const schemaBilling = { + type: 'array', + items: { + type: 'object', + required: ['hostName', 'country', 'provider'], + additionalProperties: true, + properties: { + hostName: { type: 'string' }, + country: { type: 'string' }, + provider: { type: 'string' } + } + } +}; + +const validateDomainsNew = ajv.compile(schemaDomainsNew); +const validateAsns = ajv.compile(schemaAsns); +const validateIpRanges = ajv.compile(schemaIpRanges); +const validateFilters = ajv.compile(schemaFilters); +const validateServers = ajv.compile(schemaServers); +const validateBilling = ajv.compile(schemaBilling); + // Helper: build MikroTik nested if/else blocks (RouterOS v7 filter language does not support 'else if') function buildNestedGatewayBlocks(gatewayGroups, baseIndentSpaces = 4) { const indent = (n) => ' '.repeat(n); @@ -73,6 +165,57 @@ async function headS3ObjectEtag(key) { return head.ETag ? String(head.ETag).replace(/\"/g, '"') : undefined; } +// Helper: stream and paginate big text files (line-based) +async function streamPaginatedText({ key, mapLine, q, offset = 0, limit = 0 }) { + return new Promise(async (resolve, reject) => { + let total = 0; + const items = []; + let sent = 0; + let buffered = ''; + const matchesQuery = (line) => { + if (!q) return true; + return line.toLowerCase().includes(String(q).toLowerCase()); + }; + const stream = s3.getObject({ Bucket: BUCKET_NAME, Key: key }).createReadStream(); + stream.on('data', (chunk) => { + buffered += chunk.toString('utf-8'); + let lines = buffered.split('\n'); + buffered = lines.pop(); + for (const lnRaw of lines) { + const line = lnRaw.trim(); + if (!line) continue; + if (!matchesQuery(line)) continue; + total++; + const pos = total - 1; // position among matches + if (limit > 0) { + if (pos >= offset && sent < limit) { + items.push(mapLine(line)); + sent++; + } + } else { + items.push(mapLine(line)); + } + } + }); + stream.on('end', () => { + const last = (buffered || '').trim(); + if (last) { + if (!q || last.toLowerCase().includes(String(q).toLowerCase())) { + total++; + if (limit > 0) { + const pos = total - 1; + if (pos >= offset && items.length < limit) items.push(mapLine(last)); + } else { + items.push(mapLine(last)); + } + } + } + resolve({ items, total }); + }); + stream.on('error', reject); + }); +} + // Simple in-memory soft locks with TTL const locks = new Map(); // key -> { owner, expiresAt } function cleanupExpiredLocks() { @@ -161,24 +304,41 @@ app.post('/api/domains', async (req, res) => { // Get ASNs from S3 app.get('/api/asns', async (req, res) => { + const { q = '', offset, limit } = req.query || {}; const params = { Bucket: BUCKET_NAME, Key: 'bgp_data/asns.txt', }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); - const asns = fileContent.split('\n').filter(line => line).map(line => { - const parts = line.trim().split(/\s+/); - const domain = parts[0] || ''; // Keep name 'domain' for consistency in component - const type = parts[1] || ''; - return { domain, type }; - }); - if (data.ETag) res.set('ETag', String(data.ETag)); - if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString()); - if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength)); - res.json(asns); + if (limit !== undefined) { + const { items, total } = await streamPaginatedText({ + key: 'bgp_data/asns.txt', + q, + offset: Number(offset) || 0, + limit: Number(limit) || 0, + mapLine: (line) => { + const parts = line.trim().split(/\s+/); + return { domain: parts[0] || '', type: parts[1] || '' }; + } + }); + if (!validateAsns(items)) return res.status(500).json({ message: 'Invalid data format' }); + return res.json({ items, total }); + } else { + const data = await s3.getObject(params).promise(); + const fileContent = data.Body.toString('utf-8'); + const asns = fileContent.split('\n').filter(line => line).map(line => { + const parts = line.trim().split(/\s+/); + const domain = parts[0] || ''; + const type = parts[1] || ''; + return { domain, type }; + }); + if (!validateAsns(asns)) return res.status(500).json({ message: 'Invalid data format' }); + if (data.ETag) res.set('ETag', String(data.ETag)); + if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString()); + if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength)); + res.json(asns); + } } catch (error) { if (error.code === 'NoSuchKey') { res.json([]); @@ -195,6 +355,9 @@ app.post('/api/asns', async (req, res) => { const fileContent = (asns || []).map(a => `${String(a.domain || '').trim()} ${String(a.type || '').trim()}`.trim()).filter(Boolean).join('\n'); try { + if (!validateAsns(asns || [])) { + return res.status(400).json({ message: 'Invalid payload format for asns' }); + } if (etag) { const current = await headS3ObjectEtag('bgp_data/asns.txt').catch(() => undefined); if (current && current.replace(/\"/g, '"') !== String(etag)) { @@ -224,24 +387,41 @@ app.post('/api/asns', async (req, res) => { // Get domains-new from S3 app.get('/api/domains-new', async (req, res) => { + const { q = '', offset, limit } = req.query || {}; const params = { Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt', }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); - const domains = fileContent.split('\n').filter(line => line).map(line => { - const parts = line.trim().split(/\s+/); - const domain = parts[0] || ''; - const community = parts[1] || ''; - return { domain, community }; - }); - if (data.ETag) res.set('ETag', String(data.ETag)); - if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString()); - if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength)); - res.json(domains); + if (limit !== undefined) { + const { items, total } = await streamPaginatedText({ + key: 'bgp_data/domains_community.txt', + q, + offset: Number(offset) || 0, + limit: Number(limit) || 0, + mapLine: (line) => { + const parts = line.trim().split(/\s+/); + return { domain: parts[0] || '', community: parts[1] || '' }; + } + }); + if (!validateDomainsNew(items)) return res.status(500).json({ message: 'Invalid data format' }); + return res.json({ items, total }); + } else { + const data = await s3.getObject(params).promise(); + const fileContent = data.Body.toString('utf-8'); + const domains = fileContent.split('\n').filter(line => line).map(line => { + const parts = line.trim().split(/\s+/); + const domain = parts[0] || ''; + const community = parts[1] || ''; + return { domain, community }; + }); + if (!validateDomainsNew(domains)) return res.status(500).json({ message: 'Invalid data format' }); + if (data.ETag) res.set('ETag', String(data.ETag)); + if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString()); + if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength)); + res.json(domains); + } } catch (error) { if (error.code === 'NoSuchKey') { res.json([]); // Return empty array if file does not exist @@ -258,6 +438,9 @@ app.post('/api/domains-new', async (req, res) => { const fileContent = (domains || []).map(d => `${String(d.domain || '').trim()} ${String(d.community || '').trim()}`.trim()).filter(Boolean).join('\n'); try { + if (!validateDomainsNew(domains || [])) { + return res.status(400).json({ message: 'Invalid payload format for domains-new' }); + } if (etag) { const current = await headS3ObjectEtag('bgp_data/domains_community.txt').catch(() => undefined); if (current && current.replace(/\"/g, '"') !== String(etag)) { @@ -287,24 +470,41 @@ app.post('/api/domains-new', async (req, res) => { // Get IP ranges from S3 app.get('/api/ip-ranges', async (req, res) => { + const { q = '', offset, limit } = req.query || {}; const params = { Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt', }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); - const ipRanges = fileContent.split('\n').filter(line => line).map(line => { - const parts = line.trim().split(/\s+/); - const ipRange = parts[0] || ''; - const community = parts[1] || ''; - return { ipRange, community }; - }); - if (data.ETag) res.set('ETag', String(data.ETag)); - if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString()); - if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength)); - res.json(ipRanges); + if (limit !== undefined) { + const { items, total } = await streamPaginatedText({ + key: 'bgp_data/ips.txt', + q, + offset: Number(offset) || 0, + limit: Number(limit) || 0, + mapLine: (line) => { + const parts = line.trim().split(/\s+/); + return { ipRange: parts[0] || '', community: parts[1] || '' }; + } + }); + if (!validateIpRanges(items)) return res.status(500).json({ message: 'Invalid data format' }); + return res.json({ items, total }); + } else { + const data = await s3.getObject(params).promise(); + const fileContent = data.Body.toString('utf-8'); + const ipRanges = fileContent.split('\n').filter(line => line).map(line => { + const parts = line.trim().split(/\s+/); + const ipRange = parts[0] || ''; + const community = parts[1] || ''; + return { ipRange, community }; + }); + if (!validateIpRanges(ipRanges)) return res.status(500).json({ message: 'Invalid data format' }); + if (data.ETag) res.set('ETag', String(data.ETag)); + if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString()); + if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength)); + res.json(ipRanges); + } } catch (error) { if (error.code === 'NoSuchKey') { res.json([]); // Return empty array if file does not exist @@ -321,6 +521,9 @@ app.post('/api/ip-ranges', async (req, res) => { const fileContent = (ipRanges || []).map(ip => `${String(ip.ipRange || '').trim()} ${String(ip.community || '').trim()}`.trim()).filter(Boolean).join('\n'); try { + if (!validateIpRanges(ipRanges || [])) { + return res.status(400).json({ message: 'Invalid payload format for ip-ranges' }); + } if (etag) { const current = await headS3ObjectEtag('bgp_data/ips.txt').catch(() => undefined); if (current && current.replace(/\"/g, '"') !== String(etag)) { diff --git a/frontend/src/ASNsNewManager.jsx b/frontend/src/ASNsNewManager.jsx index 6de69a0..f2c4fba 100644 --- a/frontend/src/ASNsNewManager.jsx +++ b/frontend/src/ASNsNewManager.jsx @@ -95,8 +95,9 @@ function ASNsNewManager() { const fetchItems = async () => { setLoading(true); try { - const response = await axios.get(`${API_URL}/asns`); - const mapped = response.data.map(item => ({ asn: item.domain, community: item.type })); + const response = await axios.get(`${API_URL}/asns`, { params: { offset: 0, limit: 0 } }); + const payload = Array.isArray(response.data?.items) ? response.data.items : (Array.isArray(response.data) ? response.data : []); + const mapped = payload.map(item => ({ asn: item.domain, community: item.type })); setItems(mapped); setOriginalItems(mapped); setEtag(response.headers?.etag || ''); diff --git a/frontend/src/DomainsNewManager.jsx b/frontend/src/DomainsNewManager.jsx index 4b86894..ea98bd9 100644 --- a/frontend/src/DomainsNewManager.jsx +++ b/frontend/src/DomainsNewManager.jsx @@ -100,9 +100,10 @@ function DomainsNewManager() { const fetchItems = async () => { setLoading(true); try { - const response = await axios.get(`${API_URL}/domains-new`); - setItems(response.data); - setOriginalItems(response.data); + const response = await axios.get(`${API_URL}/domains-new`, { params: { offset: 0, limit: 0 } }); + const payload = Array.isArray(response.data?.items) ? response.data.items : (Array.isArray(response.data) ? response.data : []); + setItems(payload); + setOriginalItems(payload); setEtag(response.headers?.etag || ''); setLastModified(response.headers?.['last-modified'] || ''); const lengthHeader = response.headers?.['content-length-source']; diff --git a/frontend/src/IPRangesManager.jsx b/frontend/src/IPRangesManager.jsx index 1ea6f90..320d667 100644 --- a/frontend/src/IPRangesManager.jsx +++ b/frontend/src/IPRangesManager.jsx @@ -111,9 +111,10 @@ function IPRangesManager() { const fetchItems = async () => { setLoading(true); try { - const response = await axios.get(`${API_URL}/ip-ranges`); - setItems(response.data); - setOriginalItems(response.data); + const response = await axios.get(`${API_URL}/ip-ranges`, { params: { offset: 0, limit: 0 } }); + const payload = Array.isArray(response.data?.items) ? response.data.items : (Array.isArray(response.data) ? response.data : []); + setItems(payload); + setOriginalItems(payload); setEtag(response.headers?.etag || ''); setLastModified(response.headers?.['last-modified'] || ''); const lengthHeader = response.headers?.['content-length-source'];