feat: Enhance backend API with pagination and validation for ASNs, Domains, and IP Ranges, and update frontend managers to support new API structure for improved data handling and user experience

backend/package.json

@@ -19,7 +19,9 @@
     "aws-sdk": "^2.1692.0",
     "cors": "^2.8.5",
     "dotenv": "^17.0.1",
-    "express": "^4.19.2"
+    "express": "^4.19.2",
+    "compression": "^1.7.4",
+    "ajv": "^8.17.1"
   },
   "devDependencies": {
     "nodemon": "^3.1.10"

backend/server.js

@@ -3,12 +3,15 @@ const express = require('express');
 const AWS = require('aws-sdk');
 const cors = require('cors');
 const path = require('path');
+const compression = require('compression');
+const Ajv = require('ajv');
 
 const app = express();
 const port = 3001;
 
 app.use(cors());
 app.use(express.json());
+app.use(compression());
 
 // Serve static files from the React app
 app.use(express.static(path.join(__dirname, 'public')));
@@ -26,6 +29,95 @@ const s3 = new AWS.S3({
 const BUCKET_NAME = process.env.S3_BUCKET_NAME;
 const FILE_KEY = 'bgp_data/domains.txt';
 
+// AJV setup and schemas
+const ajv = new Ajv({ allErrors: true, removeAdditional: 'failing' });
+
+const schemaDomainsNew = {
+  type: 'array',
+  items: {
+    type: 'object',
+    required: ['domain', 'community'],
+    additionalProperties: false,
+    properties: {
+      domain: { type: 'string' },
+      community: { type: 'string' }
+    }
+  }
+};
+const schemaAsns = {
+  type: 'array',
+  items: {
+    type: 'object',
+    required: ['domain', 'type'],
+    additionalProperties: false,
+    properties: {
+      domain: { type: 'string' },
+      type: { type: 'string' }
+    }
+  }
+};
+const schemaIpRanges = {
+  type: 'array',
+  items: {
+    type: 'object',
+    required: ['ipRange', 'community'],
+    additionalProperties: false,
+    properties: {
+      ipRange: { type: 'string' },
+      community: { type: 'string' }
+    }
+  }
+};
+const schemaFilters = {
+  type: 'array',
+  items: {
+    type: 'object',
+    required: ['community', 'gateway'],
+    additionalProperties: true,
+    properties: {
+      community: { type: 'string' },
+      gateway: { type: 'string' },
+      description: { type: 'string' }
+    }
+  }
+};
+const schemaServers = {
+  type: 'array',
+  items: {
+    type: 'object',
+    required: ['ip', 'dns', 'country', 'provider', 'tunnel'],
+    additionalProperties: true,
+    properties: {
+      ip: { type: 'string' },
+      dns: { type: 'string' },
+      country: { type: 'string' },
+      provider: { type: 'string' },
+      tunnel: { type: 'string' },
+      gateway: { type: 'string' }
+    }
+  }
+};
+const schemaBilling = {
+  type: 'array',
+  items: {
+    type: 'object',
+    required: ['hostName', 'country', 'provider'],
+    additionalProperties: true,
+    properties: {
+      hostName: { type: 'string' },
+      country: { type: 'string' },
+      provider: { type: 'string' }
+    }
+  }
+};
+
+const validateDomainsNew = ajv.compile(schemaDomainsNew);
+const validateAsns = ajv.compile(schemaAsns);
+const validateIpRanges = ajv.compile(schemaIpRanges);
+const validateFilters = ajv.compile(schemaFilters);
+const validateServers = ajv.compile(schemaServers);
+const validateBilling = ajv.compile(schemaBilling);
+
 // Helper: build MikroTik nested if/else blocks (RouterOS v7 filter language does not support 'else if')
 function buildNestedGatewayBlocks(gatewayGroups, baseIndentSpaces = 4) {
     const indent = (n) => ' '.repeat(n);
@@ -73,6 +165,57 @@ async function headS3ObjectEtag(key) {
     return head.ETag ? String(head.ETag).replace(/\"/g, '"') : undefined;
 }
 
+// Helper: stream and paginate big text files (line-based)
+async function streamPaginatedText({ key, mapLine, q, offset = 0, limit = 0 }) {
+  return new Promise(async (resolve, reject) => {
+    let total = 0;
+    const items = [];
+    let sent = 0;
+    let buffered = '';
+    const matchesQuery = (line) => {
+      if (!q) return true;
+      return line.toLowerCase().includes(String(q).toLowerCase());
+    };
+    const stream = s3.getObject({ Bucket: BUCKET_NAME, Key: key }).createReadStream();
+    stream.on('data', (chunk) => {
+      buffered += chunk.toString('utf-8');
+      let lines = buffered.split('\n');
+      buffered = lines.pop();
+      for (const lnRaw of lines) {
+        const line = lnRaw.trim();
+        if (!line) continue;
+        if (!matchesQuery(line)) continue;
+        total++;
+        const pos = total - 1; // position among matches
+        if (limit > 0) {
+          if (pos >= offset && sent < limit) {
+            items.push(mapLine(line));
+            sent++;
+          }
+        } else {
+          items.push(mapLine(line));
+        }
+      }
+    });
+    stream.on('end', () => {
+      const last = (buffered || '').trim();
+      if (last) {
+        if (!q || last.toLowerCase().includes(String(q).toLowerCase())) {
+          total++;
+          if (limit > 0) {
+            const pos = total - 1;
+            if (pos >= offset && items.length < limit) items.push(mapLine(last));
+          } else {
+            items.push(mapLine(last));
+          }
+        }
+      }
+      resolve({ items, total });
+    });
+    stream.on('error', reject);
+  });
+}
+
 // Simple in-memory soft locks with TTL
 const locks = new Map(); // key -> { owner, expiresAt }
 function cleanupExpiredLocks() {
@@ -161,24 +304,41 @@ app.post('/api/domains', async (req, res) => {
 
 // Get ASNs from S3
 app.get('/api/asns', async (req, res) => {
+    const { q = '', offset, limit } = req.query || {};
     const params = {
         Bucket: BUCKET_NAME,
         Key: 'bgp_data/asns.txt',
     };
 
     try {
-        const data = await s3.getObject(params).promise();
-        const fileContent = data.Body.toString('utf-8');
-        const asns = fileContent.split('\n').filter(line => line).map(line => {
-            const parts = line.trim().split(/\s+/);
-            const domain = parts[0] || ''; // Keep name 'domain' for consistency in component
-            const type = parts[1] || '';
-            return { domain, type };
-        });
-        if (data.ETag) res.set('ETag', String(data.ETag));
-        if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString());
-        if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength));
-        res.json(asns);
+        if (limit !== undefined) {
+            const { items, total } = await streamPaginatedText({
+                key: 'bgp_data/asns.txt',
+                q,
+                offset: Number(offset) || 0,
+                limit: Number(limit) || 0,
+                mapLine: (line) => {
+                    const parts = line.trim().split(/\s+/);
+                    return { domain: parts[0] || '', type: parts[1] || '' };
+                }
+            });
+            if (!validateAsns(items)) return res.status(500).json({ message: 'Invalid data format' });
+            return res.json({ items, total });
+        } else {
+            const data = await s3.getObject(params).promise();
+            const fileContent = data.Body.toString('utf-8');
+            const asns = fileContent.split('\n').filter(line => line).map(line => {
+                const parts = line.trim().split(/\s+/);
+                const domain = parts[0] || '';
+                const type = parts[1] || '';
+                return { domain, type };
+            });
+            if (!validateAsns(asns)) return res.status(500).json({ message: 'Invalid data format' });
+            if (data.ETag) res.set('ETag', String(data.ETag));
+            if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString());
+            if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength));
+            res.json(asns);
+        }
     } catch (error) {
         if (error.code === 'NoSuchKey') {
             res.json([]);
@@ -195,6 +355,9 @@ app.post('/api/asns', async (req, res) => {
     const fileContent = (asns || []).map(a => `${String(a.domain || '').trim()} ${String(a.type || '').trim()}`.trim()).filter(Boolean).join('\n');
 
     try {
+        if (!validateAsns(asns || [])) {
+            return res.status(400).json({ message: 'Invalid payload format for asns' });
+        }
         if (etag) {
             const current = await headS3ObjectEtag('bgp_data/asns.txt').catch(() => undefined);
             if (current && current.replace(/\"/g, '"') !== String(etag)) {
@@ -224,24 +387,41 @@ app.post('/api/asns', async (req, res) => {
 
 // Get domains-new from S3
 app.get('/api/domains-new', async (req, res) => {
+    const { q = '', offset, limit } = req.query || {};
     const params = {
         Bucket: BUCKET_NAME,
         Key: 'bgp_data/domains_community.txt',
     };
 
     try {
-        const data = await s3.getObject(params).promise();
-        const fileContent = data.Body.toString('utf-8');
-        const domains = fileContent.split('\n').filter(line => line).map(line => {
-            const parts = line.trim().split(/\s+/);
-            const domain = parts[0] || '';
-            const community = parts[1] || '';
-            return { domain, community };
-        });
-        if (data.ETag) res.set('ETag', String(data.ETag));
-        if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString());
-        if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength));
-        res.json(domains);
+        if (limit !== undefined) {
+            const { items, total } = await streamPaginatedText({
+                key: 'bgp_data/domains_community.txt',
+                q,
+                offset: Number(offset) || 0,
+                limit: Number(limit) || 0,
+                mapLine: (line) => {
+                    const parts = line.trim().split(/\s+/);
+                    return { domain: parts[0] || '', community: parts[1] || '' };
+                }
+            });
+            if (!validateDomainsNew(items)) return res.status(500).json({ message: 'Invalid data format' });
+            return res.json({ items, total });
+        } else {
+            const data = await s3.getObject(params).promise();
+            const fileContent = data.Body.toString('utf-8');
+            const domains = fileContent.split('\n').filter(line => line).map(line => {
+                const parts = line.trim().split(/\s+/);
+                const domain = parts[0] || '';
+                const community = parts[1] || '';
+                return { domain, community };
+            });
+            if (!validateDomainsNew(domains)) return res.status(500).json({ message: 'Invalid data format' });
+            if (data.ETag) res.set('ETag', String(data.ETag));
+            if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString());
+            if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength));
+            res.json(domains);
+        }
     } catch (error) {
         if (error.code === 'NoSuchKey') {
             res.json([]); // Return empty array if file does not exist
@@ -258,6 +438,9 @@ app.post('/api/domains-new', async (req, res) => {
     const fileContent = (domains || []).map(d => `${String(d.domain || '').trim()} ${String(d.community || '').trim()}`.trim()).filter(Boolean).join('\n');
 
     try {
+        if (!validateDomainsNew(domains || [])) {
+            return res.status(400).json({ message: 'Invalid payload format for domains-new' });
+        }
         if (etag) {
             const current = await headS3ObjectEtag('bgp_data/domains_community.txt').catch(() => undefined);
             if (current && current.replace(/\"/g, '"') !== String(etag)) {
@@ -287,24 +470,41 @@ app.post('/api/domains-new', async (req, res) => {
 
 // Get IP ranges from S3
 app.get('/api/ip-ranges', async (req, res) => {
+    const { q = '', offset, limit } = req.query || {};
     const params = {
         Bucket: BUCKET_NAME,
         Key: 'bgp_data/ips.txt',
     };
 
     try {
-        const data = await s3.getObject(params).promise();
-        const fileContent = data.Body.toString('utf-8');
-        const ipRanges = fileContent.split('\n').filter(line => line).map(line => {
-            const parts = line.trim().split(/\s+/);
-            const ipRange = parts[0] || '';
-            const community = parts[1] || '';
-            return { ipRange, community };
-        });
-        if (data.ETag) res.set('ETag', String(data.ETag));
-        if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString());
-        if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength));
-        res.json(ipRanges);
+        if (limit !== undefined) {
+            const { items, total } = await streamPaginatedText({
+                key: 'bgp_data/ips.txt',
+                q,
+                offset: Number(offset) || 0,
+                limit: Number(limit) || 0,
+                mapLine: (line) => {
+                    const parts = line.trim().split(/\s+/);
+                    return { ipRange: parts[0] || '', community: parts[1] || '' };
+                }
+            });
+            if (!validateIpRanges(items)) return res.status(500).json({ message: 'Invalid data format' });
+            return res.json({ items, total });
+        } else {
+            const data = await s3.getObject(params).promise();
+            const fileContent = data.Body.toString('utf-8');
+            const ipRanges = fileContent.split('\n').filter(line => line).map(line => {
+                const parts = line.trim().split(/\s+/);
+                const ipRange = parts[0] || '';
+                const community = parts[1] || '';
+                return { ipRange, community };
+            });
+            if (!validateIpRanges(ipRanges)) return res.status(500).json({ message: 'Invalid data format' });
+            if (data.ETag) res.set('ETag', String(data.ETag));
+            if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString());
+            if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength));
+            res.json(ipRanges);
+        }
     } catch (error) {
         if (error.code === 'NoSuchKey') {
             res.json([]); // Return empty array if file does not exist
@@ -321,6 +521,9 @@ app.post('/api/ip-ranges', async (req, res) => {
     const fileContent = (ipRanges || []).map(ip => `${String(ip.ipRange || '').trim()} ${String(ip.community || '').trim()}`.trim()).filter(Boolean).join('\n');
 
     try {
+        if (!validateIpRanges(ipRanges || [])) {
+            return res.status(400).json({ message: 'Invalid payload format for ip-ranges' });
+        }
         if (etag) {
             const current = await headS3ObjectEtag('bgp_data/ips.txt').catch(() => undefined);
             if (current && current.replace(/\"/g, '"') !== String(etag)) {

frontend/src/ASNsNewManager.jsx

@@ -95,8 +95,9 @@ function ASNsNewManager() {
   const fetchItems = async () => {
     setLoading(true);
     try {
-      const response = await axios.get(`${API_URL}/asns`);
-      const mapped = response.data.map(item => ({ asn: item.domain, community: item.type }));
+      const response = await axios.get(`${API_URL}/asns`, { params: { offset: 0, limit: 0 } });
+      const payload = Array.isArray(response.data?.items) ? response.data.items : (Array.isArray(response.data) ? response.data : []);
+      const mapped = payload.map(item => ({ asn: item.domain, community: item.type }));
       setItems(mapped);
       setOriginalItems(mapped);
       setEtag(response.headers?.etag || '');

frontend/src/DomainsNewManager.jsx

@@ -100,9 +100,10 @@ function DomainsNewManager() {
   const fetchItems = async () => {
     setLoading(true);
     try {
-      const response = await axios.get(`${API_URL}/domains-new`);
-      setItems(response.data);
-      setOriginalItems(response.data);
+      const response = await axios.get(`${API_URL}/domains-new`, { params: { offset: 0, limit: 0 } });
+      const payload = Array.isArray(response.data?.items) ? response.data.items : (Array.isArray(response.data) ? response.data : []);
+      setItems(payload);
+      setOriginalItems(payload);
       setEtag(response.headers?.etag || '');
       setLastModified(response.headers?.['last-modified'] || '');
       const lengthHeader = response.headers?.['content-length-source'];

frontend/src/IPRangesManager.jsx

@@ -111,9 +111,10 @@ function IPRangesManager() {
   const fetchItems = async () => {
     setLoading(true);
     try {
-      const response = await axios.get(`${API_URL}/ip-ranges`);
-      setItems(response.data);
-      setOriginalItems(response.data);
+      const response = await axios.get(`${API_URL}/ip-ranges`, { params: { offset: 0, limit: 0 } });
+      const payload = Array.isArray(response.data?.items) ? response.data.items : (Array.isArray(response.data) ? response.data : []);
+      setItems(payload);
+      setOriginalItems(payload);
       setEtag(response.headers?.etag || '');
       setLastModified(response.headers?.['last-modified'] || '');
       const lengthHeader = response.headers?.['content-length-source'];