From bae2d6880351074fb611d71e012f2320053279c4 Mon Sep 17 00:00:00 2001 From: Denozordec Date: Mon, 6 Apr 2026 18:58:52 +0700 Subject: [PATCH] feat: add CSV import and export functionality for module entries. Implement endpoints for exporting and importing module entries in CSV format, supporting types AS_PREFIXES, DOMAINS, and IP_RANGES. Enhance UI with buttons for CSV operations, improving user experience in managing module data. --- docs/openapi.yaml | 68 +++++ internal/httpapi/routes_crud.go | 246 ++++++++++++++++++ internal/httpapi/routes_crud_csv_test.go | 105 ++++++++ .../routes/modules/[moduleId]/+page.svelte | 160 +++++++++++- 4 files changed, 575 insertions(+), 4 deletions(-) create mode 100644 internal/httpapi/routes_crud_csv_test.go diff --git a/docs/openapi.yaml b/docs/openapi.yaml index cdfabb8..83efebe 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1515,6 +1515,74 @@ paths: default: $ref: "#/components/responses/DefaultProblem" + /v1/modules/{module_id}/entries.csv: + parameters: + - $ref: "#/components/parameters/TenantId" + - $ref: "#/components/parameters/ModuleId" + get: + tags: [Modules] + summary: Экспорт записей модуля в CSV + description: | + Доступно для типов модулей `AS_PREFIXES`, `DOMAINS`, `IP_RANGES`. + Возвращает CSV с колонками: + - AS: `asn,community` + - Домены: `domain,community` + - IP ranges: `ipRange,community` + operationId: exportModuleEntriesCsv + responses: + "200": + description: CSV-файл записей модуля. + content: + text/csv: + schema: + type: string + "404": + $ref: "#/components/responses/NotFound" + "422": + $ref: "#/components/responses/UnprocessableEntity" + default: + $ref: "#/components/responses/DefaultProblem" + post: + tags: [Modules] + summary: Импорт записей модуля из CSV + description: | + Импортирует CSV в модуль типов `AS_PREFIXES`, `DOMAINS`, `IP_RANGES`. + Поддерживаемые заголовки: + - `asn,community` + - `domain,community` + - `ipRange,community` + + В поле `community` можно передавать либо ID community, либо её значение. + operationId: importModuleEntriesCsv + parameters: + - $ref: "#/components/parameters/IdempotencyKey" + requestBody: + required: true + content: + text/csv: + schema: + type: string + responses: + "200": + description: Импорт завершён. + content: + application/json: + schema: + type: object + required: [imported, module_type] + properties: + imported: + type: integer + minimum: 0 + module_type: + type: string + "404": + $ref: "#/components/responses/NotFound" + "422": + $ref: "#/components/responses/UnprocessableEntity" + default: + $ref: "#/components/responses/DefaultProblem" + /v1/doh-profiles: get: tags: [DoH profiles] diff --git a/internal/httpapi/routes_crud.go b/internal/httpapi/routes_crud.go index c9871fc..d852540 100644 --- a/internal/httpapi/routes_crud.go +++ b/internal/httpapi/routes_crud.go @@ -1,6 +1,7 @@ package httpapi import ( + "encoding/csv" "encoding/json" "fmt" "io" @@ -38,6 +39,8 @@ func (s *Server) registerCRUDRoutes(m *http.ServeMux) { m.HandleFunc("POST /modules/{module_id}/ip-range-entries", s.handlePostIPRange) m.HandleFunc("PATCH /modules/{module_id}/ip-range-entries/{entry_id}", s.handlePatchIPRange) m.HandleFunc("DELETE /modules/{module_id}/ip-range-entries/{entry_id}", s.handleDeleteIPRange) + m.HandleFunc("GET /modules/{module_id}/entries.csv", s.handleExportModuleEntriesCSV) + m.HandleFunc("POST /modules/{module_id}/entries.csv", s.handleImportModuleEntriesCSV) m.HandleFunc("GET /doh-profiles", s.handleListDoh) m.HandleFunc("POST /doh-profiles", s.handlePostDoh) @@ -535,6 +538,249 @@ func (s *Server) handleDeleteIPRange(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } +func (s *Server) handleExportModuleEntriesCSV(w http.ResponseWriter, r *http.Request) { + a, ok := authFromContext(r.Context()) + if !ok || !s.requireAtLeast(w, a, "viewer") { + return + } + moduleID := r.PathValue("module_id") + mod, err := s.store.GetModule(a.TenantID, moduleID) + if err != nil { + writeStoreErr(w, err) + return + } + communities, err := s.store.ListCommunities(a.TenantID) + if err != nil { + writeStoreErr(w, err) + return + } + communityByID := make(map[string]string, len(communities)) + for _, c := range communities { + communityByID[c.ID] = strings.TrimSpace(c.Community) + } + + records := make([][]string, 0, 64) + + switch mod.Type { + case "AS_PREFIXES": + records = append(records, []string{"asn", "community"}) + list, err := s.store.ListASEntries(a.TenantID, moduleID) + if err != nil { + writeStoreErr(w, err) + return + } + for _, x := range list { + community := "" + if x.CommunityID != nil { + community = communityByID[*x.CommunityID] + } + records = append(records, []string{strconv.FormatInt(x.ASN, 10), community}) + } + case "DOMAINS": + records = append(records, []string{"domain", "community"}) + list, err := s.store.ListDomainEntries(a.TenantID, moduleID) + if err != nil { + writeStoreErr(w, err) + return + } + for _, x := range list { + community := "" + if x.CommunityID != nil { + community = communityByID[*x.CommunityID] + } + records = append(records, []string{x.FQDN, community}) + } + case "IP_RANGES": + records = append(records, []string{"ipRange", "community"}) + list, err := s.store.ListIPRangeEntries(a.TenantID, moduleID) + if err != nil { + writeStoreErr(w, err) + return + } + for _, x := range list { + community := "" + if x.CommunityID != nil { + community = communityByID[*x.CommunityID] + } + records = append(records, []string{x.Prefix, community}) + } + default: + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv import/export is supported only for AS_PREFIXES, DOMAINS, IP_RANGES") + return + } + + w.Header().Set("Content-Type", "text/csv; charset=utf-8") + w.Header().Set("Content-Disposition", `attachment; filename="module-entries.csv"`) + w.WriteHeader(http.StatusOK) + cw := csv.NewWriter(w) + for _, rec := range records { + if err := cw.Write(rec); err != nil { + return + } + } + cw.Flush() +} + +func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Request) { + a, ok := authFromContext(r.Context()) + if !ok || !s.requireAtLeast(w, a, "editor") { + return + } + moduleID := r.PathValue("module_id") + mod, err := s.store.GetModule(a.TenantID, moduleID) + if err != nil { + writeStoreErr(w, err) + return + } + + cr := csv.NewReader(io.LimitReader(r.Body, 8<<20)) + cr.TrimLeadingSpace = true + cr.FieldsPerRecord = -1 + rows, err := cr.ReadAll() + if err != nil { + writeProblem(w, http.StatusBadRequest, "Bad Request", "invalid csv") + return + } + if len(rows) == 0 { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv is empty") + return + } + + communities, err := s.store.ListCommunities(a.TenantID) + if err != nil { + writeStoreErr(w, err) + return + } + communityIDByID := make(map[string]string, len(communities)) + communityIDByValue := make(map[string]string, len(communities)) + for _, c := range communities { + communityIDByID[c.ID] = c.ID + communityIDByValue[strings.TrimSpace(c.Community)] = c.ID + } + + resolveCommunity := func(raw string, required bool) (*string, error) { + v := strings.TrimSpace(raw) + if v == "" { + if required { + return nil, fmt.Errorf("community is required") + } + return nil, nil + } + if id, ok := communityIDByID[v]; ok { + return &id, nil + } + if id, ok := communityIDByValue[v]; ok { + return &id, nil + } + return nil, fmt.Errorf("unknown community %q", v) + } + + start := 0 + if len(rows[0]) >= 2 { + key := strings.ToLower(strings.TrimSpace(rows[0][0])) + switch key { + case "asn", "domain", "iprange": + start = 1 + } + } + + imported := 0 + switch mod.Type { + case "AS_PREFIXES": + for i := start; i < len(rows); i++ { + rec := rows[i] + if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) { + continue + } + if len(rec) < 2 { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: expected 2 columns", i+1)) + return + } + asn, err := strconv.ParseInt(strings.TrimSpace(rec[0]), 10, 64) + if err != nil || asn <= 0 { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: invalid asn", i+1)) + return + } + cid, err := resolveCommunity(rec[1], false) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: %v", i+1, err)) + return + } + _, err = s.store.CreateASEntry(a.TenantID, moduleID, &store.ASEntry{ASN: asn, CommunityID: cid}) + if err != nil { + writeStoreErr(w, err) + return + } + imported++ + } + case "DOMAINS": + for i := start; i < len(rows); i++ { + rec := rows[i] + if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) { + continue + } + if len(rec) < 2 { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: expected 2 columns", i+1)) + return + } + fqdn := strings.TrimSpace(rec[0]) + if fqdn == "" { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: domain is required", i+1)) + return + } + cid, err := resolveCommunity(rec[1], false) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: %v", i+1, err)) + return + } + _, err = s.store.CreateDomainEntry(a.TenantID, moduleID, &store.DomainEntry{FQDN: fqdn, CommunityID: cid}) + if err != nil { + writeStoreErr(w, err) + return + } + imported++ + } + case "IP_RANGES": + for i := start; i < len(rows); i++ { + rec := rows[i] + if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) { + continue + } + if len(rec) < 2 { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: expected 2 columns", i+1)) + return + } + prefix := strings.TrimSpace(rec[0]) + if prefix == "" { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: ipRange is required", i+1)) + return + } + cid, err := resolveCommunity(rec[1], true) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: %v", i+1, err)) + return + } + _, err = s.store.CreateIPRangeEntry(a.TenantID, moduleID, &store.IPRangeEntry{Prefix: prefix, CommunityID: cid}) + if err != nil { + writeStoreErr(w, err) + return + } + imported++ + } + if imported > 0 { + s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "ip_range_import_csv") + } + default: + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv import/export is supported only for AS_PREFIXES, DOMAINS, IP_RANGES") + return + } + + writeJSON(w, http.StatusOK, map[string]any{ + "imported": imported, + "module_type": mod.Type, + }) +} + func (s *Server) handleListDoh(w http.ResponseWriter, r *http.Request) { a, ok := authFromContext(r.Context()) if !ok || !s.requireAtLeast(w, a, "viewer") { diff --git a/internal/httpapi/routes_crud_csv_test.go b/internal/httpapi/routes_crud_csv_test.go new file mode 100644 index 0000000..2d5fb4f --- /dev/null +++ b/internal/httpapi/routes_crud_csv_test.go @@ -0,0 +1,105 @@ +package httpapi + +import ( + "encoding/csv" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestModuleEntriesCSVImportExportIPRanges(t *testing.T) { + srv, err := New(Options{ + InsecureDev: true, + SeedDemo: true, + }) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + + tenant, _, modIP, _, _ := srv.Store().DemoIDs() + srv.apiKeys = parseAPIKeysSpec("opkey|" + tenant + "|operator") + + ts := httptest.NewServer(srv.Handler()) + defer ts.Close() + client := ts.Client() + base := ts.URL + + reqList, _ := http.NewRequest(http.MethodGet, base+"/v1/communities?limit=10", nil) + reqList.Header.Set("Authorization", "Bearer opkey") + respList, err := client.Do(reqList) + if err != nil { + t.Fatal(err) + } + defer respList.Body.Close() + if respList.StatusCode != http.StatusOK { + b, _ := io.ReadAll(respList.Body) + t.Fatalf("communities status %d: %s", respList.StatusCode, b) + } + var listBody struct { + Items []struct { + Community string `json:"community"` + } `json:"items"` + } + if err := json.NewDecoder(respList.Body).Decode(&listBody); err != nil { + t.Fatal(err) + } + if len(listBody.Items) == 0 { + t.Fatal("expected seeded community") + } + + csvBody := "\"ipRange\",\"community\"\n\"10.254.1.254/32\",\"" + listBody.Items[0].Community + "\"\n\"160.79.104.0/23\",\"" + listBody.Items[0].Community + "\"\n" + reqImport, _ := http.NewRequest(http.MethodPost, base+"/v1/modules/"+modIP+"/entries.csv", strings.NewReader(csvBody)) + reqImport.Header.Set("Authorization", "Bearer opkey") + reqImport.Header.Set("Content-Type", "text/csv") + respImport, err := client.Do(reqImport) + if err != nil { + t.Fatal(err) + } + defer respImport.Body.Close() + if respImport.StatusCode != http.StatusOK { + b, _ := io.ReadAll(respImport.Body) + t.Fatalf("import status %d: %s", respImport.StatusCode, b) + } + var importBody struct { + Imported int `json:"imported"` + } + if err := json.NewDecoder(respImport.Body).Decode(&importBody); err != nil { + t.Fatal(err) + } + if importBody.Imported != 2 { + t.Fatalf("expected imported=2, got %d", importBody.Imported) + } + + reqExport, _ := http.NewRequest(http.MethodGet, base+"/v1/modules/"+modIP+"/entries.csv", nil) + reqExport.Header.Set("Authorization", "Bearer opkey") + respExport, err := client.Do(reqExport) + if err != nil { + t.Fatal(err) + } + defer respExport.Body.Close() + if respExport.StatusCode != http.StatusOK { + b, _ := io.ReadAll(respExport.Body) + t.Fatalf("export status %d: %s", respExport.StatusCode, b) + } + if got := respExport.Header.Get("Content-Type"); !strings.Contains(got, "text/csv") { + t.Fatalf("unexpected content-type: %q", got) + } + raw, err := io.ReadAll(respExport.Body) + if err != nil { + t.Fatal(err) + } + records, err := csv.NewReader(strings.NewReader(string(raw))).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(records) < 3 { + t.Fatalf("expected at least 3 csv rows, got %d", len(records)) + } + if records[0][0] != "ipRange" || records[0][1] != "community" { + t.Fatalf("unexpected header: %#v", records[0]) + } +} diff --git a/web/src/routes/modules/[moduleId]/+page.svelte b/web/src/routes/modules/[moduleId]/+page.svelte index 2d565e7..3c179e2 100644 --- a/web/src/routes/modules/[moduleId]/+page.svelte +++ b/web/src/routes/modules/[moduleId]/+page.svelte @@ -3,7 +3,7 @@ import { page } from '$app/state'; import { goto } from '$app/navigation'; import { resolve } from '$app/paths'; - import { apiJSON, apiMutate } from '$lib/api/client.js'; + import { apiFetch, apiJSON, apiMutate } from '$lib/api/client.js'; import type { ModuleRow, ModulePatch, @@ -72,6 +72,8 @@ import Trash2 from '@lucide/svelte/icons/trash-2'; import RefreshCw from '@lucide/svelte/icons/refresh-cw'; import Save from '@lucide/svelte/icons/save'; + import Upload from '@lucide/svelte/icons/upload'; + import Download from '@lucide/svelte/icons/download'; const moduleId = $derived(page.params.moduleId); @@ -132,6 +134,28 @@ // Refresh let refreshing = $state(false); + let csvImporting = $state(false); + let csvExporting = $state(false); + let csvFileInput = $state(null); + + function supportsCsvIO(type: ModuleRow['type'] | null | undefined): boolean { + return type === 'AS_PREFIXES' || type === 'DOMAINS' || type === 'IP_RANGES'; + } + + function sanitizeFilenamePart(v: string): string { + const cleaned = v + .trim() + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, '-') + .replace(/-+/g, '-') + .replace(/^[-_.]+|[-_.]+$/g, ''); + return cleaned || 'module'; + } + + async function readErrorText(res: Response): Promise { + const body = (await res.text()).trim(); + return body || `HTTP ${res.status}`; + } async function loadMod() { loadingMod = true; @@ -245,6 +269,67 @@ } } + async function exportEntriesCsv() { + if (!mod || !supportsCsvIO(mod.type) || csvExporting) return; + csvExporting = true; + try { + const res = await apiFetch(`/v1/modules/${moduleId}/entries.csv`, { + method: 'GET', + headers: { Accept: 'text/csv' } + }); + if (!res.ok) { + toast.error(await readErrorText(res)); + return; + } + const blob = await res.blob(); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + const safeModuleName = sanitizeFilenamePart(mod.name); + a.href = url; + a.download = `${safeModuleName}-${mod.type.toLowerCase()}-entries.csv`; + document.body.appendChild(a); + a.click(); + a.remove(); + URL.revokeObjectURL(url); + } catch (e) { + toast.error(e instanceof Error ? e.message : String(e)); + } finally { + csvExporting = false; + } + } + + function openImportCsvPicker() { + if (!mod || !supportsCsvIO(mod.type) || csvImporting) return; + csvFileInput?.click(); + } + + async function handleImportCsvChange(event: Event) { + const input = event.currentTarget as HTMLInputElement | null; + const file = input?.files?.[0]; + if (!mod || !supportsCsvIO(mod.type) || !file || csvImporting) return; + csvImporting = true; + try { + const fileText = await file.text(); + const res = await apiFetch(`/v1/modules/${moduleId}/entries.csv`, { + method: 'POST', + headers: { 'Content-Type': 'text/csv' }, + body: fileText + }); + if (!res.ok) { + toast.error(await readErrorText(res)); + return; + } + const payload = (await res.json()) as { imported?: number }; + toast.success(`Импортировано записей: ${payload.imported ?? 0}`); + await loadEntries(); + } catch (e) { + toast.error(e instanceof Error ? e.message : String(e)); + } finally { + csvImporting = false; + if (input) input.value = ''; + } + } + // --- AS Entries --- function openAsCreate() { asEdit = null; @@ -497,6 +582,13 @@
Загрузка…
{:else if mod}
+
@@ -564,7 +656,27 @@ Номер AS и community; имя, число префиксов и дата обновляются при успешном обновлении модуля (RIPEstat)
- +
+ + + +
@@ -672,7 +784,27 @@ ДоменыFQDN для резолвинга через DoH - +
+ + + +
@@ -711,7 +843,27 @@ IP-диапазоныСтатические CIDR для анонса - +
+ + + +