From e20c9f3113e6e933cf650f9e4ebfd2b18bd6a5f2 Mon Sep 17 00:00:00 2001 From: Denozordec Date: Mon, 6 Apr 2026 15:08:17 +0700 Subject: [PATCH] feat: implement CDN source preview functionality and enhance data handling. Add a new endpoint for previewing CDN sources, allowing users to fetch and parse CIDR prefixes from specified URLs. Update OpenAPI documentation to include new request and response schemas, and modify internal logic to support JSON parsing with prefix path traversal. Enhance UI components to accommodate new preview features, improving user experience and data management. --- .gitea/workflows/ci.yaml | 51 +++++- deploy/docker/evobgp-agent/Dockerfile | 6 +- deploy/docker/gobinary/Dockerfile | 5 +- docs/openapi.yaml | 68 ++++++++ internal/httpapi/routes_crud.go | 92 +++++++++- internal/pipeline/parse.go | 100 +++++++++++ internal/pipeline/refresh.go | 6 +- internal/repository/postgres_entities.go | 23 +-- internal/store/backend.go | 64 +++---- internal/store/memory_crud.go | 26 +-- .../000006_cdn_source_json_path.down.sql | 2 + .../000006_cdn_source_json_path.up.sql | 2 + .../000006_cdn_source_json_path.down.sql | 2 + .../sqlite/000006_cdn_source_json_path.up.sql | 2 + web/src/lib/api/types.ts | 9 + .../routes/modules/[moduleId]/+page.svelte | 159 +++++++++++++++++- 16 files changed, 543 insertions(+), 74 deletions(-) create mode 100644 migrations/postgres/000006_cdn_source_json_path.down.sql create mode 100644 migrations/postgres/000006_cdn_source_json_path.up.sql create mode 100644 migrations/sqlite/000006_cdn_source_json_path.down.sql create mode 100644 migrations/sqlite/000006_cdn_source_json_path.up.sql diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 0433e76..c41c661 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -186,8 +186,11 @@ jobs: # Docker: Go-бинарники (api, all, scheduler, ingest, render, deploy, node, agent). # Запускается если изменился Go-код или Go-Dockerfile. # Если Go-код менялся — требуем успех go-тестов; если только Dockerfile — go skipped, ОК. + # + # docker-go-prime: один раз собирает stage `deps` (go mod download) и пишет BuildKit cache + # в registry — матрица docker-go не качает модули восемь раз подряд. # --------------------------------------------------------------------------- - docker-go: + docker-go-prime: needs: [changes, go] if: >- always() && @@ -197,6 +200,49 @@ jobs: (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') && (needs.changes.outputs.go == 'true' || needs.changes.outputs.docker_go == 'true') runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Prepare image metadata + id: meta + run: | + set -euo pipefail + owner_lc="$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" + echo "owner_lc=$owner_lc" >> "$GITHUB_OUTPUT" + - name: Log in to Gitea Registry + uses: docker/login-action@v3 + with: + registry: git.shts.su + username: ${{ gitea.actor }} + password: ${{ secrets.ACTIONS_PAT || gitea.token }} + - name: Prime Go module layer (deps) + env: + OWNER_LC: ${{ steps.meta.outputs.owner_lc }} + run: | + set -euxo pipefail + WS="${{ github.workspace }}" + CACHE_REF="git.shts.su/${OWNER_LC}/evobgp-buildcache:go-buildcache" + cd "$WS" + docker buildx build \ + --platform linux/amd64 \ + --file deploy/docker/gobinary/Dockerfile \ + --target deps \ + --cache-from "type=registry,ref=${CACHE_REF}" \ + --cache-to "type=registry,ref=${CACHE_REF},mode=max" \ + "$WS" + + docker-go: + needs: [changes, go, docker-go-prime] + if: >- + always() && + needs.changes.result == 'success' && + needs.go.result != 'failure' && + needs.docker-go-prime.result == 'success' && + github.event_name == 'push' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') && + (needs.changes.outputs.go == 'true' || needs.changes.outputs.docker_go == 'true') + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -271,6 +317,7 @@ jobs: WS="${{ github.workspace }}" cd "$WS" + CACHE_REF="git.shts.su/${OWNER_LC}/evobgp-buildcache:go-buildcache" IMG="git.shts.su/${OWNER_LC}/${IMAGE}" BUILD_ARGS=() if [ -n "${BIN:-}" ]; then @@ -285,6 +332,8 @@ jobs: --platform linux/amd64 \ --file "$DF" \ "${BUILD_ARGS[@]}" \ + --cache-from "type=registry,ref=${CACHE_REF}" \ + --cache-to "type=registry,ref=${CACHE_REF},mode=max" \ --tag "${IMG}:latest" \ --tag "${IMG}:${SHORT_SHA}" \ --tag "${IMG}:sha-${{ github.sha }}" \ diff --git a/deploy/docker/evobgp-agent/Dockerfile b/deploy/docker/evobgp-agent/Dockerfile index cc44dfc..f92d4a7 100644 --- a/deploy/docker/evobgp-agent/Dockerfile +++ b/deploy/docker/evobgp-agent/Dockerfile @@ -1,8 +1,12 @@ # evobgp-agent + bird2 из репозитория Ubuntu Noble (тот же стек, что evobgp-bird2). # См. gobinary/Dockerfile — ECR Public вместо прямого pull с Docker Hub. -FROM public.ecr.aws/docker/library/golang:1.22-bookworm AS build +# Порядок слоёв как в gobinary: кэш модулей отдельно от исходников (CI/CD BuildKit). +FROM public.ecr.aws/docker/library/golang:1.22-bookworm AS deps WORKDIR /src COPY go.mod go.sum ./ +RUN go mod download + +FROM deps AS build COPY . . RUN go build -trimpath -ldflags="-s -w" -o /out/evobgp-agent ./cmd/evobgp-agent diff --git a/deploy/docker/gobinary/Dockerfile b/deploy/docker/gobinary/Dockerfile index 125d126..c1a508f 100644 --- a/deploy/docker/gobinary/Dockerfile +++ b/deploy/docker/gobinary/Dockerfile @@ -1,10 +1,13 @@ # Универсальная сборка бинаря из cmd/* (ARG BIN=evobgp-api | evobgp-all). # INSTALL_BIRDC=1 собирает BIRD 2.14 из исходников (birdc) для EVOBGP_BIRDC_SOCKET; иначе пакет Debian не используется. # Базовые образы из ECR Public (официальное зеркало library/*), чтобы CI не зависел от auth.docker.io. -FROM public.ecr.aws/docker/library/golang:1.22-bookworm AS build +# Отдельный stage для кэша модулей (CI: один раз --target deps, затем параллельные сборки с cache-from). +FROM public.ecr.aws/docker/library/golang:1.22-bookworm AS deps WORKDIR /src COPY go.mod go.sum ./ RUN go mod download + +FROM deps AS build COPY . . ARG BIN=evobgp-api RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/evobgp ./cmd/${BIN} diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e3b4186..cdfabb8 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -397,6 +397,9 @@ components: source_kind: type: string description: Формат скачанного списка / парсер. + prefix_path: + type: string + description: Путь до поля с префиксами для source_kind=json (например data.items[].cidr). community_id: type: ["string", "null"] refresh_interval_sec: @@ -413,6 +416,8 @@ components: format: uri source_kind: type: string + prefix_path: + type: string community_id: type: ["string", "null"] @@ -424,11 +429,40 @@ components: format: uri source_kind: type: string + prefix_path: + type: string community_id: type: ["string", "null"] refresh_interval_sec: type: ["integer", "null"] + CdnPreviewRequest: + type: object + required: [url, source_kind] + properties: + url: + type: string + format: uri + source_kind: + type: string + prefix_path: + type: string + + CdnPreviewResponse: + type: object + required: [items, total, truncated, source_url] + properties: + items: + type: array + items: + type: string + total: + type: integer + truncated: + type: boolean + source_url: + type: string + AsEntry: type: object required: @@ -1535,6 +1569,40 @@ paths: default: $ref: "#/components/responses/DefaultProblem" + /v1/modules/{module_id}/cdn-sources/preview: + parameters: + - $ref: "#/components/parameters/TenantId" + - $ref: "#/components/parameters/ModuleId" + post: + tags: [Modules] + summary: Предпросмотр префиксов из CDN-источника + description: > + Загружает URL, парсит как plaintext или json и возвращает список извлечённых префиксов (до 100 записей). + operationId: previewCdnSource + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CdnPreviewRequest" + responses: + "200": + description: Успешно. + content: + application/json: + schema: + $ref: "#/components/schemas/CdnPreviewResponse" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "422": + $ref: "#/components/responses/UnprocessableEntity" + default: + $ref: "#/components/responses/DefaultProblem" + /v1/doh-profiles/{id}: parameters: - $ref: "#/components/parameters/TenantId" diff --git a/internal/httpapi/routes_crud.go b/internal/httpapi/routes_crud.go index 775b9a8..c9871fc 100644 --- a/internal/httpapi/routes_crud.go +++ b/internal/httpapi/routes_crud.go @@ -2,11 +2,14 @@ package httpapi import ( "encoding/json" + "fmt" + "io" "net/http" "strconv" "strings" "time" + "evobgp/internal/pipeline" "evobgp/internal/store" ) @@ -17,6 +20,7 @@ func (s *Server) registerCRUDRoutes(m *http.ServeMux) { m.HandleFunc("GET /modules/{module_id}/cdn-sources", s.handleListCDNSources) m.HandleFunc("POST /modules/{module_id}/cdn-sources", s.handlePostCDNSource) + m.HandleFunc("POST /modules/{module_id}/cdn-sources/preview", s.handlePreviewCDNSource) m.HandleFunc("PATCH /modules/{module_id}/cdn-sources/{source_id}", s.handlePatchCDNSource) m.HandleFunc("DELETE /modules/{module_id}/cdn-sources/{source_id}", s.handleDeleteCDNSource) @@ -68,14 +72,14 @@ func (s *Server) handlePostModule(w http.ResponseWriter, r *http.Request) { return } var body struct { - Type string `json:"type"` - Name string `json:"name"` - Enabled bool `json:"enabled"` - Priority int `json:"priority"` - RefreshIntervalSec int `json:"refresh_interval_sec"` - CronExpr string `json:"cron_expr"` - DefaultCommunityID *string `json:"default_community_id"` - DohProfileID *string `json:"doh_profile_id"` + Type string `json:"type"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + Priority int `json:"priority"` + RefreshIntervalSec int `json:"refresh_interval_sec"` + CronExpr string `json:"cron_expr"` + DefaultCommunityID *string `json:"default_community_id"` + DohProfileID *string `json:"doh_profile_id"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeProblem(w, http.StatusBadRequest, "Bad Request", "invalid json") @@ -153,7 +157,7 @@ func (s *Server) handleListCDNSources(w http.ResponseWriter, r *http.Request) { } func cdnSourceJSON(x *store.CDNSource) map[string]any { - m := map[string]any{"id": x.ID, "source_kind": x.SourceKind, "url": x.URL, "etag": x.Etag} + m := map[string]any{"id": x.ID, "source_kind": x.SourceKind, "url": x.URL, "prefix_path": x.PrefixPath, "etag": x.Etag} if x.RefreshIntervalSec != nil { m["refresh_interval_sec"] = *x.RefreshIntervalSec } else { @@ -167,6 +171,76 @@ func cdnSourceJSON(x *store.CDNSource) map[string]any { return m } +func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request) { + a, ok := authFromContext(r.Context()) + if !ok || !s.requireAtLeast(w, a, "editor") { + return + } + var body struct { + URL string `json:"url"` + SourceKind string `json:"source_kind"` + PrefixPath string `json:"prefix_path"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeProblem(w, http.StatusBadRequest, "Bad Request", "invalid json") + return + } + u := strings.TrimSpace(body.URL) + if u == "" { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "url is required") + return + } + mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id")) + if err != nil { + writeStoreErr(w, err) + return + } + if mod.Type != "CDN_CIDRS" { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "module type must be CDN_CIDRS") + return + } + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, u, nil) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "invalid url") + return + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + writeProblem(w, http.StatusBadGateway, "Bad Gateway", err.Error()) + return + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + writeProblem(w, http.StatusBadGateway, "Bad Gateway", fmt.Sprintf("upstream status: %s", resp.Status)) + return + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + writeProblem(w, http.StatusBadGateway, "Bad Gateway", err.Error()) + return + } + pfxs, err := pipeline.ExtractCIDRs(string(raw), body.SourceKind, body.PrefixPath) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", err.Error()) + return + } + items := make([]string, 0, len(pfxs)) + const previewLimit = 100 + for i, p := range pfxs { + if i >= previewLimit { + break + } + items = append(items, p.String()) + } + writeJSON(w, http.StatusOK, map[string]any{ + "items": items, + "total": len(pfxs), + "truncated": len(pfxs) > previewLimit, + "source_url": u, + }) +} + func (s *Server) handlePostCDNSource(w http.ResponseWriter, r *http.Request) { a, ok := authFromContext(r.Context()) if !ok || !s.requireAtLeast(w, a, "editor") { diff --git a/internal/pipeline/parse.go b/internal/pipeline/parse.go index d1b495c..6927c2d 100644 --- a/internal/pipeline/parse.go +++ b/internal/pipeline/parse.go @@ -2,6 +2,7 @@ package pipeline import ( "bufio" + "encoding/json" "net/netip" "strings" ) @@ -31,6 +32,105 @@ func ParseCIDRLines(body string) []netip.Prefix { return out } +// ExtractCIDRs parses CIDRs from either plaintext lines or JSON payload. +// For sourceKind="json", prefixPath supports dotted traversal, with [] for arrays: +// e.g. "prefixes[]", "data.items[].cidr". +func ExtractCIDRs(body, sourceKind, prefixPath string) ([]netip.Prefix, error) { + if strings.EqualFold(strings.TrimSpace(sourceKind), "json") { + return parseCIDRsFromJSON(body, prefixPath) + } + return ParseCIDRLines(body), nil +} + +func parseCIDRsFromJSON(body, prefixPath string) ([]netip.Prefix, error) { + var root any + if err := json.Unmarshal([]byte(body), &root); err != nil { + return nil, err + } + values := jsonValuesAtPath(root, prefixPath) + seen := make(map[string]struct{}) + var out []netip.Prefix + for _, raw := range values { + pfx := parseOneCIDR(raw) + if !pfx.IsValid() { + continue + } + m := pfx.Masked() + s := m.String() + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + out = append(out, m) + } + return out, nil +} + +func jsonValuesAtPath(root any, prefixPath string) []string { + path := strings.TrimSpace(prefixPath) + if path == "" { + return flattenJSONStrings(root) + } + parts := strings.Split(path, ".") + nodes := []any{root} + for _, p := range parts { + part := strings.TrimSpace(p) + if part == "" { + continue + } + iter := strings.HasSuffix(part, "[]") + key := strings.TrimSuffix(part, "[]") + var next []any + for _, n := range nodes { + obj, ok := n.(map[string]any) + if !ok { + continue + } + child, ok := obj[key] + if !ok { + continue + } + if iter { + if arr, ok := child.([]any); ok { + next = append(next, arr...) + } + continue + } + next = append(next, child) + } + nodes = next + if len(nodes) == 0 { + return nil + } + } + var out []string + for _, n := range nodes { + out = append(out, flattenJSONStrings(n)...) + } + return out +} + +func flattenJSONStrings(v any) []string { + switch x := v.(type) { + case string: + return []string{strings.TrimSpace(x)} + case []any: + var out []string + for _, item := range x { + out = append(out, flattenJSONStrings(item)...) + } + return out + case map[string]any: + var out []string + for _, item := range x { + out = append(out, flattenJSONStrings(item)...) + } + return out + default: + return nil + } +} + func parseOneCIDR(s string) netip.Prefix { if p, err := netip.ParsePrefix(s); err == nil { return p diff --git a/internal/pipeline/refresh.go b/internal/pipeline/refresh.go index b55c9ec..7930474 100644 --- a/internal/pipeline/refresh.go +++ b/internal/pipeline/refresh.go @@ -182,7 +182,11 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli e := etag _, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, &store.CDNSourcePatch{Etag: &e}) } - for _, pfx := range ParseCIDRLines(string(body)) { + pfxs, err := ExtractCIDRs(string(body), src.SourceKind, src.PrefixPath) + if err != nil { + return nil, fmt.Errorf("cdn parse %s: %w", u, err) + } + for _, pfx := range pfxs { comm := src.CommunityID if comm == nil && mod.DefaultCommunityID != nil { c := *mod.DefaultCommunityID diff --git a/internal/repository/postgres_entities.go b/internal/repository/postgres_entities.go index 4545534..56b892e 100644 --- a/internal/repository/postgres_entities.go +++ b/internal/repository/postgres_entities.go @@ -23,7 +23,7 @@ func (p *Postgres) ListCDNSources(tenantID, moduleID string) ([]*store.CDNSource } ctx := context.Background() rows, err := p.pool.Query(ctx, ` - SELECT id::text, source_kind, url, COALESCE(etag,''), refresh_interval_sec, community_id::text + SELECT id::text, source_kind, url, COALESCE(prefix_path,''), COALESCE(etag,''), refresh_interval_sec, community_id::text FROM module_cdn_source WHERE module_id=$1 ORDER BY url`, moduleID) if err != nil { return nil, err @@ -35,7 +35,7 @@ func (p *Postgres) ListCDNSources(tenantID, moduleID string) ([]*store.CDNSource s.ModuleID = moduleID var ri *int32 var comm *string - if err := rows.Scan(&s.ID, &s.SourceKind, &s.URL, &s.Etag, &ri, &comm); err != nil { + if err := rows.Scan(&s.ID, &s.SourceKind, &s.URL, &s.PrefixPath, &s.Etag, &ri, &comm); err != nil { continue } if ri != nil { @@ -62,9 +62,9 @@ func (p *Postgres) CreateCDNSource(tenantID, moduleID string, in *store.CDNSourc ctx := context.Background() id := uuid.NewString() _, err = p.pool.Exec(ctx, ` - INSERT INTO module_cdn_source (id, module_id, source_kind, url, etag, refresh_interval_sec, community_id) - VALUES ($1,$2,$3,$4,$5,$6, NULLIF($7::uuid, '00000000-0000-0000-0000-000000000000'::uuid))`, - id, moduleID, in.SourceKind, strings.TrimSpace(in.URL), in.Etag, nullInt32Ptr(in.RefreshIntervalSec), uuidOrNilPtr(in.CommunityID)) + INSERT INTO module_cdn_source (id, module_id, source_kind, url, prefix_path, etag, refresh_interval_sec, community_id) + VALUES ($1,$2,$3,$4,$5,$6,$7, NULLIF($8::uuid, '00000000-0000-0000-0000-000000000000'::uuid))`, + id, moduleID, in.SourceKind, strings.TrimSpace(in.URL), strings.TrimSpace(in.PrefixPath), in.Etag, nullInt32Ptr(in.RefreshIntervalSec), uuidOrNilPtr(in.CommunityID)) if err != nil { return nil, err } @@ -77,8 +77,8 @@ func (p *Postgres) getCDNSource(ctx context.Context, moduleID, id string) (*stor var ri *int32 var comm *string err := p.pool.QueryRow(ctx, ` - SELECT id::text, source_kind, url, COALESCE(etag,''), refresh_interval_sec, community_id::text - FROM module_cdn_source WHERE id=$1 AND module_id=$2`, id, moduleID).Scan(&s.ID, &s.SourceKind, &s.URL, &s.Etag, &ri, &comm) + SELECT id::text, source_kind, url, COALESCE(prefix_path,''), COALESCE(etag,''), refresh_interval_sec, community_id::text + FROM module_cdn_source WHERE id=$1 AND module_id=$2`, id, moduleID).Scan(&s.ID, &s.SourceKind, &s.URL, &s.PrefixPath, &s.Etag, &ri, &comm) if err != nil { return nil, err } @@ -114,6 +114,9 @@ func (p *Postgres) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *s if patch.URL != nil { cur.URL = strings.TrimSpace(*patch.URL) } + if patch.PrefixPath != nil { + cur.PrefixPath = strings.TrimSpace(*patch.PrefixPath) + } if patch.Etag != nil { cur.Etag = *patch.Etag } @@ -130,10 +133,10 @@ func (p *Postgres) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *s } ctx := context.Background() _, err = p.pool.Exec(ctx, ` - UPDATE module_cdn_source SET source_kind=$3, url=$4, etag=$5, refresh_interval_sec=$6, - community_id=NULLIF($7::uuid, '00000000-0000-0000-0000-000000000000'::uuid), updated_at=now() + UPDATE module_cdn_source SET source_kind=$3, url=$4, prefix_path=$5, etag=$6, refresh_interval_sec=$7, + community_id=NULLIF($8::uuid, '00000000-0000-0000-0000-000000000000'::uuid), updated_at=now() WHERE id=$1 AND module_id=$2`, - sourceID, moduleID, cur.SourceKind, cur.URL, cur.Etag, nullInt32Ptr(cur.RefreshIntervalSec), uuidOrNilPtr(cur.CommunityID)) + sourceID, moduleID, cur.SourceKind, cur.URL, cur.PrefixPath, cur.Etag, nullInt32Ptr(cur.RefreshIntervalSec), uuidOrNilPtr(cur.CommunityID)) if err != nil { return nil, err } diff --git a/internal/store/backend.go b/internal/store/backend.go index 04d4136..38c197e 100644 --- a/internal/store/backend.go +++ b/internal/store/backend.go @@ -82,29 +82,31 @@ type Backend interface { // ModulePatch is a partial update for module. type ModulePatch struct { - Name *string `json:"name,omitempty"` - Enabled *bool `json:"enabled,omitempty"` - Priority *int `json:"priority,omitempty"` - RefreshIntervalSec *int `json:"refresh_interval_sec,omitempty"` - CronExpr *string `json:"cron_expr,omitempty"` - DefaultCommunityID *string `json:"default_community_id,omitempty"` - DohProfileID *string `json:"doh_profile_id,omitempty"` + Name *string `json:"name,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + Priority *int `json:"priority,omitempty"` + RefreshIntervalSec *int `json:"refresh_interval_sec,omitempty"` + CronExpr *string `json:"cron_expr,omitempty"` + DefaultCommunityID *string `json:"default_community_id,omitempty"` + DohProfileID *string `json:"doh_profile_id,omitempty"` } // CDNSource is a row under a CDN module. type CDNSource struct { - ID string `json:"id,omitempty"` - ModuleID string `json:"module_id,omitempty"` - SourceKind string `json:"source_kind"` - URL string `json:"url"` - Etag string `json:"etag"` - RefreshIntervalSec *int `json:"refresh_interval_sec"` - CommunityID *string `json:"community_id"` + ID string `json:"id,omitempty"` + ModuleID string `json:"module_id,omitempty"` + SourceKind string `json:"source_kind"` + URL string `json:"url"` + PrefixPath string `json:"prefix_path,omitempty"` + Etag string `json:"etag"` + RefreshIntervalSec *int `json:"refresh_interval_sec"` + CommunityID *string `json:"community_id"` } type CDNSourcePatch struct { SourceKind *string `json:"source_kind,omitempty"` URL *string `json:"url,omitempty"` + PrefixPath *string `json:"prefix_path,omitempty"` Etag *string `json:"etag,omitempty"` RefreshIntervalSec *int `json:"refresh_interval_sec,omitempty"` CommunityID *string `json:"community_id,omitempty"` @@ -131,10 +133,10 @@ func ValidASN(n int64) bool { } type DomainEntry struct { - ID string `json:"id,omitempty"` - ModuleID string `json:"module_id,omitempty"` - FQDN string `json:"fqdn"` - CommunityID *string `json:"community_id"` + ID string `json:"id,omitempty"` + ModuleID string `json:"module_id,omitempty"` + FQDN string `json:"fqdn"` + CommunityID *string `json:"community_id"` } type DomainEntryPatch struct { @@ -143,10 +145,10 @@ type DomainEntryPatch struct { } type IPRangeEntry struct { - ID string `json:"id,omitempty"` - ModuleID string `json:"module_id,omitempty"` - Prefix string `json:"prefix"` - CommunityID *string `json:"community_id"` + ID string `json:"id,omitempty"` + ModuleID string `json:"module_id,omitempty"` + Prefix string `json:"prefix"` + CommunityID *string `json:"community_id"` } type IPRangePatch struct { @@ -155,12 +157,12 @@ type IPRangePatch struct { } type DohProfile struct { - ID string `json:"id,omitempty"` - TenantID string `json:"tenant_id,omitempty"` - Name string `json:"name"` - URL string `json:"url"` - TimeoutMs *int `json:"timeout_ms"` - SecretRef *string `json:"vault_secret_ref"` + ID string `json:"id,omitempty"` + TenantID string `json:"tenant_id,omitempty"` + Name string `json:"name"` + URL string `json:"url"` + TimeoutMs *int `json:"timeout_ms"` + SecretRef *string `json:"vault_secret_ref"` } type DohProfilePatch struct { @@ -202,7 +204,7 @@ type SpeakerPatch struct { // PrefixRow is one materialized prefix for GET /revisions/.../prefixes. type PrefixRow struct { - Prefix string - CommunityID *string - Source string + Prefix string + CommunityID *string + Source string } diff --git a/internal/store/memory_crud.go b/internal/store/memory_crud.go index 7a35b11..93168d9 100644 --- a/internal/store/memory_crud.go +++ b/internal/store/memory_crud.go @@ -22,16 +22,16 @@ func (m *Memory) CreateModule(tenantID string, in *Module) (*Module, error) { } id := uuid.NewString() mod := &Module{ - ID: id, - TenantID: tenantID, - Type: in.Type, - Name: strings.TrimSpace(in.Name), - Enabled: in.Enabled, - Priority: in.Priority, - RefreshIntervalSec: in.RefreshIntervalSec, - CronExpr: in.CronExpr, - DefaultCommunityID: in.DefaultCommunityID, - DohProfileID: in.DohProfileID, + ID: id, + TenantID: tenantID, + Type: in.Type, + Name: strings.TrimSpace(in.Name), + Enabled: in.Enabled, + Priority: in.Priority, + RefreshIntervalSec: in.RefreshIntervalSec, + CronExpr: in.CronExpr, + DefaultCommunityID: in.DefaultCommunityID, + DohProfileID: in.DohProfileID, } m.modules[id] = mod return mod, nil @@ -139,6 +139,7 @@ func (m *Memory) CreateCDNSource(tenantID, moduleID string, in *CDNSource) (*CDN ModuleID: moduleID, SourceKind: in.SourceKind, URL: strings.TrimSpace(in.URL), + PrefixPath: strings.TrimSpace(in.PrefixPath), Etag: in.Etag, RefreshIntervalSec: in.RefreshIntervalSec, CommunityID: in.CommunityID, @@ -166,6 +167,9 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN if patch.URL != nil { s.URL = strings.TrimSpace(*patch.URL) } + if patch.PrefixPath != nil { + s.PrefixPath = strings.TrimSpace(*patch.PrefixPath) + } if patch.Etag != nil { s.Etag = *patch.Etag } @@ -654,7 +658,7 @@ func (m *Memory) CreatePeer(tenantID string, in *BGPPeer) (*BGPPeer, error) { p := &BGPPeer{ ID: id, TenantID: tenantID, SpeakerID: in.SpeakerID, Name: in.Name, Neighbor: neighbor, RemoteASN: in.RemoteASN, - Enabled: EffectivePeerEnabledOnCreate(in.Enabled, in.SessionState), + Enabled: EffectivePeerEnabledOnCreate(in.Enabled, in.SessionState), SessionState: in.SessionState, PoliciesJSON: in.PoliciesJSON, } m.peers[id] = p diff --git a/migrations/postgres/000006_cdn_source_json_path.down.sql b/migrations/postgres/000006_cdn_source_json_path.down.sql new file mode 100644 index 0000000..a453364 --- /dev/null +++ b/migrations/postgres/000006_cdn_source_json_path.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE module_cdn_source +DROP COLUMN prefix_path; diff --git a/migrations/postgres/000006_cdn_source_json_path.up.sql b/migrations/postgres/000006_cdn_source_json_path.up.sql new file mode 100644 index 0000000..354cc86 --- /dev/null +++ b/migrations/postgres/000006_cdn_source_json_path.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE module_cdn_source +ADD COLUMN prefix_path TEXT NOT NULL DEFAULT ''; diff --git a/migrations/sqlite/000006_cdn_source_json_path.down.sql b/migrations/sqlite/000006_cdn_source_json_path.down.sql new file mode 100644 index 0000000..a453364 --- /dev/null +++ b/migrations/sqlite/000006_cdn_source_json_path.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE module_cdn_source +DROP COLUMN prefix_path; diff --git a/migrations/sqlite/000006_cdn_source_json_path.up.sql b/migrations/sqlite/000006_cdn_source_json_path.up.sql new file mode 100644 index 0000000..354cc86 --- /dev/null +++ b/migrations/sqlite/000006_cdn_source_json_path.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE module_cdn_source +ADD COLUMN prefix_path TEXT NOT NULL DEFAULT ''; diff --git a/web/src/lib/api/types.ts b/web/src/lib/api/types.ts index 76c6ed7..04723cf 100644 --- a/web/src/lib/api/types.ts +++ b/web/src/lib/api/types.ts @@ -60,17 +60,26 @@ export type CdnSource = { id: string; url: string; source_kind: string; + prefix_path: string; community_id: string | null; refresh_interval_sec: number | null; }; export type CdnSourceCreate = { url: string; source_kind: string; + prefix_path?: string; community_id?: string | null; }; export type CdnSourcePatch = Partial & { refresh_interval_sec?: number | null }; export type CdnSourcesResponse = Page; +export type CdnPreviewResponse = { + items: string[]; + total: number; + truncated: boolean; + source_url: string; +}; + // ---- Domain Entries ---- export type DomainEntry = { id: string; diff --git a/web/src/routes/modules/[moduleId]/+page.svelte b/web/src/routes/modules/[moduleId]/+page.svelte index 91be897..ed0a85c 100644 --- a/web/src/routes/modules/[moduleId]/+page.svelte +++ b/web/src/routes/modules/[moduleId]/+page.svelte @@ -14,6 +14,7 @@ CdnSource, CdnSourceCreate, CdnSourcesResponse, + CdnPreviewResponse, DomainEntry, DomainEntryCreate, DomainEntriesResponse, @@ -91,9 +92,20 @@ let cdnSources = $state([]); let cdnDialog = $state(false); let cdnEdit = $state(null); - let cdnForm = $state({ url: '', source_kind: '', community_id: null }); + let cdnForm = $state({ + url: '', + source_kind: 'plaintext', + prefix_path: '', + community_id: null + }); let cdnSaving = $state(false); let cdnDeleteTarget = $state(null); + let cdnPreviewLoading = $state(false); + let cdnPreviewItems = $state([]); + let cdnPreviewTotal = $state(0); + let cdnPreviewTruncated = $state(false); + let cdnPreviewError = $state(null); + let cdnPreviewOk = $state(false); // Domain entries let domainEntries = $state([]); @@ -164,6 +176,19 @@ onMount(loadMod); + function normalizeCdnSourceKind(k: string): 'plaintext' | 'json' { + return k.trim().toLowerCase() === 'json' ? 'json' : 'plaintext'; + } + + function clearCdnPreview() { + cdnPreviewLoading = false; + cdnPreviewItems = []; + cdnPreviewTotal = 0; + cdnPreviewTruncated = false; + cdnPreviewError = null; + cdnPreviewOk = false; + } + // --- Module Actions --- async function openEditMod() { if (!mod) return; @@ -270,24 +295,77 @@ // --- CDN Sources --- function openCdnCreate() { cdnEdit = null; - cdnForm = { url: '', source_kind: '', community_id: null }; + clearCdnPreview(); + cdnForm = { url: '', source_kind: 'plaintext', prefix_path: '', community_id: null }; cdnDialog = true; } function openCdnEdit(src: CdnSource) { cdnEdit = src; - cdnForm = { url: src.url, source_kind: src.source_kind, community_id: src.community_id, refresh_interval_sec: src.refresh_interval_sec }; + clearCdnPreview(); + cdnForm = { + url: src.url, + source_kind: normalizeCdnSourceKind(src.source_kind), + prefix_path: src.prefix_path ?? '', + community_id: src.community_id, + refresh_interval_sec: src.refresh_interval_sec + }; cdnDialog = true; } + async function previewCdn() { + const urlTrim = cdnForm.url.trim(); + if (!urlTrim) { + toast.error('Укажите URL'); + return; + } + cdnPreviewLoading = true; + cdnPreviewError = null; + cdnPreviewOk = false; + try { + const res = await apiMutate( + `/v1/modules/${moduleId}/cdn-sources/preview`, + 'POST', + { + url: urlTrim, + source_kind: cdnForm.source_kind, + prefix_path: cdnForm.prefix_path?.trim() ?? '' + } + ); + cdnPreviewItems = res.items; + cdnPreviewTotal = res.total; + cdnPreviewTruncated = res.truncated; + cdnPreviewOk = true; + } catch (e) { + cdnPreviewError = e instanceof Error ? e.message : String(e); + cdnPreviewItems = []; + cdnPreviewTotal = 0; + cdnPreviewTruncated = false; + cdnPreviewOk = false; + } finally { + cdnPreviewLoading = false; + } + } async function saveCdn() { + const urlTrim = cdnForm.url.trim(); + if (!urlTrim) { + toast.error('Укажите URL'); + return; + } cdnSaving = true; try { + const body = { + ...cdnForm, + url: urlTrim, + source_kind: cdnForm.source_kind, + prefix_path: cdnForm.prefix_path?.trim() ?? '' + }; if (cdnEdit) { - await apiMutate(`/v1/modules/${moduleId}/cdn-sources/${cdnEdit.id}`, 'PATCH', cdnForm); + await apiMutate(`/v1/modules/${moduleId}/cdn-sources/${cdnEdit.id}`, 'PATCH', body); toast.success('Источник обновлён'); } else { - await apiMutate(`/v1/modules/${moduleId}/cdn-sources`, 'POST', cdnForm); + await apiMutate(`/v1/modules/${moduleId}/cdn-sources`, 'POST', body); toast.success('Источник добавлен'); } + clearCdnPreview(); cdnDialog = false; await loadEntries(); } catch (e) { @@ -559,7 +637,16 @@ {#each cdnSources as src (src.id)} {src.url} - {src.source_kind} + +
+ {normalizeCdnSourceKind(src.source_kind)} + {#if src.prefix_path?.trim()} + {src.prefix_path} + {/if} +
+ {communityLabel(src.community_id)} {src.refresh_interval_sec ? `${src.refresh_interval_sec}с` : '—'} @@ -805,8 +892,13 @@ - - + { + if (!open) clearCdnPreview(); + }} +> + {cdnEdit ? 'Редактировать источник' : 'Новый CDN-источник'} @@ -817,7 +909,34 @@
- + +
+
+ + + {#if cdnForm.source_kind === 'json' && !cdnForm.prefix_path?.trim()} +

+ Для JSON укажите путь к полям с CIDR; пустой путь может не дать префиксов. +

+ {/if}
@@ -837,6 +956,28 @@
+
+
+ + {#if cdnPreviewError} + {cdnPreviewError} + {:else if cdnPreviewOk} + + Всего: {cdnPreviewTotal}{#if cdnPreviewTruncated} + (обрезано){/if} + + {/if} +
+ {#if cdnPreviewItems.length} +
    + {#each cdnPreviewItems as item, i (`${i}-${item}`)} +
  • {item}
    • + {/each} +
+ {/if} +