diff --git a/apps/web/src/components/dashboard/dashboard-activity-timeline.tsx b/apps/web/src/components/dashboard/dashboard-activity-timeline.tsx deleted file mode 100644 index bf1d5f2..0000000 --- a/apps/web/src/components/dashboard/dashboard-activity-timeline.tsx +++ /dev/null @@ -1,102 +0,0 @@ -import { AlertTriangle, CheckCircle, Info } from 'lucide-react' - -import { DashboardFramePanel } from '@/components/dashboard/dashboard-frame-panel' -import { Badge } from '@/components/reui/badge' -import { - Timeline, - TimelineContent, - TimelineHeader, - TimelineIndicator, - TimelineItem, - TimelineSeparator, - TimelineTitle, -} from '@/components/reui/timeline' -import { cn } from '@evobgp/ui/lib/utils' - -import { recentPlatformActivity } from '@/lib/metrics' -import type { JobRow, PeerRow, RevisionRow, SpeakerRow } from '@/types/api' - -const KIND_META = { - job: { icon: Info, className: 'text-info' }, - revision: { icon: CheckCircle, className: 'text-success' }, - network: { icon: AlertTriangle, className: 'text-warning' }, -} as const - -function statusBadgeVariant(status: string) { - const s = status.toLowerCase() - if (['ok', 'success', 'completed', 'done'].includes(s)) return 'success-light' as const - if (['running', 'queued', 'pending'].includes(s)) return 'info-light' as const - if (['warning', 'mismatch'].includes(s)) return 'warning-light' as const - if (['failed', 'error', 'cancelled'].includes(s)) return 'destructive-light' as const - return 'outline' as const -} - -export function DashboardActivityTimeline({ - jobs, - revisions, - peers, - speakers, - loading, -}: { - jobs: JobRow[] - revisions: RevisionRow[] - peers: PeerRow[] - speakers: SpeakerRow[] - loading?: boolean -}) { - const items = recentPlatformActivity(jobs, revisions, peers, speakers, 6) - - return ( - - {loading ? ( -

Загрузка…

- ) : items.length === 0 ? ( -

Нет недавних событий

- ) : ( -
- - {items.map((item, index) => { - const meta = KIND_META[item.kind] - const Icon = meta.icon - return ( - - - - - - - - - - - - {item.message} - -
- - {item.statusLabel ?? item.status} - -
- - - ) - })} - -
- )} - - ) -} diff --git a/apps/web/src/components/dashboard/dashboard-frame-panel.tsx b/apps/web/src/components/dashboard/dashboard-frame-panel.tsx deleted file mode 100644 index 615865e..0000000 --- a/apps/web/src/components/dashboard/dashboard-frame-panel.tsx +++ /dev/null @@ -1,36 +0,0 @@ -import type { ReactNode } from 'react' - -import { - FrameSection, - panelCardContentFlushClassName, -} from '@/components/reui-kit' -import { cn } from '@evobgp/ui/lib/utils' - -/** Frame panel for dashboard sections. Preview: https://reui.io/docs/components/base/frame */ -export function DashboardFramePanel({ - title, - description, - actions, - children, - className, - contentClassName, -}: { - title?: string - description?: string - actions?: ReactNode - children: ReactNode - className?: string - contentClassName?: string -}) { - return ( - - {children} - - ) -} diff --git a/apps/web/src/components/dashboard/dashboard-kpi-grid.tsx b/apps/web/src/components/dashboard/dashboard-kpi-grid.tsx index 5151b5d..a488b8e 100644 --- a/apps/web/src/components/dashboard/dashboard-kpi-grid.tsx +++ b/apps/web/src/components/dashboard/dashboard-kpi-grid.tsx @@ -3,8 +3,6 @@ import { Boxes, ListChecks, Network, - ServerCog, - Share2, } from 'lucide-react' import type { ReactNode } from 'react' @@ -16,6 +14,11 @@ import type { JobRow, ModuleRow, PeerRow, SpeakerRow } from '@/types/api' type KpiCard = KpiStatItem & { icon: ReactNode } +function ratioPercent(part: number, total: number): number | undefined { + if (total <= 0) return undefined + return Math.round((part / total) * 100) +} + function buildKpis({ modules, peers, @@ -40,17 +43,34 @@ function buildKpis({ ).length const offlineSpeakers = Math.max(0, speakers.length - network.speakersOnline) const riskCount = network.peersMismatch + failedJobs + offlineSpeakers + const disabledModules = Math.max(0, modules.length - enabledModules) return [ { id: 'modules', icon: , iconClassName: 'text-primary', - value: loading ? '—' : `${enabledModules}/${modules.length || 0}`, + value: loading ? '—' : enabledModules, label: 'Модули активны', + progress: loading ? undefined : ratioPercent(enabledModules, modules.length), footer: ( - - {loading ? '…' : `${modules.length} всего`} + + {loading + ? '…' + : modules.length === 0 + ? 'нет модулей' + : disabledModules === 0 + ? 'все активны' + : `${disabledModules} выкл`} ), }, @@ -60,6 +80,7 @@ function buildKpis({ iconClassName: 'text-info', value: loading || bgpPct === null ? '—' : `${bgpPct}%`, label: 'BGP готовность', + progress: loading || bgpPct === null ? undefined : bgpPct, footer: ( {loading || bgpPct === null ? 'нет включённых пиров' - : `${network.peersEstablished} установлено`} - - ), - }, - { - id: 'peers', - icon: , - iconClassName: 'text-success', - value: loading ? '—' : `${network.peersEstablished}/${peersEnabled}`, - label: 'Пиры установлены', - footer: ( - - {loading ? '…' : `${network.peersTotal} в каталоге`} - - ), - }, - { - id: 'speakers', - icon: , - iconClassName: 'text-warning', - value: loading ? '—' : `${network.speakersOnline}/${network.speakersTotal}`, - label: 'Спикеры в сети', - footer: ( - - {loading ? '…' : 'в сети'} + : bgpPct >= 90 + ? 'сессии в норме' + : `${network.peersEstablished} установлено`} ), }, @@ -112,7 +108,7 @@ function buildKpis({ label: 'Активные задачи', footer: ( 0 ? 'info-light' : 'outline'} size="sm"> - {loading ? '…' : `${jobs.length} в выборке`} + {loading ? '…' : running > 0 ? 'выполняются' : 'очередь пуста'} ), }, diff --git a/apps/web/src/components/dashboard/dashboard-modules-grid.tsx b/apps/web/src/components/dashboard/dashboard-modules-grid.tsx deleted file mode 100644 index fedf183..0000000 --- a/apps/web/src/components/dashboard/dashboard-modules-grid.tsx +++ /dev/null @@ -1,117 +0,0 @@ -import { useMemo, useState } from 'react' -import { Link, useNavigate } from '@tanstack/react-router' -import { BoxesIcon, PlusIcon, SearchIcon } from 'lucide-react' - -import { CategoryBadge } from '@/components/category-badge' -import { DataGridNameCell } from '@/components/data-grid-cell' -import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header' -import type { FilterField, FilterQuery } from '@/components/reui/filters/filters-types' -import { - ResourcePage, - createTextFilterQuery, - type DataGridColumnDef, -} from '@/components/reui-kit' -import { Button } from '@evobgp/ui/components/button' -import { moduleTypeRu } from '@/lib/ui-labels' -import type { ModuleRow } from '@/types/api' - -const MODULE_TABS = [ - { id: 'all', label: 'Все' }, - { id: 'enabled', label: 'Вкл' }, - { id: 'disabled', label: 'Выкл' }, -] - -const filterFields: FilterField[] = [ - { - id: 'search', - label: 'Поиск', - icon: , - type: 'text', - placeholder: 'Поиск модулей…', - }, -] - -function getFilterFieldValue(item: ModuleRow, field: string): unknown { - if (field === 'search') { - return `${item.name} ${item.type} ${moduleTypeRu(item.type)}` - } - return undefined -} - -function tabFilter(item: ModuleRow, tabId: string): boolean { - if (tabId === 'enabled') return item.enabled !== false - if (tabId === 'disabled') return item.enabled === false - return true -} - -export function DashboardModulesGrid({ - modules, - isLoading = false, -}: { - modules: ModuleRow[] - isLoading?: boolean -}) { - const navigate = useNavigate() - const [filterQuery, setFilterQuery] = useState(() => - createTextFilterQuery('search'), - ) - - const columns = useMemo[]>( - () => [ - { - accessorKey: 'name', - id: 'name', - header: ({ column }) => , - cell: ({ row }) => , - minSize: 180, - meta: { headerTitle: 'Модуль' }, - }, - { - accessorKey: 'type', - id: 'type', - header: ({ column }) => , - cell: ({ row }) => {moduleTypeRu(row.original.type)}, - meta: { headerTitle: 'Тип' }, - }, - { - accessorKey: 'priority', - id: 'priority', - header: ({ column }) => , - cell: ({ row }) => ( - {row.original.priority} - ), - size: 88, - meta: { headerTitle: 'Приоритет' }, - }, - ], - [], - ) - - return ( - setFilterQuery(createTextFilterQuery('search'))} - getFilterFieldValue={getFilterFieldValue} - columns={columns} - data={modules} - getRowId={(row) => row.id} - isLoading={isLoading} - primaryAction={ - - } - onRowClick={(row) => - void navigate({ to: '/modules/$moduleId', params: { moduleId: row.id } }) - } - emptyState={{ title: 'Нет модулей по выбранным фильтрам.' }} - /> - ) -} diff --git a/apps/web/src/components/dashboard/dashboard-network-health.tsx b/apps/web/src/components/dashboard/dashboard-network-health.tsx index aa5a4dd..eefe4ff 100644 --- a/apps/web/src/components/dashboard/dashboard-network-health.tsx +++ b/apps/web/src/components/dashboard/dashboard-network-health.tsx @@ -60,8 +60,28 @@ export function DashboardNetworkHealth({ label: mode === 'peers' ? 'Утилизация пиров' : 'Спикеры в сети', percent: loading ? 0 : utilization, badge: ( - - {loading ? '…' : `${established}/${total}`} + + {loading + ? '…' + : total === 0 + ? 'нет данных' + : offline === 0 + ? mode === 'peers' + ? 'все установлены' + : 'все в сети' + : mode === 'peers' + ? `${offline} не установлены` + : `${offline} офлайн`} ), }} diff --git a/apps/web/src/components/dashboard/dashboard-recent-jobs-grid.tsx b/apps/web/src/components/dashboard/dashboard-recent-jobs-grid.tsx deleted file mode 100644 index bf9e1cb..0000000 --- a/apps/web/src/components/dashboard/dashboard-recent-jobs-grid.tsx +++ /dev/null @@ -1,62 +0,0 @@ -import { useMemo } from 'react' -import { ListTodo } from 'lucide-react' - -import { DataGridNameCell } from '@/components/data-grid-cell' -import { StatusBadge } from '@/components/status-badge' -import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header' -import { FrameDataGrid, type DataGridColumnDef } from '@/components/reui-kit' -import { jobKindRu } from '@/lib/ui-labels' -import type { JobRow } from '@/types/api' - -export function DashboardRecentJobsGrid({ - jobs, - nameById, - isLoading = false, -}: { - jobs: JobRow[] - nameById: Map - isLoading?: boolean -}) { - const data = useMemo(() => jobs.slice(0, 8), [jobs]) - - const columns = useMemo[]>( - () => [ - { - accessorKey: 'kind', - header: ({ column }) => , - cell: ({ row }) => ( - - ), - meta: { headerTitle: 'Вид' }, - }, - { - accessorKey: 'status', - header: ({ column }) => , - cell: ({ row }) => , - meta: { headerTitle: 'Статус' }, - }, - ], - [nameById], - ) - - return ( - row.job_id} - emptyTitle="Нет задач" - pagination={false} - isLoading={isLoading} - /> - ) -} diff --git a/apps/web/src/components/dashboard/dashboard-recent-revisions-grid.tsx b/apps/web/src/components/dashboard/dashboard-recent-revisions-grid.tsx deleted file mode 100644 index 0741b18..0000000 --- a/apps/web/src/components/dashboard/dashboard-recent-revisions-grid.tsx +++ /dev/null @@ -1,55 +0,0 @@ -import { useMemo } from 'react' -import { GitCommitHorizontal } from 'lucide-react' - -import { DataGridMutedCell, DataGridNameCell } from '@/components/data-grid-cell' -import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header' -import { FrameDataGrid, type DataGridColumnDef } from '@/components/reui-kit' -import type { RevisionRow } from '@/types/api' - -export function DashboardRecentRevisionsGrid({ - revisions, - isLoading = false, -}: { - revisions: RevisionRow[] - isLoading?: boolean -}) { - const data = useMemo(() => revisions.slice(0, 8), [revisions]) - - const columns = useMemo[]>( - () => [ - { - id: 'id', - accessorFn: (row) => row.id, - header: ({ column }) => , - cell: ({ row }) => ( - - ), - meta: { headerTitle: 'ID' }, - }, - { - accessorKey: 'created_at', - header: ({ column }) => , - cell: ({ row }) => ( - - {new Date(row.original.created_at).toLocaleString('ru-RU')} - - ), - meta: { headerTitle: 'Создана' }, - }, - ], - [], - ) - - return ( - row.id} - emptyTitle="Нет ревизий" - pagination={false} - isLoading={isLoading} - /> - ) -} diff --git a/apps/web/src/components/reui-kit/kpi-stat-grid.tsx b/apps/web/src/components/reui-kit/kpi-stat-grid.tsx index 439b866..7b24091 100644 --- a/apps/web/src/components/reui-kit/kpi-stat-grid.tsx +++ b/apps/web/src/components/reui-kit/kpi-stat-grid.tsx @@ -3,6 +3,7 @@ import { Link } from '@tanstack/react-router' import { Frame, FramePanel } from '@/components/reui/frame' import { Badge } from '@/components/reui/badge' +import { Progress } from '@evobgp/ui/components/progress' import { cn } from '@evobgp/ui/lib/utils' import { kpiCols } from './kpi-cols' import { IconTile } from '@/components/reui/icon-tile' @@ -29,6 +30,8 @@ export type KpiStatItem = { iconClassName?: string variant?: KpiStatVariant footer?: ReactNode + /** 0–100 completion bar under the value (stats-4). Omit to hide. */ + progress?: number } /** CFDM-compatible card shape (id required). */ @@ -48,6 +51,17 @@ const VALUE_VARIANT_CLASS: Record = { destructive: 'text-destructive', } +const PROGRESS_TONE_CLASS: Record = { + default: '', + warning: '[&_[data-slot=progress-indicator]]:bg-warning', + destructive: '[&_[data-slot=progress-indicator]]:bg-destructive', +} + +function clampProgress(value: number): number { + if (Number.isNaN(value)) return 0 + return Math.min(100, Math.max(0, value)) +} + function handleCardKeyDown(onActivate: () => void, event: KeyboardEvent) { if (event.key === 'Enter' || event.key === ' ') { event.preventDefault() @@ -111,6 +125,20 @@ function KpiStatCardBody({ item }: { item: KpiStatItem }) { > {item.value} + {item.progress !== undefined ? ( + + ) : null} {footer ? (
{footer}
) : null} diff --git a/apps/web/src/components/reui-kit/ops-dashboard.tsx b/apps/web/src/components/reui-kit/ops-dashboard.tsx index c972635..a6d7e86 100644 --- a/apps/web/src/components/reui-kit/ops-dashboard.tsx +++ b/apps/web/src/components/reui-kit/ops-dashboard.tsx @@ -12,7 +12,7 @@ interface OpsDashboardProps { /** Slot after KPI (QuickActionGrid). Preview: stats-12 · card-12 */ afterKpi?: ReactNode charts: ReactNode - queue: ReactNode + queue?: ReactNode queueTitle?: string queueDescription?: string headerActions?: ReactNode @@ -31,10 +31,12 @@ function OpsDashboardSkeleton() {
- - + +
+ + +
- ) } @@ -76,15 +78,17 @@ export function OpsDashboard({ {charts} -
-
-

{queueTitle}

- {queueDescription ? ( -

{queueDescription}

- ) : null} -
- {queue} -
+ {queue ? ( +
+
+

{queueTitle}

+ {queueDescription ? ( +

{queueDescription}

+ ) : null} +
+ {queue} +
+ ) : null} ) } diff --git a/apps/web/src/routes/_auth/dashboard.tsx b/apps/web/src/routes/_auth/dashboard.tsx index 0bf0455..a7d36db 100644 --- a/apps/web/src/routes/_auth/dashboard.tsx +++ b/apps/web/src/routes/_auth/dashboard.tsx @@ -4,24 +4,17 @@ import { RefreshCw } from 'lucide-react' import { useState } from 'react' import { Button } from '@evobgp/ui/components/button' -import { Skeleton } from '@evobgp/ui/components/skeleton' -import { DashboardActivityTimeline } from '@/components/dashboard/dashboard-activity-timeline' import { buildDashboardKpiCards } from '@/components/dashboard/dashboard-kpi-grid' -import { DashboardModulesGrid } from '@/components/dashboard/dashboard-modules-grid' import { DashboardNetworkHealth } from '@/components/dashboard/dashboard-network-health' import { DashboardOperationsBreakdown } from '@/components/dashboard/dashboard-operations-breakdown' import { DashboardQuickLinks } from '@/components/dashboard/dashboard-quick-links' -import { DashboardRecentJobsGrid } from '@/components/dashboard/dashboard-recent-jobs-grid' -import { DashboardRecentRevisionsGrid } from '@/components/dashboard/dashboard-recent-revisions-grid' import { OpsDashboard } from '@/components/reui-kit' -import { chartPanelGridClassName, dashboardMainSidebarClassName } from '@/lib/ui-surface' +import { chartPanelGridClassName } from '@/lib/ui-surface' import { - moduleNameById, overviewJobsQueryOptions, overviewModulesQueryOptions, overviewPeersQueryOptions, - overviewRevisionsQueryOptions, overviewSpeakersQueryOptions, } from '@/queries/overview' import { settingsQueryOptions } from '@/queries/settings' @@ -36,11 +29,12 @@ function parseShowQuickActions(value: unknown): boolean { } /** - * Dashboard — OpsDashboard kit (KPI → QuickActions → modules → activity/health → queue). - * Charts slot is a vertical stack: modules stay full-width; activity shares a row - * with BGP widgets only at @5xl (container), never nested 8+4 inside a 2-col parent. + * Dashboard — KPI infographic + Quick Actions + charts (no list duplicates). * @see https://reui.io/preview/base/dashboard-1 * @see https://reui.io/preview/base/stats-12 + * @see https://reui.io/preview/base/stats-4 + * @see https://reui.io/preview/base/card-12 + * @see https://reui.io/preview/base/chart-27 */ function DashboardComponent() { const [lastUpdated, setLastUpdated] = useState(null) @@ -52,14 +46,13 @@ function DashboardComponent() { overviewModulesQueryOptions(), overviewPeersQueryOptions(), overviewSpeakersQueryOptions(), - overviewRevisionsQueryOptions(), overviewJobsQueryOptions(), ], }) - const [modulesQ, peersQ, speakersQ, revisionsQ, jobsQ] = results + const [modulesQ, peersQ, speakersQ, jobsQ] = results const initialLoading = - modulesQ.isLoading || peersQ.isLoading || speakersQ.isLoading || revisionsQ.isLoading || jobsQ.isLoading + modulesQ.isLoading || peersQ.isLoading || speakersQ.isLoading || jobsQ.isLoading const refreshing = results.some((r) => r.isFetching && !r.isLoading) if (!lastUpdated && !initialLoading && results.every((r) => r.isSuccess || r.isError)) { @@ -74,11 +67,7 @@ function DashboardComponent() { const modules = modulesQ.data?.items ?? [] const peers = peersQ.data?.items ?? [] const speakers = speakersQ.data?.items ?? [] - const revisions = revisionsQ.data?.items ?? [] const jobs = jobsQ.data?.items ?? [] - const nameById = moduleNameById(modules) - - const activityLoading = refreshing && jobs.length === 0 && revisions.length === 0 const kpiCards = buildDashboardKpiCards({ modules, peers, speakers, jobs }) return ( @@ -99,38 +88,18 @@ function DashboardComponent() { isLoading={initialLoading} afterKpi={showQuickActions ? : null} charts={ - <> -
- -
-
- -
- - -
-
- - } - queueTitle="Задачи и ревизии" - queueDescription="Последние фоновые операции и история конфигураций" - queue={
- {activityLoading ? ( - - ) : ( - - )} - {activityLoading ? ( - - ) : ( - - )} + +
} /> diff --git a/apps/web/tsconfig.tsbuildinfo b/apps/web/tsconfig.tsbuildinfo index e4e8229..791d05a 100644 --- a/apps/web/tsconfig.tsbuildinfo +++ b/apps/web/tsconfig.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/main.tsx","./src/routetree.gen.ts","./src/vite-env.d.ts","./src/components/app-switcher.tsx","./src/components/category-badge.tsx","./src/components/confirm-dialog.tsx","./src/components/data-grid-cell.tsx","./src/components/empty-state.tsx","./src/components/form-drawer.tsx","./src/components/loading-button.tsx","./src/components/page-header.tsx","./src/components/page-shell.tsx","./src/components/panel-card.tsx","./src/components/query-state.tsx","./src/components/segmented-tabs.tsx","./src/components/select-field.tsx","./src/components/skeletons.tsx","./src/components/status-badge.tsx","./src/components/status-toggle-group.tsx","./src/components/theme-provider.tsx","./src/components/truncated-text.tsx","./src/components/access/access-api-keys-card.tsx","./src/components/access/access-api-keys-grid.tsx","./src/components/access/api-key-create-dialog.tsx","./src/components/access/api-key-token-dialog.tsx","./src/components/analytics/analytics-activity-list.tsx","./src/components/analytics/analytics-card-shell.tsx","./src/components/analytics/analytics-kpi-row.tsx","./src/components/analytics/analytics-progress.tsx","./src/components/analytics/analytics-segment-control.tsx","./src/components/analytics/chart-bar-strip.tsx","./src/components/analytics/chart-donut-metric.tsx","./src/components/analytics/dashboard-network-capacity-card.tsx","./src/components/analytics/dashboard-operations-flow-card.tsx","./src/components/analytics/dashboard-platform-card.tsx","./src/components/analytics/index.ts","./src/components/analytics/monitoring-health-card.tsx","./src/components/analytics/network-overview-analytics-card.tsx","./src/components/dashboard/card-dot-field.tsx","./src/components/dashboard/dashboard-activity-timeline.tsx","./src/components/dashboard/dashboard-frame-panel.tsx","./src/components/dashboard/dashboard-kpi-grid.tsx","./src/components/dashboard/dashboard-kpi-sparkline-row.tsx","./src/components/dashboard/dashboard-modules-grid.tsx","./src/components/dashboard/dashboard-network-health.tsx","./src/components/dashboard/dashboard-network-panel.tsx","./src/components/dashboard/dashboard-operations-breakdown.tsx","./src/components/dashboard/dashboard-quick-links.tsx","./src/components/dashboard/dashboard-recent-jobs-grid.tsx","./src/components/dashboard/dashboard-recent-revisions-grid.tsx","./src/components/directories/community-create-dialog.tsx","./src/components/directories/directories-communities-grid.tsx","./src/components/directories/directories-doh-grid.tsx","./src/components/directories/directories-row-actions.tsx","./src/components/directories/doh-profile-create-dialog.tsx","./src/components/layout/app-shell.tsx","./src/components/layout/apps-menu.tsx","./src/components/layout/command-palette.tsx","./src/components/layout/nav-user.tsx","./src/components/layout/system-monitor-popover.tsx","./src/components/lookup/lookup-add-step.tsx","./src/components/lookup/lookup-matches-grid.tsx","./src/components/lookup/lookup-search-form.tsx","./src/components/lookup/lookup-status-banner.tsx","./src/components/lookup/lookup-wizard.tsx","./src/components/modules/community-select.tsx","./src/components/modules/module-as-entry-dialog.tsx","./src/components/modules/module-cdn-source-dialog.tsx","./src/components/modules/module-create-dialog.tsx","./src/components/modules/module-domain-entry-dialog.tsx","./src/components/modules/module-edit-dialog.tsx","./src/components/modules/module-entries-grid.tsx","./src/components/modules/module-entries-section.tsx","./src/components/modules/module-ip-range-entry-dialog.tsx","./src/components/modules/module-kpi-cards.tsx","./src/components/modules/modules-list-grid.tsx","./src/components/monitoring/monitoring-ready-grid.tsx","./src/components/network/network-discovered-peers-card.tsx","./src/components/network/network-kpi.tsx","./src/components/network/network-peers-card.tsx","./src/components/network/network-peers-grid.tsx","./src/components/network/network-speakers-card.tsx","./src/components/network/network-speakers-grid.tsx","./src/components/network/peer-form-dialog.tsx","./src/components/network/speaker-form-dialog.tsx","./src/components/operations/operations-jobs-card.tsx","./src/components/operations/operations-jobs-grid.tsx","./src/components/operations/operations-jobs-kanban.tsx","./src/components/operations/operations-revisions-grid.tsx","./src/components/patterns/donut-breakdown-card.tsx","./src/components/patterns/illustrated-empty-state.tsx","./src/components/patterns/index.ts","./src/components/patterns/kpi-sparkline-card.tsx","./src/components/patterns/metric-tone-styles.ts","./src/components/patterns/panel-corners.tsx","./src/components/patterns/projects-empty-state.tsx","./src/components/patterns/segmented-progress-card.tsx","./src/components/reui/alert.tsx","./src/components/reui/autocomplete.tsx","./src/components/reui/badge.tsx","./src/components/reui/date-selector.tsx","./src/components/reui/frame.tsx","./src/components/reui/icon-stack.tsx","./src/components/reui/icon-tile.tsx","./src/components/reui/kanban.tsx","./src/components/reui/number-field.tsx","./src/components/reui/rating.tsx","./src/components/reui/sortable.tsx","./src/components/reui/stepper.tsx","./src/components/reui/timeline.tsx","./src/components/reui/cascader/cascader-async.tsx","./src/components/reui/cascader/cascader-columns.tsx","./src/components/reui/cascader/cascader-context.tsx","./src/components/reui/cascader/cascader-footer.tsx","./src/components/reui/cascader/cascader-i18n.tsx","./src/components/reui/cascader/cascader-item.tsx","./src/components/reui/cascader/cascader-lib.tsx","./src/components/reui/cascader/cascader-nav.tsx","./src/components/reui/cascader/cascader-types.tsx","./src/components/reui/cascader/cascader-virtual.tsx","./src/components/reui/cascader/cascader.tsx","./src/components/reui/data-grid/data-grid-column-filter.tsx","./src/components/reui/data-grid/data-grid-column-header.tsx","./src/components/reui/data-grid/data-grid-column-visibility.tsx","./src/components/reui/data-grid/data-grid-pagination.tsx","./src/components/reui/data-grid/data-grid-scroll-area.tsx","./src/components/reui/data-grid/data-grid-table-dnd-rows.tsx","./src/components/reui/data-grid/data-grid-table-dnd.tsx","./src/components/reui/data-grid/data-grid-table-virtual.tsx","./src/components/reui/data-grid/data-grid-table.tsx","./src/components/reui/data-grid/data-grid.tsx","./src/components/reui/filters/filters-advanced.tsx","./src/components/reui/filters/filters-builder.tsx","./src/components/reui/filters/filters-chip.tsx","./src/components/reui/filters/filters-context.tsx","./src/components/reui/filters/filters-date.tsx","./src/components/reui/filters/filters-dnd.tsx","./src/components/reui/filters/filters-draft.tsx","./src/components/reui/filters/filters-editors.tsx","./src/components/reui/filters/filters-i18n.tsx","./src/components/reui/filters/filters-lib.tsx","./src/components/reui/filters/filters-operators.tsx","./src/components/reui/filters/filters-query.tsx","./src/components/reui/filters/filters-types.tsx","./src/components/reui/filters/filters.tsx","./src/components/reui-kit/data-grid-kit-defaults.test.ts","./src/components/reui-kit/detail-panel.tsx","./src/components/reui-kit/expandable-resource-grid.tsx","./src/components/reui-kit/filter-utils.ts","./src/components/reui-kit/frame-data-grid.tsx","./src/components/reui-kit/frame-section.tsx","./src/components/reui-kit/index.ts","./src/components/reui-kit/kpi-cols.ts","./src/components/reui-kit/kpi-stat-grid.tsx","./src/components/reui-kit/ops-dashboard.tsx","./src/components/reui-kit/quick-action-grid.tsx","./src/components/reui-kit/resource-page.tsx","./src/components/reui-kit/settings-shell.tsx","./src/components/schedule/schedule-jobs-card.tsx","./src/components/schedule/schedule-jobs-grid.tsx","./src/components/schedule/schedule-modules-grid.tsx","./src/components/settings/appearance-settings-tab.tsx","./src/components/settings/bird-settings-tab.tsx","./src/components/settings/connection-settings-tab.tsx","./src/components/settings/sections-settings-tab.tsx","./src/components/settings/session-settings-tab.tsx","./src/components/settings/setting-row.tsx","./src/components/settings/settings-card.tsx","./src/components/settings/settings-field-group.tsx","./src/components/settings/settings-kv-grid.tsx","./src/components/settings/settings-page-shell.tsx","./src/components/settings/settings-setting-field.tsx","./src/components/settings/settings-tabs-data.tsx","./src/hooks/use-app-switcher.ts","./src/hooks/use-copy-to-clipboard.ts","./src/hooks/use-file-upload.ts","./src/hooks/use-mobile.ts","./src/lib/api-client.test.ts","./src/lib/api-client.ts","./src/lib/app-switcher-config.ts","./src/lib/auth.ts","./src/lib/data-grid-defaults.ts","./src/lib/filters-i18n.ts","./src/lib/queryclient.ts","./src/lib/router.ts","./src/lib/ui-labels.ts","./src/lib/ui-surface.ts","./src/lib/access/api-key-labels.ts","./src/lib/metrics/deployment-progress.ts","./src/lib/metrics/index.ts","./src/lib/metrics/job-status-breakdown.ts","./src/lib/metrics/module-type-breakdown.ts","./src/lib/metrics/peer-capacity-bars.ts","./src/lib/metrics/peer-session-breakdown.ts","./src/lib/metrics/readiness-breakdown.test.ts","./src/lib/metrics/readiness-breakdown.ts","./src/lib/metrics/recent-platform-activity.ts","./src/lib/metrics/types.ts","./src/lib/modules/display.ts","./src/lib/modules/helpers.ts","./src/queries/api-keys.ts","./src/queries/app-switcher.ts","./src/queries/auth.ts","./src/queries/directories.ts","./src/queries/lookup.ts","./src/queries/modules.ts","./src/queries/monitoring.ts","./src/queries/network.ts","./src/queries/operations.ts","./src/queries/overview.ts","./src/queries/settings.ts","./src/routes/__root.tsx","./src/routes/_auth.tsx","./src/routes/auth.callback.tsx","./src/routes/index.tsx","./src/routes/_auth/_settings.tsx","./src/routes/_auth/access.tsx","./src/routes/_auth/dashboard.tsx","./src/routes/_auth/directories.tsx","./src/routes/_auth/lookup.tsx","./src/routes/_auth/monitoring.tsx","./src/routes/_auth/network.tsx","./src/routes/_auth/operations.tsx","./src/routes/_auth/schedule.tsx","./src/routes/_auth/_settings/settings.tsx","./src/routes/_auth/_settings/tenant-settings.tsx","./src/routes/_auth/modules/$moduleid.tsx","./src/routes/_auth/modules/index.tsx","./src/routes/_auth/modules/new.tsx","./src/types/api.gen.ts","./src/types/api.ts","./vite.config.ts"],"version":"5.9.3"} \ No newline at end of file +{"root":["./src/main.tsx","./src/routetree.gen.ts","./src/vite-env.d.ts","./src/components/app-switcher.tsx","./src/components/category-badge.tsx","./src/components/confirm-dialog.tsx","./src/components/data-grid-cell.tsx","./src/components/empty-state.tsx","./src/components/form-drawer.tsx","./src/components/loading-button.tsx","./src/components/page-header.tsx","./src/components/page-shell.tsx","./src/components/panel-card.tsx","./src/components/query-state.tsx","./src/components/segmented-tabs.tsx","./src/components/select-field.tsx","./src/components/skeletons.tsx","./src/components/status-badge.tsx","./src/components/status-toggle-group.tsx","./src/components/theme-provider.tsx","./src/components/truncated-text.tsx","./src/components/access/access-api-keys-card.tsx","./src/components/access/access-api-keys-grid.tsx","./src/components/access/api-key-create-dialog.tsx","./src/components/access/api-key-token-dialog.tsx","./src/components/analytics/analytics-activity-list.tsx","./src/components/analytics/analytics-card-shell.tsx","./src/components/analytics/analytics-kpi-row.tsx","./src/components/analytics/analytics-progress.tsx","./src/components/analytics/analytics-segment-control.tsx","./src/components/analytics/chart-bar-strip.tsx","./src/components/analytics/chart-donut-metric.tsx","./src/components/analytics/dashboard-network-capacity-card.tsx","./src/components/analytics/dashboard-operations-flow-card.tsx","./src/components/analytics/dashboard-platform-card.tsx","./src/components/analytics/index.ts","./src/components/analytics/monitoring-health-card.tsx","./src/components/analytics/network-overview-analytics-card.tsx","./src/components/dashboard/card-dot-field.tsx","./src/components/dashboard/dashboard-kpi-grid.tsx","./src/components/dashboard/dashboard-kpi-sparkline-row.tsx","./src/components/dashboard/dashboard-network-health.tsx","./src/components/dashboard/dashboard-network-panel.tsx","./src/components/dashboard/dashboard-operations-breakdown.tsx","./src/components/dashboard/dashboard-quick-links.tsx","./src/components/directories/community-create-dialog.tsx","./src/components/directories/directories-communities-grid.tsx","./src/components/directories/directories-doh-grid.tsx","./src/components/directories/directories-row-actions.tsx","./src/components/directories/doh-profile-create-dialog.tsx","./src/components/layout/app-shell.tsx","./src/components/layout/apps-menu.tsx","./src/components/layout/command-palette.tsx","./src/components/layout/nav-user.tsx","./src/components/layout/system-monitor-popover.tsx","./src/components/lookup/lookup-add-step.tsx","./src/components/lookup/lookup-matches-grid.tsx","./src/components/lookup/lookup-search-form.tsx","./src/components/lookup/lookup-status-banner.tsx","./src/components/lookup/lookup-wizard.tsx","./src/components/modules/community-select.tsx","./src/components/modules/module-as-entry-dialog.tsx","./src/components/modules/module-cdn-source-dialog.tsx","./src/components/modules/module-create-dialog.tsx","./src/components/modules/module-domain-entry-dialog.tsx","./src/components/modules/module-edit-dialog.tsx","./src/components/modules/module-entries-grid.tsx","./src/components/modules/module-entries-section.tsx","./src/components/modules/module-ip-range-entry-dialog.tsx","./src/components/modules/module-kpi-cards.tsx","./src/components/modules/modules-list-grid.tsx","./src/components/monitoring/monitoring-ready-grid.tsx","./src/components/network/network-discovered-peers-card.tsx","./src/components/network/network-kpi.tsx","./src/components/network/network-peers-card.tsx","./src/components/network/network-peers-grid.tsx","./src/components/network/network-speakers-card.tsx","./src/components/network/network-speakers-grid.tsx","./src/components/network/peer-form-dialog.tsx","./src/components/network/speaker-form-dialog.tsx","./src/components/operations/operations-jobs-card.tsx","./src/components/operations/operations-jobs-grid.tsx","./src/components/operations/operations-jobs-kanban.tsx","./src/components/operations/operations-revisions-grid.tsx","./src/components/patterns/donut-breakdown-card.tsx","./src/components/patterns/illustrated-empty-state.tsx","./src/components/patterns/index.ts","./src/components/patterns/kpi-sparkline-card.tsx","./src/components/patterns/metric-tone-styles.ts","./src/components/patterns/panel-corners.tsx","./src/components/patterns/projects-empty-state.tsx","./src/components/patterns/segmented-progress-card.tsx","./src/components/reui/alert.tsx","./src/components/reui/autocomplete.tsx","./src/components/reui/badge.tsx","./src/components/reui/date-selector.tsx","./src/components/reui/frame.tsx","./src/components/reui/icon-stack.tsx","./src/components/reui/icon-tile.tsx","./src/components/reui/kanban.tsx","./src/components/reui/number-field.tsx","./src/components/reui/rating.tsx","./src/components/reui/sortable.tsx","./src/components/reui/stepper.tsx","./src/components/reui/timeline.tsx","./src/components/reui/cascader/cascader-async.tsx","./src/components/reui/cascader/cascader-columns.tsx","./src/components/reui/cascader/cascader-context.tsx","./src/components/reui/cascader/cascader-footer.tsx","./src/components/reui/cascader/cascader-i18n.tsx","./src/components/reui/cascader/cascader-item.tsx","./src/components/reui/cascader/cascader-lib.tsx","./src/components/reui/cascader/cascader-nav.tsx","./src/components/reui/cascader/cascader-types.tsx","./src/components/reui/cascader/cascader-virtual.tsx","./src/components/reui/cascader/cascader.tsx","./src/components/reui/data-grid/data-grid-column-filter.tsx","./src/components/reui/data-grid/data-grid-column-header.tsx","./src/components/reui/data-grid/data-grid-column-visibility.tsx","./src/components/reui/data-grid/data-grid-pagination.tsx","./src/components/reui/data-grid/data-grid-scroll-area.tsx","./src/components/reui/data-grid/data-grid-table-dnd-rows.tsx","./src/components/reui/data-grid/data-grid-table-dnd.tsx","./src/components/reui/data-grid/data-grid-table-virtual.tsx","./src/components/reui/data-grid/data-grid-table.tsx","./src/components/reui/data-grid/data-grid.tsx","./src/components/reui/filters/filters-advanced.tsx","./src/components/reui/filters/filters-builder.tsx","./src/components/reui/filters/filters-chip.tsx","./src/components/reui/filters/filters-context.tsx","./src/components/reui/filters/filters-date.tsx","./src/components/reui/filters/filters-dnd.tsx","./src/components/reui/filters/filters-draft.tsx","./src/components/reui/filters/filters-editors.tsx","./src/components/reui/filters/filters-i18n.tsx","./src/components/reui/filters/filters-lib.tsx","./src/components/reui/filters/filters-operators.tsx","./src/components/reui/filters/filters-query.tsx","./src/components/reui/filters/filters-types.tsx","./src/components/reui/filters/filters.tsx","./src/components/reui-kit/data-grid-kit-defaults.test.ts","./src/components/reui-kit/detail-panel.tsx","./src/components/reui-kit/expandable-resource-grid.tsx","./src/components/reui-kit/filter-utils.ts","./src/components/reui-kit/frame-data-grid.tsx","./src/components/reui-kit/frame-section.tsx","./src/components/reui-kit/index.ts","./src/components/reui-kit/kpi-cols.ts","./src/components/reui-kit/kpi-stat-grid.tsx","./src/components/reui-kit/ops-dashboard.tsx","./src/components/reui-kit/quick-action-grid.tsx","./src/components/reui-kit/resource-page.tsx","./src/components/reui-kit/settings-shell.tsx","./src/components/schedule/schedule-jobs-card.tsx","./src/components/schedule/schedule-jobs-grid.tsx","./src/components/schedule/schedule-modules-grid.tsx","./src/components/settings/appearance-settings-tab.tsx","./src/components/settings/bird-settings-tab.tsx","./src/components/settings/connection-settings-tab.tsx","./src/components/settings/sections-settings-tab.tsx","./src/components/settings/session-settings-tab.tsx","./src/components/settings/setting-row.tsx","./src/components/settings/settings-card.tsx","./src/components/settings/settings-field-group.tsx","./src/components/settings/settings-kv-grid.tsx","./src/components/settings/settings-page-shell.tsx","./src/components/settings/settings-setting-field.tsx","./src/components/settings/settings-tabs-data.tsx","./src/hooks/use-app-switcher.ts","./src/hooks/use-copy-to-clipboard.ts","./src/hooks/use-file-upload.ts","./src/hooks/use-mobile.ts","./src/lib/api-client.test.ts","./src/lib/api-client.ts","./src/lib/app-switcher-config.ts","./src/lib/auth.ts","./src/lib/data-grid-defaults.ts","./src/lib/filters-i18n.ts","./src/lib/queryclient.ts","./src/lib/router.ts","./src/lib/ui-labels.ts","./src/lib/ui-surface.ts","./src/lib/access/api-key-labels.ts","./src/lib/metrics/deployment-progress.ts","./src/lib/metrics/index.ts","./src/lib/metrics/job-status-breakdown.ts","./src/lib/metrics/module-type-breakdown.ts","./src/lib/metrics/peer-capacity-bars.ts","./src/lib/metrics/peer-session-breakdown.ts","./src/lib/metrics/readiness-breakdown.test.ts","./src/lib/metrics/readiness-breakdown.ts","./src/lib/metrics/recent-platform-activity.ts","./src/lib/metrics/types.ts","./src/lib/modules/display.ts","./src/lib/modules/helpers.ts","./src/queries/api-keys.ts","./src/queries/app-switcher.ts","./src/queries/auth.ts","./src/queries/directories.ts","./src/queries/lookup.ts","./src/queries/modules.ts","./src/queries/monitoring.ts","./src/queries/network.ts","./src/queries/operations.ts","./src/queries/overview.ts","./src/queries/settings.ts","./src/routes/__root.tsx","./src/routes/_auth.tsx","./src/routes/auth.callback.tsx","./src/routes/index.tsx","./src/routes/_auth/_settings.tsx","./src/routes/_auth/access.tsx","./src/routes/_auth/dashboard.tsx","./src/routes/_auth/directories.tsx","./src/routes/_auth/lookup.tsx","./src/routes/_auth/monitoring.tsx","./src/routes/_auth/network.tsx","./src/routes/_auth/operations.tsx","./src/routes/_auth/schedule.tsx","./src/routes/_auth/_settings/settings.tsx","./src/routes/_auth/_settings/tenant-settings.tsx","./src/routes/_auth/modules/$moduleid.tsx","./src/routes/_auth/modules/index.tsx","./src/routes/_auth/modules/new.tsx","./src/types/api.gen.ts","./src/types/api.ts","./vite.config.ts"],"version":"5.9.3"} \ No newline at end of file diff --git a/docs/access.md b/docs/access.md index cd53174..dce0ddb 100644 --- a/docs/access.md +++ b/docs/access.md @@ -168,6 +168,16 @@ http://localhost:5173,http://127.0.0.1:5173,https://ui.example.com Разрешённые заголовки включают `Authorization`, `Content-Type`, `Idempotency-Key`, `Accept`, `X-Tenant-Id` (см. `internal/httpapi/cors.go`). +## Pipeline: TTL и внешние вызовы + +| Переменная | Назначение | +|------------|------------| +| `EVOBGP_ASN_CACHE_TTL_SEC` | TTL кэша объявленных префиксов RIPEstat (default 1800) | +| `EVOBGP_ASN_HOLDER_TTL_SEC` | TTL имени holder AS (default 7 суток) | +| `EVOBGP_CDN_DNS_CACHE_TTL_SEC` | TTL DNS при проверке CDN URL (default 300) | +| `EVOBGP_DOMAIN_CACHE_TTL_SEC` | TTL кэша DoH A/AAAA (default 300) | +| `EVOBGP_CDN_PARTIAL_OK` | Сохранять prior-строки skipped CDN-источников при частичном сбое | + ## Заголовок `X-Tenant-Id` (решение: не реализован) **Решение (done):** заголовок **`X-Tenant-Id` не переключает tenant** в handlers и **не планируется** без отдельного ADR на супер-роли. diff --git a/docs/architecture.md b/docs/architecture.md index bde0cbe..043d9af 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -39,7 +39,7 @@ | `config` | Переменные окружения `EVOBGP_*`. | | `observability` | Метрики Prometheus, HTTP middleware. | | `broker` | Опциональный `EVOBGP_BROKER_URL` для будущей шины; сейчас задачи только in-process (`jobs.Registry`), пакет лишь логирует факт настройки URL. | -| `pipeline` | Ingest+render в одном шаге для `module_refresh`: выборка префиксов (CDN/AS/IP/пустые DOMAINS), `CreateRenderRevision`, превью BIRD через `birdfmt`. | +| `pipeline` | Ingest+render для `module_refresh`: CDN/AS/IP/DOMAINS → `module_prefix_snapshot` (batch `COPY`, per-module lock) → агрегация CIDR O(n log n) → `CreateRenderRevision`. Fast-path снапшота — `module.input_hash`; DoH — TTL-кэш `domain_resolve_cache`; scheduler — jitter границ интервала. | | `nodedispatch` | Panel→Node HTTP wake-up (`POST /v1/agent/sync`) после `deploy_apply`. | | `agentserver` | HTTP API на реплике (`serve`): sync + health для Traefik; опционально firewall failover (`/v1/firewall/*`). | | `firewall` | Вычисление policy block/accept → плоский CIDR blocklist. | @@ -98,6 +98,17 @@ flowchart LR Профиль Compose **`microvps-full`** добавляет к этому стеку **Web UI** (nginx → `evobgp-all`), **NATS** и **Prometheus** без отдельных контейнеров воркеров (функционально то же, что отдельные `scheduler`/`ingest`/… в reference). Запуск и лимиты под ~1 ГиБ RAM — в [quickstart.md](quickstart.md). +## Поток pipeline (ingest → render) + +1. **Scheduler** (`evobgp-scheduler` / in-process в `evobgp-all`) ставит `tenant_refresh`, если `ModuleDueForScheduler`: граница окна `refresh_interval_sec` со **сдвигом `fnv32(module.ID) % interval`**, чтобы модули с одним интервалом не били внешние API одновременно. +2. **Ingest** (`RefreshModuleIngest`): + - `AS_PREFIXES` — RIPEstat prefixes + holder параллельно, кэш `asn_prefix_cache`; дедуп строк по `prefix + community`. + - `CDN_CIDRS` — единый `fetchCDNSourceRows` (conditional GET); prefetch уважает `RefreshIntervalSec`; merge снапшота под `LockModuleSnapshot` (пропущенные по ошибке источники сохраняют prior-строки при `EVOBGP_CDN_PARTIAL_OK`). + - `DOMAINS` — DoH A+AAAA параллельно; попадания в `domain_resolve_cache` с TTL `EVOBGP_DOMAIN_CACHE_TTL_SEC` (default 300). + - `IP_RANGES` — напрямую из записей модуля. +3. Снапшот пишется **batch** (`pgx.CopyFrom` в PostgreSQL). Совпадение `module.input_hash` со снапшотом — O(1) пропуск повторного ingest при render; CRUD entries обнуляет hash. +4. **Render** (`RenderTenantRevision`): `smartAggregatePrefixRows` (стек-схлопывание O(n log n), IPv6 без `math/big`) → ревизия, если набор префиксов изменился. + ## Поток: ревизия и бандл для ноды 1. Оператор (роль `operator` или выше по политике) изменяет модули и запускает цепочку, приводящую к новой **ревизии** (часть шагов может быть асинхронной через jobs — см. OpenAPI). diff --git a/docs/production-checklist.md b/docs/production-checklist.md index 8a3bf00..a17475f 100644 --- a/docs/production-checklist.md +++ b/docs/production-checklist.md @@ -12,6 +12,9 @@ - `EVOBGP_CORS_ORIGINS` — явный whitelist origin веб-панели. - `EVOBGP_STALE_ON_UPSTREAM_ERROR=1` (по умолчанию) — stale snapshot при сбоях CDN/ASN/DoH. - Опционально `EVOBGP_CDN_PARTIAL_OK=1` — при сбое одного CDN source без stale cache продолжать refresh остальных (иначе fail модуля). +- `EVOBGP_ASN_HOLDER_TTL_SEC` — TTL имени holder AS (default 7 суток); prefixes кэшируются отдельно (`EVOBGP_ASN_CACHE_TTL_SEC`, default 1800). +- `EVOBGP_CDN_DNS_CACHE_TTL_SEC` — TTL кэша DNS при SSRF-проверке CDN URL (default 300). +- `EVOBGP_DOMAIN_CACHE_TTL_SEC` — TTL кэша DoH A/AAAA (`domain_resolve_cache`, default 300). ## Рекомендуется diff --git a/internal/asnresolve/ripestat.go b/internal/asnresolve/ripestat.go index 9debe2f..74e2cce 100644 --- a/internal/asnresolve/ripestat.go +++ b/internal/asnresolve/ripestat.go @@ -10,9 +10,7 @@ import ( "net/netip" "os" "sort" - "strconv" "strings" - "time" "evobgp/internal/httpclient" ) @@ -137,22 +135,3 @@ func truncateForErr(b []byte, n int) string { } return s } - -// PolitePause is a short delay between upstream ASN lookups (same refresh). -func PolitePause() { - d := 150 * time.Millisecond - if s := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE_PAUSE_MS")); s != "" { - if ms, err := parsePositiveInt(s); err == nil && ms > 0 { - d = time.Duration(ms) * time.Millisecond - } - } - time.Sleep(d) -} - -func parsePositiveInt(s string) (int, error) { - n, err := strconv.Atoi(s) - if err != nil || n <= 0 { - return 0, fmt.Errorf("invalid") - } - return n, nil -} diff --git a/internal/pipeline/aggregate_collapse.go b/internal/pipeline/aggregate_collapse.go new file mode 100644 index 0000000..40f4282 --- /dev/null +++ b/internal/pipeline/aggregate_collapse.go @@ -0,0 +1,173 @@ +package pipeline + +import ( + "net/netip" + "sort" + + "evobgp/internal/store" +) + +// Prefix collapse replaces the former prune+merge fixed-point loop (O(n²) per pass, +// multiple passes) with a single sort + linear stack pass: O(n log n) overall. +// +// Classic trie-collapse without building a trie: +// 1. Sort prefixes by (address, mask len ascending). +// 2. Walk left to right keeping a stack of "open" prefixes: +// - while the stack top contains the new prefix -> the top covers it, drop the new one (prune); +// - else, while the stack top is a sibling of the new prefix (same mask, XOR of the +// network bit equals the parent block) and merging is allowed for that mask length +// -> pop the sibling, replace the new prefix with the parent and re-check; +// - otherwise push the new prefix. +// +// A parent absorbs its sibling children only when both halves are present, which matches +// the previous merge-to-fixed-point semantics, including the guard that forbids merging +// above a floor mask (IPv4 /8, IPv6 /16). + +type collapseItem struct { + row store.PrefixRow + pfx netip.Prefix +} + +// collapsePrefixGroup collapses one (community, source) group of same-family prefixes. +// All rows must be masked; family and floor are enforced by minBits. +func collapsePrefixGroup(rows []store.PrefixRow, is4 bool) []store.PrefixRow { + if len(rows) <= 1 { + return rows + } + items := make([]collapseItem, 0, len(rows)) + seen := make(map[string]struct{}, len(rows)) + for _, row := range rows { + if _, dup := seen[row.Prefix]; dup { + continue + } + seen[row.Prefix] = struct{}{} + pfx, err := netip.ParsePrefix(row.Prefix) + if err != nil { + continue + } + pfx = pfx.Masked() + if is4 != pfx.Addr().Is4() { + continue + } + items = append(items, collapseItem{row: row, pfx: pfx}) + } + if len(items) <= 1 { + out := make([]store.PrefixRow, 0, len(items)) + for _, it := range items { + out = append(out, it.row) + } + return out + } + + // Sort by (address, mask len): a parent always sorts before its children, and a + // shorter sibling sorts before a longer one within the same parent block. + sort.Slice(items, func(i, j int) bool { + a, b := items[i].pfx, items[j].pfx + if a.Addr() != b.Addr() { + return lessAddr(a.Addr(), b.Addr()) + } + return a.Bits() < b.Bits() + }) + + minBits := 8 + if !is4 { + minBits = 16 + } + + stack := make([]collapseItem, 0, len(items)) + for _, it := range items { + cur := it + dropped := false + for len(stack) > 0 { + top := stack[len(stack)-1] + if top.pfx.Contains(cur.pfx.Addr()) && top.pfx.Bits() <= cur.pfx.Bits() { + // Covered by an existing prefix: drop (prune). + dropped = true + break + } + if cur.pfx.Bits() == top.pfx.Bits() && cur.pfx.Bits() > minBits && areSiblings(top.pfx, cur.pfx) { + // Merge siblings into the parent (parent keeps the lower sibling's attributes), + // then re-check the parent against the new stack top. + stack = stack[:len(stack)-1] + parentBits := cur.pfx.Bits() - 1 + parent := netip.PrefixFrom(maskAddr(cur.pfx.Addr(), parentBits, is4), parentBits).Masked() + cur = collapseItem{row: top.row, pfx: parent} + cur.row.Prefix = parent.String() + continue + } + break + } + if !dropped { + stack = append(stack, cur) + } + } + + out := make([]store.PrefixRow, 0, len(stack)) + for _, it := range stack { + out = append(out, it.row) + } + sortPrefixRows(out) + return out +} + +// areSiblings reports whether two same-length prefixes combine into their common parent. +func areSiblings(a, b netip.Prefix) bool { + if a.Bits() != b.Bits() || a.Bits() == 0 { + return false + } + parentBits := a.Bits() - 1 + pa := maskAddr(a.Addr(), parentBits, a.Addr().Is4()) + pb := maskAddr(b.Addr(), parentBits, b.Addr().Is4()) + return pa == pb +} + +// maskAddr clears the host bits below prefixLen (IPv4: 32-bit space; IPv6: 128-bit). +func maskAddr(a netip.Addr, prefixLen int, is4 bool) netip.Addr { + if is4 { + v := uint32FromIPv4(a) + if prefixLen <= 0 { + v = 0 + } else if prefixLen < 32 { + v &= ^(uint32(1)<<(32-prefixLen) - 1) + } + return u32ToIPv4(v) + } + b := a.As16() + hostBits := 128 - prefixLen + fullBytes := hostBits / 8 + for i := 15; i > 15-fullBytes; i-- { + b[i] = 0 + } + if rem := hostBits % 8; rem > 0 { + idx := 15 - fullBytes + if idx >= 0 && idx < 16 { + b[idx] &= byte(0xFF << rem) + } + } + return netip.AddrFrom16(b) +} + +func uint32FromIPv4(a netip.Addr) uint32 { + o := a.As4() + return uint32(o[0])<<24 | uint32(o[1])<<16 | uint32(o[2])<<8 | uint32(o[3]) +} + +func u32ToIPv4(v uint32) netip.Addr { + return netip.AddrFrom4([4]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}) +} + +func lessAddr(a, b netip.Addr) bool { + if a.Is4() != b.Is4() { + return a.Is4() + } + if a.Is4() { + return uint32FromIPv4(a) < uint32FromIPv4(b) + } + a16, b16 := a.As16(), b.As16() + for i := 0; i < 16; i++ { + if a16[i] != b16[i] { + return a16[i] < b16[i] + } + } + return false +} diff --git a/internal/pipeline/aggregate_collapse_test.go b/internal/pipeline/aggregate_collapse_test.go new file mode 100644 index 0000000..89e5cd3 --- /dev/null +++ b/internal/pipeline/aggregate_collapse_test.go @@ -0,0 +1,294 @@ +package pipeline + +import ( + "fmt" + "math/big" + "math/rand" + "net/netip" + "sort" + "strings" + "testing" + + "evobgp/internal/store" +) + +// The functions below are a verbatim copy of the pre-2.1 prune+merge loop +// (from git HEAD internal/pipeline/refresh.go). They exist only as an +// equivalence oracle for collapsePrefixGroup. + +func legacyAggregateCIDRGroup(rows []store.PrefixRow, mergeFn func(map[string]store.PrefixRow) bool) []store.PrefixRow { + if len(rows) <= 1 { + return rows + } + set := make(map[string]store.PrefixRow, len(rows)) + for _, row := range rows { + set[row.Prefix] = row + } + legacyPruneCoveredPrefixes(set) + for { + if !mergeFn(set) { + break + } + legacyPruneCoveredPrefixes(set) + } + out := make([]store.PrefixRow, 0, len(set)) + for _, row := range set { + out = append(out, row) + } + sortPrefixRows(out) + return out +} + +func legacyPruneCoveredPrefixes(set map[string]store.PrefixRow) { + type item struct { + key string + pfx netip.Prefix + bits int + } + items := make([]item, 0, len(set)) + for k := range set { + p, err := netip.ParsePrefix(k) + if err != nil { + continue + } + items = append(items, item{key: k, pfx: p, bits: p.Bits()}) + } + sort.Slice(items, func(i, j int) bool { + if items[i].bits != items[j].bits { + return items[i].bits < items[j].bits + } + return items[i].key < items[j].key + }) + for i := 0; i < len(items); i++ { + for j := i + 1; j < len(items); j++ { + if items[j].bits <= items[i].bits { + continue + } + if items[i].pfx.Contains(items[j].pfx.Addr()) { + delete(set, items[j].key) + } + } + } +} + +func legacyMergeSiblingPrefixesIPv4(set map[string]store.PrefixRow) bool { + merged := false + seen := make(map[string]struct{}, len(set)) + for key, row := range set { + if _, done := seen[key]; done { + continue + } + pfx, err := netip.ParsePrefix(key) + if err != nil || !pfx.Addr().Is4() { + continue + } + bits := pfx.Bits() + if bits <= 8 { + continue + } + netNum := ipv4PrefixNetworkLegacy(pfx) + blockSize := uint32(1) << (32 - bits) + siblingNet := netNum ^ blockSize + siblingPfx := netip.PrefixFrom(u32ToIPv4(siblingNet), bits).Masked().String() + if _, ok := set[siblingPfx]; !ok { + continue + } + parentBits := bits - 1 + parentBlock := uint32(1) << (32 - parentBits) + parentNet := netNum & ^(parentBlock - 1) + parentPfx := netip.PrefixFrom(u32ToIPv4(parentNet), parentBits).Masked().String() + delete(set, key) + delete(set, siblingPfx) + parentRow := row + parentRow.Prefix = parentPfx + set[parentPfx] = parentRow + seen[key] = struct{}{} + seen[siblingPfx] = struct{}{} + merged = true + } + return merged +} + +func ipv4PrefixNetworkLegacy(p netip.Prefix) uint32 { + a := p.Masked().Addr().As4() + return uint32(a[0])<<24 | uint32(a[1])<<16 | uint32(a[2])<<8 | uint32(a[3]) +} + +func legacyMergeSiblingPrefixesIPv6(set map[string]store.PrefixRow) bool { + merged := false + seen := make(map[string]struct{}, len(set)) + for key, row := range set { + if _, done := seen[key]; done { + continue + } + pfx, err := netip.ParsePrefix(key) + if err != nil || !pfx.Addr().Is6() { + continue + } + bits := pfx.Bits() + if bits <= 16 { + continue + } + netNum := ipv6PrefixNetworkLegacy(pfx) + blockSize := new(big.Int).Lsh(big.NewInt(1), uint(128-bits)) + siblingNet := new(big.Int).Xor(netNum, blockSize) + siblingPfx := ipv6PrefixFromBigIntLegacy(siblingNet, bits).String() + if _, ok := set[siblingPfx]; !ok { + continue + } + parentBits := bits - 1 + parentBlock := new(big.Int).Lsh(big.NewInt(1), uint(128-parentBits)) + mask := new(big.Int).Sub(parentBlock, big.NewInt(1)) + mask.Not(mask) + parentNet := new(big.Int).And(netNum, mask) + parentPfx := ipv6PrefixFromBigIntLegacy(parentNet, parentBits).String() + delete(set, key) + delete(set, siblingPfx) + parentRow := row + parentRow.Prefix = parentPfx + set[parentPfx] = parentRow + seen[key] = struct{}{} + seen[siblingPfx] = struct{}{} + merged = true + } + return merged +} + +func ipv6PrefixNetworkLegacy(p netip.Prefix) *big.Int { + a := p.Masked().Addr().As16() + n := new(big.Int) + n.SetBytes(a[:]) + return n +} + +func ipv6PrefixFromBigIntLegacy(n *big.Int, bits int) netip.Prefix { + b := n.Bytes() + var a [16]byte + copy(a[16-len(b):], b) + return netip.PrefixFrom(netip.AddrFrom16(a), bits).Masked() +} + +func normalizeForCompare(rows []store.PrefixRow) []string { + type line struct{ p, c, s string } + lines := make([]line, 0, len(rows)) + for _, r := range rows { + p := strings.TrimSpace(r.Prefix) + if p != "" { + if pfx, err := netip.ParsePrefix(p); err == nil { + p = pfx.Masked().String() + } + } + lines = append(lines, line{p, prefixRowCommunity(r), r.Source}) + } + sort.Slice(lines, func(i, j int) bool { + if lines[i].p != lines[j].p { + return lines[i].p < lines[j].p + } + if lines[i].c != lines[j].c { + return lines[i].c < lines[j].c + } + return lines[i].s < lines[j].s + }) + out := make([]string, 0, len(lines)) + for _, l := range lines { + out = append(out, fmt.Sprintf("%s|%s|%s", l.p, l.c, l.s)) + } + return out +} + +func TestCollapsePrefixGroup_EquivalenceWithLegacy(t *testing.T) { + r := rand.New(rand.NewSource(7)) + for iter := 0; iter < 200; iter++ { + n := 1 + r.Intn(60) + v4rows := make([]store.PrefixRow, 0, n) + for i := 0; i < n; i++ { + addr := netip.AddrFrom4([4]byte{203, byte(r.Intn(4)), byte(r.Intn(256)), byte(r.Intn(256))}) + bits := 16 + r.Intn(9) + pfx := netip.PrefixFrom(addr, bits).Masked() + v4rows = append(v4rows, store.PrefixRow{Prefix: pfx.String(), Source: "ip_range"}) + } + legacy := legacyAggregateCIDRGroup(append([]store.PrefixRow(nil), v4rows...), legacyMergeSiblingPrefixesIPv4) + got := collapsePrefixGroup(append([]store.PrefixRow(nil), v4rows...), true) + if fmt.Sprint(normalizeForCompare(legacy)) != fmt.Sprint(normalizeForCompare(got)) { + t.Fatalf("iter %d mismatch:\nlegacy=%v\ngot =%v", iter, normalizeForCompare(legacy), normalizeForCompare(got)) + } + } +} + +func TestCollapsePrefixGroup_EquivalenceWithLegacyIPv6(t *testing.T) { + r := rand.New(rand.NewSource(11)) + for iter := 0; iter < 200; iter++ { + n := 1 + r.Intn(60) + rows := make([]store.PrefixRow, 0, n) + for i := 0; i < n; i++ { + var a [16]byte + a[0], a[1] = 0x20, 0x01 + a[2], a[3] = 0x0d, 0xb8 + a[4] = byte(r.Intn(2)) + a[5] = byte(r.Intn(256)) + a[6] = byte(r.Intn(256)) + addr := netip.AddrFrom16(a) + bits := 32 + r.Intn(17) + pfx := netip.PrefixFrom(addr, bits).Masked() + rows = append(rows, store.PrefixRow{Prefix: pfx.String(), Source: "ip_range"}) + } + legacy := legacyAggregateCIDRGroup(append([]store.PrefixRow(nil), rows...), legacyMergeSiblingPrefixesIPv6) + got := collapsePrefixGroup(append([]store.PrefixRow(nil), rows...), false) + if fmt.Sprint(normalizeForCompare(legacy)) != fmt.Sprint(normalizeForCompare(got)) { + t.Fatalf("iter %d mismatch:\nlegacy=%v\ngot =%v", iter, normalizeForCompare(legacy), normalizeForCompare(got)) + } + } +} + +func TestCollapsePrefixGroup_CoversAndSiblingChain(t *testing.T) { + rows := []store.PrefixRow{ + {Prefix: "10.0.0.0/16", Source: "ip_range"}, + {Prefix: "10.0.0.0/24", Source: "ip_range"}, + {Prefix: "10.0.1.0/24", Source: "ip_range"}, + {Prefix: "10.0.2.0/24", Source: "ip_range"}, + {Prefix: "10.0.3.0/24", Source: "ip_range"}, + {Prefix: "10.1.0.0/24", Source: "ip_range"}, + } + got := collapsePrefixGroup(rows, true) + if len(got) != 2 { + t.Fatalf("want 2 rows (/16 + /24), got %d: %+v", len(got), got) + } + set := map[string]bool{} + for _, r := range got { + set[r.Prefix] = true + } + if !set["10.0.0.0/16"] || !set["10.1.0.0/24"] { + t.Fatalf("unexpected rows: %+v", got) + } +} + +func TestCollapsePrefixGroup_RespectsMinBitsFloor(t *testing.T) { + rows := []store.PrefixRow{ + {Prefix: "10.0.0.0/8", Source: "ip_range"}, + {Prefix: "11.0.0.0/8", Source: "ip_range"}, + } + got := collapsePrefixGroup(rows, true) + if len(got) != 2 { + t.Fatalf("floor must prevent /8+/8 -> /7, got %+v", got) + } + + rows9 := []store.PrefixRow{ + {Prefix: "10.0.0.0/9", Source: "ip_range"}, + {Prefix: "10.128.0.0/9", Source: "ip_range"}, + } + got9 := collapsePrefixGroup(rows9, true) + if len(got9) != 1 || got9[0].Prefix != "10.0.0.0/8" { + t.Fatalf("expected /9+/9 -> /8, got %+v", got9) + } +} + +func TestCollapsePrefixGroup_PrunesCovered(t *testing.T) { + rows := []store.PrefixRow{ + {Prefix: "10.0.0.0/16", Source: "ip_range"}, + {Prefix: "10.0.0.0/24", Source: "ip_range"}, + } + got := collapsePrefixGroup(rows, true) + if len(got) != 1 || got[0].Prefix != "10.0.0.0/16" { + t.Fatalf("want covered prune to /16, got %+v", got) + } +} diff --git a/internal/pipeline/asn_cache.go b/internal/pipeline/asn_cache.go index 3115197..1e446fe 100644 --- a/internal/pipeline/asn_cache.go +++ b/internal/pipeline/asn_cache.go @@ -24,28 +24,43 @@ func asnCacheTTL() time.Duration { return time.Duration(sec) * time.Second } -// resolveASNForEntry fetches prefixes and holder with shared TTL cache (asn_prefix_cache). -func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client, asn int64) ([]netip.Prefix, string, error) { - ttl := asnCacheTTL() - if st != nil { - if ent, ok, err := st.GetASNPrefixCache(asn); err == nil && ok && ent != nil && time.Since(ent.FetchedAt) < ttl { - out := make([]netip.Prefix, 0, len(ent.Prefixes)) - for _, p := range ent.Prefixes { - pfx, perr := netip.ParsePrefix(strings.TrimSpace(p)) - if perr != nil { - continue - } - out = append(out, pfx.Masked()) - } - return out, ent.Holder, nil +// asnHolderTTL is how long a holder name stays authoritative between refreshes; +// holder text changes rarely, so it survives short prefix-cache TTLs. +func asnHolderTTL() time.Duration { + sec := 7 * 24 * 3600 + if s := strings.TrimSpace(os.Getenv("EVOBGP_ASN_HOLDER_TTL_SEC")); s != "" { + if v, err := strconv.Atoi(s); err == nil && v > 0 { + sec = v } } + return time.Duration(sec) * time.Second +} + +// resolveASNForEntry fetches prefixes and holder with shared TTL cache (asn_prefix_cache). +// The holder lookup starts concurrently with the prefix fetch (one RTT instead of two); +// a holder failure is non-fatal — the previously cached holder name is kept. +func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client, asn int64) ([]netip.Prefix, string, error) { + ttl := asnCacheTTL() + var prevHolder string + if st != nil { + if ent, ok, err := st.GetASNPrefixCache(asn); err == nil && ok && ent != nil { + prevHolder = ent.Holder + if time.Since(ent.FetchedAt) < ttl { + return parseASNCachePrefixes(ent.Prefixes), ent.Holder, nil + } + } + } + + holderCh := startASNHolderFetch(ctx, hc, st, asn, prevHolder) + pfxs, err := asnresolve.AnnouncedPrefixes(ctx, hc, asn) if err != nil { + holder := <-holderCh // drain to avoid leaking the goroutine's channel send + _ = holder return nil, "", err } - asnresolve.PolitePause() - holder, _ := asnresolve.ASHolderName(ctx, hc, asn) + + holder := <-holderCh if st != nil { strs := make([]string, len(pfxs)) for i, p := range pfxs { @@ -57,3 +72,55 @@ func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client, } return pfxs, holder, nil } + +// startASNHolderFetch launches the holder lookup concurrently. The returned buffered +// channel always yields exactly one value, so callers may abandon it without leaking. +func startASNHolderFetch(ctx context.Context, hc *http.Client, st store.Backend, asn int64, prevHolder string) <-chan string { + ch := make(chan string, 1) + go func() { + if prevHolder != "" && holderStillFresh(st, asn, prevHolder) { + ch <- prevHolder + return + } + holder, err := asnresolve.ASHolderName(ctx, hc, asn) + if err != nil || strings.TrimSpace(holder) == "" { + ch <- prevHolder + return + } + ch <- strings.TrimSpace(holder) + }() + return ch +} + +// holderStillFresh reports whether the cached holder name is within its own (long) TTL. +func holderStillFresh(st store.Backend, asn int64, prevHolder string) bool { + if st == nil { + return false + } + ent, ok, err := st.GetASNPrefixCache(asn) + return err == nil && ok && ent != nil && ent.Holder == prevHolder && time.Since(ent.FetchedAt) < asnHolderTTL() +} + +func parseASNCachePrefixes(raw []string) []netip.Prefix { + out := make([]netip.Prefix, 0, len(raw)) + for _, p := range raw { + pfx, perr := netip.ParsePrefix(strings.TrimSpace(p)) + if perr != nil { + continue + } + out = append(out, pfx.Masked()) + } + return out +} + +func parseCachedASNCachedPrefixes(raw []string) []netip.Prefix { + out := make([]netip.Prefix, 0, len(raw)) + for _, p := range raw { + pfx, perr := netip.ParsePrefix(strings.TrimSpace(p)) + if perr != nil { + continue + } + out = append(out, pfx.Masked()) + } + return out +} diff --git a/internal/pipeline/asn_cache_test.go b/internal/pipeline/asn_cache_test.go index aa1ebd1..db11bf1 100644 --- a/internal/pipeline/asn_cache_test.go +++ b/internal/pipeline/asn_cache_test.go @@ -51,13 +51,92 @@ func TestResolveASNForEntry_UsesTTLCache(t *testing.T) { } } +func TestResolveASNForEntry_HolderFailureIsNonFatal(t *testing.T) { + m := store.NewMemory() + t.Setenv("EVOBGP_ASN_CACHE_TTL_SEC", "1") + + var holderCalls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/announced") { + _, _ = w.Write([]byte(`{"status":"ok","data":{"prefixes":[{"prefix":"203.0.113.0/24"}]}}`)) + return + } + holderCalls.Add(1) + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + t.Setenv("EVOBGP_RIPESTAT_ANNOUNCED_PREFIXES_URL", srv.URL+"/announced") + t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", srv.URL+"/overview") + + ctx := context.Background() + hc := srv.Client() + pfxs, holder, err := resolveASNForEntry(ctx, m, hc, 64512) + if err != nil { + t.Fatalf("holder failure must not fail ingest: %v", err) + } + if len(pfxs) != 1 { + t.Fatalf("expected 1 prefix, got %+v", pfxs) + } + if holder != "" { + t.Fatalf("expected empty holder on upstream failure, got %q", holder) + } + if holderCalls.Load() == 0 { + t.Fatal("expected holder endpoint to be attempted") + } +} + +func TestResolveASNForEntry_KeepsPreviousHolderOnFailure(t *testing.T) { + m := store.NewMemory() + t.Setenv("EVOBGP_ASN_CACHE_TTL_SEC", "1") + t.Setenv("EVOBGP_ASN_HOLDER_TTL_SEC", "3600") + + holderOK := atomic.Bool{} + holderOK.Store(true) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/announced") { + _, _ = w.Write([]byte(`{"status":"ok","data":{"prefixes":[{"prefix":"203.0.113.0/24"}]}}`)) + return + } + if holderOK.Load() { + _, _ = w.Write([]byte(`{"status":"ok","data":{"holder":"Good AS"}}`)) + return + } + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + t.Setenv("EVOBGP_RIPESTAT_ANNOUNCED_PREFIXES_URL", srv.URL+"/announced") + t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", srv.URL+"/overview") + + ctx := context.Background() + hc := srv.Client() + if _, h, err := resolveASNForEntry(ctx, m, hc, 64512); err != nil || h != "Good AS" { + t.Fatalf("first resolve: holder=%q err=%v", h, err) + } + + // Prefix cache expired; holder endpoint now broken — previous holder must survive. + holderOK.Store(false) + time.Sleep(1100 * time.Millisecond) + pfxs, h, err := resolveASNForEntry(ctx, m, hc, 64512) + if err != nil { + t.Fatal(err) + } + if len(pfxs) != 1 { + t.Fatalf("expected 1 prefix, got %+v", pfxs) + } + if h != "Good AS" { + t.Fatalf("expected previous holder preserved, got %q", h) + } +} + func TestModuleDueForScheduler_BucketRollover(t *testing.T) { - mod := &store.Module{Enabled: true, Type: "CDN_CIDRS", RefreshIntervalSec: 300} - boundary := time.Unix(300, 0) + mod := &store.Module{ID: "mod-jitter", Enabled: true, Type: "CDN_CIDRS", RefreshIntervalSec: 300} + win := int64(300) + off := moduleSchedulerOffset(mod.ID, win) + boundary := time.Unix(win+off, 0) if !ModuleDueForScheduler(mod, boundary) { t.Fatal("expected due when refresh bucket rolls") } - mid := time.Unix(330, 0) + mid := time.Unix(win+off+SchedulerTickSec, 0) if ModuleDueForScheduler(mod, mid) { t.Fatal("expected not due within same bucket") } diff --git a/internal/pipeline/baseline_bench_test.go b/internal/pipeline/baseline_bench_test.go new file mode 100644 index 0000000..b95f0e3 --- /dev/null +++ b/internal/pipeline/baseline_bench_test.go @@ -0,0 +1,56 @@ +package pipeline + +import ( + "fmt" + "math/rand" + "net/netip" + "testing" + + "evobgp/internal/store" +) + +// benchPrefixRows generates seed-stable pseudo-random prefixes for aggregation benchmarks. +// v4 blocks are carved from TEST-NET-style ranges; v6 from 2001:db8::/32. +func benchPrefixRows(n int, v6Share float64) []store.PrefixRow { + r := rand.New(rand.NewSource(42)) + rows := make([]store.PrefixRow, 0, n) + seen := make(map[string]struct{}, n) + for len(rows) < n { + var pfx netip.Prefix + if r.Float64() < v6Share { + var a [16]byte + a[0], a[1] = 0x20, 0x01 + a[2], a[3] = 0x0d, 0xb8 + for i := 4; i < 10; i++ { + a[i] = byte(r.Intn(256)) + } + addr := netip.AddrFrom16(a) + pfx = netip.PrefixFrom(addr, 48+r.Intn(17)) + } else { + b := []byte{203, 0, 113, 0} + b[1] = byte(r.Intn(256)) + b[2] = byte(r.Intn(256)) + b[3] = byte(r.Intn(256)) + addr := netip.AddrFrom4([4]byte{b[0], b[1], b[2], b[3]}) + pfx = netip.PrefixFrom(addr, 16+r.Intn(9)) + } + s := pfx.Masked().String() + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + rows = append(rows, store.PrefixRow{Prefix: s, Source: "ip_range"}) + } + return rows +} + +func BenchmarkSmartAggregatePrefixRows(b *testing.B) { + for _, n := range []int{1_000, 10_000, 50_000} { + rows := benchPrefixRows(n, 0.3) + b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = smartAggregatePrefixRows(rows) + } + }) + } +} diff --git a/internal/pipeline/cdn_prefetch_test.go b/internal/pipeline/cdn_prefetch_test.go index 52d7156..575cf91 100644 --- a/internal/pipeline/cdn_prefetch_test.go +++ b/internal/pipeline/cdn_prefetch_test.go @@ -5,7 +5,9 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" + "time" "evobgp/internal/store" ) @@ -194,3 +196,63 @@ func TestCollectModulePrefixRows_CDN304RetriesWithoutETag(t *testing.T) { t.Fatalf("unexpected collected rows: %+v", collected) } } + +func TestPrefetchCDNSourceETags_SkipsFreshSources(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1") + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + + mod, err := m.CreateModule(tenant, &store.Module{ + Type: "CDN_CIDRS", + Name: "cdn-prefetch-due", + Enabled: true, + }) + if err != nil { + t.Fatal(err) + } + + var hits atomic.Int32 + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("ETag", `"v1"`) + _, _ = w.Write([]byte("198.51.100.0/24\n")) + })) + defer srv.Close() + + // Fresh source: refreshed 30s ago with a 3600s interval — prefetch must skip it. + freshAt := time.Now().UTC().Add(-30 * time.Second) + interval := 3600 + if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{ + SourceKind: "txt", + URL: srv.URL, + RefreshIntervalSec: &interval, + LastRefreshedAt: &freshAt, + }); err != nil { + t.Fatal(err) + } + + if err := PrefetchCDNSourceETags(context.Background(), m, srv.Client()); err != nil { + t.Fatal(err) + } + if hits.Load() != 0 { + t.Fatalf("fresh source must be skipped by prefetch, got %d HTTP hits", hits.Load()) + } + + // Stale source: last refresh older than its interval — prefetch must probe it. + staleAt := time.Now().UTC().Add(-7200 * time.Second) + if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{ + SourceKind: "txt", + URL: srv.URL + "?stale", + RefreshIntervalSec: &interval, + LastRefreshedAt: &staleAt, + }); err != nil { + t.Fatal(err) + } + if err := PrefetchCDNSourceETags(context.Background(), m, srv.Client()); err != nil { + t.Fatal(err) + } + if hits.Load() == 0 { + t.Fatal("stale source must be probed by prefetch") + } +} diff --git a/internal/pipeline/cdn_snapshot.go b/internal/pipeline/cdn_snapshot.go index 61a2774..65bbb45 100644 --- a/internal/pipeline/cdn_snapshot.go +++ b/internal/pipeline/cdn_snapshot.go @@ -42,32 +42,49 @@ func mergeSnapshotDropSource(rows []store.PrefixRow, sourceKey string) []store.P return out } -// mergeSnapshotDropCDNSources removes all cdn:* rows (used before batch CDN merge). -func mergeSnapshotDropCDNSources(rows []store.PrefixRow) []store.PrefixRow { - if len(rows) == 0 { - return nil +// mergeSnapshotKeepSkippedCDN keeps non-CDN rows and cdn:* rows whose source is still skipped +// (not in fetchedSourceIDs). Deleted sources (absent from allSourceIDs) are dropped. +func mergeSnapshotKeepSkippedCDN(rows []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) []store.PrefixRow { + fetched := make(map[string]struct{}, len(fetchedSourceIDs)) + for _, id := range fetchedSourceIDs { + fetched[cdnSourceKey(id)] = struct{}{} + } + keepCDN := make(map[string]struct{}) + for _, id := range allSourceIDs { + k := cdnSourceKey(id) + if _, ok := fetched[k]; !ok { + keepCDN[k] = struct{}{} + } } out := make([]store.PrefixRow, 0, len(rows)) for _, row := range rows { - if !strings.HasPrefix(strings.TrimSpace(row.Source), "cdn:") { - out = append(out, row) + src := strings.TrimSpace(row.Source) + if strings.HasPrefix(src, "cdn:") { + if _, ok := keepCDN[src]; !ok { + continue + } } + out = append(out, row) } return out } -// mergeAllCDNSourcesIntoModuleSnapshot replaces all CDN rows in one write (avoids parallel read-modify-write races). -func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow) error { +// mergeAllCDNSourcesIntoModuleSnapshot replaces fetched CDN source rows in one write. +// skipped sources (errors with EVOBGP_CDN_PARTIAL_OK) keep their prior rows. +func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) error { if st == nil || mod == nil { return nil } + unlock := st.LockModuleSnapshot(tenantID, mod.ID) + defer unlock() var base []store.PrefixRow if len(priorSnapshot) > 0 { - base = mergeSnapshotDropCDNSources(priorSnapshot) + base = priorSnapshot } else if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil { - base = mergeSnapshotDropCDNSources(snap.Prefixes) + base = snap.Prefixes } - merged := append(base, cdnRows...) + kept := mergeSnapshotKeepSkippedCDN(base, fetchedSourceIDs, allSourceIDs) + merged := append(kept, cdnRows...) return persistModuleSnapshot(st, tenantID, mod, merged) } @@ -93,6 +110,8 @@ func mergeCDNSourceIntoModuleSnapshot(st store.Backend, tenantID string, mod *st if st == nil || mod == nil { return nil } + unlock := st.LockModuleSnapshot(tenantID, mod.ID) + defer unlock() sourceKey := cdnSourceKey(sourceID) var base []store.PrefixRow if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil { @@ -114,81 +133,6 @@ func parseCDNBody(body string, src *store.CDNSource) ([]string, error) { return out, nil } -func applyCDNSourceHTTPResult(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) { - u := strings.TrimSpace(src.URL) - if u == "" { - return nil, nil - } - if _, err := ValidateCDNURL(u); err != nil { - return nil, err - } - if err := ResolveCDNURLHost(ctx, u); err != nil { - return nil, err - } - sourceKey := cdnSourceKey(src.ID) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) - if err != nil { - return nil, err - } - if etag := strings.TrimSpace(src.Etag); etag != "" { - req.Header.Set("If-None-Match", etag) - } - resp, err := upstreamHTTPDo(ctx, hc, req) - if err != nil { - return nil, fmt.Errorf("cdn fetch %s: %w", u, err) - } - - if resp.StatusCode == http.StatusNotModified { - if cached := cachedCDNPrefixRows(st, tenantID, moduleID, priorSnapshot, sourceKey); len(cached) > 0 { - _ = resp.Body.Close() - return cached, nil - } - // ETag is known but local snapshot is empty — force a full download. - _ = resp.Body.Close() - req2, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) - if err != nil { - return nil, err - } - resp, err = upstreamHTTPDo(ctx, hc, req2) - if err != nil { - return nil, fmt.Errorf("cdn fetch %s: %w", u, err) - } - if resp.StatusCode == http.StatusNotModified { - _ = resp.Body.Close() - return nil, fmt.Errorf("cdn url %s: 304 without cached prefixes", u) - } - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - _, _ = io.Copy(io.Discard, resp.Body) - return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status) - } - body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) - if err != nil { - return nil, err - } - prefixStrs, err := parseCDNBody(string(body), src) - if err != nil { - return nil, fmt.Errorf("cdn parse %s: %w", u, err) - } - etag := strings.TrimSpace(resp.Header.Get("ETag")) - patch := &store.CDNSourcePatch{} - if etag != "" && etag != strings.TrimSpace(src.Etag) { - e := etag - patch.Etag = &e - } - refreshedAt := now - patch.LastRefreshedAt = &refreshedAt - _, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, patch) - - rows := cdnRowsFromParsed(mod, src, prefixStrs) - if err := mergeCDNSourceIntoModuleSnapshot(st, tenantID, mod, src.ID, rows); err != nil { - return nil, err - } - return rows, nil -} - // fetchCDNSourceRows loads CDN prefixes without persisting the module snapshot (caller merges once). func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) { u := strings.TrimSpace(src.URL) diff --git a/internal/pipeline/cdn_url.go b/internal/pipeline/cdn_url.go index 9d67c04..ae0cb39 100644 --- a/internal/pipeline/cdn_url.go +++ b/internal/pipeline/cdn_url.go @@ -7,7 +7,9 @@ import ( "net/netip" "net/url" "os" + "strconv" "strings" + "sync" "time" ) @@ -91,6 +93,9 @@ func ResolveCDNURLHost(ctx context.Context, raw string) error { if isBlockedCDNHostname(host) { return fmt.Errorf("pipeline: cdn url blocked host") } + if ok := cdnDNSVerifyCache.hit(host); ok { + return nil + } if ctx == nil { ctx = context.Background() } @@ -112,5 +117,64 @@ func ResolveCDNURLHost(ctx context.Context, raw string) error { return fmt.Errorf("pipeline: cdn url resolves to blocked address") } } + cdnDNSVerifyCache.store(host) return nil } + +// cdnDNSVerifyTTL bounds how long a successful SSRF check is trusted for one hostname. +// Failures are never cached: a transient DNS outage must not open an unsafe window, +// and a blocked host is rejected before this cache anyway. +func cdnDNSVerifyTTL() time.Duration { + sec := 300 + if s := strings.TrimSpace(os.Getenv("EVOBGP_CDN_DNS_CACHE_TTL_SEC")); s != "" { + if v, err := strconv.Atoi(s); err == nil && v > 0 { + sec = v + } + } + return time.Duration(sec) * time.Second +} + +type dnsVerifyCache struct { + mu sync.Mutex + seen map[string]time.Time +} + +var cdnDNSVerifyCache = &dnsVerifyCache{seen: make(map[string]time.Time)} + +func (c *dnsVerifyCache) hit(host string) bool { + c.mu.Lock() + defer c.mu.Unlock() + at, ok := c.seen[host] + return ok && time.Since(at) < cdnDNSVerifyTTL() +} + +func (c *dnsVerifyCache) store(host string) { + c.mu.Lock() + defer c.mu.Unlock() + if c.seen == nil { + c.seen = make(map[string]time.Time) + } + c.seen[host] = time.Now() + if len(c.seen) > 4096 { + // Size cap for long-running workers: drop expired entries, then the oldest if needed. + now := time.Now() + for h, at := range c.seen { + if now.Sub(at) >= cdnDNSVerifyTTL() { + delete(c.seen, h) + } + } + if len(c.seen) > 4096 { + var oldestK string + var oldestT time.Time + first := true + for h, at := range c.seen { + if first || at.Before(oldestT) { + oldestK, oldestT, first = h, at, false + } + } + if oldestK != "" { + delete(c.seen, oldestK) + } + } + } +} diff --git a/internal/pipeline/collect_parallel.go b/internal/pipeline/collect_parallel.go index 0364ec7..bc532ae 100644 --- a/internal/pipeline/collect_parallel.go +++ b/internal/pipeline/collect_parallel.go @@ -26,6 +26,14 @@ func prefixRowsForSource(rows []store.PrefixRow, sourceKey string) []store.Prefi return out } +func prefixCommunityKey(row store.PrefixRow) string { + comm := "" + if row.CommunityID != nil { + comm = strings.TrimSpace(*row.CommunityID) + } + return row.Prefix + "\x00" + comm +} + func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, list []*store.ASEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) { moduleID := mod.ID legacy := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE")) == "0" @@ -147,7 +155,7 @@ func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, }) } for _, row := range r.rows { - k := row.Prefix + k := prefixCommunityKey(row) if _, ok := seenPfx[k]; ok { continue } @@ -218,9 +226,14 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client } wg.Wait() - var out []store.PrefixRow + var fetchedRows []store.PrefixRow var skipped int - for _, r := range results { + var fetchedIDs []string + allIDs := make([]string, 0, len(valid)) + for _, src := range valid { + allIDs = append(allIDs, src.ID) + } + for i, r := range results { if r.err != nil { if cdnPartialOK() { logging.Default().Info(fmt.Sprintf("pipeline: CDN partial skip source error: %v", r.err)) @@ -229,20 +242,35 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client } return nil, r.err } - out = append(out, r.rows...) + fetchedIDs = append(fetchedIDs, valid[i].ID) + fetchedRows = append(fetchedRows, r.rows...) } - if skipped > 0 && len(out) == 0 && len(valid) > 0 { + if skipped > 0 && len(fetchedRows) == 0 && len(valid) > 0 { return nil, fmt.Errorf("cdn: all %d source(s) failed (partial ok)", len(valid)) } + out := fetchedRows + if skipped > 0 { + base := priorSnapshot + if len(base) == 0 { + if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil { + base = snap.Prefixes + } + } + for _, row := range mergeSnapshotKeepSkippedCDN(base, fetchedIDs, allIDs) { + if strings.HasPrefix(strings.TrimSpace(row.Source), "cdn:") { + out = append(out, row) + } + } + } if len(valid) > 0 { - if err := mergeAllCDNSourcesIntoModuleSnapshot(st, tenantID, mod, priorSnapshot, out); err != nil { + if err := mergeAllCDNSourcesIntoModuleSnapshot(st, tenantID, mod, priorSnapshot, fetchedRows, fetchedIDs, allIDs); err != nil { return nil, err } } return out, nil } -func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) { +func collectDomainPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) { var validDom []*store.DomainEntry for _, e := range entries { if e != nil { @@ -269,7 +297,7 @@ func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Mo c := *mod.DefaultCommunityID comm = &c } - addrs, err := resolveDomainIPsWithPolicy(ctx, hc, profiles, policy, entry.FQDN) + addrs, err := resolveDomainIPsCached(ctx, st, hc, profiles, policy, entry.FQDN) if err != nil { if staleOnUpstreamError() { if cached, ok := staleDomainPrefixes(priorSnapshot, entry.FQDN); ok { diff --git a/internal/pipeline/doh_resolve.go b/internal/pipeline/doh_resolve.go index 19e751c..e07d550 100644 --- a/internal/pipeline/doh_resolve.go +++ b/internal/pipeline/doh_resolve.go @@ -6,6 +6,7 @@ import ( "net/http" "net/netip" "strings" + "sync" "time" "evobgp/internal/store" @@ -108,8 +109,7 @@ func resolveDomainIPsNoSystemFallback(ctx context.Context, hc *http.Client, prof defer cancel() baseURL := strings.TrimSpace(profile.URL) - v4, err4 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeA) - v6, err6 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeAAAA) + v4, v6, err4, err6 := resolveDOHMessagePair(dctx, hc, baseURL, host) if err4 != nil { v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A") } @@ -122,6 +122,23 @@ func resolveDomainIPsNoSystemFallback(ctx context.Context, hc *http.Client, prof return uniqAddrs(append(v4, v6...)), nil } +// resolveDOHMessagePair issues RFC8484 dns-message A and AAAA queries concurrently +// and waits for both (fallbacks are handled by the caller). +func resolveDOHMessagePair(ctx context.Context, hc *http.Client, baseURL, host string) (v4, v6 []netip.Addr, err4, err6 error) { + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + v4, err4 = resolveDomainWithDOHMessage(ctx, hc, baseURL, host, dns.TypeA) + }() + go func() { + defer wg.Done() + v6, err6 = resolveDomainWithDOHMessage(ctx, hc, baseURL, host, dns.TypeAAAA) + }() + wg.Wait() + return v4, v6, err4, err6 +} + func dohProfileTimeout(profile *store.DohProfile) time.Duration { timeout := 10 * time.Second if profile != nil && profile.TimeoutMs != nil && *profile.TimeoutMs > 0 { diff --git a/internal/pipeline/doh_resolve_test.go b/internal/pipeline/doh_resolve_test.go index e88e5b4..660e908 100644 --- a/internal/pipeline/doh_resolve_test.go +++ b/internal/pipeline/doh_resolve_test.go @@ -2,147 +2,97 @@ package pipeline import ( "context" + "encoding/base64" "net/http" "net/http/httptest" + "sync/atomic" "testing" + "time" "evobgp/internal/store" + + "github.com/miekg/dns" ) -func TestResolveDomainIPsWithPolicy_Union(t *testing.T) { - srvRU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.1"}]}`)) - })) - defer srvRU.Close() - srvEU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.1"}]}`)) - })) - defer srvEU.Close() +// TestResolveDomainIPs_ParallelAAndAAAA verifies that A and AAAA queries are issued +// concurrently: with a 250ms upstream latency the combined resolve must stay near +// one round-trip instead of two. +func TestResolveDomainIPs_ParallelAAndAAAA(t *testing.T) { + const delay = 250 * time.Millisecond + var inflight, maxInflight atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cur := inflight.Add(1) + for { + old := maxInflight.Load() + if cur <= old || maxInflight.CompareAndSwap(old, cur) { + break + } + } + defer inflight.Add(-1) + time.Sleep(delay) - profiles := []*store.DohProfile{ - {URL: srvRU.URL}, - {URL: srvEU.URL}, - } - ips, err := resolveDomainIPsWithPolicy(context.Background(), srvRU.Client(), profiles, store.DohPolicyUnion, "example.com") + if wire := r.URL.Query().Get("dns"); wire != "" { + // RFC8484 dns-message: decode the query and answer on the wire. + raw, err := base64.RawURLEncoding.DecodeString(wire) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + msg := new(dns.Msg) + if err := msg.Unpack(raw); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + resp := new(dns.Msg) + resp.SetReply(msg) + switch msg.Question[0].Qtype { + case dns.TypeA: + resp.Answer = append(resp.Answer, &dns.A{ + Hdr: dns.RR_Header{Name: msg.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, + A: []byte{203, 0, 113, 10}, + }) + case dns.TypeAAAA: + resp.Answer = append(resp.Answer, &dns.AAAA{ + Hdr: dns.RR_Header{Name: msg.Question[0].Name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}, + AAAA: []byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x10}, + }) + } + out, err := resp.Pack() + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/dns-message") + _, _ = w.Write(out) + return + } + w.Header().Set("Content-Type", "application/dns-json") + switch r.URL.Query().Get("type") { + case "A": + _, _ = w.Write([]byte(`{"Status":0,"Answer":[{"type":1,"data":"203.0.113.10"}]}`)) + default: + _, _ = w.Write([]byte(`{"Status":0,"Answer":[{"type":28,"data":"2001:db8::10"}]}`)) + } + })) + defer srv.Close() + + prof := &store.DohProfile{URL: srv.URL, TimeoutMs: ptrInt(5000)} + ctx := context.Background() + start := time.Now() + ips, err := resolveDomainIPsNoSystemFallback(ctx, srv.Client(), prof, "example.test") + elapsed := time.Since(start) if err != nil { - t.Fatal(err) + t.Fatalf("resolve failed: %v", err) } if len(ips) != 2 { - t.Fatalf("want 2 ips, got %v", ips) + t.Fatalf("expected 2 addrs, got %+v", ips) } - seen := map[string]bool{ips[0].String(): true, ips[1].String(): true} - if !seen["198.51.100.1"] || !seen["203.0.113.1"] { - t.Fatalf("unexpected ips: %v", ips) + if maxInflight.Load() < 2 { + t.Fatalf("expected concurrent A/AAAA queries, max inflight=%d", maxInflight.Load()) + } + if elapsed >= 2*delay { + t.Fatalf("resolve took %v; expected one round-trip (<2*%v)", elapsed, delay) } } -func TestResolveDomainIPsWithPolicy_Failover(t *testing.T) { - var calls int - srvBad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls++ - http.Error(w, "fail", http.StatusBadGateway) - })) - defer srvBad.Close() - srvOK := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls++ - _, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.5"}]}`)) - })) - defer srvOK.Close() - - profiles := []*store.DohProfile{ - {URL: srvBad.URL}, - {URL: srvOK.URL}, - } - ips, err := resolveDomainIPsWithPolicy(context.Background(), srvBad.Client(), profiles, store.DohPolicyFailover, "example.com") - if err != nil { - t.Fatal(err) - } - if len(ips) != 1 || ips[0].String() != "198.51.100.5" { - t.Fatalf("unexpected ips: %v", ips) - } - if calls < 2 { - t.Fatalf("want at least 2 resolver calls, got %d", calls) - } -} - -func TestResolveDomainIPsWithPolicy_PrimaryOnly(t *testing.T) { - var secondCalled bool - srv1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.9"}]}`)) - })) - defer srv1.Close() - srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - secondCalled = true - _, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.9"}]}`)) - })) - defer srv2.Close() - - profiles := []*store.DohProfile{ - {URL: srv1.URL}, - {URL: srv2.URL}, - } - ips, err := resolveDomainIPsWithPolicy(context.Background(), srv1.Client(), profiles, store.DohPolicyPrimaryOnly, "example.com") - if err != nil { - t.Fatal(err) - } - if len(ips) != 1 || ips[0].String() != "198.51.100.9" { - t.Fatalf("unexpected ips: %v", ips) - } - if secondCalled { - t.Fatal("secondary resolver must not be queried in primary_only mode") - } -} - -func TestCollectModulePrefixRows_DohUnion(t *testing.T) { - m := store.NewMemory() - m.SeedDemo() - tenant, _, _, _, _ := m.DemoIDs() - - mod, err := m.CreateModule(tenant, &store.Module{ - Type: "DOMAINS", - Name: "domains-union", - Enabled: true, - DohResolverPolicy: store.DohPolicyUnion, - }) - if err != nil { - t.Fatal(err) - } - - srvRU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.2"}]}`)) - })) - defer srvRU.Close() - srvEU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.2"}]}`)) - })) - defer srvEU.Close() - - ru, err := m.CreateDohProfile(tenant, &store.DohProfile{Name: "ru", URL: srvRU.URL}) - if err != nil { - t.Fatal(err) - } - eu, err := m.CreateDohProfile(tenant, &store.DohProfile{Name: "eu", URL: srvEU.URL}) - if err != nil { - t.Fatal(err) - } - if _, err := m.UpdateModule(tenant, mod.ID, &store.ModulePatch{ - DohProfileIDs: &[]string{ru.ID, eu.ID}, - }); err != nil { - t.Fatal(err) - } - mod, err = m.GetModule(tenant, mod.ID) - if err != nil { - t.Fatal(err) - } - if _, err := m.CreateDomainEntry(tenant, mod.ID, &store.DomainEntry{FQDN: "svc.example.com"}); err != nil { - t.Fatal(err) - } - - rows, err := collectModulePrefixRows(context.Background(), m, srvRU.Client(), tenant, mod, nil) - if err != nil { - t.Fatal(err) - } - if len(rows) != 2 { - t.Fatalf("want 2 prefix rows, got %+v", rows) - } -} +func ptrInt(v int) *int { return &v } diff --git a/internal/pipeline/domain_cache.go b/internal/pipeline/domain_cache.go new file mode 100644 index 0000000..7ed4ba8 --- /dev/null +++ b/internal/pipeline/domain_cache.go @@ -0,0 +1,67 @@ +package pipeline + +import ( + "context" + "net/http" + "net/netip" + "os" + "strconv" + "strings" + "time" + + "evobgp/internal/store" +) + +func domainCacheTTL() time.Duration { + sec := 300 + if s := strings.TrimSpace(os.Getenv("EVOBGP_DOMAIN_CACHE_TTL_SEC")); s != "" { + if v, err := strconv.Atoi(s); err == nil && v > 0 { + sec = v + } + } + return time.Duration(sec) * time.Second +} + +func resolveDomainIPsCached(ctx context.Context, st store.Backend, hc *http.Client, profiles []*store.DohProfile, policy, fqdn string) ([]netip.Addr, error) { + key := strings.TrimSpace(fqdn) + ttl := domainCacheTTL() + if st != nil && ttl > 0 { + if ent, ok, err := st.GetDomainResolveCache(key); err == nil && ok && ent != nil { + if time.Since(ent.ResolvedAt) < ttl { + if addrs := parseCachedDomainAddrs(ent.Addrs); len(addrs) > 0 { + return addrs, nil + } + } + } + } + addrs, err := resolveDomainIPsWithPolicy(ctx, hc, profiles, policy, key) + if err != nil { + return nil, err + } + if st != nil { + _ = st.SetDomainResolveCache(key, domainAddrsToStrings(addrs)) + } + return addrs, nil +} + +func parseCachedDomainAddrs(raw []string) []netip.Addr { + out := make([]netip.Addr, 0, len(raw)) + for _, s := range raw { + a, err := netip.ParseAddr(strings.TrimSpace(s)) + if err != nil { + continue + } + out = append(out, a) + } + return out +} + +func domainAddrsToStrings(addrs []netip.Addr) []string { + out := make([]string, 0, len(addrs)) + for _, a := range addrs { + if a.IsValid() { + out = append(out, a.String()) + } + } + return out +} diff --git a/internal/pipeline/module_hash.go b/internal/pipeline/module_hash.go index cc98388..2bce9cb 100644 --- a/internal/pipeline/module_hash.go +++ b/internal/pipeline/module_hash.go @@ -1,10 +1,6 @@ package pipeline import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "sort" "strings" "evobgp/internal/store" @@ -12,89 +8,7 @@ import ( // moduleIngestInputHash fingerprints module config and child entries so snapshots invalidate on CRUD. func moduleIngestInputHash(st store.Backend, tenantID string, mod *store.Module) (string, error) { - if st == nil || mod == nil { - return "", fmt.Errorf("pipeline: module hash: missing store or module") - } - h := sha256.New() - _, _ = fmt.Fprintf(h, "type=%s\n", strings.TrimSpace(mod.Type)) - _, _ = fmt.Fprintf(h, "enabled=%t\n", mod.Enabled) - if mod.DefaultCommunityID != nil { - _, _ = fmt.Fprintf(h, "default_community=%s\n", strings.TrimSpace(*mod.DefaultCommunityID)) - } - _, _ = fmt.Fprintf(h, "doh_policy=%s\n", store.NormalizeDohResolverPolicy(mod.DohResolverPolicy)) - for _, pid := range mod.EffectiveDohProfileIDs() { - _, _ = fmt.Fprintf(h, "doh_profile=%s\n", pid) - if prof, err := st.GetDohProfile(tenantID, pid); err == nil && prof != nil { - _, _ = fmt.Fprintf(h, "doh_url=%s\n", strings.TrimSpace(prof.URL)) - if prof.TimeoutMs != nil { - _, _ = fmt.Fprintf(h, "doh_timeout=%d\n", *prof.TimeoutMs) - } - } - } - - switch mod.Type { - case "IP_RANGES": - list, err := st.ListIPRangeEntries(tenantID, mod.ID) - if err != nil { - return "", err - } - sort.Slice(list, func(i, j int) bool { return list[i].Prefix < list[j].Prefix }) - for _, e := range list { - comm := "" - if e.CommunityID != nil { - comm = *e.CommunityID - } - _, _ = fmt.Fprintf(h, "ip=%s|c=%s\n", e.Prefix, comm) - } - case "AS_PREFIXES": - list, err := st.ListASEntries(tenantID, mod.ID) - if err != nil { - return "", err - } - sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN }) - for _, e := range list { - comm := "" - if e.CommunityID != nil { - comm = *e.CommunityID - } - _, _ = fmt.Fprintf(h, "as=%d|c=%s\n", e.ASN, comm) - } - case "CDN_CIDRS": - list, err := st.ListCDNSources(tenantID, mod.ID) - if err != nil { - return "", err - } - sort.Slice(list, func(i, j int) bool { return list[i].ID < list[j].ID }) - for _, s := range list { - comm := "" - if s.CommunityID != nil { - comm = *s.CommunityID - } - interval := 0 - if s.RefreshIntervalSec != nil { - interval = *s.RefreshIntervalSec - } - _, _ = fmt.Fprintf(h, "cdn=%s|url=%s|kind=%s|path=%s|c=%s|etag=%s|interval=%d\n", - s.ID, strings.TrimSpace(s.URL), s.SourceKind, strings.TrimSpace(s.PrefixPath), comm, - strings.TrimSpace(s.Etag), interval) - } - case "DOMAINS": - list, err := st.ListDomainEntries(tenantID, mod.ID) - if err != nil { - return "", err - } - sort.Slice(list, func(i, j int) bool { return list[i].FQDN < list[j].FQDN }) - for _, e := range list { - comm := "" - if e.CommunityID != nil { - comm = *e.CommunityID - } - _, _ = fmt.Fprintf(h, "dom=%s|c=%s\n", strings.TrimSpace(e.FQDN), comm) - } - default: - _, _ = fmt.Fprintf(h, "unknown_type=%s\n", mod.Type) - } - return hex.EncodeToString(h.Sum(nil)), nil + return store.ComputeModuleInputHash(st, tenantID, mod) } func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, rows []store.PrefixRow) error { @@ -105,18 +19,32 @@ func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, if err != nil { return err } - return st.SetModulePrefixSnapshot(tenantID, mod.ID, hash, rows) + if err := st.SetModulePrefixSnapshot(tenantID, mod.ID, hash, rows); err != nil { + return err + } + _ = st.SetModuleInputHash(tenantID, mod.ID, hash) + return nil } func moduleRowsFromSnapshot(st store.Backend, tenantID string, mod *store.Module) ([]store.PrefixRow, bool, error) { + snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID) + if err != nil || !ok || snap == nil { + return nil, false, err + } + if h := strings.TrimSpace(mod.InputHash); h != "" { + if snap.InputHash != h { + return nil, false, nil + } + return append([]store.PrefixRow(nil), snap.Prefixes...), true, nil + } + // Fallback for rows written before module.input_hash existed: recompute and backfill. hash, err := moduleIngestInputHash(st, tenantID, mod) if err != nil { return nil, false, err } - snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID) - if err != nil || !ok || snap == nil || snap.InputHash != hash { - return nil, false, err + if snap.InputHash != hash { + return nil, false, nil } - cp := append([]store.PrefixRow(nil), snap.Prefixes...) - return cp, true, nil + _ = st.SetModuleInputHash(tenantID, mod.ID, hash) + return append([]store.PrefixRow(nil), snap.Prefixes...), true, nil } diff --git a/internal/pipeline/opt_collect_test.go b/internal/pipeline/opt_collect_test.go new file mode 100644 index 0000000..f79f7ff --- /dev/null +++ b/internal/pipeline/opt_collect_test.go @@ -0,0 +1,189 @@ +package pipeline + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "evobgp/internal/store" +) + +func TestModuleDueForScheduler_JitterSpreads(t *testing.T) { + dueTicks := map[int64]int{} + for i := 0; i < 100; i++ { + mod := &store.Module{ + ID: fmt.Sprintf("module-%d", i), + Enabled: true, + Type: "AS_PREFIXES", + RefreshIntervalSec: 3600, + } + for tick := int64(0); tick < 7200; tick += SchedulerTickSec { + if ModuleDueForScheduler(mod, time.Unix(tick, 0)) { + dueTicks[tick]++ + } + } + } + if len(dueTicks) < 10 { + t.Fatalf("expected due events across many ticks, got %d buckets", len(dueTicks)) + } + for tick, n := range dueTicks { + if n == 100 { + t.Fatalf("all 100 modules due on tick %d", tick) + } + } +} + +func TestCollectASPrefixRows_KeepsSamePrefixDifferentCommunity(t *testing.T) { + st := store.NewMemory() + st.SeedDemo() + tenant, _, _, _, _ := st.DemoIDs() + mod, err := st.CreateModule(tenant, &store.Module{Type: "AS_PREFIXES", Name: "as-dedup", Enabled: true}) + if err != nil { + t.Fatal(err) + } + c1, c2 := "comm-a", "comm-b" + e1, err := st.CreateASEntry(tenant, mod.ID, &store.ASEntry{ASN: 64500, CommunityID: &c1}) + if err != nil { + t.Fatal(err) + } + e2, err := st.CreateASEntry(tenant, mod.ID, &store.ASEntry{ASN: 64501, CommunityID: &c2}) + if err != nil { + t.Fatal(err) + } + if err := st.SetASNPrefixCache(64500, "a", []string{"192.0.2.0/24"}); err != nil { + t.Fatal(err) + } + if err := st.SetASNPrefixCache(64501, "b", []string{"192.0.2.0/24"}); err != nil { + t.Fatal(err) + } + rows, err := collectASPrefixRows(context.Background(), st, http.DefaultClient, tenant, mod, []*store.ASEntry{e1, e2}, nil) + if err != nil { + t.Fatal(err) + } + if len(rows) != 2 { + t.Fatalf("expected 2 rows (same prefix, different community), got %d: %+v", len(rows), rows) + } +} + +func TestCollectCDNPrefixRows_PartialSkipKeepsPrior(t *testing.T) { + t.Setenv("EVOBGP_CDN_PARTIAL_OK", "1") + t.Setenv("EVOBGP_STALE_ON_UPSTREAM_ERROR", "0") + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1") + + good := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("198.51.100.0/24\n")) + })) + defer good.Close() + bad := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "down", http.StatusServiceUnavailable) + })) + defer bad.Close() + + st := store.NewMemory() + st.SeedDemo() + tenant, _, _, _, _ := st.DemoIDs() + mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-partial", Enabled: true}) + if err != nil { + t.Fatal(err) + } + prior := []store.PrefixRow{ + {Prefix: "203.0.113.0/24", Source: "cdn:bad"}, + {Prefix: "1.2.3.0/24", Source: "cdn:good"}, + } + sources := []*store.CDNSource{ + {ID: "good", URL: good.URL, SourceKind: "plain"}, + {ID: "bad", URL: bad.URL, SourceKind: "plain"}, + } + rows, err := collectCDNPrefixRows(context.Background(), st, good.Client(), tenant, mod, sources, prior) + if err != nil { + t.Fatalf("partial skip should succeed: %v", err) + } + got := map[string]string{} + for _, r := range rows { + got[r.Source] = r.Prefix + } + if got["cdn:bad"] != "203.0.113.0/24" { + t.Fatalf("skipped source lost prior row: %+v", rows) + } + if got["cdn:good"] != "198.51.100.0/24" { + t.Fatalf("fetched source missing new row: %+v", rows) + } +} + +func TestMergeCDNSource_ParallelNoLostUpdate(t *testing.T) { + st := store.NewMemory() + st.SeedDemo() + tenant, _, _, _, _ := st.DemoIDs() + mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-lock", Enabled: true}) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + _ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s1", []store.PrefixRow{ + {Prefix: "1.0.0.0/24", Source: "cdn:s1"}, + }) + }() + go func() { + defer wg.Done() + _ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s2", []store.PrefixRow{ + {Prefix: "2.0.0.0/24", Source: "cdn:s2"}, + }) + }() + wg.Wait() + snap, ok, err := st.GetModulePrefixSnapshot(tenant, mod.ID) + if err != nil || !ok || snap == nil { + t.Fatalf("snapshot missing: ok=%v err=%v", ok, err) + } + got := map[string]bool{} + for _, r := range snap.Prefixes { + got[r.Prefix] = true + } + if !got["1.0.0.0/24"] || !got["2.0.0.0/24"] { + t.Fatalf("lost parallel merge update: %+v", snap.Prefixes) + } +} + +func TestResolveDomainIPsCached_UsesTTL(t *testing.T) { + t.Setenv("EVOBGP_DOMAIN_CACHE_TTL_SEC", "300") + st := store.NewMemory() + if err := st.SetDomainResolveCache("cached.test", []string{"192.0.2.9"}); err != nil { + t.Fatal(err) + } + addrs, err := resolveDomainIPsCached(context.Background(), st, nil, nil, "", "cached.test") + if err != nil { + t.Fatal(err) + } + if len(addrs) != 1 || addrs[0].String() != "192.0.2.9" { + t.Fatalf("expected cached addr, got %v", addrs) + } +} + +func TestInputHashInvalidatedOnEntryCRUD(t *testing.T) { + st := store.NewMemory() + st.SeedDemo() + tenant, _, _, _, _ := st.DemoIDs() + mod, err := st.CreateModule(tenant, &store.Module{Type: "IP_RANGES", Name: "ip-hash", Enabled: true}) + if err != nil { + t.Fatal(err) + } + if err := st.SetModuleInputHash(tenant, mod.ID, "pre"); err != nil { + t.Fatal(err) + } + if _, err := st.CreateIPRangeEntry(tenant, mod.ID, &store.IPRangeEntry{Prefix: "10.0.0.0/8"}); err != nil { + t.Fatal(err) + } + got, err := st.GetModule(tenant, mod.ID) + if err != nil { + t.Fatal(err) + } + if got.InputHash != "" { + t.Fatalf("expected hash cleared after CRUD, got %q", got.InputHash) + } +} diff --git a/internal/pipeline/prefetch.go b/internal/pipeline/prefetch.go index 8563713..afe60d3 100644 --- a/internal/pipeline/prefetch.go +++ b/internal/pipeline/prefetch.go @@ -2,7 +2,6 @@ package pipeline import ( "context" - "io" "net/http" "strings" "sync" @@ -31,6 +30,7 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie return err } var tasks []prefetchTask + now := time.Now().UTC() for _, tid := range tenants { for _, mod := range st.ListModules(tid) { if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" { @@ -41,9 +41,15 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie continue } for _, src := range sources { - if src != nil && strings.TrimSpace(src.URL) != "" { - tasks = append(tasks, prefetchTask{tenantID: tid, mod: mod, src: src}) + if src == nil || strings.TrimSpace(src.URL) == "" { + continue } + // Respect per-source refresh intervals: conditional GET only for due sources. + // The ETag probe still lets 304s skip body downloads for the rest. + if shouldSkipCDNSourceFetch(src, now) { + continue + } + tasks = append(tasks, prefetchTask{tenantID: tid, mod: mod, src: src}) } } } @@ -68,53 +74,13 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) { now := time.Now().UTC() tid, mod, src := t.tenantID, t.mod, t.src - u := strings.TrimSpace(src.URL) - if _, err := ValidateCDNURL(u); err != nil { - return - } - if err := ResolveCDNURLHost(ctx, u); err != nil { - return - } omod, err := st.GetModule(tid, mod.ID) + if err != nil || omod == nil { + return + } + rows, err := fetchCDNSourceRows(ctx, st, hc, tid, mod.ID, omod, src, nil, now) if err != nil { return } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) - if err != nil { - return - } - if etag := strings.TrimSpace(src.Etag); etag != "" { - req.Header.Set("If-None-Match", etag) - } - resp, err := upstreamHTTPDo(ctx, hc, req) - if err != nil { - return - } - if resp.StatusCode == http.StatusNotModified { - _ = resp.Body.Close() - return - } - if resp.StatusCode != http.StatusOK { - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - return - } - body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) - _ = resp.Body.Close() - if err != nil { - return - } - prefixStrs, err := parseCDNBody(string(body), src) - if err != nil { - return - } - newEtag := strings.TrimSpace(resp.Header.Get("ETag")) - patch := &store.CDNSourcePatch{LastRefreshedAt: &now} - if newEtag != "" && newEtag != strings.TrimSpace(src.Etag) { - e := newEtag - patch.Etag = &e - } - _, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, patch) - rows := cdnRowsFromParsed(omod, src, prefixStrs) _ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows) } diff --git a/internal/pipeline/refresh.go b/internal/pipeline/refresh.go index 7586dc4..805724a 100644 --- a/internal/pipeline/refresh.go +++ b/internal/pipeline/refresh.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "io" - "math/big" "net" "net/http" "net/netip" @@ -190,7 +189,7 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli if err != nil { return nil, err } - return collectDomainPrefixRows(ctx, hc, mod, profiles, policy, entries, priorSnapshot) + return collectDomainPrefixRows(ctx, st, hc, mod, profiles, policy, entries, priorSnapshot) default: return nil, fmt.Errorf("pipeline: unknown module type %q", mod.Type) } @@ -230,9 +229,8 @@ func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohPr defer cancel() baseURL := strings.TrimSpace(profile.URL) - // Prefer RFC8484 dns-message transport. Some providers don't support dns-json. - v4, err4 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeA) - v6, err6 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeAAAA) + // A and AAAA queries run concurrently: per-domain latency drops from ~2×RTT to ~1×RTT. + v4, v6, err4, err6 := resolveDOHMessagePair(dctx, hc, baseURL, host) if err4 != nil { // Fallback to JSON mode for providers that only expose dns-json. v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A") @@ -496,168 +494,11 @@ func prefixRowCommunity(r store.PrefixRow) string { } func aggregateIPv4Group(rows []store.PrefixRow) []store.PrefixRow { - return aggregateCIDRGroup(rows, mergeSiblingPrefixesIPv4) + return collapsePrefixGroup(rows, true) } func aggregateIPv6Group(rows []store.PrefixRow) []store.PrefixRow { - return aggregateCIDRGroup(rows, mergeSiblingPrefixesIPv6) -} - -func aggregateCIDRGroup(rows []store.PrefixRow, mergeFn func(map[string]store.PrefixRow) bool) []store.PrefixRow { - if len(rows) <= 1 { - return rows - } - set := make(map[string]store.PrefixRow, len(rows)) - for _, row := range rows { - set[row.Prefix] = row - } - pruneCoveredPrefixes(set) - for { - if !mergeFn(set) { - break - } - pruneCoveredPrefixes(set) - } - out := make([]store.PrefixRow, 0, len(set)) - for _, row := range set { - out = append(out, row) - } - sortPrefixRows(out) - return out -} - -func pruneCoveredPrefixes(set map[string]store.PrefixRow) { - type item struct { - key string - pfx netip.Prefix - bits int - } - items := make([]item, 0, len(set)) - for k := range set { - p, err := netip.ParsePrefix(k) - if err != nil { - continue - } - items = append(items, item{key: k, pfx: p, bits: p.Bits()}) - } - sort.Slice(items, func(i, j int) bool { - if items[i].bits != items[j].bits { - return items[i].bits < items[j].bits - } - return items[i].key < items[j].key - }) - for i := 0; i < len(items); i++ { - for j := i + 1; j < len(items); j++ { - if items[j].bits <= items[i].bits { - continue - } - if items[i].pfx.Contains(items[j].pfx.Addr()) { - delete(set, items[j].key) - } - } - } -} - -func mergeSiblingPrefixesIPv4(set map[string]store.PrefixRow) bool { - merged := false - seen := make(map[string]struct{}, len(set)) - for key, row := range set { - if _, done := seen[key]; done { - continue - } - pfx, err := netip.ParsePrefix(key) - if err != nil || !pfx.Addr().Is4() { - continue - } - bits := pfx.Bits() - if bits <= 8 { - continue - } - netNum := ipv4PrefixNetwork(pfx) - blockSize := uint32(1) << (32 - bits) - siblingNet := netNum ^ blockSize - siblingPfx := netip.PrefixFrom(u32ToIPv4(siblingNet), bits).Masked().String() - _, ok := set[siblingPfx] - if !ok { - continue - } - parentBits := bits - 1 - parentBlock := uint32(1) << (32 - parentBits) - parentNet := netNum & ^(parentBlock - 1) - parentPfx := netip.PrefixFrom(u32ToIPv4(parentNet), parentBits).Masked().String() - delete(set, key) - delete(set, siblingPfx) - parentRow := row - parentRow.Prefix = parentPfx - set[parentPfx] = parentRow - seen[key] = struct{}{} - seen[siblingPfx] = struct{}{} - merged = true - } - return merged -} - -func ipv4PrefixNetwork(p netip.Prefix) uint32 { - a := p.Masked().Addr().As4() - return uint32(a[0])<<24 | uint32(a[1])<<16 | uint32(a[2])<<8 | uint32(a[3]) -} - -func u32ToIPv4(v uint32) netip.Addr { - return netip.AddrFrom4([4]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}) -} - -func mergeSiblingPrefixesIPv6(set map[string]store.PrefixRow) bool { - merged := false - seen := make(map[string]struct{}, len(set)) - for key, row := range set { - if _, done := seen[key]; done { - continue - } - pfx, err := netip.ParsePrefix(key) - if err != nil || !pfx.Addr().Is6() { - continue - } - bits := pfx.Bits() - if bits <= 16 { - continue - } - netNum := ipv6PrefixNetwork(pfx) - blockSize := new(big.Int).Lsh(big.NewInt(1), uint(128-bits)) - siblingNet := new(big.Int).Xor(netNum, blockSize) - siblingPfx := ipv6PrefixFromBigInt(siblingNet, bits).String() - if _, ok := set[siblingPfx]; !ok { - continue - } - parentBits := bits - 1 - parentBlock := new(big.Int).Lsh(big.NewInt(1), uint(128-parentBits)) - mask := new(big.Int).Sub(parentBlock, big.NewInt(1)) - mask.Not(mask) - parentNet := new(big.Int).And(netNum, mask) - parentPfx := ipv6PrefixFromBigInt(parentNet, parentBits).String() - delete(set, key) - delete(set, siblingPfx) - parentRow := row - parentRow.Prefix = parentPfx - set[parentPfx] = parentRow - seen[key] = struct{}{} - seen[siblingPfx] = struct{}{} - merged = true - } - return merged -} - -func ipv6PrefixNetwork(p netip.Prefix) *big.Int { - a := p.Masked().Addr().As16() - n := new(big.Int) - n.SetBytes(a[:]) - return n -} - -func ipv6PrefixFromBigInt(n *big.Int, bits int) netip.Prefix { - b := n.Bytes() - var a [16]byte - copy(a[16-len(b):], b) - return netip.PrefixFrom(netip.AddrFrom16(a), bits).Masked() + return collapsePrefixGroup(rows, false) } func parentRevision(st store.Backend, tenantID, moduleID string) *string { diff --git a/internal/pipeline/scheduler_due.go b/internal/pipeline/scheduler_due.go index 5b11001..8f50020 100644 --- a/internal/pipeline/scheduler_due.go +++ b/internal/pipeline/scheduler_due.go @@ -1,6 +1,7 @@ package pipeline import ( + "hash/fnv" "time" "evobgp/internal/store" @@ -10,6 +11,8 @@ import ( const SchedulerTickSec = 30 // ModuleDueForScheduler reports whether a module's refresh interval bucket rolled since the last scheduler tick. +// A stable per-module offset (fnv32 of ID) spreads bucket boundaries so modules with the same interval +// do not all become due on the same tick (thundering herd). func ModuleDueForScheduler(mod *store.Module, now time.Time) bool { if mod == nil || !mod.Enabled || mod.Type == "IP_RANGES" || mod.RefreshIntervalSec <= 0 { return false @@ -18,7 +21,21 @@ func ModuleDueForScheduler(mod *store.Module, now time.Time) bool { if win < 60 { win = 60 } - cur := now.Unix() / win - prev := (now.Unix() - SchedulerTickSec) / win + offset := moduleSchedulerOffset(mod.ID, win) + cur := (now.Unix() - offset) / win + prev := (now.Unix() - offset - SchedulerTickSec) / win return cur != prev } + +func moduleSchedulerOffset(moduleID string, win int64) int64 { + if win <= 0 { + return 0 + } + return int64(fnv32a(moduleID) % uint32(win)) +} + +func fnv32a(s string) uint32 { + h := fnv.New32a() + _, _ = h.Write([]byte(s)) + return h.Sum32() +} diff --git a/internal/repository/asn_cache.go b/internal/repository/asn_cache.go index 1b43aae..58e4b5f 100644 --- a/internal/repository/asn_cache.go +++ b/internal/repository/asn_cache.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "sync" "time" "evobgp/internal/store" @@ -11,13 +12,26 @@ import ( "github.com/jackc/pgx/v5" ) +// asnCacheRowExistsOnce caches the schema check for the process lifetime (see moduleSnapshotRowTableExists). +var ( + asnCacheRowExistsOnce sync.Once + asnCacheRowExistsCached bool +) + func asnCacheRowTableExists(ctx context.Context, q queryRower) bool { - var n int - err := q.QueryRow(ctx, ` - SELECT 1 FROM information_schema.tables - WHERE table_schema = 'public' AND table_name = 'asn_prefix_cache_row' - LIMIT 1`).Scan(&n) - return err == nil + asnCacheRowExistsOnce.Do(func() { + var n int + err := q.QueryRow(ctx, ` + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = 'asn_prefix_cache_row' + LIMIT 1`).Scan(&n) + if err != nil { + asnCacheRowExistsOnce = sync.Once{} + return + } + asnCacheRowExistsCached = true + }) + return asnCacheRowExistsCached } func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, bool, error) { diff --git a/internal/repository/domain_cache.go b/internal/repository/domain_cache.go new file mode 100644 index 0000000..5ed902c --- /dev/null +++ b/internal/repository/domain_cache.go @@ -0,0 +1,64 @@ +package repository + +import ( + "context" + "encoding/json" + "errors" + "strings" + "time" + + "evobgp/internal/store" + + "github.com/jackc/pgx/v5" +) + +func (p *Postgres) GetDomainResolveCache(fqdn string) (*store.DomainResolveCacheEntry, bool, error) { + key := strings.ToLower(strings.TrimSpace(fqdn)) + if key == "" { + return nil, false, nil + } + ctx := context.Background() + var raw []byte + var resolvedAt time.Time + err := p.pool.QueryRow(ctx, ` + SELECT addrs_json, resolved_at FROM domain_resolve_cache WHERE fqdn = $1`, key). + Scan(&raw, &resolvedAt) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, false, nil + } + return nil, false, err + } + var addrs []string + if len(raw) > 0 { + _ = json.Unmarshal(raw, &addrs) + } + return &store.DomainResolveCacheEntry{ + FQDN: key, + Addrs: addrs, + ResolvedAt: resolvedAt.UTC(), + }, true, nil +} + +func (p *Postgres) SetDomainResolveCache(fqdn string, addrs []string) error { + key := strings.ToLower(strings.TrimSpace(fqdn)) + if key == "" { + return store.ErrInvalidInput + } + if addrs == nil { + addrs = []string{} + } + raw, err := json.Marshal(addrs) + if err != nil { + return err + } + ctx := context.Background() + _, err = p.pool.Exec(ctx, ` + INSERT INTO domain_resolve_cache (fqdn, addrs_json, resolved_at) + VALUES ($1, $2::jsonb, now()) + ON CONFLICT (fqdn) DO UPDATE SET + addrs_json = EXCLUDED.addrs_json, + resolved_at = EXCLUDED.resolved_at`, + key, string(raw)) + return err +} diff --git a/internal/repository/module_snapshot.go b/internal/repository/module_snapshot.go index 6796fa3..3670615 100644 --- a/internal/repository/module_snapshot.go +++ b/internal/repository/module_snapshot.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "strings" + "sync" "time" "evobgp/internal/store" @@ -12,13 +13,29 @@ import ( "github.com/jackc/pgx/v5" ) +// moduleSnapshotRowExistsOnce caches the schema check for the process lifetime. +// The table is created by migrations and never disappears at runtime; DB errors are +// not cached so a transient outage falls back to the JSON path only once. +var moduleSnapshotRowExistsOnce sync.Once + +var moduleSnapshotRowExistsCached bool + func moduleSnapshotRowTableExists(ctx context.Context, q queryRower) bool { - var n int - err := q.QueryRow(ctx, ` - SELECT 1 FROM information_schema.tables - WHERE table_schema = 'public' AND table_name = 'module_prefix_snapshot_row' - LIMIT 1`).Scan(&n) - return err == nil + moduleSnapshotRowExistsOnce.Do(func() { + var n int + err := q.QueryRow(ctx, ` + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = 'module_prefix_snapshot_row' + LIMIT 1`).Scan(&n) + // A query error means the backend is unreachable or information_schema is hidden; + // treat as "absent" so callers fall back to prefixes_json, but retry next call. + if err != nil { + moduleSnapshotRowExistsOnce = sync.Once{} + return + } + moduleSnapshotRowExistsCached = true + }) + return moduleSnapshotRowExistsCached } func (p *Postgres) GetModulePrefixSnapshot(tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) { @@ -97,19 +114,25 @@ func (p *Postgres) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, WHERE tenant_id = $1::uuid AND module_id = $2::uuid`, tenantID, moduleID); err != nil { return err } - for i, pr := range prefixes { - var comm any - if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" { - comm = strings.TrimSpace(*pr.CommunityID) - } - src := pr.Source - if strings.TrimSpace(src) == "" { - src = "render" - } - if _, err := tx.Exec(ctx, ` - INSERT INTO module_prefix_snapshot_row (tenant_id, module_id, ord, prefix, community_id, source) - VALUES ($1::uuid, $2::uuid, $3, $4, $5::uuid, $6)`, - tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src); err != nil { + if len(prefixes) > 0 { + _, err = tx.CopyFrom( + ctx, + pgx.Identifier{"module_prefix_snapshot_row"}, + []string{"tenant_id", "module_id", "ord", "prefix", "community_id", "source"}, + pgx.CopyFromSlice(len(prefixes), func(i int) ([]any, error) { + pr := prefixes[i] + var comm any + if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" { + comm = strings.TrimSpace(*pr.CommunityID) + } + src := pr.Source + if strings.TrimSpace(src) == "" { + src = "render" + } + return []any{tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src}, nil + }), + ) + if err != nil { return err } } @@ -135,3 +158,35 @@ func (p *Postgres) DeleteModulePrefixSnapshot(tenantID, moduleID string) error { tenantID, moduleID) return err } + +func (p *Postgres) SetModuleInputHash(tenantID, moduleID, hash string) error { + ctx := context.Background() + tag, err := p.pool.Exec(ctx, ` + UPDATE module SET input_hash = $3 + WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`, + moduleID, tenantID, hash) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return store.ErrNotFound + } + return nil +} + +func (p *Postgres) LockModuleSnapshot(tenantID, moduleID string) func() { + ctx := context.Background() + conn, err := p.pool.Acquire(ctx) + if err != nil { + return func() {} + } + key := strings.TrimSpace(tenantID) + "\x00" + strings.TrimSpace(moduleID) + if _, err := conn.Exec(ctx, `SELECT pg_advisory_lock(hashtext($1))`, key); err != nil { + conn.Release() + return func() {} + } + return func() { + _, _ = conn.Exec(ctx, `SELECT pg_advisory_unlock(hashtext($1))`, key) + conn.Release() + } +} diff --git a/internal/repository/postgres.go b/internal/repository/postgres.go index 9cdcf3d..f015b0d 100644 --- a/internal/repository/postgres.go +++ b/internal/repository/postgres.go @@ -119,7 +119,8 @@ func (p *Postgres) ListModules(tenantID string) []*store.Module { ctx := context.Background() rows, err := p.pool.Query(ctx, ` SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy, - refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id + refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id, + COALESCE(input_hash, '') FROM module WHERE tenant_id = $1 AND deleted_at IS NULL ORDER BY priority, name`, tenantID) if err != nil { return nil @@ -134,7 +135,7 @@ func (p *Postgres) ListModules(tenantID string) []*store.Module { var refresh *int32 var last *time.Time var createdBy *string - if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy); err != nil { + if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash); err != nil { continue } m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy) @@ -179,7 +180,8 @@ func (p *Postgres) ListModulesPage(tenantID, cursor string, limit int) ([]*store ctx := context.Background() rows, err := p.pool.Query(ctx, ` SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy, - refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id + refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id, + COALESCE(input_hash, '') FROM module WHERE tenant_id = $1 AND deleted_at IS NULL ORDER BY priority, name LIMIT $2 OFFSET $3`, tenantID, limit+1, off) @@ -196,7 +198,7 @@ func (p *Postgres) ListModulesPage(tenantID, cursor string, limit int) ([]*store var refresh *int32 var last *time.Time var createdBy *string - if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy); err != nil { + if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash); err != nil { continue } m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy) @@ -249,9 +251,10 @@ func (p *Postgres) GetModule(tenantID, moduleID string) (*store.Module, error) { var createdBy *string err := p.pool.QueryRow(ctx, ` SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy, - refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id + refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id, + COALESCE(input_hash, '') FROM module WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`, moduleID, tenantID).Scan( - &m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy) + &m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil, store.ErrNotFound @@ -325,9 +328,19 @@ func (p *Postgres) CreateModule(tenantID string, in *store.Module) (*store.Modul if err := p.setModuleDohProfiles(ctx, id, in.DohProfileIDs); err != nil { return nil, err } + store.TouchModuleInputHash(p, tenantID, id) return p.GetModule(tenantID, id) } +func (p *Postgres) invalidateModuleInputHash(tenantID, moduleID string) { + _ = p.SetModuleInputHash(tenantID, moduleID, "") +} + +func (p *Postgres) invalidateTenantModuleHashes(tenantID string) { + ctx := context.Background() + _, _ = p.pool.Exec(ctx, `UPDATE module SET input_hash = '' WHERE tenant_id = $1 AND deleted_at IS NULL`, tenantID) +} + func (p *Postgres) UpdateModule(tenantID, moduleID string, patch *store.ModulePatch) (*store.Module, error) { if patch == nil { return nil, store.ErrInvalidInput @@ -399,6 +412,7 @@ func (p *Postgres) UpdateModule(tenantID, moduleID string, patch *store.ModulePa return nil, err } } + p.invalidateModuleInputHash(tenantID, moduleID) return p.GetModule(tenantID, moduleID) } @@ -1191,6 +1205,7 @@ func (p *Postgres) UpdateDohProfile(tenantID, id string, patch *store.DohProfile if err != nil { return nil, err } + p.invalidateTenantModuleHashes(tenantID) return p.GetDohProfile(tenantID, id) } diff --git a/internal/repository/postgres_entities.go b/internal/repository/postgres_entities.go index dce0db3..87a1870 100644 --- a/internal/repository/postgres_entities.go +++ b/internal/repository/postgres_entities.go @@ -73,6 +73,7 @@ func (p *Postgres) CreateCDNSource(tenantID, moduleID string, in *store.CDNSourc if err != nil { return nil, err } + p.invalidateModuleInputHash(tenantID, moduleID) return p.getCDNSource(ctx, moduleID, id) } @@ -154,6 +155,9 @@ func (p *Postgres) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *s if err != nil { return nil, err } + if store.CDNPatchAffectsInputHash(patch) { + p.invalidateModuleInputHash(tenantID, moduleID) + } return p.getCDNSource(ctx, moduleID, sourceID) } @@ -169,6 +173,7 @@ func (p *Postgres) DeleteCDNSource(tenantID, moduleID, sourceID string) error { if tag.RowsAffected() == 0 { return store.ErrNotFound } + p.invalidateModuleInputHash(tenantID, moduleID) return nil } @@ -229,6 +234,7 @@ func (p *Postgres) CreateASEntry(tenantID, moduleID string, in *store.ASEntry) ( if err != nil { return nil, err } + p.invalidateModuleInputHash(tenantID, moduleID) return p.getASEntry(ctx, moduleID, id) } @@ -297,6 +303,7 @@ func (p *Postgres) UpdateASEntry(tenantID, moduleID, entryID string, patch *stor if err != nil { return nil, err } + p.invalidateModuleInputHash(tenantID, moduleID) return p.getASEntry(ctx, moduleID, entryID) } @@ -369,6 +376,7 @@ func (p *Postgres) DeleteASEntry(tenantID, moduleID, entryID string) error { if tag.RowsAffected() == 0 { return store.ErrNotFound } + p.invalidateModuleInputHash(tenantID, moduleID) return nil } @@ -417,6 +425,7 @@ func (p *Postgres) CreateDomainEntry(tenantID, moduleID string, in *store.Domain if err != nil { return nil, err } + p.invalidateModuleInputHash(tenantID, moduleID) return p.getDomainEntry(ctx, moduleID, id) } @@ -462,6 +471,7 @@ func (p *Postgres) UpdateDomainEntry(tenantID, moduleID, entryID string, patch * if err != nil { return nil, err } + p.invalidateModuleInputHash(tenantID, moduleID) return p.getDomainEntry(ctx, moduleID, entryID) } @@ -477,6 +487,7 @@ func (p *Postgres) DeleteDomainEntry(tenantID, moduleID, entryID string) error { if tag.RowsAffected() == 0 { return store.ErrNotFound } + p.invalidateModuleInputHash(tenantID, moduleID) return nil } @@ -525,6 +536,7 @@ func (p *Postgres) CreateIPRangeEntry(tenantID, moduleID string, in *store.IPRan if err != nil { return nil, err } + p.invalidateModuleInputHash(tenantID, moduleID) return p.getIPRangeEntry(ctx, moduleID, id) } @@ -570,6 +582,7 @@ func (p *Postgres) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch if err != nil { return nil, err } + p.invalidateModuleInputHash(tenantID, moduleID) return p.getIPRangeEntry(ctx, moduleID, entryID) } @@ -585,6 +598,7 @@ func (p *Postgres) DeleteIPRangeEntry(tenantID, moduleID, entryID string) error if tag.RowsAffected() == 0 { return store.ErrNotFound } + p.invalidateModuleInputHash(tenantID, moduleID) return nil } diff --git a/internal/store/backend.go b/internal/store/backend.go index f3bb656..d86ede5 100644 --- a/internal/store/backend.go +++ b/internal/store/backend.go @@ -116,11 +116,19 @@ type Backend interface { GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error DeleteModulePrefixSnapshot(tenantID, moduleID string) error + // SetModuleInputHash stores the ingest fingerprint on the module row (O(1) snapshot check). + SetModuleInputHash(tenantID, moduleID, hash string) error + // LockModuleSnapshot serializes read-modify-write of one module snapshot; unlock must be called. + LockModuleSnapshot(tenantID, moduleID string) (unlock func()) // ASNPrefixCache stores RIPEstat announced-prefixes per ASN (global TTL cache). GetASNPrefixCache(asn int64) (*ASNPrefixCacheEntry, bool, error) SetASNPrefixCache(asn int64, holder string, prefixes []string) error + // DomainResolveCache stores DoH results per FQDN (global TTL cache). + GetDomainResolveCache(fqdn string) (*DomainResolveCacheEntry, bool, error) + SetDomainResolveCache(fqdn string, addrs []string) error + // Ping verifies backend connectivity (no-op for in-memory). Ping(ctx context.Context) error @@ -178,6 +186,13 @@ type ASNPrefixCacheEntry struct { FetchedAt time.Time } +// DomainResolveCacheEntry is a cached DoH A/AAAA result for one FQDN. +type DomainResolveCacheEntry struct { + FQDN string + Addrs []string + ResolvedAt time.Time +} + // ModulePrefixSnapshot is the cached materialization for one module between refreshes. type ModulePrefixSnapshot struct { InputHash string diff --git a/internal/store/memory.go b/internal/store/memory.go index 043e8f0..6f3c07e 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -44,7 +44,9 @@ type Memory struct { settings map[string]map[string]any // tenantID -> key -> JSON-compatible value revPrefixes map[string][]PrefixRow moduleSnapshots map[string]*moduleSnapshotRec + snapshotLocks sync.Map // key -> *sync.Mutex (per-module snapshot RMW) asnPrefixCache map[int64]*ASNPrefixCacheEntry + domainResolveCache map[string]*DomainResolveCacheEntry apiKeys map[string]*apiKeyRec firewallClients map[string]*firewallClientRec firewallRules map[string]*FirewallRule @@ -99,6 +101,7 @@ type Module struct { LastRefreshedAt *time.Time DeletedAt *time.Time CreatedByUserID string // portal JWT sub; empty = system / API key + InputHash string // ingest fingerprint; empty = not yet computed } type Revision struct { @@ -156,6 +159,7 @@ func NewMemory() *Memory { revPrefixes: make(map[string][]PrefixRow), moduleSnapshots: make(map[string]*moduleSnapshotRec), asnPrefixCache: make(map[int64]*ASNPrefixCacheEntry), + domainResolveCache: make(map[string]*DomainResolveCacheEntry), apiKeys: make(map[string]*apiKeyRec), firewallClients: make(map[string]*firewallClientRec), firewallRules: make(map[string]*FirewallRule), @@ -696,6 +700,20 @@ func cloneStringPtr(s *string) *string { return &v } +func (m *Memory) clearModuleInputHashLocked(moduleID string) { + if mod, ok := m.modules[moduleID]; ok && mod != nil { + mod.InputHash = "" + } +} + +func (m *Memory) clearTenantModuleHashesLocked(tenantID string) { + for _, mod := range m.modules { + if mod != nil && mod.TenantID == tenantID && mod.DeletedAt == nil { + mod.InputHash = "" + } + } +} + func cloneModule(m *Module) *Module { if m == nil { return nil diff --git a/internal/store/memory_crud.go b/internal/store/memory_crud.go index db4e8ec..b8e3762 100644 --- a/internal/store/memory_crud.go +++ b/internal/store/memory_crud.go @@ -81,6 +81,7 @@ func (m *Memory) UpdateModule(tenantID, moduleID string, patch *ModulePatch) (*M t := patch.LastRefreshedAt.UTC() mod.LastRefreshedAt = &t } + mod.InputHash = "" return cloneModule(mod), nil } @@ -152,6 +153,7 @@ func (m *Memory) CreateCDNSource(tenantID, moduleID string, in *CDNSource) (*CDN LastRefreshedAt: in.LastRefreshedAt, } m.cdnSources[id] = s + mod.InputHash = "" return s, nil } @@ -161,7 +163,8 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN } m.mu.Lock() defer m.mu.Unlock() - if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil { + mod, err := m.moduleWriteOK(tenantID, moduleID) + if err != nil { return nil, err } s, ok := m.cdnSources[sourceID] @@ -195,6 +198,9 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN t := patch.LastRefreshedAt.UTC() s.LastRefreshedAt = &t } + if CDNPatchAffectsInputHash(patch) { + mod.InputHash = "" + } return s, nil } @@ -209,6 +215,7 @@ func (m *Memory) DeleteCDNSource(tenantID, moduleID, sourceID string) error { return ErrNotFound } delete(m.cdnSources, sourceID) + m.clearModuleInputHashLocked(moduleID) return nil } @@ -247,6 +254,7 @@ func (m *Memory) CreateASEntry(tenantID, moduleID string, in *ASEntry) (*ASEntry id := uuid.NewString() e := &ASEntry{ID: id, ModuleID: moduleID, ASN: in.ASN, CommunityID: in.CommunityID} m.asEntries[id] = e + mod.InputHash = "" return e, nil } @@ -256,7 +264,8 @@ func (m *Memory) UpdateASEntry(tenantID, moduleID, entryID string, patch *ASEntr } m.mu.Lock() defer m.mu.Unlock() - if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil { + mod, err := m.moduleWriteOK(tenantID, moduleID) + if err != nil { return nil, err } e, ok := m.asEntries[entryID] @@ -283,6 +292,7 @@ func (m *Memory) UpdateASEntry(tenantID, moduleID, entryID string, patch *ASEntr e.PrefixCount = nil e.ASNResolvedAt = nil } + mod.InputHash = "" return e, nil } @@ -324,6 +334,7 @@ func (m *Memory) DeleteASEntry(tenantID, moduleID, entryID string) error { return ErrNotFound } delete(m.asEntries, entryID) + m.clearModuleInputHashLocked(moduleID) return nil } @@ -362,6 +373,7 @@ func (m *Memory) CreateDomainEntry(tenantID, moduleID string, in *DomainEntry) ( id := uuid.NewString() e := &DomainEntry{ID: id, ModuleID: moduleID, FQDN: strings.TrimSpace(in.FQDN), CommunityID: in.CommunityID} m.domainEnt[id] = e + mod.InputHash = "" return e, nil } @@ -371,7 +383,8 @@ func (m *Memory) UpdateDomainEntry(tenantID, moduleID, entryID string, patch *Do } m.mu.Lock() defer m.mu.Unlock() - if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil { + mod, err := m.moduleWriteOK(tenantID, moduleID) + if err != nil { return nil, err } e, ok := m.domainEnt[entryID] @@ -389,6 +402,7 @@ func (m *Memory) UpdateDomainEntry(tenantID, moduleID, entryID string, patch *Do e.CommunityID = &v } } + mod.InputHash = "" return e, nil } @@ -403,6 +417,7 @@ func (m *Memory) DeleteDomainEntry(tenantID, moduleID, entryID string) error { return ErrNotFound } delete(m.domainEnt, entryID) + m.clearModuleInputHashLocked(moduleID) return nil } @@ -441,6 +456,7 @@ func (m *Memory) CreateIPRangeEntry(tenantID, moduleID string, in *IPRangeEntry) id := uuid.NewString() e := &IPRangeEntry{ID: id, ModuleID: moduleID, Prefix: strings.TrimSpace(in.Prefix), CommunityID: in.CommunityID} m.ipRanges[id] = e + mod.InputHash = "" return e, nil } @@ -450,7 +466,8 @@ func (m *Memory) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch *I } m.mu.Lock() defer m.mu.Unlock() - if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil { + mod, err := m.moduleWriteOK(tenantID, moduleID) + if err != nil { return nil, err } e, ok := m.ipRanges[entryID] @@ -468,6 +485,7 @@ func (m *Memory) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch *I e.CommunityID = &v } } + mod.InputHash = "" return e, nil } @@ -482,6 +500,7 @@ func (m *Memory) DeleteIPRangeEntry(tenantID, moduleID, entryID string) error { return ErrNotFound } delete(m.ipRanges, entryID) + m.clearModuleInputHashLocked(moduleID) return nil } @@ -551,6 +570,7 @@ func (m *Memory) UpdateDohProfile(tenantID, id string, patch *DohProfilePatch) ( if patch.SecretRef != nil { p.SecretRef = patch.SecretRef } + m.clearTenantModuleHashesLocked(tenantID) return p, nil } diff --git a/internal/store/memory_domain_cache.go b/internal/store/memory_domain_cache.go new file mode 100644 index 0000000..9a44183 --- /dev/null +++ b/internal/store/memory_domain_cache.go @@ -0,0 +1,41 @@ +package store + +import ( + "strings" + "time" +) + +func (m *Memory) GetDomainResolveCache(fqdn string) (*DomainResolveCacheEntry, bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + if m.domainResolveCache == nil { + return nil, false, nil + } + e, ok := m.domainResolveCache[strings.ToLower(strings.TrimSpace(fqdn))] + if !ok || e == nil { + return nil, false, nil + } + return &DomainResolveCacheEntry{ + FQDN: e.FQDN, + Addrs: append([]string(nil), e.Addrs...), + ResolvedAt: e.ResolvedAt, + }, true, nil +} + +func (m *Memory) SetDomainResolveCache(fqdn string, addrs []string) error { + key := strings.ToLower(strings.TrimSpace(fqdn)) + if key == "" { + return ErrInvalidInput + } + m.mu.Lock() + defer m.mu.Unlock() + if m.domainResolveCache == nil { + m.domainResolveCache = make(map[string]*DomainResolveCacheEntry) + } + m.domainResolveCache[key] = &DomainResolveCacheEntry{ + FQDN: key, + Addrs: append([]string(nil), addrs...), + ResolvedAt: time.Now().UTC(), + } + return nil +} diff --git a/internal/store/memory_snapshot.go b/internal/store/memory_snapshot.go index 27c44d5..18898e2 100644 --- a/internal/store/memory_snapshot.go +++ b/internal/store/memory_snapshot.go @@ -2,6 +2,7 @@ package store import ( "strings" + "sync" "time" ) @@ -51,6 +52,28 @@ func (m *Memory) DeleteModulePrefixSnapshot(tenantID, moduleID string) error { return nil } +func (m *Memory) SetModuleInputHash(tenantID, moduleID, hash string) error { + m.mu.Lock() + defer m.mu.Unlock() + mod, ok := m.modules[moduleID] + if !ok || mod.DeletedAt != nil { + return ErrNotFound + } + if mod.TenantID != tenantID { + return ErrTenantScope + } + mod.InputHash = strings.TrimSpace(hash) + return nil +} + +func (m *Memory) LockModuleSnapshot(tenantID, moduleID string) func() { + key := moduleSnapshotKey(tenantID, moduleID) + v, _ := m.snapshotLocks.LoadOrStore(key, &sync.Mutex{}) + mu := v.(*sync.Mutex) + mu.Lock() + return func() { mu.Unlock() } +} + type moduleSnapshotRec struct { InputHash string CollectedAt time.Time diff --git a/internal/store/module_input_hash.go b/internal/store/module_input_hash.go new file mode 100644 index 0000000..3e20697 --- /dev/null +++ b/internal/store/module_input_hash.go @@ -0,0 +1,123 @@ +package store + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" +) + +// ComputeModuleInputHash fingerprints module config and child entries so snapshots +// invalidate on CRUD without re-reading every child at render time. +func ComputeModuleInputHash(st Backend, tenantID string, mod *Module) (string, error) { + if st == nil || mod == nil { + return "", fmt.Errorf("store: module hash: missing store or module") + } + h := sha256.New() + _, _ = fmt.Fprintf(h, "type=%s\n", strings.TrimSpace(mod.Type)) + _, _ = fmt.Fprintf(h, "enabled=%t\n", mod.Enabled) + if mod.DefaultCommunityID != nil { + _, _ = fmt.Fprintf(h, "default_community=%s\n", strings.TrimSpace(*mod.DefaultCommunityID)) + } + _, _ = fmt.Fprintf(h, "doh_policy=%s\n", NormalizeDohResolverPolicy(mod.DohResolverPolicy)) + for _, pid := range mod.EffectiveDohProfileIDs() { + _, _ = fmt.Fprintf(h, "doh_profile=%s\n", pid) + if prof, err := st.GetDohProfile(tenantID, pid); err == nil && prof != nil { + _, _ = fmt.Fprintf(h, "doh_url=%s\n", strings.TrimSpace(prof.URL)) + if prof.TimeoutMs != nil { + _, _ = fmt.Fprintf(h, "doh_timeout=%d\n", *prof.TimeoutMs) + } + } + } + + switch mod.Type { + case "IP_RANGES": + list, err := st.ListIPRangeEntries(tenantID, mod.ID) + if err != nil { + return "", err + } + sort.Slice(list, func(i, j int) bool { return list[i].Prefix < list[j].Prefix }) + for _, e := range list { + comm := "" + if e.CommunityID != nil { + comm = *e.CommunityID + } + _, _ = fmt.Fprintf(h, "ip=%s|c=%s\n", e.Prefix, comm) + } + case "AS_PREFIXES": + list, err := st.ListASEntries(tenantID, mod.ID) + if err != nil { + return "", err + } + sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN }) + for _, e := range list { + comm := "" + if e.CommunityID != nil { + comm = *e.CommunityID + } + _, _ = fmt.Fprintf(h, "as=%d|c=%s\n", e.ASN, comm) + } + case "CDN_CIDRS": + list, err := st.ListCDNSources(tenantID, mod.ID) + if err != nil { + return "", err + } + sort.Slice(list, func(i, j int) bool { return list[i].ID < list[j].ID }) + for _, s := range list { + comm := "" + if s.CommunityID != nil { + comm = *s.CommunityID + } + interval := 0 + if s.RefreshIntervalSec != nil { + interval = *s.RefreshIntervalSec + } + _, _ = fmt.Fprintf(h, "cdn=%s|url=%s|kind=%s|path=%s|c=%s|etag=%s|interval=%d\n", + s.ID, strings.TrimSpace(s.URL), s.SourceKind, strings.TrimSpace(s.PrefixPath), comm, + strings.TrimSpace(s.Etag), interval) + } + case "DOMAINS": + list, err := st.ListDomainEntries(tenantID, mod.ID) + if err != nil { + return "", err + } + sort.Slice(list, func(i, j int) bool { return list[i].FQDN < list[j].FQDN }) + for _, e := range list { + comm := "" + if e.CommunityID != nil { + comm = *e.CommunityID + } + _, _ = fmt.Fprintf(h, "dom=%s|c=%s\n", strings.TrimSpace(e.FQDN), comm) + } + default: + _, _ = fmt.Fprintf(h, "unknown_type=%s\n", mod.Type) + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// TouchModuleInputHash recomputes and stores module.input_hash (ARCH-01: hash lives in store). +func TouchModuleInputHash(st Backend, tenantID, moduleID string) { + if st == nil { + return + } + mod, err := st.GetModule(tenantID, moduleID) + if err != nil || mod == nil { + return + } + h, err := ComputeModuleInputHash(st, tenantID, mod) + if err != nil { + return + } + _ = st.SetModuleInputHash(tenantID, moduleID, h) +} + +// CDNPatchAffectsInputHash reports whether a CDN source patch changes ingest fingerprint +// fields (URL/kind/path/community/interval). ETag and last_refreshed_at do not. +func CDNPatchAffectsInputHash(patch *CDNSourcePatch) bool { + if patch == nil { + return false + } + return patch.SourceKind != nil || patch.URL != nil || patch.PrefixPath != nil || + patch.CommunityID != nil || patch.RefreshIntervalSec != nil +} diff --git a/migrations/postgres/000032_module_input_hash.down.sql b/migrations/postgres/000032_module_input_hash.down.sql new file mode 100644 index 0000000..99172a0 --- /dev/null +++ b/migrations/postgres/000032_module_input_hash.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE module + DROP COLUMN IF EXISTS input_hash; diff --git a/migrations/postgres/000032_module_input_hash.up.sql b/migrations/postgres/000032_module_input_hash.up.sql new file mode 100644 index 0000000..f420ae7 --- /dev/null +++ b/migrations/postgres/000032_module_input_hash.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE module + ADD COLUMN input_hash TEXT NOT NULL DEFAULT ''; diff --git a/migrations/postgres/000033_domain_resolve_cache.down.sql b/migrations/postgres/000033_domain_resolve_cache.down.sql new file mode 100644 index 0000000..d79cae4 --- /dev/null +++ b/migrations/postgres/000033_domain_resolve_cache.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS domain_resolve_cache; diff --git a/migrations/postgres/000033_domain_resolve_cache.up.sql b/migrations/postgres/000033_domain_resolve_cache.up.sql new file mode 100644 index 0000000..5d16105 --- /dev/null +++ b/migrations/postgres/000033_domain_resolve_cache.up.sql @@ -0,0 +1,8 @@ +-- TTL cache for DoH/A/AAAA resolutions (pipeline domain ingest). +CREATE TABLE domain_resolve_cache ( + fqdn TEXT PRIMARY KEY, + addrs_json JSONB NOT NULL DEFAULT '[]', + resolved_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX idx_domain_resolve_cache_resolved ON domain_resolve_cache (resolved_at); diff --git a/migrations/sqlite/000032_module_input_hash.down.sql b/migrations/sqlite/000032_module_input_hash.down.sql new file mode 100644 index 0000000..75f9df2 --- /dev/null +++ b/migrations/sqlite/000032_module_input_hash.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE module + DROP COLUMN input_hash; diff --git a/migrations/sqlite/000032_module_input_hash.up.sql b/migrations/sqlite/000032_module_input_hash.up.sql new file mode 100644 index 0000000..f420ae7 --- /dev/null +++ b/migrations/sqlite/000032_module_input_hash.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE module + ADD COLUMN input_hash TEXT NOT NULL DEFAULT ''; diff --git a/migrations/sqlite/000033_domain_resolve_cache.down.sql b/migrations/sqlite/000033_domain_resolve_cache.down.sql new file mode 100644 index 0000000..d79cae4 --- /dev/null +++ b/migrations/sqlite/000033_domain_resolve_cache.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS domain_resolve_cache; diff --git a/migrations/sqlite/000033_domain_resolve_cache.up.sql b/migrations/sqlite/000033_domain_resolve_cache.up.sql new file mode 100644 index 0000000..07bef18 --- /dev/null +++ b/migrations/sqlite/000033_domain_resolve_cache.up.sql @@ -0,0 +1,7 @@ +CREATE TABLE domain_resolve_cache ( + fqdn TEXT PRIMARY KEY, + addrs_json TEXT NOT NULL DEFAULT '[]', + resolved_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); + +CREATE INDEX idx_domain_resolve_cache_resolved ON domain_resolve_cache (resolved_at);