diff --git a/.gitea/workflows/quality.yaml b/.gitea/workflows/quality.yaml index 69704af..ec78133 100644 --- a/.gitea/workflows/quality.yaml +++ b/.gitea/workflows/quality.yaml @@ -322,6 +322,66 @@ jobs: go build -o "$out/$name" "./$d" done + postgres-integration: + needs: [changes] + if: needs.changes.outputs.go == 'true' + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: evobgp + POSTGRES_PASSWORD: evobgp_ci + POSTGRES_DB: evobgp_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U evobgp -d evobgp_test" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: "1.24" + cache: false + - name: Export cache paths + run: sh scripts/ci/export-cache-env.sh + - id: go-hash + run: echo "key=$(sha256sum go.sum | awk '{print $1}')" >> "$GITHUB_OUTPUT" + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: | + ${{ env.GOMODCACHE }} + ${{ env.GOCACHE }} + key: go-${{ runner.os }}-1.24-postgres-${{ steps.go-hash.outputs.key }} + restore-keys: | + go-${{ runner.os }}-1.24-postgres- + - name: Run migrations and verify PostgreSQL integration tests executed + env: + GOFLAGS: -mod=readonly + EVOBGP_TEST_DATABASE_URL: postgres://evobgp:evobgp_ci@127.0.0.1:5432/evobgp_test?sslmode=disable + run: bash scripts/ci/postgres-integration.sh + + legacy-compatibility: + needs: [changes] + if: needs.changes.outputs.go == 'true' + runs-on: ubuntu-latest + strategy: + matrix: + legacy_version: ["2.56.2", "1.0.0"] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: "1.24" + cache: false + - name: Build pinned legacy executables and test current control plane + env: + EVOBGP_LEGACY_VERSION: ${{ matrix.legacy_version }} + run: bash scripts/ci/legacy-node-compatibility.sh + bird2: runs-on: ubuntu-latest needs: [changes, go] @@ -335,6 +395,10 @@ jobs: needs.changes.outputs.docker_go == 'true') steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: "1.24" + cache: false - name: Install bird2 (репозиторий Ubuntu runner, как в образе evobgp-bird2) run: | set -euxo pipefail @@ -358,6 +422,11 @@ jobs: echo "==> $conf" bird -c "$WS/$conf" -p done + - name: Parse staged birddeploy generation with real BIRD + env: + GOFLAGS: -mod=readonly + EVOBGP_TEST_BIRD_BIN: bird + run: go test ./internal/birddeploy -run '^TestRealBirdParseGeneration$' -count=1 -v commitlint: if: inputs.is_pull_request @@ -405,6 +474,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Offline mirror behavior tests + run: bash scripts/tests/mirror-base-images.test.sh - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 with: name: evobgp diff --git a/.gitignore b/.gitignore index c5fdc9e..4ac7fca 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ deploy/compose/runtime-logs/ .cursor/mcp.json .mcp.json .codegraph/daemon.pid + +# Local backend resource snapshots (host metadata, not source artifacts) +artifacts/backend-baseline/ diff --git a/cmd/evobgp-agent/main.go b/cmd/evobgp-agent/main.go index f660403..8360eb5 100644 --- a/cmd/evobgp-agent/main.go +++ b/cmd/evobgp-agent/main.go @@ -6,10 +6,14 @@ import ( "fmt" "log" "os" + "os/signal" + "strings" "sync" + "syscall" "time" "evobgp/internal/agentserver" + "evobgp/internal/agentwatch" "evobgp/internal/birdfmt" ) @@ -18,19 +22,22 @@ func main() { birdc := flag.String("birdc", "", "path to birdc binary (default: birdc from PATH)") socket := flag.String("socket", "", "optional birdc control socket (-s)") timeout := flag.Duration("timeout", 30*time.Second, "timeout for bird/birdc") - watchEvery := flag.Duration("watch-interval", 30*time.Second, "for watch: interval between birdc configure") + watchEvery := flag.Duration("watch-interval", 30*time.Second, "for watch: interval between configuration checks") listen := flag.String("listen", "", "for serve: listen address (default :8443 or EVOBGP_AGENT_LISTEN)") flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s [flags] \n", os.Args[0]) fmt.Fprintf(os.Stderr, "Commands:\n") fmt.Fprintf(os.Stderr, " parse-check run bird -c -p (syntax check)\n") fmt.Fprintf(os.Stderr, " configure run birdc configure (reload running BIRD)\n") - fmt.Fprintf(os.Stderr, " watch periodically run birdc configure (compose sidecar)\n") + fmt.Fprintf(os.Stderr, " watch reload changed BIRD configuration and periodically reconcile\n") fmt.Fprintf(os.Stderr, " serve Panel→Node HTTP API (POST /v1/agent/sync)\n") flag.PrintDefaults() } - flag.Parse() - args := flag.Args() + args, err := parseAgentArgs(flag.CommandLine, os.Args[1:]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } if len(args) < 1 { flag.Usage() os.Exit(2) @@ -47,10 +54,12 @@ func main() { switch args[0] { case "serve": runServe(*listen, *timeout) - case "parse-check", "configure", "watch": + case "watch": + runWatch(ctl, *watchEvery, *timeout) + case "parse-check", "configure": ctx, cancel := context.WithTimeout(context.Background(), *timeout) defer cancel() - runBirdCommand(ctx, args, ctl, *watchEvery, *timeout) + runBirdCommand(ctx, args, ctl) default: fmt.Fprintf(os.Stderr, "unknown command: %s\n", args[0]) flag.Usage() @@ -58,7 +67,47 @@ func main() { } } -func runBirdCommand(ctx context.Context, args []string, ctl *birdfmt.BirdCtl, watchEvery, timeout time.Duration) { +// parseAgentArgs accepts flags before or immediately after the subcommand. +// A second parse preserves previously supplied flag values and rejects unknown +// flags instead of silently passing them to watch or serve as positional args. +func parseAgentArgs(fs *flag.FlagSet, argv []string) ([]string, error) { + if err := fs.Parse(argv); err != nil { + return nil, err + } + args := fs.Args() + if len(args) == 0 { + return args, nil + } + command := args[0] + globalStop := false + for i, consumed := 0, len(argv)-len(args); i < consumed; i++ { + if argv[i] == "--" { + globalStop = true + break + } + name, _, hasValue := strings.Cut(strings.TrimLeft(argv[i], "-"), "=") + if !hasValue { + if f := fs.Lookup(name); f != nil { + boolean, ok := f.Value.(interface{ IsBoolFlag() bool }) + if !ok || !boolean.IsBoolFlag() { + i++ + } + } + } + } + if !globalStop { + if err := fs.Parse(args[1:]); err != nil { + return nil, err + } + args = append([]string{command}, fs.Args()...) + } + if (command == "watch" || command == "serve" || command == "configure") && len(args) != 1 { + return nil, fmt.Errorf("%s takes no positional arguments", command) + } + return args, nil +} + +func runBirdCommand(ctx context.Context, args []string, ctl *birdfmt.BirdCtl) { switch args[0] { case "parse-check": if len(args) != 2 { @@ -78,21 +127,33 @@ func runBirdCommand(ctx context.Context, args []string, ctl *birdfmt.BirdCtl, wa fmt.Fprintln(os.Stderr, err) os.Exit(1) } - case "watch": - if watchEvery <= 0 { - fmt.Fprintln(os.Stderr, "watch-interval must be > 0") + } +} + +func runWatch(ctl *birdfmt.BirdCtl, every, timeout time.Duration) { + if every <= 0 || timeout <= 0 { + fmt.Fprintln(os.Stderr, "watch-interval and timeout must be > 0") + os.Exit(2) + } + reconcile := 5 * time.Minute + if raw := os.Getenv("EVOBGP_AGENT_WATCH_RECONCILE_INTERVAL"); raw != "" { + var err error + reconcile, err = time.ParseDuration(raw) + if err != nil || reconcile <= 0 { + fmt.Fprintln(os.Stderr, "EVOBGP_AGENT_WATCH_RECONCILE_INTERVAL must be a positive duration") os.Exit(2) } - log.Printf("evobgp-agent watch: birdc configure every %s (socket=%q)", watchEvery, ctl.Socket) - for { - cctx, cancel := context.WithTimeout(context.Background(), timeout) - err := ctl.Configure(cctx) - cancel() - if err != nil { - log.Printf("evobgp-agent watch: configure: %v", err) - } - time.Sleep(watchEvery) - } + } + active := os.Getenv("EVOBGP_BIRD_ACTIVE_DIR") + if active == "" { + active = "/etc/bird" + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + log.Printf("evobgp-agent watch: checking %s every %s, forced reconciliation every %s (socket=%q)", active, every, reconcile, ctl.Socket) + if err := agentwatch.Run(ctx, ctl, agentwatch.Config{ActiveDir: active, Every: every, ReconcileEvery: reconcile, CommandTimeout: timeout}); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) } } diff --git a/cmd/evobgp-agent/main_test.go b/cmd/evobgp-agent/main_test.go new file mode 100644 index 0000000..6f534fb --- /dev/null +++ b/cmd/evobgp-agent/main_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "flag" + "io" + "reflect" + "testing" + "time" +) + +func TestAgentFlagsBeforeAndAfterCommand(t *testing.T) { + for _, tc := range []struct { + name string + args []string + want []string + socket, listen string + every time.Duration + }{ + {"legacy flags first", []string{"-socket=/custom/bird.ctl", "-watch-interval=45s", "watch"}, []string{"watch"}, "/custom/bird.ctl", "", 45 * time.Second}, + {"compose command first", []string{"watch", "-socket=/run/bird/bird.ctl", "-watch-interval=30s"}, []string{"watch"}, "/run/bird/bird.ctl", "", 30 * time.Second}, + {"serve", []string{"serve", "-listen=127.0.0.1:9999"}, []string{"serve"}, "", "127.0.0.1:9999", time.Minute}, + {"mixed", []string{"-socket=/first", "watch", "-watch-interval=5s"}, []string{"watch"}, "/first", "", 5 * time.Second}, + {"parse check", []string{"parse-check", "bird.conf"}, []string{"parse-check", "bird.conf"}, "", "", time.Minute}, + {"global delimiter", []string{"--", "parse-check", "-bird.conf"}, []string{"parse-check", "-bird.conf"}, "", "", time.Minute}, + {"flags then delimiter", []string{"-socket=/first", "--", "parse-check", "-bird.conf"}, []string{"parse-check", "-bird.conf"}, "/first", "", time.Minute}, + {"delimiter as value", []string{"-socket", "--", "watch", "-watch-interval=5s"}, []string{"watch"}, "--", "", 5 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + fs := flag.NewFlagSet("agent", flag.ContinueOnError) + fs.SetOutput(io.Discard) + socket := fs.String("socket", "", "") + listen := fs.String("listen", "", "") + every := fs.Duration("watch-interval", time.Minute, "") + args, err := parseAgentArgs(fs, tc.args) + if err != nil || !reflect.DeepEqual(args, tc.want) || *socket != tc.socket || *listen != tc.listen || *every != tc.every { + t.Fatalf("args=%v socket=%s listen=%s every=%s err=%v", args, *socket, *listen, *every, err) + } + }) + } +} +func TestAgentRejectsIgnoredFlagsAndUnexpectedArguments(t *testing.T) { + for _, argv := range [][]string{{"watch", "-unknown=1"}, {"serve", "-unknown=1"}, {"watch", "unused"}, {"serve", "unused"}, {"configure", "unused"}} { + fs := flag.NewFlagSet("agent", flag.ContinueOnError) + fs.SetOutput(io.Discard) + if _, err := parseAgentArgs(fs, argv); err == nil { + t.Fatalf("ignored argv: %v", argv) + } + } +} diff --git a/cmd/evobgp-scheduler/main.go b/cmd/evobgp-scheduler/main.go index c42b2d1..4151a5a 100644 --- a/cmd/evobgp-scheduler/main.go +++ b/cmd/evobgp-scheduler/main.go @@ -41,23 +41,41 @@ func main() { apiBase := strings.TrimSpace(os.Getenv("EVOBGP_CONTROL_PLANE_URL")) apiTok := strings.TrimSpace(os.Getenv("EVOBGP_SCHEDULER_BEARER")) - if os.Getenv("EVOBGP_SCHEDULER_STANDALONE") == "1" && apiBase == "" { - log.Fatal("evobgp-scheduler: EVOBGP_SCHEDULER_STANDALONE=1 requires EVOBGP_CONTROL_PLANE_URL and EVOBGP_SCHEDULER_BEARER (split deploy must not use in-process jobs.Registry)") + mode := strings.ToLower(strings.TrimSpace(os.Getenv("EVOBGP_SCHEDULER_MODE"))) + if mode == "" { + switch { + case apiBase != "" || apiTok != "": + mode = "http" + case pool != nil: + mode = "db" + default: + mode = "local" + } } - deps := &scheduler.Deps{ - Store: st, - HTTP: &http.Client{Timeout: 45 * time.Second}, - } - if (apiBase != "") != (apiTok != "") { - log.Fatal("evobgp-scheduler: set both EVOBGP_CONTROL_PLANE_URL and EVOBGP_SCHEDULER_BEARER for HTTP mode, or leave both empty for in-process job registry") - } - if apiBase != "" { - deps.APIBase = apiBase - deps.APIToken = apiTok - log.Printf("evobgp-scheduler: control plane HTTP mode (%s)", apiBase) - } else { + deps := &scheduler.Deps{Store: st, HTTP: &http.Client{Timeout: 45 * time.Second}} + switch mode { + case "db": + if pool == nil { + log.Fatal("evobgp-scheduler: db mode requires EVOBGP_DATABASE_URL") + } deps.Jobs = reg - log.Printf("evobgp-scheduler: in-process job registry mode (use EVOBGP_CONTROL_PLANE_URL + EVOBGP_SCHEDULER_BEARER for split containers)") + log.Print("evobgp-scheduler: trusted DB enqueue mode; API workers consume the shared durable queue") + case "http": + tenant := strings.TrimSpace(os.Getenv("EVOBGP_SCHEDULER_TENANT_ID")) + if apiBase == "" || apiTok == "" || tenant == "" { + log.Fatal("evobgp-scheduler: http mode requires EVOBGP_CONTROL_PLANE_URL, EVOBGP_SCHEDULER_BEARER and EVOBGP_SCHEDULER_TENANT_ID") + } + deps.APIBase, deps.APIToken, deps.APITenantID = apiBase, apiTok, tenant + log.Print("evobgp-scheduler: single-tenant HTTP enqueue mode") + case "local": + if os.Getenv("EVOBGP_SCHEDULER_STANDALONE") == "1" || pool != nil { + log.Fatal("evobgp-scheduler: local mode is for in-process development only; use db for a shared PostgreSQL queue") + } + deps.Jobs = reg + defer func() { reg.RequestCancelAll(); _ = reg.Drain(context.Background()) }() + log.Print("evobgp-scheduler: local development mode") + default: + log.Fatal("evobgp-scheduler: EVOBGP_SCHEDULER_MODE must be db, http or local") } scheduler.Run(ctx, deps) } diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 9d28fe1..ec40406 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -10,6 +10,20 @@ info: **Соглашения:** префикс `/v1`; идентификаторы - UUID v7 или ULID (строки); время - ISO 8601 UTC. Ошибки - `application/problem+json` ([RFC 9457](https://www.rfc-editor.org/rfc/rfc9457)). Пагинация списков - `cursor` + `limit`; ответ содержит `items`, `next_cursor`, `has_more`. + **Общий предел request body:** после аутентификации каждый непубличный `/v1/...` + запрос проверяется целиком до выполнения handler: максимум 8 MiB (8388608 bytes), + независимо от HTTP-метода, Content-Type и наличия Content-Length. Превышение, + включая whitespace после JSON, возвращает `413 application/problem+json` + до мутаций. Ошибка чтения тела возвращает `400`, отмена/таймаут чтения — `408`. + Пустые node GET/pull и streaming SSE responses не меняются. CSV дополнительно + сохраняет ограничения числа строк и полей своей операции. + Запросы с телом занимают ограниченный слот API instance до окончания handler: + по умолчанию 4 одновременно, `EVOBGP_API_BODY_MAX_CONCURRENT` задаёт 1..64. + Свободных слотов нет — немедленный `429` с `Retry-After: 1`, без очереди ожидания. + Пустые запросы (`http.NoBody`) слота не занимают. + Если CRUD-изменение сохранено, но автоматическая refresh/reconcile задача не поставлена, + успешный ответ содержит `X-EvoBGP-Job-Status: enqueue-failed`; ошибка также попадает в аудит. + Изменение уже сохранено: следует явно запустить refresh/reconcile, а не повторять создание объекта. **Аутентификация (dual):** - **API key** — `Authorization: Bearer ` из `EVOBGP_API_KEYS` / таблицы `api_key` (роли `viewer`/`editor`/`operator`/`node`/`firewall`). @@ -226,6 +240,16 @@ components: Основные значения: `module_refresh`, `peer_reconcile`, `deploy_apply`, `revision_rollback`, `bird_reload`. responses: + ExpensiveOperationLimited: + description: Локальная квота процесса API — по умолчанию 4 активных preview/lookup, 2 на tenant; 120 lookup и по 30 CDN/revision preview в минуту на tenant. Квоты умножаются на число реплик. Повторить после Retry-After, без очереди ожидания. + headers: + Retry-After: + description: Минимальная задержка повторной попытки в секундах; не гарантия свободного места. + schema: {type: integer, minimum: 1} + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Problem" Unauthorized: description: Нет или невалидный Bearer-токен. content: @@ -263,7 +287,10 @@ components: schema: $ref: "#/components/schemas/Problem" DefaultProblem: - description: Ошибка (см. тело Problem). + description: > + Ошибка (см. тело Problem). Общие для authenticated `/v1` ответы middleware: + 413 при request body свыше 8 MiB, 400 при ошибке чтения тела, 408 при отмене/таймауте чтения. + 429 с Retry-After: 1 при занятых слотах обработки request body. content: application/problem+json: schema: @@ -903,6 +930,21 @@ components: connected_speaker_label: type: string description: Человекочитаемая метка ноды из live-опроса. + last_known_session_state: + type: ["string", "null"] + description: Последнее сохранённое состояние; не является текущим наблюдением. + session_state: + type: ["string", "null"] + description: > + При live-ответе текущее наблюдение: BGP state, absent после успешного + пустого опроса или null, если актуальное состояние неизвестно. + live_observation: + $ref: "#/components/schemas/LiveObservation" + description: > + Качество BGP-наблюдений при GET /v1/peers?live=1. Для назначенного + пира учитывается только bgp_speaker_id; для общего — все speakers. + Смешанный успех и сбой не дают quality=ok или session_state=absent. + Timestamps относятся только к ожидаемым speakers. session_on_speakers: type: array description: Состояние протокола пира на каждой опрошенной ноде. @@ -915,6 +957,10 @@ components: type: string state: type: string + poll_error: + type: string + observation: + $ref: "#/components/schemas/LiveObservation" established_on_speakers: type: array description: Ноды, где сессия в состоянии Established (один пир может быть на нескольких). @@ -1001,7 +1047,11 @@ components: description: Человекочитаемая метка ноды (agent domain или CP master). agent_ok: type: boolean - description: true если agent /v1/agent/health успешен (master — local birdc poll). + description: true только после успешного agent /v1/agent/health (master — local birdc poll). + agent_observation: + $ref: "#/components/schemas/LiveObservation" + bgp_observation: + $ref: "#/components/schemas/LiveObservation" agent_error: type: string agent_last_sync_at: @@ -1015,15 +1065,49 @@ components: bgp_poll_error: type: string bgp_sessions_total: - type: integer + type: ["integer", "null"] + description: null при отсутствии успешного опроса. bgp_established: - type: integer + type: ["integer", "null"] + description: null при отсутствии успешного опроса. sessions: type: array items: $ref: "#/components/schemas/BgpSessionLive" additionalProperties: true + LiveObservation: + type: object + description: > + Метаданные конкретного опроса. observed_at — время завершения опроса; + received_at — время получения результата CP; last_success_at — время + последнего успешного опроса. Старые агенты не обязаны передавать эти поля: + CP формирует их по собственным часам. При отсутствии опроса timestamps null. + required: [observed_at, received_at, last_success_at, source, quality] + properties: + observed_at: + type: ["string", "null"] + format: date-time + received_at: + type: ["string", "null"] + format: date-time + last_success_at: + type: ["string", "null"] + format: date-time + expires_at: + type: ["string", "null"] + format: date-time + description: Срок актуальности snapshot по часам CP и настроенному интервалу collector; клиент не должен считать cached quality=ok актуальным после этого времени. Для агрегата — самый ранний срок источников. + source: + type: string + enum: [none, cp_birdc, agent_bird_protocols, agent_health, mixed] + quality: + type: string + enum: [ok, stale, unreachable, unknown, disabled] + error_code: + type: string + enum: [not_configured, not_polled, poll_failed, reported_unhealthy, snapshot_expired, configuration_changed, ambiguous_authority] + BgpSessionLive: type: object properties: @@ -1052,9 +1136,12 @@ components: ok: type: boolean session_count: - type: integer + type: ["integer", "null"] + description: null, если успешного опроса не было. poll_error: type: string + observation: + $ref: "#/components/schemas/LiveObservation" additionalProperties: true PostgresOverview: @@ -2108,6 +2195,8 @@ paths: application/json: schema: $ref: "#/components/schemas/LookupResponse" + "429": + $ref: "#/components/responses/ExpensiveOperationLimited" "400": $ref: "#/components/responses/BadRequest" "401": @@ -2115,6 +2204,29 @@ paths: default: $ref: "#/components/responses/DefaultProblem" + /v1/telemetry/refresh: + parameters: + - $ref: "#/components/parameters/TenantId" + post: + tags: [Speakers] + summary: Запросить фоновое обновление телеметрии tenant + description: Требует bgp:network:read. Объединяет повторные запросы в пределах 10 секунд; HTTP handler не опрашивает агентов или BIRD. GET speakers/peers/discoveries читает сохранённые snapshots, включая live=1. Запрос во время выполняющегося poll сохраняется для следующего цикла. Не изменяет конфигурацию speaker и не требует нового протокола агента. + operationId: requestTelemetryRefresh + responses: + "202": + description: Запрос принят либо объединён с уже принятым. + content: + application/json: + schema: + type: object + required: [requested] + properties: + requested: + type: boolean + description: true при постановке нового запроса; false при объединении или отсутствии целей. + default: + $ref: "#/components/responses/DefaultProblem" + /v1/router-lists/catalog: get: tags: [Modules] @@ -2789,6 +2901,9 @@ paths: - `ipRange,community` В поле `community` можно передавать либо ID community, либо её значение. + Предел запроса — 8 MiB (8388608 байт), 10000 строк данных без заголовка и 64 поля в записи. + Превышение возвращает 413 до создания записей; большой импорт следует разбить на части. + Ошибка в данных или БД после начала создания записей может оставить частичный импорт. operationId: importModuleEntriesCsv parameters: - $ref: "#/components/parameters/IdempotencyKey" @@ -2812,6 +2927,12 @@ paths: minimum: 0 module_type: type: string + "413": + description: Превышен лимит байт, строк или полей CSV; записи не созданы. + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Problem" "404": $ref: "#/components/responses/NotFound" "422": @@ -2898,6 +3019,8 @@ paths: application/json: schema: $ref: "#/components/schemas/CdnPreviewResponse" + "429": + $ref: "#/components/responses/ExpensiveOperationLimited" "401": $ref: "#/components/responses/Unauthorized" "403": @@ -3146,7 +3269,7 @@ paths: type: string enum: ["1"] description: > - Опрос birdc на CP и GET /v1/agent/bird/protocols на репликах; обогащает session_state и connected_speaker_*. + Сохранённые результаты фонового опроса birdc на CP и agent protocols на репликах; обогащает session_state и connected_speaker_*. GET не запускает опрос; live=1 сохранён для совместимости. responses: "200": description: Успешно. @@ -3167,7 +3290,7 @@ paths: live_speaker_poll: type: array description: > - При live=1 — результат опроса каждой ноды (CP birdc + agent protocols). + Сохранённый результат фонового опроса каждой ноды (CP birdc + agent protocols); live=1 не запускает новый опрос. items: $ref: "#/components/schemas/LiveSpeakerPoll" default: @@ -3204,7 +3327,7 @@ paths: summary: Список обнаруженных (неодобренных) пиров description: > Dynamic BGP-сессии (`evobgp_dyn_*`), которых ещё нет в `bgp_peer`. - По умолчанию возвращает `status=pending`. При листинге выполняет live-опрос birdc/agent и upsert pending. + По умолчанию возвращает `status=pending`. Читает сохранённые discoveries. Фоновый collector выполняет опрос и fenced upsert; GET не опрашивает ноды и не изменяет данные. operationId: listDiscoveredPeers parameters: - $ref: "#/components/parameters/TenantId" @@ -3380,7 +3503,7 @@ paths: type: string enum: ["1"] description: > - Live-опрос agent /v1/agent/health и BGP protocols на репликах; CP — local birdc. + Сохранённые результаты фоновых опросов agent health/BGP и local birdc. GET не запускает опрос. Обогащает каждый item полем `live`. responses: "200": @@ -3640,6 +3763,8 @@ paths: description: Только чтение, без apply. operationId: getRevisionPreview responses: + "429": + $ref: "#/components/responses/ExpensiveOperationLimited" "200": description: Текст или структурированное представление - формат задаётся реализацией. content: @@ -3881,6 +4006,58 @@ paths: default: $ref: "#/components/responses/DefaultProblem" + /v1/jobs/{job_id}/report: + parameters: + - $ref: "#/components/parameters/TenantId" + - $ref: "#/components/parameters/JobId" + get: + tags: [Jobs] + summary: Отчёт задачи и сохранённый прогресс доставки + description: Требует bgp:operations:read. Прогресс хранится независимо от meta задачи. Для reconciliation без собственного прогресса возвращается прогресс исходной задачи в том же tenant; после начала восстановления приоритет имеет собственный прогресс. agent_reported не означает проверку состояния BIRD. + operationId: getJobReport + responses: + "200": + description: Успешно. + content: + application/json: + schema: + type: object + required: [job_id, kind, status, deployment_progress] + properties: + job_id: {type: string} + kind: {type: string} + status: {type: string} + meta: {type: [object, "null"], additionalProperties: true} + error: {type: [string, "null"]} + created_at: {type: string, format: date-time} + revision_id: {type: string} + log_entries: {type: array, items: {type: object, additionalProperties: true}} + log_total: {type: integer} + source_deployment_job_id: {type: string} + deployment_progress: + type: array + items: + type: object + required: [job_id, speaker_id, revision_id, generation, state, confirmation, dispatch_attempts, retry_round] + properties: + job_id: {type: string} + speaker_id: {type: string} + revision_id: {type: string} + generation: {type: integer, format: int64} + state: {type: string} + confirmation: {type: string} + error: {type: string} + dispatch_attempts: {type: integer} + retry_round: {type: integer} + health_received_at: {type: [string, "null"], format: date-time} + health_revision_id: {type: string} + health_source_at: {type: string} + health_error: {type: string} + "404": + $ref: "#/components/responses/NotFound" + default: + $ref: "#/components/responses/DefaultProblem" + /v1/jobs/{job_id}/cancel: parameters: - $ref: "#/components/parameters/TenantId" @@ -3904,6 +4081,33 @@ paths: default: $ref: "#/components/responses/DefaultProblem" + /v1/jobs/{job_id}/retry: + parameters: + - $ref: "#/components/parameters/TenantId" + - $ref: "#/components/parameters/JobId" + post: + tags: [Jobs] + summary: Повторить завершившуюся ошибкой задачу + description: Операторский повтор сохраняет ID и исходный запрос, сбрасывая бюджет попыток. + operationId: postJobRetry + responses: + "202": + description: Повтор поставлен в очередь. + content: + application/json: + schema: + $ref: "#/components/schemas/Job" + "409": + $ref: "#/components/responses/Conflict" + "429": + description: Очередь задач заполнена. + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Problem" + default: + $ref: "#/components/responses/DefaultProblem" + /v1/speakers/{speaker_id}/revisions/latest: parameters: - $ref: "#/components/parameters/SpeakerId" diff --git a/internal/agentwatch/snapshot.go b/internal/agentwatch/snapshot.go new file mode 100644 index 0000000..dcf240d --- /dev/null +++ b/internal/agentwatch/snapshot.go @@ -0,0 +1,312 @@ +package agentwatch + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "hash" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +const ( + maxConfigFiles = 512 + maxConfigBytes = 64 << 20 + maxFileBytes = 16 << 20 + maxIncludeDepth = 16 + maxDirectoryEntries = 4096 +) + +var errMainNotReady = errors.New("main bird.conf is not ready") + +// Only standalone literal include statements are hashable. Other valid BIRD +// grammar falls back to the legacy configure-on-every-tick path. +var includeLine = regexp.MustCompile(`(?m)^[\t ]*include[\t ]+"([^"\r\n]+)"[\t ]*;[\t ]*(?:#[^\r\n]*)?\r?$`) + +type configScanner struct { + ctx context.Context + root string + h hash.Hash + seen map[string]bool + files int + bytes int64 +} + +func hashConfigTree(ctx context.Context, activeDir string) ([sha256.Size]byte, error) { + var zero [sha256.Size]byte + root, err := filepath.Abs(activeDir) + if err != nil { + return zero, fmt.Errorf("agentwatch: active directory: %w", err) + } + root, err = filepath.EvalSymlinks(root) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return zero, fmt.Errorf("agentwatch: %w: %v", errMainNotReady, err) + } + return zero, fmt.Errorf("agentwatch: resolve active directory: %w", err) + } + s := &configScanner{ctx: ctx, root: filepath.Clean(root), h: sha256.New(), seen: make(map[string]bool)} + if err := s.visit(filepath.Join(s.root, "bird.conf"), 0); err != nil { + return zero, err + } + var digest [sha256.Size]byte + copy(digest[:], s.h.Sum(nil)) + return digest, nil +} + +func (s *configScanner) visit(path string, depth int) error { + if err := s.ctx.Err(); err != nil { + return err + } + if depth > maxIncludeDepth { + return fmt.Errorf("agentwatch: include depth exceeds %d", maxIncludeDepth) + } + path = filepath.Clean(path) + rel, err := filepath.Rel(s.root, path) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return fmt.Errorf("agentwatch: include %q escapes active directory", path) + } + if s.seen[path] { + return nil + } + info, err := os.Lstat(path) + if err != nil { + if depth == 0 && errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("agentwatch: %w: %v", errMainNotReady, err) + } + return fmt.Errorf("agentwatch: inspect %q: %w", path, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("agentwatch: include %q is not a regular file", path) + } + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return fmt.Errorf("agentwatch: resolve %q: %w", path, err) + } + if _, err := s.withinRoot(resolved); err != nil { + return err + } + if info.Size() > maxFileBytes { + return fmt.Errorf("agentwatch: include %q exceeds %d bytes", path, maxFileBytes) + } + if s.files >= maxConfigFiles || s.bytes+info.Size() > maxConfigBytes { + return fmt.Errorf("agentwatch: configuration tree exceeds %d files or %d bytes", maxConfigFiles, maxConfigBytes) + } + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("agentwatch: open %q: %w", path, err) + } + defer func() { _ = f.Close() }() + limit := int64(maxFileBytes) + if remaining := int64(maxConfigBytes) - s.bytes; remaining < limit { + limit = remaining + } + content, err := readBounded(f, limit) + if err != nil { + return fmt.Errorf("agentwatch: read %q: %w", path, err) + } + s.seen[path] = true + s.files++ + s.bytes += int64(len(content)) + writeHash(s.h, filepath.ToSlash(rel), content) + patterns, err := includePatterns(string(content)) + if err != nil { + return fmt.Errorf("agentwatch: %s: %w", rel, err) + } + for _, pattern := range patterns { + if strings.ContainsAny(pattern, `[\`) { + return fmt.Errorf("agentwatch: unsupported escaped include %q", pattern) + } + candidate := pattern + if !filepath.IsAbs(candidate) { + candidate = filepath.Join(filepath.Dir(path), filepath.FromSlash(candidate)) + } + candidate = filepath.Clean(candidate) + // The pattern itself changes the digest even when its glob currently matches nothing. + writeHash(s.h, "include", []byte(filepath.ToSlash(candidate))) + if _, err := s.withinRoot(candidate); err != nil { + return err + } + if strings.ContainsAny(candidate, "*?") { + if strings.ContainsAny(filepath.Dir(candidate), "*?") { + return fmt.Errorf("agentwatch: unsupported include directory glob %q", pattern) + } + resolvedDir, err := filepath.EvalSymlinks(filepath.Dir(candidate)) + if err != nil { + return fmt.Errorf("agentwatch: resolve include directory %q: %w", pattern, err) + } + if _, err := s.withinRoot(resolvedDir); err != nil { + return err + } + matches, err := boundedGlob(candidate) + if err != nil { + return fmt.Errorf("agentwatch: include glob %q: %w", pattern, err) + } + for _, match := range matches { + if err := s.visit(match, depth+1); err != nil { + return err + } + } + continue + } + if err := s.visit(candidate, depth+1); err != nil { + return err + } + } + return nil +} + +func readBounded(r io.Reader, limit int64) ([]byte, error) { + content, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(content)) > limit { + return nil, fmt.Errorf("configuration tree changed or exceeded size limit") + } + return content, nil +} + +// boundedGlob only handles * and ? in a filename. Broader BIRD glob grammar +// falls back to every-tick configure instead of risking a false unchanged hash. +func boundedGlob(pattern string) ([]string, error) { + dir, base := filepath.Dir(pattern), filepath.Base(pattern) + f, err := os.Open(dir) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + names, err := f.Readdirnames(maxDirectoryEntries + 1) + if err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + if len(names) > maxDirectoryEntries { + return nil, fmt.Errorf("directory exceeds %d entries", maxDirectoryEntries) + } + matches := make([]string, 0) + for _, name := range names { + match, err := filepath.Match(base, name) + if err != nil { + return nil, err + } + if match { + matches = append(matches, filepath.Join(dir, name)) + if len(matches) > maxConfigFiles { + return nil, fmt.Errorf("glob exceeds %d matches", maxConfigFiles) + } + } + } + sort.Strings(matches) + return matches, nil +} + +func (s *configScanner) withinRoot(path string) (string, error) { + rel, err := filepath.Rel(s.root, path) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("agentwatch: include %q escapes active directory", path) + } + return rel, nil +} + +func writeHash(h hash.Hash, name string, content []byte) { + var size [8]byte + binary.LittleEndian.PutUint64(size[:], uint64(len(name))) + _, _ = h.Write(size[:]) + _, _ = h.Write([]byte(name)) + binary.LittleEndian.PutUint64(size[:], uint64(len(content))) + _, _ = h.Write(size[:]) + _, _ = h.Write(content) +} + +func includePatterns(content string) ([]string, error) { + tokens, unsupportedQuotedText := includeTokenOffsets(content) + if unsupportedQuotedText { + return nil, fmt.Errorf("unsupported quoted text or escape") + } + tokenSet := make(map[int]bool, len(tokens)) + for _, token := range tokens { + tokenSet[token] = true + } + var patterns []string + for _, match := range includeLine.FindAllStringSubmatchIndex(content, -1) { + keyword := match[0] + strings.Index(content[match[0]:match[1]], "include") + if !tokenSet[keyword] { + continue + } // text inside a block comment + delete(tokenSet, keyword) + patterns = append(patterns, content[match[2]:match[3]]) + } + if len(tokenSet) != 0 { + return nil, fmt.Errorf("unsupported include syntax") + } + return patterns, nil +} + +// Recognize include keywords outside BIRD comments and quoted strings. +func includeTokenOffsets(config string) ([]int, bool) { + var offsets []int + quote := byte(0) + lineComment, blockComment := false, false + for i := 0; i < len(config); { + c := config[i] + if lineComment { + if c == '\n' { + lineComment = false + } + i++ + continue + } + if blockComment { + if c == '*' && i+1 < len(config) && config[i+1] == '/' { + blockComment = false + i += 2 + } else { + i++ + } + continue + } + if quote != 0 { + if c == '\\' { + return offsets, true + } + if c == quote { + quote = 0 + } + i++ + continue + } + if c == '#' || (c == '/' && i+1 < len(config) && config[i+1] == '/') { + lineComment = true + i++ + continue + } + if c == '/' && i+1 < len(config) && config[i+1] == '*' { + blockComment = true + i += 2 + continue + } + if c == '"' || c == '\'' { + quote = c + i++ + continue + } + if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' { + start := i + for i < len(config) && ((config[i] >= 'a' && config[i] <= 'z') || (config[i] >= 'A' && config[i] <= 'Z') || (config[i] >= '0' && config[i] <= '9') || config[i] == '_') { + i++ + } + if config[start:i] == "include" { + offsets = append(offsets, start) + } + continue + } + i++ + } + return offsets, quote != 0 || blockComment +} diff --git a/internal/agentwatch/watch.go b/internal/agentwatch/watch.go new file mode 100644 index 0000000..8dc7c77 --- /dev/null +++ b/internal/agentwatch/watch.go @@ -0,0 +1,109 @@ +// Package agentwatch reloads BIRD when its active configuration tree changes. +package agentwatch + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "log" + "path/filepath" + "time" +) + +// Control is the subset of BIRD operations required by the watch loop. +type Control interface { + ParseCheck(context.Context, string) error + Configure(context.Context) error +} + +// Config sets bounded polling and reconciliation behavior. +type Config struct { + ActiveDir string + Every time.Duration + ReconcileEvery time.Duration + CommandTimeout time.Duration + Logf func(string, ...any) +} + +type watcher struct { + ctl Control + cfg Config + hash [sha256.Size]byte + haveHash bool + lastConfigured time.Time +} + +// Run starts with one check, then polls until ctx is cancelled. A successful +// birdc exit advances the remembered hash; it is not proof of BIRD ACK. +func Run(ctx context.Context, ctl Control, cfg Config) error { + if ctl == nil || cfg.Every <= 0 || cfg.ReconcileEvery <= 0 || cfg.CommandTimeout <= 0 { + return fmt.Errorf("agentwatch: control and positive intervals/timeouts are required") + } + if cfg.ActiveDir == "" { + cfg.ActiveDir = "/etc/bird" + } + if cfg.Logf == nil { + cfg.Logf = log.Printf + } + w := &watcher{ctl: ctl, cfg: cfg} + if err := w.step(ctx, time.Now()); err != nil && ctx.Err() == nil { + cfg.Logf("evobgp-agent watch: %v", err) + } + ticker := time.NewTicker(cfg.Every) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case now := <-ticker.C: + if err := w.step(ctx, now); err != nil && ctx.Err() == nil { + cfg.Logf("evobgp-agent watch: %v", err) + } + } + } +} + +func (w *watcher) step(ctx context.Context, now time.Time) error { + if err := ctx.Err(); err != nil { + return err + } + digest, scanErr := hashConfigTree(ctx, w.cfg.ActiveDir) + if scanErr != nil { + if errors.Is(scanErr, errMainNotReady) { + return fmt.Errorf("configuration not ready: %w", scanErr) + } + // Unsupported/unhashable trees retain the old every-tick configure cadence. + w.cfg.Logf("evobgp-agent watch: cannot hash complete config tree (%v); using every-tick parse/configure", scanErr) + } else if w.haveHash && digest == w.hash && now.Sub(w.lastConfigured) < w.cfg.ReconcileEvery { + return nil + } + main := filepath.Join(w.cfg.ActiveDir, "bird.conf") + if err := w.command(ctx, func(cctx context.Context) error { return w.ctl.ParseCheck(cctx, main) }); err != nil { + return fmt.Errorf("parse check: %w", err) + } + if scanErr == nil { + stable, err := hashConfigTree(ctx, w.cfg.ActiveDir) + if err != nil { + return fmt.Errorf("configuration changed during parse check: %w", err) + } + if stable != digest { + return fmt.Errorf("configuration changed during parse check; retrying") + } + } + if err := w.command(ctx, w.ctl.Configure); err != nil { + return fmt.Errorf("configure: %w", err) + } + w.haveHash = scanErr == nil + if w.haveHash { + w.hash = digest + } + w.lastConfigured = now + return nil +} + +func (w *watcher) command(parent context.Context, run func(context.Context) error) error { + ctx, cancel := context.WithTimeout(parent, w.cfg.CommandTimeout) + defer cancel() + return run(ctx) +} diff --git a/internal/agentwatch/watch_test.go b/internal/agentwatch/watch_test.go new file mode 100644 index 0000000..8c7a48a --- /dev/null +++ b/internal/agentwatch/watch_test.go @@ -0,0 +1,311 @@ +package agentwatch + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +type fakeControl struct { + parsed int + configured int + failConfigure int + onParse func() +} + +func (f *fakeControl) ParseCheck(ctx context.Context, _ string) error { + if err := ctx.Err(); err != nil { + return err + } + f.parsed++ + if f.onParse != nil { + f.onParse() + } + return nil +} + +func (f *fakeControl) Configure(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + f.configured++ + if f.failConfigure > 0 { + f.failConfigure-- + return errors.New("birdc unavailable") + } + return nil +} + +func testConfig(t *testing.T) (string, string) { + t.Helper() + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "bird.d"), 0o755); err != nil { + t.Fatal(err) + } + main := filepath.Join(root, "bird.conf") + if err := os.WriteFile(main, []byte("router id 192.0.2.1;\ninclude \"bird.d/*.conf\";\n"), 0o644); err != nil { + t.Fatal(err) + } + fragment := filepath.Join(root, "bird.d", "peer.conf") + if err := os.WriteFile(fragment, []byte("# peer v1\n"), 0o644); err != nil { + t.Fatal(err) + } + return root, fragment +} + +func testWatcher(root string, ctl Control) *watcher { + return &watcher{ctl: ctl, cfg: Config{ActiveDir: root, Every: 30 * time.Second, ReconcileEvery: 5 * time.Minute, CommandTimeout: time.Second, Logf: func(string, ...any) {}}} +} + +func TestHashConfigTreeTracksMainIncludesAndGlobMembership(t *testing.T) { + root, fragment := testConfig(t) + first, err := hashConfigTree(context.Background(), root) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(fragment, []byte("# peer v2\n"), 0o644); err != nil { + t.Fatal(err) + } + second, err := hashConfigTree(context.Background(), root) + if err != nil || second == first { + t.Fatalf("included edit not detected: digest=%x err=%v", second, err) + } + if err := os.WriteFile(filepath.Join(root, "bird.d", "manual.conf"), []byte("# manual\n"), 0o644); err != nil { + t.Fatal(err) + } + third, err := hashConfigTree(context.Background(), root) + if err != nil || third == second { + t.Fatalf("new glob member not detected: digest=%x err=%v", third, err) + } + if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("router id 192.0.2.2;\ninclude \"bird.d/*.conf\";\n"), 0o644); err != nil { + t.Fatal(err) + } + fourth, err := hashConfigTree(context.Background(), root) + if err != nil || fourth == third { + t.Fatalf("main edit not detected: digest=%x err=%v", fourth, err) + } +} + +func TestWatchSkipsUnchangedRetriesErrorsAndForcesReconciliation(t *testing.T) { + root, fragment := testConfig(t) + ctl := &fakeControl{} + w := testWatcher(root, ctl) + now := time.Now() + if err := w.step(context.Background(), now); err != nil { + t.Fatal(err) + } + if err := w.step(context.Background(), now.Add(30*time.Second)); err != nil { + t.Fatal(err) + } + if ctl.configured != 1 || ctl.parsed != 1 { + t.Fatalf("unchanged tree configured again: %+v", ctl) + } + if err := os.WriteFile(fragment, []byte("# changed\n"), 0o644); err != nil { + t.Fatal(err) + } + ctl.failConfigure = 1 + if err := w.step(context.Background(), now.Add(time.Minute)); err == nil { + t.Fatal("expected configure failure") + } + if !w.haveHash || ctl.configured != 2 { + t.Fatalf("failure changed remembered state: %+v", w) + } + if err := w.step(context.Background(), now.Add(90*time.Second)); err != nil { + t.Fatal(err) + } + if ctl.configured != 3 { + t.Fatalf("failed change was not retried: %+v", ctl) + } + if err := w.step(context.Background(), now.Add(7*time.Minute)); err != nil { + t.Fatal(err) + } + if ctl.configured != 4 { + t.Fatalf("forced reconciliation missing: %+v", ctl) + } + // A fresh watcher must reconcile once even when another process already loaded the same bytes. + restarted := testWatcher(root, ctl) + if err := restarted.step(context.Background(), now.Add(8*time.Minute)); err != nil { + t.Fatal(err) + } + if ctl.configured != 5 { + t.Fatalf("restart did not reconcile: %+v", ctl) + } +} + +func TestWatchRejectsChangingTreeBeforeConfigure(t *testing.T) { + root, fragment := testConfig(t) + ctl := &fakeControl{onParse: func() { _ = os.WriteFile(fragment, []byte("# raced\n"), 0o644) }} + w := testWatcher(root, ctl) + if err := w.step(context.Background(), time.Now()); err == nil || !strings.Contains(err.Error(), "changed during parse check") { + t.Fatalf("want race retry, got %v", err) + } + if ctl.configured != 0 { + t.Fatal("configured a tree that changed after parse check") + } + ctl.onParse = nil + if err := w.step(context.Background(), time.Now()); err != nil { + t.Fatal(err) + } + if ctl.configured != 1 { + t.Fatal("stable tree was not configured") + } +} + +func TestUnsupportedIncludeUsesLegacyEveryTickCadence(t *testing.T) { + root, _ := testConfig(t) + if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include variable;\n"), 0o644); err != nil { + t.Fatal(err) + } + ctl := &fakeControl{} + w := testWatcher(root, ctl) + now := time.Now() + for i := 0; i < 3; i++ { + if err := w.step(context.Background(), now.Add(time.Duration(i)*30*time.Second)); err != nil { + t.Fatal(err) + } + } + if ctl.parsed != 3 || ctl.configured != 3 || w.haveHash { + t.Fatalf("unsupported tree was skipped: %+v", ctl) + } +} + +func TestMissingIncludedFileFallsBackButMissingMainWaits(t *testing.T) { + root, fragment := testConfig(t) + if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \"bird.d/peer.conf\";\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Remove(fragment); err != nil { + t.Fatal(err) + } + ctl := &fakeControl{} + w := testWatcher(root, ctl) + now := time.Now() + for i := 0; i < 2; i++ { + if err := w.step(context.Background(), now.Add(time.Duration(i)*30*time.Second)); err != nil { + t.Fatal(err) + } + } + if ctl.configured != 2 { + t.Fatalf("missing include skipped legacy cadence: %+v", ctl) + } + if err := os.Remove(filepath.Join(root, "bird.conf")); err != nil { + t.Fatal(err) + } + if err := w.step(context.Background(), now.Add(time.Minute)); !errors.Is(err, errMainNotReady) { + t.Fatalf("missing main should wait, got %v", err) + } + if ctl.configured != 2 { + t.Fatal("configured without main bird.conf") + } +} + +func TestHashConfigTreeBoundsAndEscapes(t *testing.T) { + root, fragment := testConfig(t) + if err := os.WriteFile(fragment, make([]byte, maxFileBytes+1), 0o644); err != nil { + t.Fatal(err) + } + if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("want size limit, got %v", err) + } + if err := os.WriteFile(fragment, []byte("# small\n"), 0o644); err != nil { + t.Fatal(err) + } + outside := filepath.Join(t.TempDir(), "outside.conf") + if err := os.WriteFile(outside, []byte("# outside\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \""+filepath.ToSlash(outside)+"\";\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "escapes active directory") { + t.Fatalf("want outside include rejection, got %v", err) + } +} + +func TestHashConfigTreeRejectsSymlinkInclude(t *testing.T) { + root, fragment := testConfig(t) + if err := os.Remove(fragment); err != nil { + t.Fatal(err) + } + outside := filepath.Join(t.TempDir(), "outside.conf") + if err := os.WriteFile(outside, []byte("# outside\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, fragment); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "not a regular file") { + t.Fatalf("want symlink rejection, got %v", err) + } +} + +func TestIncludeCommentsAndUnsupportedForms(t *testing.T) { + patterns, err := includePatterns("# include \"ignored.conf\";\n/*\ninclude \"also-ignored.conf\";\n*/\ninclude \"bird.d/*.conf\";\n") + if err != nil || len(patterns) != 1 || patterns[0] != "bird.d/*.conf" { + t.Fatalf("comment handling failed: patterns=%v err=%v", patterns, err) + } + if _, err := includePatterns("if true then include \"dynamic.conf\";\n"); err == nil { + t.Fatal("inline include should fall back to legacy cadence") + } + if _, err := includePatterns("include \"bird.d/[!a].conf\";\n"); err != nil { + // Bracket glob is rejected by the scanner after lexical parsing. + t.Fatalf("literal include statement should parse before glob validation: %v", err) + } + root, _ := testConfig(t) + if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \"bird.d/[!a].conf\";\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "unsupported") { + t.Fatalf("bracket glob should fall back, got %v", err) + } + if _, err := includePatterns("define note = \"single\\\";\ninclude \"bird.d/real.conf\";\n"); err == nil { + t.Fatal("backslash in quoted text must force conservative fallback") + } +} + +func TestBoundedGlobLimitsMatches(t *testing.T) { + root := t.TempDir() + for i := 0; i <= maxConfigFiles; i++ { + name := filepath.Join(root, fmt.Sprintf("peer-%03d.conf", i)) + if err := os.WriteFile(name, nil, 0o644); err != nil { + t.Fatal(err) + } + } + if _, err := boundedGlob(filepath.Join(root, "*.conf")); err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("want bounded glob rejection, got %v", err) + } +} + +func TestReadBoundedStopsAfterLimit(t *testing.T) { + r := strings.NewReader("abcdefghij") + if _, err := readBounded(r, 3); err == nil { + t.Fatal("growing file exceeded limit") + } + if got := r.Len(); got != 6 { + t.Fatalf("reader consumed beyond limit+1: %d bytes remain", got) + } +} + +func TestRunStopsOnCancellation(t *testing.T) { + root, _ := testConfig(t) + ctx, cancel := context.WithCancel(context.Background()) + ctl := &fakeControl{} + done := make(chan error, 1) + go func() { + done <- Run(ctx, ctl, Config{ActiveDir: root, Every: time.Hour, ReconcileEvery: 5 * time.Minute, CommandTimeout: time.Second, Logf: func(string, ...any) {}}) + }() + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("watch did not stop after cancellation") + } +} diff --git a/internal/asnresolve/ripestat.go b/internal/asnresolve/ripestat.go index 74e2cce..96862aa 100644 --- a/internal/asnresolve/ripestat.go +++ b/internal/asnresolve/ripestat.go @@ -23,6 +23,8 @@ const DefaultASOverviewURL = "https://stat.ripe.net/data/as-overview/data.json" // AnnouncedPrefixes returns currently announced IPv4/IPv6 prefixes for the ASN (best-effort via RIPEstat). func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip.Prefix, error) { + ctx, cancel := ripeRequestContext(ctx) + defer cancel() if hc == nil { hc = httpclient.New(httpclient.DefaultTimeout) } @@ -43,13 +45,14 @@ func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip return nil, fmt.Errorf("ripestat fetch AS%d: %w", asn, err) } defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20)) - if err != nil { - return nil, err - } if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) return nil, fmt.Errorf("ripestat AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200)) } + body, err := httpclient.ReadResponseBody(resp, 32<<20) + if err != nil { + return nil, fmt.Errorf("ripestat AS%d response: %w", asn, err) + } var wrap struct { Status string `json:"status"` @@ -85,6 +88,8 @@ func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip // ASHolderName returns the holder / organization label for the ASN from RIPEstat as-overview (best-effort). func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, error) { + ctx, cancel := ripeRequestContext(ctx) + defer cancel() if hc == nil { hc = httpclient.New(httpclient.DefaultTimeout) } @@ -105,13 +110,14 @@ func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, erro return "", fmt.Errorf("ripestat as-overview AS%d: %w", asn, err) } defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) - if err != nil { - return "", err - } if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) return "", fmt.Errorf("ripestat as-overview AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200)) } + body, err := httpclient.ReadResponseBody(resp, 4<<20) + if err != nil { + return "", fmt.Errorf("ripestat as-overview AS%d response: %w", asn, err) + } var wrap struct { Status string `json:"status"` @@ -135,3 +141,10 @@ func truncateForErr(b []byte, n int) string { } return s } + +func ripeRequestContext(ctx context.Context) (context.Context, context.CancelFunc) { + if _, ok := ctx.Deadline(); ok { + return context.WithCancel(ctx) + } + return context.WithTimeout(ctx, httpclient.DefaultTimeout) +} diff --git a/internal/asnresolve/ripestat_limits_test.go b/internal/asnresolve/ripestat_limits_test.go new file mode 100644 index 0000000..6c8f71f --- /dev/null +++ b/internal/asnresolve/ripestat_limits_test.go @@ -0,0 +1,103 @@ +package asnresolve + +import ( + "compress/gzip" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "evobgp/internal/httpclient" +) + +type ripeRoundTrip func(*http.Request) (*http.Response, error) + +func (f ripeRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +type ripeCountingBody struct { + io.Reader + read int + closed bool +} + +func (b *ripeCountingBody) Read(p []byte) (int, error) { + n, e := b.Reader.Read(p) + b.read += n + return n, e +} +func (b *ripeCountingBody) Close() error { b.closed = true; return nil } + +func TestRIPECompleteResponseLimits(t *testing.T) { + cases := []struct { + name string + limit int + prefix string + call func(context.Context, *http.Client) error + }{ + {"announced", 32 << 20, `{"status":"ok","data":{"prefixes":[]}}`, func(ctx context.Context, hc *http.Client) error { _, e := AnnouncedPrefixes(ctx, hc, 64496); return e }}, + {"holder", 4 << 20, `{"status":"ok","data":{"holder":"Example"}}`, func(ctx context.Context, hc *http.Client) error { _, e := ASHolderName(ctx, hc, 64496); return e }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + for _, known := range []bool{true, false} { + for _, extra := range []int{0, 1} { + body := &ripeCountingBody{Reader: strings.NewReader(tc.prefix + strings.Repeat(" ", tc.limit-len(tc.prefix)+extra))} + hc := &http.Client{Transport: ripeRoundTrip(func(r *http.Request) (*http.Response, error) { + if _, ok := r.Context().Deadline(); !ok { + t.Error("missing request deadline") + } + length := int64(-1) + if known { + length = int64(tc.limit + extra) + } + return &http.Response{StatusCode: 200, Body: body, ContentLength: length, Header: make(http.Header)}, nil + })} + err := tc.call(context.Background(), hc) + if extra == 0 && err != nil { + t.Fatalf("exact limit rejected known=%v: %v", known, err) + } + if extra == 1 && !errors.Is(err, httpclient.ErrResponseTooLarge) { + t.Fatalf("oversized valid prefix accepted known=%v: %v", known, err) + } + if body.read > tc.limit+1 || !body.closed { + t.Fatalf("read=%d closed=%v", body.read, body.closed) + } + } + } + }) + } +} + +func TestRIPEErrorBodySmallAndContextPreserved(t *testing.T) { + body := &ripeCountingBody{Reader: strings.NewReader(strings.Repeat("x", 1<<20))} + hc := &http.Client{Transport: ripeRoundTrip(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 400, Status: "400 Bad Request", Body: body, Header: make(http.Header)}, nil + })} + if _, err := AnnouncedPrefixes(context.Background(), hc, 1); err == nil || body.read > 4<<10 || !body.closed { + t.Fatalf("non200 unbounded read=%d close=%v err=%v", body.read, body.closed, err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + hc.Transport = ripeRoundTrip(func(r *http.Request) (*http.Response, error) { <-r.Context().Done(); return nil, r.Context().Err() }) + if _, err := ASHolderName(ctx, hc, 1); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("caller deadline lost: %v", err) + } +} + +func TestRIPEGzipExpandedLimit(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Encoding", "gzip") + gz := gzip.NewWriter(w) + _, _ = io.WriteString(gz, `{"status":"ok","data":{"holder":"Example"}}`+strings.Repeat(" ", 4<<20)) + _ = gz.Close() + })) + defer server.Close() + t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", server.URL) + if _, err := ASHolderName(context.Background(), server.Client(), 1); !errors.Is(err, httpclient.ErrResponseTooLarge) { + t.Fatalf("gzip bypassed decoded limit: %v", err) + } +} diff --git a/internal/birddeploy/apply.go b/internal/birddeploy/apply.go index e8a6fee..a2f5afe 100644 --- a/internal/birddeploy/apply.go +++ b/internal/birddeploy/apply.go @@ -1,152 +1,433 @@ -// Package birddeploy applies revision preview fragments to a shared BIRD volume with parse-check and LKG (plan §13). package birddeploy import ( "context" + "errors" "fmt" "io/fs" "os" "path/filepath" + "regexp" + "sort" "strings" + "time" "evobgp/internal/birdfmt" "evobgp/internal/store" ) -// Config drives two-phase deploy from a revision in the store. +const recoveryTimeout = 30 * time.Second + +var includeLine = regexp.MustCompile(`(?m)^([ \t]*include[ \t]+")([^"]+)("[ \t]*;[ \t]*(?:#[^\n]*)?$)`) + +// Config identifies the BIRD instance. StagingDir remains for caller compatibility; +// generations live inside ActiveDir so referenced files survive process restart. type Config struct { - ActiveDir string // e.g. /etc/bird (live config tree) - StagingDir string // e.g. /tmp/evobgp-staging (same host/volume as ActiveDir) + ActiveDir string + StagingDir string BirdBin string BirdcBin string - Socket string // birdc -s + Socket string + writeFile func(string, []byte) error // test-only fault injection; nil uses synced writes } -// ApplyRevision writes preview fragments to staging, runs bird -p, swaps into active, runs birdc configure; restores LKG on failure. +// ApplyRevision installs a checked generation through one atomic bird.conf rename. func ApplyRevision(ctx context.Context, ctl *birdfmt.BirdCtl, rev *store.Revision, cfg Config) error { if rev == nil || len(rev.PreviewFragments) == 0 { return fmt.Errorf("birddeploy: no preview fragments") } - active := strings.TrimSpace(cfg.ActiveDir) - staging := strings.TrimSpace(cfg.StagingDir) - if active == "" || staging == "" { - return fmt.Errorf("birddeploy: ActiveDir and StagingDir required") + if strings.TrimSpace(cfg.ActiveDir) == "" { + return fmt.Errorf("birddeploy: ActiveDir required") } - if err := os.MkdirAll(staging, 0o755); err != nil { - return err + active, err := filepath.Abs(cfg.ActiveDir) + if err != nil { + return fmt.Errorf("birddeploy: resolve active directory: %w", err) } - if err := os.MkdirAll(filepath.Join(active, "bird.d"), 0o755); err != nil { - return err + if err := os.MkdirAll(active, 0o755); err != nil { + return fmt.Errorf("birddeploy: create active directory: %w", err) } - lkg := filepath.Join(active, ".evobgp_lkg") - - // Phase 1: write staging tree - for rel, content := range rev.PreviewFragments { - rel = strings.TrimPrefix(rel, "/") - if !isDeployableBirdFragment(rel) { - continue + lock, err := lockTarget(ctx, filepath.Join(active, ".evobgp-apply.lock")) + if err != nil { + return fmt.Errorf("birddeploy: acquire apply lock: %w", err) + } + defer func() { + if err := lock.Close(); err != nil { + fmt.Fprintf(os.Stderr, "birddeploy: release apply lock: %v\n", err) } - dst := filepath.Join(staging, rel) - if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { - return err - } - if err := os.WriteFile(dst, []byte(content), 0o644); err != nil { - return err - } - } - mainConf := filepath.Join(staging, "bird.conf") - if _, err := os.Stat(mainConf); err != nil { - return fmt.Errorf("birddeploy: staging missing bird.conf: %w", err) + }() + if err := ctx.Err(); err != nil { + return fmt.Errorf("birddeploy: apply cancelled: %w", err) } bc := ctl if bc == nil { bc = &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket} } - if err := bc.ParseCheck(ctx, mainConf); err != nil { - return fmt.Errorf("birddeploy: bird -p failed: %w", err) + writeFile := cfg.writeFile + if writeFile == nil { + writeFile = writeSynced } - - // Snapshot LKG (best-effort copy of current active bird.conf + bird.d) - _ = os.RemoveAll(lkg) - _ = snapshotBirdTree(active, lkg) - - // Phase 2: atomic swap into active - for rel, content := range rev.PreviewFragments { - rel = strings.TrimPrefix(rel, "/") - if !isDeployableBirdFragment(rel) { - continue + mainPath := filepath.Join(active, "bird.conf") + mainInfo, err := os.Lstat(mainPath) + if err != nil { + return fmt.Errorf("birddeploy: inspect current bird.conf: %w", err) + } + if !mainInfo.Mode().IsRegular() { + return fmt.Errorf("birddeploy: current bird.conf must be a regular file; refusing to replace %s", mainInfo.Mode()) + } + oldMain, err := os.ReadFile(mainPath) + if err != nil { + return fmt.Errorf("birddeploy: read current bird.conf: %w", err) + } + if err := bc.ParseCheck(ctx, mainPath); err != nil { + return fmt.Errorf("birddeploy: current configuration is not a usable LKG: %w", err) + } + gen, err := os.MkdirTemp(active, ".evobgp-generation-") + if err != nil { + return fmt.Errorf("birddeploy: create generation: %w", err) + } + activated := false + defer func() { + if !activated { + if err := os.RemoveAll(gen); err != nil { + fmt.Fprintf(os.Stderr, "birddeploy: remove unactivated generation %s: %v\n", gen, err) + } } - dst := filepath.Join(active, rel) - if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { - _ = restoreLKG(lkg, active) - return err - } - tmp := dst + ".tmp" - if err := os.WriteFile(tmp, []byte(content), 0o644); err != nil { - _ = restoreLKG(lkg, active) - return err - } - if err := os.Rename(tmp, dst); err != nil { - _ = restoreLKG(lkg, active) - return err + }() + if err := writeGenerationWith(active, gen, rev.PreviewFragments, writeFile); err != nil { + return err + } + newMain, err := os.ReadFile(filepath.Join(gen, "bird.conf")) + if err != nil { + return fmt.Errorf("birddeploy: read generated bird.conf: %w", err) + } + if err := bc.ParseCheck(ctx, filepath.Join(gen, "bird.conf")); err != nil { + return fmt.Errorf("birddeploy: staged bird -p failed: %w", err) + } + if err := writeFile(filepath.Join(gen, "previous-bird.conf"), oldMain); err != nil { + return fmt.Errorf("birddeploy: save LKG: %w", err) + } + if err := syncDir(gen); err != nil { + return fmt.Errorf("birddeploy: sync LKG generation: %w", err) + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("birddeploy: apply cancelled: %w", err) + } + swapped, err := replaceMain(active, newMain) + activated = swapped + if err != nil { + if swapped { + return recoverPrevious(bc, active, gen, oldMain, fmt.Errorf("activate generation: %w", err)) } + return fmt.Errorf("birddeploy: activate generation: %w", err) } if err := bc.Configure(ctx); err != nil { - _ = restoreLKG(lkg, active) - _ = bc.Configure(ctx) - return fmt.Errorf("birddeploy: birdc configure failed, restored LKG: %w", err) + return recoverPrevious(bc, active, gen, oldMain, err) + } + if err := pruneGenerations(active, gen, oldMain); err != nil { + fmt.Fprintf(os.Stderr, "birddeploy: configuration applied; generation cleanup failed: %v\n", err) } return nil } -// isDeployableBirdFragment skips UI-only preview keys (e.g. flattened full config) not referenced from bird.conf. -func isDeployableBirdFragment(rel string) bool { - return rel != "" && !strings.HasPrefix(rel, "_") +func recoverPrevious(bc *birdfmt.BirdCtl, active, gen string, oldMain []byte, cause error) error { + ctx, cancel := context.WithTimeout(context.Background(), recoveryTimeout) + defer cancel() + if _, err := replaceMain(active, oldMain); err != nil { + return fmt.Errorf("birddeploy: configure failed: %w; restore failed: %v", cause, err) + } + if err := bc.Configure(ctx); err != nil { + return fmt.Errorf("birddeploy: configure failed: %w; LKG restored on disk but recovery configure failed: %v", cause, err) + } + if err := os.RemoveAll(gen); err != nil { + fmt.Fprintf(os.Stderr, "birddeploy: LKG restored; remove failed generation %s: %v\n", gen, err) + } + return fmt.Errorf("birddeploy: configure failed: %w; LKG restored and recovery configure succeeded", cause) } -func snapshotBirdTree(activeRoot, dstRoot string) error { - if err := os.MkdirAll(dstRoot, 0o755); err != nil { - return err - } - mc := filepath.Join(activeRoot, "bird.conf") - if b, err := os.ReadFile(mc); err == nil { - _ = os.WriteFile(filepath.Join(dstRoot, "bird.conf"), b, 0o644) - } - bd := filepath.Join(activeRoot, "bird.d") - if st, err := os.Stat(bd); err == nil && st.IsDir() { - _ = filepath.WalkDir(bd, func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return err - } - suffix, _ := filepath.Rel(activeRoot, path) - out := filepath.Join(dstRoot, suffix) - _ = os.MkdirAll(filepath.Dir(out), 0o755) - b, err := os.ReadFile(path) - if err != nil { - return nil - } - return os.WriteFile(out, b, 0o644) - }) - } - return nil +func writeGeneration(active, gen string, fragments map[string]string) error { + return writeGenerationWith(active, gen, fragments, writeSynced) } -func restoreLKG(lkg, active string) error { - if _, err := os.Stat(filepath.Join(lkg, "bird.conf")); err != nil { - return err +func writeGenerationWith(active, gen string, fragments map[string]string, writeFile func(string, []byte) error) error { + main, ok := fragments["bird.conf"] + if !ok { + return fmt.Errorf("birddeploy: preview missing bird.conf") } - _ = filepath.WalkDir(lkg, func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return err - } - suffix, _ := filepath.Rel(lkg, path) - out := filepath.Join(active, suffix) - _ = os.MkdirAll(filepath.Dir(out), 0o755) - b, err := os.ReadFile(path) - if err != nil { + oldBirdD := filepath.Join(active, birdfmt.DirBirdD) + newBirdD := filepath.Join(gen, birdfmt.DirBirdD) + if err := os.MkdirAll(newBirdD, 0o755); err != nil { + return fmt.Errorf("birddeploy: create generation bird.d: %w", err) + } + if err := filepath.WalkDir(oldBirdD, func(path string, d fs.DirEntry, walkErr error) error { + if errors.Is(walkErr, fs.ErrNotExist) && path == oldBirdD { return nil } - return os.WriteFile(out, b, 0o644) + if walkErr != nil { + return walkErr + } + rel, err := filepath.Rel(oldBirdD, path) + if err != nil { + return err + } + if d.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("operator snippet %q is a symlink", rel) + } + if rel == "." { + if !d.IsDir() { + return fmt.Errorf("birddeploy: bird.d is not a directory") + } + return nil + } + if d.IsDir() { + return os.MkdirAll(filepath.Join(newBirdD, rel), 0o755) + } + if !d.Type().IsRegular() { + return fmt.Errorf("operator snippet %q is not a regular file", rel) + } + _, replaced := fragments[filepath.ToSlash(filepath.Join(birdfmt.DirBirdD, rel))] + if strings.HasPrefix(filepath.Base(rel), "evobgp_") || replaced { + return nil + } + b, err := os.ReadFile(path) + if err != nil { + return err + } + if countIncludeTokens(string(b)) > 0 { + return fmt.Errorf("operator snippet %q has a nested include; cannot pin the complete tree", rel) + } + return writeFile(filepath.Join(newBirdD, rel), b) + }); err != nil { + return fmt.Errorf("birddeploy: copy operator snippets: %w", err) + } + for rel, content := range fragments { + if rel == "bird.conf" || strings.HasPrefix(rel, "_") { + continue + } + if countIncludeTokens(content) > 0 { + return fmt.Errorf("birddeploy: nested include in preview fragment %q is unsupported", rel) + } + clean := filepath.Clean(filepath.FromSlash(rel)) + if filepath.IsAbs(clean) || clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) || !strings.HasPrefix(clean, birdfmt.DirBirdD+string(os.PathSeparator)) || filepath.Ext(clean) != ".conf" { + return fmt.Errorf("birddeploy: unsafe preview path %q", rel) + } + dst := filepath.Join(gen, clean) + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return fmt.Errorf("birddeploy: create fragment directory: %w", err) + } + if err := writeFile(dst, []byte(content)); err != nil { + return fmt.Errorf("birddeploy: write %s: %w", rel, err) + } + } + var rewriteErr error + if !standaloneIncludesOnly(main) { + return fmt.Errorf("birddeploy: unsupported include syntax in bird.conf") + } + rewritten := includeLine.ReplaceAllStringFunc(main, func(line string) string { + m := includeLine.FindStringSubmatch(line) + inc := m[2] + if filepath.IsAbs(inc) { + rel, err := filepath.Rel(active, inc) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + rewriteErr = fmt.Errorf("birddeploy: external absolute include %q is unsupported", inc) + return line + } + inc = rel + } + clean := filepath.Clean(filepath.FromSlash(inc)) + if clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) || !strings.HasPrefix(clean, birdfmt.DirBirdD+string(os.PathSeparator)) { + rewriteErr = fmt.Errorf("birddeploy: unsupported relative include %q", inc) + return line + } + return m[1] + filepath.ToSlash(filepath.Join(gen, clean)) + m[3] }) + if rewriteErr != nil { + return rewriteErr + } + if err := writeFile(filepath.Join(gen, ".evobgp-owned"), []byte("evobgp-generation-v1\n")); err != nil { + return fmt.Errorf("birddeploy: mark generation: %w", err) + } + if err := writeFile(filepath.Join(gen, "bird.conf"), []byte(rewritten)); err != nil { + return fmt.Errorf("birddeploy: write generated bird.conf: %w", err) + } + return syncTree(gen) +} + +func standaloneIncludesOnly(config string) bool { + tokens := includeTokenOffsets(config) + matches := includeLine.FindAllStringIndex(config, -1) + if len(tokens) != len(matches) { + return false + } + for i, match := range matches { + word := strings.Index(config[match[0]:match[1]], "include") + if word < 0 || tokens[i] != match[0]+word { + return false + } + } + return true +} + +func countIncludeTokens(config string) int { return len(includeTokenOffsets(config)) } + +// includeTokenOffsets skips BIRD comments and strings, retaining exact offsets +// so a commented standalone include cannot mask a live inline include. +func includeTokenOffsets(config string) []int { + var offsets []int + quote := byte(0) + lineComment, blockComment := false, false + for i := 0; i < len(config); { + c := config[i] + if lineComment { + if c == '\n' { + lineComment = false + } + i++ + continue + } + if blockComment { + if c == '*' && i+1 < len(config) && config[i+1] == '/' { + blockComment = false + i += 2 + } else { + i++ + } + continue + } + if quote != 0 { + // BIRD quoted text does not use C-style backslash escapes. + // Skipping the following quote could hide a live include. + if c == quote { + quote = 0 + } + i++ + continue + } + if c == '#' { + lineComment = true + i++ + continue + } + if c == '/' && i+1 < len(config) && config[i+1] == '*' { + blockComment = true + i += 2 + continue + } + if c == '"' || c == '\'' { + quote = c + i++ + continue + } + if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' { + start := i + for i < len(config) && ((config[i] >= 'a' && config[i] <= 'z') || (config[i] >= 'A' && config[i] <= 'Z') || (config[i] >= '0' && config[i] <= '9') || config[i] == '_') { + i++ + } + if config[start:i] == "include" { + offsets = append(offsets, start) + } + continue + } + i++ + } + return offsets +} + +func syncTree(root string) error { + var dirs []string + if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + dirs = append(dirs, path) + } + return nil + }); err != nil { + return err + } + sort.Slice(dirs, func(i, j int) bool { return len(dirs[i]) > len(dirs[j]) }) + for _, dir := range dirs { + if err := syncDir(dir); err != nil { + return fmt.Errorf("sync %s: %w", dir, err) + } + } return nil } + +func pruneGenerations(active, current string, previous []byte) error { + entries, err := os.ReadDir(active) + if err != nil { + return err + } + var stale []string + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), ".evobgp-generation-") { + continue + } + path := filepath.Join(active, entry.Name()) + if path == current || strings.Contains(string(previous), filepath.ToSlash(path)+"/") { + continue + } + marker, err := os.ReadFile(filepath.Join(path, ".evobgp-owned")) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + return err + } + if string(marker) != "evobgp-generation-v1\n" { + continue + } + stale = append(stale, path) + } + sort.Strings(stale) + for _, path := range stale { + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("remove %s: %w", path, err) + } + } + return nil +} + +func replaceMain(active string, content []byte) (bool, error) { + tmp, err := os.CreateTemp(active, ".evobgp-bird-conf-") + if err != nil { + return false, err + } + defer func() { _ = os.Remove(tmp.Name()) }() + if err := tmp.Chmod(0o644); err != nil { + _ = tmp.Close() + return false, err + } + if _, err := tmp.Write(content); err != nil { + _ = tmp.Close() + return false, err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return false, err + } + if err := tmp.Close(); err != nil { + return false, err + } + if err := os.Rename(tmp.Name(), filepath.Join(active, "bird.conf")); err != nil { + return false, err + } + return true, syncDir(active) +} + +func writeSynced(path string, b []byte) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return err + } + if _, err := f.Write(b); err != nil { + _ = f.Close() + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + return f.Close() +} diff --git a/internal/birddeploy/apply_linux_test.go b/internal/birddeploy/apply_linux_test.go new file mode 100644 index 0000000..922a912 --- /dev/null +++ b/internal/birddeploy/apply_linux_test.go @@ -0,0 +1,279 @@ +//go:build linux + +package birddeploy + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" + + "evobgp/internal/birdfmt" + "evobgp/internal/store" +) + +func fakeCtl(t *testing.T, active string) *birdfmt.BirdCtl { + t.Helper() + bin := t.TempDir() + parse := filepath.Join(bin, "bird") + configure := filepath.Join(bin, "birdc") + parseScript := `#!/bin/sh +if grep -q FAIL_PARSE "$2"; then exit 1; fi +exit 0 +` + configureScript := fmt.Sprintf(`#!/bin/sh +root=%q +if [ -e "$root/.configure-wait" ]; then + rm "$root/.configure-wait" + touch "$root/.configure-started" + exec sleep 5 +fi +if [ -e "$root/.configure-fail-once" ]; then + rm "$root/.configure-fail-once" + exit 1 +fi +mkdir "$root/.configuring" || exit 1 +sleep 0.05 +rmdir "$root/.configuring" +`, active) + for path, script := range map[string]string{parse: parseScript, configure: configureScript} { + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + } + return &birdfmt.BirdCtl{Bird: parse, Birdc: configure} +} + +func revision(label string) *store.Revision { + return &store.Revision{PreviewFragments: map[string]string{ + "bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/*.conf\";\n# " + label + "\n", + "bird.d/evobgp_new.conf": "# " + label + "\n", + }} +} + +func setupActive(t *testing.T) (string, *birdfmt.BirdCtl, []byte) { + t.Helper() + active := t.TempDir() + if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil { + t.Fatal(err) + } + old := []byte("router id 192.0.2.2;\ninclude \"bird.d/*.conf\";\n# old\n") + if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil { + t.Fatal(err) + } + for name, content := range map[string]string{"evobgp_removed.conf": "# removed\n", "operator.conf": "# operator\n"} { + if err := os.WriteFile(filepath.Join(active, "bird.d", name), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + return active, fakeCtl(t, active), old +} + +func TestApplyRevisionGenerationAndPrune(t *testing.T) { + active, ctl, _ := setupActive(t) + for _, label := range []string{"one", "two", "three"} { + if err := ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active}); err != nil { + t.Fatal(err) + } + main, err := os.ReadFile(filepath.Join(active, "bird.conf")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(main), "# "+label) { + t.Fatalf("wrong active generation: %s", main) + } + if strings.Contains(string(main), `include "bird.d/`) { + t.Fatalf("include not pinned: %s", main) + } + } + gens, err := filepath.Glob(filepath.Join(active, ".evobgp-generation-*")) + if err != nil { + t.Fatal(err) + } + if len(gens) != 2 { + t.Fatalf("want active and previous generations, got %d", len(gens)) + } + for _, gen := range gens { + if _, err := os.Stat(filepath.Join(gen, "bird.d", "evobgp_removed.conf")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("removed fragment retained in %s: %v", gen, err) + } + if _, err := os.Stat(filepath.Join(gen, "bird.d", "operator.conf")); err != nil { + t.Fatalf("operator fragment missing: %v", err) + } + } +} + +func TestApplyRevisionFaults(t *testing.T) { + for _, mode := range []string{"parse", "write", "disk-full", "configure", "cancel"} { + t.Run(mode, func(t *testing.T) { + active, ctl, old := setupActive(t) + rev := revision(mode) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cfg := Config{ActiveDir: active} + switch mode { + case "parse": + rev.PreviewFragments["bird.conf"] += "FAIL_PARSE\n" + case "write": + rev.PreviewFragments["../escape.conf"] = "bad" + case "disk-full": + cfg.writeFile = func(path string, b []byte) error { + if filepath.Base(path) == "previous-bird.conf" { + return syscall.ENOSPC + } + return writeSynced(path, b) + } + case "configure": + if err := os.WriteFile(filepath.Join(active, ".configure-fail-once"), nil, 0o644); err != nil { + t.Fatal(err) + } + case "cancel": + if err := os.WriteFile(filepath.Join(active, ".configure-wait"), nil, 0o644); err != nil { + t.Fatal(err) + } + } + result := make(chan error, 1) + go func() { result <- ApplyRevision(ctx, ctl, rev, cfg) }() + if mode == "cancel" { + deadline := time.After(3 * time.Second) + for { + if _, err := os.Stat(filepath.Join(active, ".configure-started")); err == nil { + break + } + select { + case <-deadline: + t.Fatal("configure did not start") + case <-time.After(10 * time.Millisecond): + } + } + cancel() + } + err := <-result + if err == nil { + t.Fatal("expected apply failure") + } + main, readErr := os.ReadFile(filepath.Join(active, "bird.conf")) + if readErr != nil { + t.Fatal(readErr) + } + if string(main) != string(old) { + t.Fatalf("old config not restored: %s; error: %v", main, err) + } + if mode == "configure" || mode == "cancel" { + if !strings.Contains(err.Error(), "recovery configure succeeded") { + t.Fatalf("unconfirmed recovery: %v", err) + } + } + }) + } +} + +func TestApplyRevisionSerializes(t *testing.T) { + active, ctl, _ := setupActive(t) + var wg sync.WaitGroup + errs := make(chan error, 2) + for _, label := range []string{"first", "second"} { + wg.Add(1) + go func(label string) { + defer wg.Done() + errs <- ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active}) + }(label) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } +} + +func TestLockHelperProcess(t *testing.T) { + active := os.Getenv("EVOBGP_LOCK_HELPER_ACTIVE") + if active == "" { + return + } + lock, err := lockTarget(context.Background(), filepath.Join(active, ".evobgp-apply.lock")) + if err != nil { + t.Fatal(err) + } + defer func() { _ = lock.Close() }() + if err := os.WriteFile(filepath.Join(active, ".helper-ready"), nil, 0o644); err != nil { + t.Fatal(err) + } + for { + if _, err := os.Stat(filepath.Join(active, ".helper-done")); err == nil { + return + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestApplyRevisionCrossProcessLock(t *testing.T) { + active, ctl, old := setupActive(t) + cmd := exec.Command(os.Args[0], "-test.run=^TestLockHelperProcess$") + cmd.Env = append(os.Environ(), "EVOBGP_LOCK_HELPER_ACTIVE="+active) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + _ = os.WriteFile(filepath.Join(active, ".helper-done"), nil, 0o644) + if err := cmd.Wait(); err != nil { + t.Errorf("lock helper: %v", err) + } + }() + deadline := time.After(3 * time.Second) + for { + if _, err := os.Stat(filepath.Join(active, ".helper-ready")); err == nil { + break + } + select { + case <-deadline: + t.Fatal("lock helper did not become ready") + case <-time.After(10 * time.Millisecond): + } + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := ApplyRevision(ctx, ctl, revision("blocked"), Config{ActiveDir: active}); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("want lock timeout, got %v", err) + } + main, err := os.ReadFile(filepath.Join(active, "bird.conf")) + if err != nil { + t.Fatal(err) + } + if string(main) != string(old) { + t.Fatalf("apply bypassed cross-process lock: %s", main) + } +} + +func TestRealBirdParseGeneration(t *testing.T) { + bin := strings.TrimSpace(os.Getenv("EVOBGP_TEST_BIRD_BIN")) + if bin == "" { + t.Skip("EVOBGP_TEST_BIRD_BIN is not set") + } + active := t.TempDir() + old := []byte("router id 192.0.2.2;\nprotocol device {}\n") + if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil { + t.Fatal(err) + } + ctl := fakeCtl(t, active) + ctl.Bird = bin + rev := &store.Revision{PreviewFragments: map[string]string{ + "bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/evobgp_new.conf\";\nprotocol device {}\n", + "bird.d/evobgp_new.conf": "# complete generated fragment\n", + }} + if err := ApplyRevision(context.Background(), ctl, rev, Config{ActiveDir: active}); err != nil { + t.Fatal(err) + } + if err := ctl.ParseCheck(context.Background(), filepath.Join(active, "bird.conf")); err != nil { + t.Fatalf("activated config fails real bird -p: %v", err) + } +} diff --git a/internal/birddeploy/apply_test.go b/internal/birddeploy/apply_test.go new file mode 100644 index 0000000..b2237e2 --- /dev/null +++ b/internal/birddeploy/apply_test.go @@ -0,0 +1,204 @@ +package birddeploy + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "evobgp/internal/store" +) + +func TestWriteGenerationPinsCompleteIncludeTree(t *testing.T) { + active := t.TempDir() + if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(active, "bird.d", "operator.conf"), []byte("# operator"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(active, "bird.d", "evobgp_removed.conf"), []byte("# removed"), 0o644); err != nil { + t.Fatal(err) + } + gen := filepath.Join(active, ".evobgp-generation-test") + if err := os.Mkdir(gen, 0o755); err != nil { + t.Fatal(err) + } + fragments := map[string]string{"bird.conf": "include \"bird.d/*.conf\";\n", "bird.d/evobgp_new.conf": "# new"} + if err := writeGeneration(active, gen, fragments); err != nil { + t.Fatal(err) + } + main, err := os.ReadFile(filepath.Join(gen, "bird.conf")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(main), filepath.ToSlash(filepath.Join(gen, "bird.d", "*.conf"))) { + t.Fatalf("include is not pinned to generation: %s", main) + } + for _, name := range []string{"operator.conf", "evobgp_new.conf"} { + if _, err := os.Stat(filepath.Join(gen, "bird.d", name)); err != nil { + t.Fatalf("%s missing: %v", name, err) + } + } + if _, err := os.Stat(filepath.Join(gen, "bird.d", "evobgp_removed.conf")); !os.IsNotExist(err) { + t.Fatalf("removed fragment retained: %v", err) + } +} + +func TestWriteGenerationRejectsUnsafePaths(t *testing.T) { + for _, bad := range []string{"../escape.conf", "bird.d/../../escape.conf", "other.conf"} { + t.Run(bad, func(t *testing.T) { + active := t.TempDir() + gen := filepath.Join(active, ".evobgp-generation-test") + if err := os.Mkdir(gen, 0o755); err != nil { + t.Fatal(err) + } + fragments := map[string]string{"bird.conf": "router id 192.0.2.1;\n", bad: "bad"} + if err := writeGeneration(active, gen, fragments); err == nil { + t.Fatal("unsafe path accepted") + } + }) + } +} + +func TestWriteGenerationRejectsExternalAndNestedIncludes(t *testing.T) { + for _, main := range []string{ + `include "/outside/bird.d/peer.conf";` + "\n", + `include "other/peer.conf";` + "\n", + `include "bird.d/peer.conf"; include "other.conf";` + "\n", + `router id 192.0.2.1; include "/outside/peer.conf";` + "\n", + "/*\ninclude \"bird.d/decoy.conf\";\n*/\nrouter id 192.0.2.1; include \"/outside/live.conf\";\n", + } { + active := t.TempDir() + gen := filepath.Join(active, ".evobgp-generation-test") + if err := os.Mkdir(gen, 0o755); err != nil { + t.Fatal(err) + } + if err := writeGeneration(active, gen, map[string]string{"bird.conf": main}); err == nil { + t.Fatalf("include accepted: %s", main) + } + } + active := t.TempDir() + if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(active, "bird.d", "operator.conf"), []byte(`define X=1; include "/outside/foo.conf";`), 0o644); err != nil { + t.Fatal(err) + } + gen := filepath.Join(active, ".evobgp-generation-test") + if err := os.Mkdir(gen, 0o755); err != nil { + t.Fatal(err) + } + if err := writeGeneration(active, gen, map[string]string{"bird.conf": "router id 192.0.2.1;\n"}); err == nil { + t.Fatal("nested operator include accepted") + } +} + +func TestCountIncludeTokensIgnoresCommentsAndStrings(t *testing.T) { + content := "# include \"ignored.conf\";\n" + + "/* include \"ignored.conf\"; */\n" + + "define note = \"include in string\";\n" + + "include \"bird.d/real.conf\";\n" + if got := countIncludeTokens(content); got != 1 { + t.Fatalf("want one include, got %d", got) + } +} + +func TestIncludeAfterBackslashStringCannotHideNestedInclude(t *testing.T) { + content := `define note = "\";` + "\n" + `include "/outside/hidden.conf";` + "\n" + if got := countIncludeTokens(content); got != 1 { + t.Fatalf("live include hidden after quoted backslash: %d", got) + } + active := t.TempDir() + gen := filepath.Join(active, ".evobgp-generation-test") + if err := os.Mkdir(gen, 0o755); err != nil { + t.Fatal(err) + } + fragments := map[string]string{"bird.conf": "router id 192.0.2.1;\n", "bird.d/evobgp_test.conf": content} + if err := writeGeneration(active, gen, fragments); err == nil { + t.Fatal("nested external include accepted after quoted backslash") + } +} + +func TestWriteGenerationRewritesAbsoluteActiveInclude(t *testing.T) { + active := t.TempDir() + gen := filepath.Join(active, ".evobgp-generation-test") + if err := os.Mkdir(gen, 0o755); err != nil { + t.Fatal(err) + } + inc := filepath.ToSlash(filepath.Join(active, "bird.d", "peer.conf")) + main := "include \"" + inc + "\";\n" + if err := writeGeneration(active, gen, map[string]string{"bird.conf": main, "bird.d/peer.conf": "# peer"}); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(filepath.Join(gen, "bird.conf")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), filepath.ToSlash(filepath.Join(gen, "bird.d", "peer.conf"))) { + t.Fatalf("absolute include not rewritten: %s", b) + } +} + +func TestWriteGenerationRejectsSymlinkBirdDRoot(t *testing.T) { + active := t.TempDir() + outside := t.TempDir() + if err := os.WriteFile(filepath.Join(outside, "operator.conf"), []byte("# operator"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(active, "bird.d")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + old := []byte("router id 192.0.2.2;\n") + if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil { + t.Fatal(err) + } + gen := filepath.Join(active, ".evobgp-generation-test") + if err := os.Mkdir(gen, 0o755); err != nil { + t.Fatal(err) + } + if err := writeGeneration(active, gen, map[string]string{"bird.conf": "router id 192.0.2.1;\n"}); err == nil { + t.Fatal("bird.d symlink accepted") + } + current, err := os.ReadFile(filepath.Join(active, "bird.conf")) + if err != nil { + t.Fatal(err) + } + if string(current) != string(old) { + t.Fatalf("active config changed: %s", current) + } + if _, err := os.ReadFile(filepath.Join(outside, "operator.conf")); err != nil { + t.Fatalf("operator snippet changed: %v", err) + } +} + +func TestApplyRevisionRejectsSymlinkMain(t *testing.T) { + active := t.TempDir() + target := filepath.Join(t.TempDir(), "operator-bird.conf") + old := []byte("router id 192.0.2.2;\n") + if err := os.WriteFile(target, old, 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(active, "bird.conf")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + err := ApplyRevision(context.Background(), nil, &store.Revision{PreviewFragments: map[string]string{"bird.conf": "router id 192.0.2.1;\n"}}, Config{ActiveDir: active}) + if err == nil || !strings.Contains(err.Error(), "regular file") { + t.Fatalf("want explicit symlink rejection, got %v", err) + } + info, err := os.Lstat(filepath.Join(active, "bird.conf")) + if err != nil { + t.Fatal(err) + } + if info.Mode()&os.ModeSymlink == 0 { + t.Fatal("active symlink replaced") + } + current, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + if string(current) != string(old) { + t.Fatalf("operator target changed: %s", current) + } +} diff --git a/internal/birddeploy/lock_linux.go b/internal/birddeploy/lock_linux.go new file mode 100644 index 0000000..8583619 --- /dev/null +++ b/internal/birddeploy/lock_linux.go @@ -0,0 +1,35 @@ +//go:build linux + +package birddeploy + +import ( + "context" + "errors" + "os" + "time" + + "golang.org/x/sys/unix" +) + +func lockTarget(ctx context.Context, path string) (*os.File, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, err + } + for { + err = unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if err == nil { + return f, nil + } + if !errors.Is(err, unix.EWOULDBLOCK) && !errors.Is(err, unix.EAGAIN) { + _ = f.Close() + return nil, err + } + select { + case <-ctx.Done(): + _ = f.Close() + return nil, ctx.Err() + case <-time.After(25 * time.Millisecond): + } + } +} diff --git a/internal/birddeploy/lock_other.go b/internal/birddeploy/lock_other.go new file mode 100644 index 0000000..64715e6 --- /dev/null +++ b/internal/birddeploy/lock_other.go @@ -0,0 +1,13 @@ +//go:build !linux && !windows + +package birddeploy + +import ( + "context" + "errors" + "os" +) + +func lockTarget(context.Context, string) (*os.File, error) { + return nil, errors.New("birddeploy: cross-process apply lock is unsupported on this platform") +} diff --git a/internal/birddeploy/lock_windows.go b/internal/birddeploy/lock_windows.go new file mode 100644 index 0000000..5ce9b20 --- /dev/null +++ b/internal/birddeploy/lock_windows.go @@ -0,0 +1,36 @@ +//go:build windows + +package birddeploy + +import ( + "context" + "errors" + "os" + "time" + + "golang.org/x/sys/windows" +) + +func lockTarget(ctx context.Context, path string) (*os.File, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, err + } + var overlapped windows.Overlapped + for { + err = windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &overlapped) + if err == nil { + return f, nil + } + if !errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + _ = f.Close() + return nil, err + } + select { + case <-ctx.Done(): + _ = f.Close() + return nil, ctx.Err() + case <-time.After(25 * time.Millisecond): + } + } +} diff --git a/internal/birddeploy/sync_unix.go b/internal/birddeploy/sync_unix.go new file mode 100644 index 0000000..029fe42 --- /dev/null +++ b/internal/birddeploy/sync_unix.go @@ -0,0 +1,14 @@ +//go:build !windows + +package birddeploy + +import "os" + +func syncDir(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} diff --git a/internal/birddeploy/sync_windows.go b/internal/birddeploy/sync_windows.go new file mode 100644 index 0000000..794a156 --- /dev/null +++ b/internal/birddeploy/sync_windows.go @@ -0,0 +1,6 @@ +//go:build windows + +package birddeploy + +// Windows does not support syncing a directory handle through os.File.Sync. +func syncDir(string) error { return nil } diff --git a/internal/bundle/extract.go b/internal/bundle/extract.go index 4a3a33f..068a408 100644 --- a/internal/bundle/extract.go +++ b/internal/bundle/extract.go @@ -3,34 +3,100 @@ package bundle import ( "fmt" "os" - pathpkg "path" "path/filepath" "strings" ) // WriteExtractedFiles writes payload files from a verified bundle under root (directories created as needed). func WriteExtractedFiles(root string, v *VerifiedContents) error { + return WriteExtractedFilesWithLimits(root, v, Limits{}) +} + +// WriteExtractedFilesWithLimits confines all filesystem operations to an open +// directory handle. Replacing existing files uses unlink + exclusive creation, +// so existing hard links and symlinks cannot redirect writes outside the root. +func WriteExtractedFilesWithLimits(root string, v *VerifiedContents, limits Limits) error { + l, err := limits.Normalized() + if err != nil { + return err + } + if v == nil { + return fmt.Errorf("bundle: nil verified contents") + } + if len(v.Manifest.Files) > l.MaxEntries-2 { + return ErrLimit + } + seen := map[string]bool{} + var total int64 + for _, fe := range v.Manifest.Files { + if err := validatePath(fe.Path, l.MaxPathBytes); err != nil { + return err + } + if seen[strings.ToLower(fe.Path)] { + return fmt.Errorf("bundle: duplicate path %q", fe.Path) + } + seen[strings.ToLower(fe.Path)] = true + data, ok := v.Files[fe.Path] + if !ok { + return fmt.Errorf("bundle: missing file %q", fe.Path) + } + if int64(len(data)) > l.MaxFileBytes || int64(len(data)) > l.MaxExpandedBytes-total { + return ErrLimit + } + total += int64(len(data)) + } root = filepath.Clean(root) + if err := os.MkdirAll(root, 0o755); err != nil { + return err + } + confined, err := os.OpenRoot(root) + if err != nil { + return err + } + defer func() { _ = confined.Close() }() for _, fe := range v.Manifest.Files { data, ok := v.Files[fe.Path] if !ok { return fmt.Errorf("bundle: missing file %q", fe.Path) } - rel := pathpkg.Clean(strings.TrimPrefix(fe.Path, "/")) - if rel == "." || strings.HasPrefix(rel, "..") { - return fmt.Errorf("bundle: unsafe path %q", fe.Path) + parts := strings.Split(fe.Path, "/") + for i := 1; i < len(parts); i++ { + dir := filepath.FromSlash(strings.Join(parts[:i], "/")) + if err := confined.Mkdir(dir, 0o755); err != nil && !os.IsExist(err) { + return err + } + info, err := confined.Lstat(dir) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("bundle: unsafe directory %q", dir) + } } - dest := filepath.Join(root, filepath.FromSlash(rel)) - relToRoot, err := filepath.Rel(root, dest) - if err != nil || strings.HasPrefix(relToRoot, "..") { - return fmt.Errorf("bundle: path escapes root: %q", fe.Path) - } - if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + dest := filepath.FromSlash(fe.Path) + info, err := confined.Lstat(dest) + if err == nil { + if !info.Mode().IsRegular() { + return fmt.Errorf("bundle: unsafe destination %q", dest) + } + if err = confined.Remove(dest); err != nil { + return err + } + } else if !os.IsNotExist(err) { return err } - if err := os.WriteFile(dest, data, 0o644); err != nil { + f, err := confined.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { return err } + _, writeErr := f.Write(data) + closeErr := f.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return closeErr + } } return nil } diff --git a/internal/bundle/legacy_compatibility_test.go b/internal/bundle/legacy_compatibility_test.go new file mode 100644 index 0000000..daeb9bf --- /dev/null +++ b/internal/bundle/legacy_compatibility_test.go @@ -0,0 +1,120 @@ +package bundle + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "os" + "reflect" + "sort" + "testing" +) + +// legacyManifestV1 mirrors the decoder fields present in the locally available +// v1.0.0 source. It deliberately has no future capability or deployment fields. +type legacyManifestV1 struct { + RevisionID string `json:"revision_id"` + SpeakerID string `json:"speaker_id"` + CreatedAt string `json:"created_at"` + Files []struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + } `json:"files"` + Algorithm string `json:"signature_algorithm"` + PublicKeyBase64 string `json:"public_key_base64"` +} + +func TestLegacyCompatibilityFrozenSignedCurrentBaseline(t *testing.T) { + raw, err := os.ReadFile("testdata/current_baseline_signed_bundle.tar.gz") + if err != nil { + t.Fatal(err) + } + pub := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{0x42}, ed25519.SeedSize)).Public().(ed25519.PublicKey) + verified, err := VerifyGzippedTar(raw, pub) + if err != nil { + t.Fatalf("frozen signed baseline rejected: %v", err) + } + if verified.Manifest.RevisionID != "revision-legacy" || verified.Manifest.SpeakerID != "speaker-legacy" || len(verified.Files) != 2 { + t.Fatalf("frozen baseline contents: %+v", verified.Manifest) + } +} + +func TestLegacyCompatibilityBundleWire(t *testing.T) { + seed := bytes.Repeat([]byte{0x42}, ed25519.SeedSize) + priv := ed25519.NewKeyFromSeed(seed) + pub := priv.Public().(ed25519.PublicKey) + fragments := map[string]string{ + "bird.conf": "router id 192.0.2.1;\n", + "bird.d/evobgp_prefixes_v4.conf": "# revision\n", + "_preview": "must not ship", + } + raw, err := BuildGzippedTar("revision-legacy", "speaker-legacy", fragments, priv) + if err != nil { + t.Fatal(err) + } + gr, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + defer func() { _ = gr.Close() }() + tr := tar.NewReader(gr) + entries := map[string][]byte{} + var order []string + for { + h, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + order = append(order, h.Name) + entries[h.Name] = body + } + wantOrder := []string{"manifest.json", "bundle.sig", "bird.conf", "bird.d/evobgp_prefixes_v4.conf"} + if !reflect.DeepEqual(order, wantOrder) { + t.Fatalf("legacy tar layout: got %v, want %v", order, wantOrder) + } + var top map[string]json.RawMessage + if err := json.Unmarshal(entries["manifest.json"], &top); err != nil { + t.Fatal(err) + } + wantFields := []string{"created_at", "files", "public_key_base64", "revision_id", "signature_algorithm", "speaker_id"} + fields := make([]string, 0, len(top)) + for k := range top { + fields = append(fields, k) + } + sort.Strings(fields) + if !reflect.DeepEqual(fields, wantFields) { + t.Fatalf("manifest fields: got %v, want %v", fields, wantFields) + } + var m legacyManifestV1 + if err := json.Unmarshal(entries["manifest.json"], &m); err != nil { + t.Fatal(err) + } + if m.RevisionID != "revision-legacy" || m.SpeakerID != "speaker-legacy" || m.CreatedAt == "" || m.Algorithm != "ed25519" || m.PublicKeyBase64 != base64.StdEncoding.EncodeToString(pub) { + t.Fatalf("legacy manifest fields: %+v", m) + } + if !ed25519.Verify(pub, entries["manifest.json"], entries["bundle.sig"]) { + t.Fatal("legacy Ed25519 verifier rejected current signed manifest") + } + if len(m.Files) != 2 { + t.Fatalf("legacy manifest file count: %d", len(m.Files)) + } + for _, f := range m.Files { + sum := sha256.Sum256(entries[f.Path]) + if hex.EncodeToString(sum[:]) != f.SHA256 { + t.Fatalf("legacy SHA-256 mismatch for %q", f.Path) + } + } +} diff --git a/internal/bundle/limits.go b/internal/bundle/limits.go new file mode 100644 index 0000000..7cb91af --- /dev/null +++ b/internal/bundle/limits.go @@ -0,0 +1,103 @@ +package bundle + +import ( + "errors" + "fmt" + "io" + "path" + "strings" +) + +// ErrLimit reports that a transport, archive or extraction budget was exceeded. +var ErrLimit = errors.New("bundle: resource limit exceeded") + +// Limits bounds both archive transport and decoded tar data, including metadata. +// Zero fields select defaults. These ceilings permit large (1M-prefix) configs; +// they are safety ceilings, not a promise of suitability for a small-RAM node. +type Limits struct { + MaxCompressedBytes int64 + MaxExpandedBytes int64 + MaxFileBytes int64 + MaxManifestBytes int64 + MaxEntries int + MaxPathBytes int +} + +// DefaultLimits permits large production configs while bounding untrusted input. +func DefaultLimits() Limits { + return Limits{MaxCompressedBytes: 128 << 20, MaxExpandedBytes: 512 << 20, MaxFileBytes: 256 << 20, MaxManifestBytes: 2 << 20, MaxEntries: 4096, MaxPathBytes: 1024} +} + +// Normalized fills zero fields with defaults and rejects invalid budgets. +func (l Limits) Normalized() (Limits, error) { + d := DefaultLimits() + values := []struct { + v *int64 + fallback int64 + }{{&l.MaxCompressedBytes, d.MaxCompressedBytes}, {&l.MaxExpandedBytes, d.MaxExpandedBytes}, {&l.MaxFileBytes, d.MaxFileBytes}, {&l.MaxManifestBytes, d.MaxManifestBytes}} + for _, x := range values { + if *x.v < 0 || *x.v == int64(^uint64(0)>>1) { + return l, fmt.Errorf("bundle: invalid byte limit") + } + if *x.v == 0 { + *x.v = x.fallback + } + } + if l.MaxEntries < 0 || l.MaxPathBytes < 0 { + return l, fmt.Errorf("bundle: invalid entry/path limit") + } + if l.MaxEntries == 0 { + l.MaxEntries = d.MaxEntries + } + if l.MaxPathBytes == 0 { + l.MaxPathBytes = d.MaxPathBytes + } + return l, nil +} + +func validatePath(name string, limit int) error { + if name == "" || len(name) > limit || path.IsAbs(name) || path.Clean(name) != name || name == "." || strings.ContainsAny(name, "\\:\x00") { + return fmt.Errorf("bundle: unsafe path %q", name) + } + for _, part := range strings.Split(name, "/") { + if part == ".." || strings.TrimRight(part, " .") != part { + return fmt.Errorf("bundle: unsafe path %q", name) + } + base := strings.ToUpper(strings.SplitN(part, ".", 2)[0]) + if base == "CON" || base == "PRN" || base == "AUX" || base == "NUL" || (len(base) == 4 && (strings.HasPrefix(base, "COM") || strings.HasPrefix(base, "LPT")) && base[3] >= '1' && base[3] <= '9') { + return fmt.Errorf("bundle: reserved path %q", name) + } + } + return nil +} + +type limitedWriter struct { + w io.Writer + remaining int64 +} + +func (w *limitedWriter) Write(p []byte) (int, error) { + if int64(len(p)) > w.remaining { + return 0, ErrLimit + } + n, err := w.w.Write(p) + w.remaining -= int64(n) + return n, err +} + +// ReadCompressed rejects oversized input while reading, including chunked HTTP +// bodies and local files whose size changes after stat. +func ReadCompressed(r io.Reader, limits Limits) ([]byte, error) { + l, err := limits.Normalized() + if err != nil { + return nil, err + } + b, err := io.ReadAll(io.LimitReader(r, l.MaxCompressedBytes+1)) + if err != nil { + return nil, err + } + if int64(len(b)) > l.MaxCompressedBytes { + return nil, ErrLimit + } + return b, nil +} diff --git a/internal/bundle/limits_test.go b/internal/bundle/limits_test.go new file mode 100644 index 0000000..3b31acd --- /dev/null +++ b/internal/bundle/limits_test.go @@ -0,0 +1,286 @@ +package bundle + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/ed25519" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +type testEntry struct { + header tar.Header + body []byte +} + +func archiveEntries(t *testing.T, entries []testEntry) []byte { + t.Helper() + var b bytes.Buffer + gz := gzip.NewWriter(&b) + tw := tar.NewWriter(gz) + for _, entry := range entries { + h := entry.header + if h.Typeflag == 0 { + h.Typeflag = tar.TypeReg + } + h.Size = int64(len(entry.body)) + if err := tw.WriteHeader(&h); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(entry.body); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return b.Bytes() +} +func testKey() ed25519.PrivateKey { + return ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, ed25519.SeedSize)) +} + +func TestBundleResourceLimits(t *testing.T) { + pub := testKey().Public().(ed25519.PublicKey) + cases := []struct { + name string + entries []testEntry + limits Limits + }{ + {"file", []testEntry{{header: tar.Header{Name: "bird.conf"}, body: bytes.Repeat([]byte{'a'}, 1025)}}, Limits{MaxFileBytes: 1024}}, + {"expanded", []testEntry{{header: tar.Header{Name: "bird.conf"}, body: bytes.Repeat([]byte{'a'}, 8192)}}, Limits{MaxExpandedBytes: 4096}}, + {"manifest", []testEntry{{header: tar.Header{Name: manifestName}, body: bytes.Repeat([]byte{'a'}, 1025)}}, Limits{MaxManifestBytes: 1024}}, + {"entries", []testEntry{{header: tar.Header{Name: "a"}}, {header: tar.Header{Name: "b"}}, {header: tar.Header{Name: "c"}}}, Limits{MaxEntries: 2}}, + {"pax-metadata", []testEntry{{header: tar.Header{Name: "bird.conf", Format: tar.FormatPAX, PAXRecords: map[string]string{"comment": strings.Repeat("a", 8192)}}}}, Limits{MaxExpandedBytes: 4096}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + raw := archiveEntries(t, tc.entries) + if _, err := VerifyGzippedTarWithLimits(raw, pub, tc.limits); !errors.Is(err, ErrLimit) { + t.Fatalf("want limit, got %v", err) + } + }) + } + if _, err := VerifyGzippedTarWithLimits(make([]byte, 65), pub, Limits{MaxCompressedBytes: 64}); !errors.Is(err, ErrLimit) { + t.Fatalf("compressed cap: %v", err) + } +} + +func TestBundleRejectsUnsafeEntries(t *testing.T) { + pub := testKey().Public().(ed25519.PublicKey) + for _, name := range []string{"../escape", "/etc/bird.conf", `bird.d\escape`, "C:/escape", "bird.d/../escape", "a//b", "a/./b", "NUL", "bird.conf:stream", "a. /b"} { + t.Run(name, func(t *testing.T) { + raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: name}, body: []byte("x")}}) + if _, err := VerifyGzippedTar(raw, pub); err == nil { + t.Fatal("unsafe path accepted") + } + }) + } + for _, kind := range []byte{tar.TypeSymlink, tar.TypeLink, tar.TypeDir, tar.TypeFifo} { + raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: "bird.conf", Typeflag: kind, Linkname: "outside"}}}) + if _, err := VerifyGzippedTar(raw, pub); err == nil { + t.Fatalf("type %d accepted", kind) + } + } + raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: "bird.conf"}}, {header: tar.Header{Name: "bird.conf"}}}) + if _, err := VerifyGzippedTar(raw, pub); err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Fatalf("duplicate accepted: %v", err) + } +} + +func TestBundleChecksTrailerAndTrailingExpansion(t *testing.T) { + priv := testKey() + pub := priv.Public().(ed25519.PublicKey) + raw, err := BuildGzippedTar("r", "s", map[string]string{"bird.conf": "router id 192.0.2.1;"}, priv) + if err != nil { + t.Fatal(err) + } + corrupt := append([]byte{}, raw...) + corrupt[len(corrupt)-8] ^= 0xff + if _, err = VerifyGzippedTar(corrupt, pub); err == nil { + t.Fatal("gzip checksum ignored after tar EOF") + } + var extra bytes.Buffer + gz := gzip.NewWriter(&extra) + if _, err = gz.Write(bytes.Repeat([]byte{'x'}, 32<<10)); err != nil { + t.Fatal(err) + } + if err = gz.Close(); err != nil { + t.Fatal(err) + } + joined := append(append([]byte{}, raw...), extra.Bytes()...) + if _, err = VerifyGzippedTarWithLimits(joined, pub, Limits{MaxExpandedBytes: 8 << 10}); !errors.Is(err, ErrLimit) { + t.Fatalf("trailing gzip member bypassed budget: %v", err) + } +} + +func TestBundleRejectsPAXSparseRegularFile(t *testing.T) { + // archive/tar expands PAX sparse holes even though the returned type is a + // regular file. Build a physical one-byte entry with a 4096-byte logical size. + record := func(key, value string) string { + body := key + "=" + value + "\n" + n := len(body) + 2 + for { + next := len(fmt.Sprint(n)) + 1 + len(body) + if next == n { + return fmt.Sprintf("%d %s", n, body) + } + n = next + } + } + pax := record("GNU.sparse.size", "4096") + record("GNU.sparse.numblocks", "1") + record("GNU.sparse.map", "0,1") + var physical bytes.Buffer + tw := tar.NewWriter(&physical) + for _, entry := range []testEntry{{header: tar.Header{Name: "PaxHeader", Size: int64(len(pax))}, body: []byte(pax)}, {header: tar.Header{Name: "bird.conf", Size: 1}, body: []byte("x")}} { + if err := tw.WriteHeader(&entry.header); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(entry.body); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + data := physical.Bytes() + data[156] = tar.TypeXHeader + for i := 148; i < 156; i++ { + data[i] = ' ' + } + sum := 0 + for _, b := range data[:512] { + sum += int(b) + } + copy(data[148:156], fmt.Sprintf("%06o\x00 ", sum)) + h, err := tar.NewReader(bytes.NewReader(data)).Next() + if err != nil || h.Typeflag != tar.TypeReg || h.Size != 4096 { + t.Fatalf("invalid sparse regression fixture %+v %v", h, err) + } + var compressed bytes.Buffer + gz := gzip.NewWriter(&compressed) + if _, err = gz.Write(data); err != nil { + t.Fatal(err) + } + if err = gz.Close(); err != nil { + t.Fatal(err) + } + if _, err = VerifyGzippedTar(compressed.Bytes(), testKey().Public().(ed25519.PublicKey)); err == nil || !strings.Contains(err.Error(), "sparse") { + t.Fatalf("PAX sparse accepted: %v", err) + } +} + +func TestBundleRejectsUnmanifestedPayload(t *testing.T) { + priv := testKey() + raw, err := BuildGzippedTar("r", "s", map[string]string{"bird.conf": "x"}, priv) + if err != nil { + t.Fatal(err) + } + gr, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + defer func() { _ = gr.Close() }() + tr := tar.NewReader(gr) + var entries []testEntry + for { + h, e := tr.Next() + if e == io.EOF { + break + } + if e != nil { + t.Fatal(e) + } + b, e := io.ReadAll(tr) + if e != nil { + t.Fatal(e) + } + entries = append(entries, testEntry{header: *h, body: b}) + } + entries = append(entries, testEntry{header: tar.Header{Name: "unsigned.conf"}, body: []byte("unsigned")}) + if _, err = VerifyGzippedTar(archiveEntries(t, entries), priv.Public().(ed25519.PublicKey)); err == nil { + t.Fatal("unmanifested payload accepted") + } +} + +func TestExtractionDoesNotFollowLinks(t *testing.T) { + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "outside") + if err := os.WriteFile(outside, []byte("preserve"), 0o600); err != nil { + t.Fatal(err) + } + dest := filepath.Join(root, "bird.conf") + if err := os.Link(outside, dest); err != nil { + if runtime.GOOS == "linux" { + t.Fatalf("Linux link regression requires hard links: %v", err) + } + t.Skipf("hard links unavailable: %v", err) + } + v := &VerifiedContents{Manifest: Manifest{Files: []FileEntry{{Path: "bird.conf"}}}, Files: map[string][]byte{"bird.conf": []byte("replacement")}} + if err := WriteExtractedFiles(root, v); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(outside) + if err != nil || string(got) != "preserve" { + t.Fatalf("hard link modified outside: %s %v", got, err) + } + if err = os.Remove(dest); err != nil { + t.Fatal(err) + } + if err = os.Symlink(outside, dest); err != nil { + if runtime.GOOS == "linux" { + t.Fatalf("Linux link regression requires symlinks: %v", err) + } + t.Skipf("symlinks unavailable: %v", err) + } + if err = WriteExtractedFiles(root, v); err == nil { + t.Fatal("symlink destination accepted") + } + got, err = os.ReadFile(outside) + if err != nil || string(got) != "preserve" { + t.Fatal("symlink modified outside") + } +} + +func TestBundleBuilderBoundsAndPortablePaths(t *testing.T) { + for _, name := range []string{"/bird.conf", "a/../bird.conf", manifestName, sigFileName, `bird.d\bad`} { + if _, err := BuildGzippedTar("r", "s", map[string]string{name: "x"}, testKey()); err == nil { + t.Fatalf("invalid builder path accepted: %q", name) + } + } + if _, err := BuildGzippedTarWithLimits("r", "s", map[string]string{"bird.conf": strings.Repeat("x", 1025)}, testKey(), Limits{MaxFileBytes: 1024}); !errors.Is(err, ErrLimit) { + t.Fatalf("builder file cap %v", err) + } + if _, err := BuildGzippedTarWithLimits("r", "s", map[string]string{"bird.conf": "x"}, testKey(), Limits{MaxExpandedBytes: 1024}); !errors.Is(err, ErrLimit) { + t.Fatalf("builder tar overhead cap %v", err) + } +} + +// This is a decoded-size regression using repeated representative route lines, +// not a unique-prefix cardinality test or a BIRD validity/performance benchmark. +func TestDefaultLimitsAllowMillionRepresentativeIPv6RouteLines(t *testing.T) { + if testing.Short() { + t.Skip("large representative config") + } + config := strings.Repeat("route 2001:db8:1234:5678::/64 blackhole;\n", 1_000_000) + raw, err := BuildGzippedTar("large", "speaker", map[string]string{"bird.conf": config}, testKey()) + if err != nil { + t.Fatal(err) + } + v, err := VerifyGzippedTar(raw, testKey().Public().(ed25519.PublicKey)) + if err != nil { + t.Fatal(err) + } + if len(v.Files["bird.conf"]) != len(config) { + t.Fatal("large config truncated") + } +} diff --git a/internal/bundle/pack.go b/internal/bundle/pack.go index 135b522..ed401eb 100644 --- a/internal/bundle/pack.go +++ b/internal/bundle/pack.go @@ -9,7 +9,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "path" "sort" "strings" "time" @@ -36,22 +35,47 @@ const manifestName = "manifest.json" // BuildGzippedTar builds a .tar.gz with manifest.json, bundle.sig (Ed25519 over manifest JSON), and payload files. func BuildGzippedTar(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) { + return BuildGzippedTarWithLimits(revisionID, speakerID, fragments, priv, Limits{}) +} + +// BuildGzippedTarWithLimits preserves the manifest/signature wire format while +// refusing payloads that exceed the receiving node's configured budgets. +func BuildGzippedTarWithLimits(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey, limits Limits) ([]byte, error) { + l, err := limits.Normalized() + if err != nil { + return nil, err + } if len(priv) != ed25519.PrivateKeySize { return nil, fmt.Errorf("bundle: invalid ed25519 private key size") } pub := priv.Public().(ed25519.PublicKey) - norm := make(map[string]string, len(fragments)) + norm := make(map[string]string) + seen := map[string]bool{} + var total int64 for p, content := range fragments { - p = path.Clean(strings.TrimPrefix(p, "/")) - if p == "." || strings.HasPrefix(p, "..") { - return nil, fmt.Errorf("bundle: invalid path %q", p) - } if strings.HasPrefix(p, "_") { continue // UI-only preview aggregate, not shipped to nodes } + if len(norm) >= l.MaxEntries-2 { + return nil, ErrLimit + } + if err := validatePath(p, l.MaxPathBytes); err != nil { + return nil, err + } + if p == manifestName || p == sigFileName || seen[strings.ToLower(p)] { + return nil, fmt.Errorf("bundle: duplicate or reserved path %q", p) + } + seen[strings.ToLower(p)] = true + if int64(len(content)) > l.MaxFileBytes || int64(len(content)) > l.MaxExpandedBytes-total { + return nil, ErrLimit + } + total += int64(len(content)) norm[p] = content } + if len(norm) > l.MaxEntries-2 { + return nil, ErrLimit + } var files []FileEntry var payload [][]byte @@ -75,11 +99,14 @@ func BuildGzippedTar(revisionID, speakerID string, fragments map[string]string, if err != nil { return nil, err } + if int64(len(manifestJSON)) > l.MaxManifestBytes { + return nil, ErrLimit + } sig := ed25519.Sign(priv, manifestJSON) var buf bytes.Buffer - gw := gzip.NewWriter(&buf) - tw := tar.NewWriter(gw) + gw := gzip.NewWriter(&limitedWriter{w: &buf, remaining: l.MaxCompressedBytes}) + tw := tar.NewWriter(&limitedWriter{w: gw, remaining: l.MaxExpandedBytes}) add := func(name string, body []byte) error { hdr := &tar.Header{ diff --git a/internal/bundle/testdata/current_baseline_signed_bundle.tar.gz b/internal/bundle/testdata/current_baseline_signed_bundle.tar.gz new file mode 100644 index 0000000..e1bf62b Binary files /dev/null and b/internal/bundle/testdata/current_baseline_signed_bundle.tar.gz differ diff --git a/internal/bundle/verify.go b/internal/bundle/verify.go index 70344ca..2237aad 100644 --- a/internal/bundle/verify.go +++ b/internal/bundle/verify.go @@ -11,44 +11,95 @@ import ( "encoding/json" "fmt" "io" - "path" "strings" ) -// VerifiedContents is the result of verifying a downloaded bundle on evobgp-node. +// VerifiedContents contains only payloads covered by a verified signed manifest. type VerifiedContents struct { Manifest Manifest Files map[string][]byte } -// VerifyGzippedTar checks Ed25519 signature over manifest.json and SHA-256 of each file. -func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents, error) { - gr, err := gzip.NewReader(bytes.NewReader(bundle)) +// VerifyGzippedTar verifies the unchanged signed manifest format with default limits. +func VerifyGzippedTar(raw []byte, pub ed25519.PublicKey) (*VerifiedContents, error) { + return VerifyGzippedTarWithLimits(raw, pub, Limits{}) +} + +// VerifyGzippedTarWithLimits verifies the signature and hashes under explicit +// compressed, expanded, per-entry and metadata budgets. +func VerifyGzippedTarWithLimits(raw []byte, pub ed25519.PublicKey, limits Limits) (*VerifiedContents, error) { + l, err := limits.Normalized() + if err != nil { + return nil, err + } + if int64(len(raw)) > l.MaxCompressedBytes { + return nil, ErrLimit + } + if len(pub) != ed25519.PublicKeySize { + return nil, fmt.Errorf("bundle: invalid public key size") + } + gr, err := gzip.NewReader(bytes.NewReader(raw)) if err != nil { return nil, err } defer func() { _ = gr.Close() }() - - var manifestRaw []byte - var sig []byte - files := make(map[string][]byte) - tr := tar.NewReader(gr) + decoded := &io.LimitedReader{R: gr, N: l.MaxExpandedBytes + 1} + tr := tar.NewReader(decoded) + var manifestRaw, sig []byte + files := map[string][]byte{} + seen := map[string]bool{} + count := 0 + var logicalBytes int64 for { hdr, err := tr.Next() + if decoded.N == 0 { + return nil, ErrLimit + } if err == io.EOF { break } if err != nil { return nil, err } - name := path.Clean(hdr.Name) - if strings.HasPrefix(name, "..") { - return nil, fmt.Errorf("bundle: illegal tar entry %q", hdr.Name) + count++ + if count > l.MaxEntries { + return nil, ErrLimit } - data, err := io.ReadAll(tr) + if err = validatePath(hdr.Name, l.MaxPathBytes); err != nil { + return nil, err + } + name := hdr.Name + folded := strings.ToLower(name) + if seen[folded] { + return nil, fmt.Errorf("bundle: duplicate tar entry %q", name) + } + seen[folded] = true + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + return nil, fmt.Errorf("bundle: nonregular tar entry %q", name) + } + for key := range hdr.PAXRecords { + if strings.HasPrefix(key, "GNU.sparse.") { + return nil, fmt.Errorf("bundle: sparse entry %q is unsupported", name) + } + } + max := l.MaxFileBytes + if name == manifestName { + max = l.MaxManifestBytes + } + if name == sigFileName { + max = ed25519.SignatureSize + } + if hdr.Size < 0 || hdr.Size > max || hdr.Size >= decoded.N || hdr.Size > l.MaxExpandedBytes-logicalBytes { + return nil, ErrLimit + } + logicalBytes += hdr.Size + data, err := io.ReadAll(io.LimitReader(tr, max+1)) if err != nil { return nil, err } + if int64(len(data)) > max || decoded.N == 0 { + return nil, ErrLimit + } switch name { case manifestName: manifestRaw = data @@ -58,17 +109,36 @@ func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents, files[name] = data } } - if len(manifestRaw) == 0 || len(sig) == 0 { + // tar EOF precedes the gzip trailer. Drain through the same byte budget so + // checksums, extra gzip members and trailing decompression are also bounded. + if _, err = io.Copy(io.Discard, decoded); err != nil { + return nil, err + } + if decoded.N == 0 { + return nil, ErrLimit + } + if len(manifestRaw) == 0 || len(sig) != ed25519.SignatureSize { return nil, fmt.Errorf("bundle: missing manifest or signature") } if !ed25519.Verify(pub, manifestRaw, sig) { return nil, fmt.Errorf("bundle: ed25519 signature mismatch") } var m Manifest - if err := json.Unmarshal(manifestRaw, &m); err != nil { + if err = json.Unmarshal(manifestRaw, &m); err != nil { return nil, err } + if len(m.Files) > l.MaxEntries-2 { + return nil, ErrLimit + } + listed := map[string]bool{} for _, fe := range m.Files { + if err = validatePath(fe.Path, l.MaxPathBytes); err != nil { + return nil, err + } + if fe.Path == manifestName || fe.Path == sigFileName || listed[strings.ToLower(fe.Path)] { + return nil, fmt.Errorf("bundle: duplicate or reserved manifest path %q", fe.Path) + } + listed[strings.ToLower(fe.Path)] = true body, ok := files[fe.Path] if !ok { return nil, fmt.Errorf("bundle: missing file %q", fe.Path) @@ -78,10 +148,13 @@ func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents, return nil, fmt.Errorf("bundle: checksum mismatch for %q", fe.Path) } } + if len(listed) != len(files) { + return nil, fmt.Errorf("bundle: payload is not covered by manifest") + } return &VerifiedContents{Manifest: m, Files: files}, nil } -// ParsePublicKeyHex decodes a 64-char hex Ed25519 public key (32 bytes). +// ParsePublicKeyHex decodes a 32-byte Ed25519 public key from hexadecimal. func ParsePublicKeyHex(s string) (ed25519.PublicKey, error) { s = strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(s, "0x"), "0X")) raw, err := hex.DecodeString(s) @@ -94,7 +167,8 @@ func ParsePublicKeyHex(s string) (ed25519.PublicKey, error) { return ed25519.PublicKey(raw), nil } -// PublicKeyFromManifestBase64 uses the key embedded in the signed manifest (cross-check with expected pubkey optional). +// PublicKeyFromManifestBase64 decodes the manifest key. Callers must still trust +// an independently provisioned key when authenticating the bundle. func PublicKeyFromManifestBase64(m *Manifest) (ed25519.PublicKey, error) { if m.PublicKeyB64 == "" { return nil, fmt.Errorf("bundle: manifest missing public_key_base64") diff --git a/internal/db/apply.go b/internal/db/apply.go index 64c4f2a..f926eb4 100644 --- a/internal/db/apply.go +++ b/internal/db/apply.go @@ -8,48 +8,292 @@ import ( "path" "sort" "strings" + "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) -// ApplyPostgresMigrations runs ordered *.up.sql from an embedded subtree (idempotent via schema_migrations). -func ApplyPostgresMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, subdir string) error { - if _, err := pool.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT NOT NULL PRIMARY KEY)`); err != nil { - return fmt.Errorf("db: schema_migrations: %w", err) - } +const ( + postgresMigrationLockKey int64 = 0x45766f4247504d47 // "EvoBGPMG", scoped to the database. + migrationLockTimeout = 30 * time.Second + migrationCleanupTimeout = 5 * time.Second +) + +// PostgreSQL migrations are transaction-only. Non-transactional statements +// need an explicit, separately reviewed execution path. + +// ApplyPostgresMigrations runs ordered *.up.sql from an embedded subtree. +// Each migration and its version row commit atomically under a session advisory lock. +func ApplyPostgresMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, subdir string) (retErr error) { entries, err := fs.ReadDir(fsys, subdir) if err != nil { - return err + return fmt.Errorf("db: read migrations: %w", err) } var ups []string for _, e := range entries { - if e.IsDir() || !strings.HasSuffix(e.Name(), ".up.sql") { - continue + if !e.IsDir() && strings.HasSuffix(e.Name(), ".up.sql") { + ups = append(ups, e.Name()) } - ups = append(ups, e.Name()) } sort.Strings(ups) + // Include pool acquisition in the lock timeout. The lock precedes even + // schema_migrations creation, because another process may start from an empty DB. + lockCtx, cancelLock := context.WithTimeout(ctx, migrationLockTimeout) + conn, err := pool.Acquire(lockCtx) + if err != nil { + cancelLock() + return fmt.Errorf("db: acquire migration connection: %w", err) + } + defer conn.Release() + if _, err := conn.Exec(lockCtx, `SELECT pg_advisory_lock($1)`, postgresMigrationLockKey); err != nil { + cancelLock() + // Cancellation may race with a successful server-side lock acquisition. + closeCtx, closeCancel := context.WithTimeout(context.Background(), migrationCleanupTimeout) + defer closeCancel() + if closeErr := conn.Conn().Close(closeCtx); closeErr != nil { + return errors.Join(fmt.Errorf("db: acquire migration lock: %w", err), fmt.Errorf("db: close migration connection: %w", closeErr)) + } + return fmt.Errorf("db: acquire migration lock: %w", err) + } + cancelLock() + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), migrationCleanupTimeout) + defer cancel() + var unlocked bool + if err := conn.QueryRow(cleanupCtx, `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey).Scan(&unlocked); err != nil || !unlocked { + if err == nil { + err = errors.New("lock was not held") + } + retErr = errors.Join(retErr, fmt.Errorf("db: release migration lock: %w", err)) + // Never return a possibly locked session to the pool. + closeCtx, closeCancel := context.WithTimeout(context.Background(), migrationCleanupTimeout) + defer closeCancel() + if closeErr := conn.Conn().Close(closeCtx); closeErr != nil { + retErr = errors.Join(retErr, fmt.Errorf("db: close migration connection: %w", closeErr)) + } + } + }() + if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT NOT NULL PRIMARY KEY)`); err != nil { + return fmt.Errorf("db: schema_migrations: %w", err) + } for _, name := range ups { - ver := strings.TrimSuffix(name, ".up.sql") - var dummy int - err := pool.QueryRow(ctx, `SELECT 1 FROM schema_migrations WHERE version = $1`, ver).Scan(&dummy) - if err == nil { - continue - } - if !errors.Is(err, pgx.ErrNoRows) { - return err - } body, err := fs.ReadFile(fsys, path.Join(subdir, name)) if err != nil { - return err + return fmt.Errorf("db: read migration %s: %w", name, err) } - if _, err := pool.Exec(ctx, string(body)); err != nil { + if bad := unsupportedMigrationStatement(string(body)); bad != "" { + return fmt.Errorf("db: migration %s uses unsupported non-transactional SQL %q", name, bad) + } + if err := applyPostgresMigration(ctx, conn, strings.TrimSuffix(name, ".up.sql"), body); err != nil { return fmt.Errorf("db: migrate %s: %w", name, err) } - if _, err := pool.Exec(ctx, `INSERT INTO schema_migrations (version) VALUES ($1)`, ver); err != nil { - return err - } } return nil } + +// unsupportedMigrationStatement inspects statement starts after masking SQL +// comments and quoted strings. In particular, a comment after a semicolon +// must not conceal COMMIT or another command that would escape our transaction. +func unsupportedMigrationStatement(sql string) string { + const ( + normal = iota + singleQuote + doubleQuote + lineComment + blockComment + dollarQuote + ) + state := normal + blockDepth := 0 + dollarTag := "" + escapeString := false + var clean strings.Builder + for i := 0; i < len(sql); { + switch state { + case normal: + switch { + case sql[i] == '\'': + escapeString = i > 0 && (sql[i-1] == 'E' || sql[i-1] == 'e') && (i == 1 || !isSQLIdentifierByte(sql[i-2])) + state = singleQuote + clean.WriteByte(' ') + i++ + case sql[i] == '"': + state = doubleQuote + clean.WriteByte(' ') + i++ + case strings.HasPrefix(sql[i:], "--"): + state = lineComment + clean.WriteByte(' ') + i += 2 + case strings.HasPrefix(sql[i:], "/*"): + state = blockComment + blockDepth = 1 + clean.WriteByte(' ') + i += 2 + case sql[i] == '$' && (i == 0 || !isSQLIdentifierByte(sql[i-1])): + end := i + 1 + if end < len(sql) && isDollarTagStartByte(sql[end]) { + end++ + for end < len(sql) && isDollarTagByte(sql[end]) { + end++ + } + } + if end < len(sql) && sql[end] == '$' { + dollarTag = sql[i : end+1] + state = dollarQuote + clean.WriteByte(' ') + i = end + 1 + } else { + clean.WriteByte(sql[i]) + i++ + } + default: + clean.WriteByte(sql[i]) + i++ + } + case singleQuote: + switch { + case escapeString && sql[i] == '\\' && i+1 < len(sql): + i += 2 + case sql[i] == '\'' && i+1 < len(sql) && sql[i+1] == '\'': + i += 2 + case sql[i] == '\'': + state = normal + i++ + default: + i++ + } + case doubleQuote: + if sql[i] == '"' { + if i+1 < len(sql) && sql[i+1] == '"' { + i += 2 + } else { + state = normal + i++ + } + } else { + i++ + } + case lineComment: + if sql[i] == '\n' || sql[i] == '\r' { + clean.WriteByte('\n') + state = normal + } + i++ + case blockComment: + switch { + case strings.HasPrefix(sql[i:], "/*"): + blockDepth++ + i += 2 + case strings.HasPrefix(sql[i:], "*/"): + blockDepth-- + i += 2 + if blockDepth == 0 { + state = normal + } + default: + i++ + } + case dollarQuote: + if strings.HasPrefix(sql[i:], dollarTag) { + i += len(dollarTag) + state = normal + } else { + i++ + } + } + } + for _, statement := range strings.Split(clean.String(), ";") { + words := strings.Fields(strings.ToUpper(statement)) + if isUnsupportedMigrationCommand(words) { + return strings.Join(words[:min(len(words), 4)], " ") + } + } + return "" +} + +func isSQLIdentifierByte(b byte) bool { + return isDollarTagByte(b) || b == '$' +} + +func isDollarTagStartByte(b byte) bool { + return b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' || b == '_' || b >= 0x80 +} + +func isDollarTagByte(b byte) bool { + return isDollarTagStartByte(b) || b >= '0' && b <= '9' +} + +func isUnsupportedMigrationCommand(words []string) bool { + if len(words) == 0 { + return false + } + second := "" + if len(words) > 1 { + second = words[1] + } + switch words[0] { + case "BEGIN", "COMMIT", "END", "ROLLBACK", "ABORT", "SAVEPOINT", "REINDEX", "VACUUM", "CLUSTER": + return true + case "START", "PREPARE": + return second == "TRANSACTION" + case "RELEASE": + return second == "SAVEPOINT" || second != "" + case "ALTER": + return second == "SYSTEM" + case "CREATE": + if second == "DATABASE" || second == "TABLESPACE" { + return true + } + idx := 2 + if second == "UNIQUE" { + idx = 3 + } + return len(words) > idx && words[idx-1] == "INDEX" && words[idx] == "CONCURRENTLY" + case "DROP": + return second == "DATABASE" || second == "TABLESPACE" || second == "INDEX" && len(words) > 2 && words[2] == "CONCURRENTLY" + case "REFRESH": + return len(words) > 3 && second == "MATERIALIZED" && words[2] == "VIEW" && words[3] == "CONCURRENTLY" + case "DISCARD": + return second == "ALL" + } + return false +} + +func applyPostgresMigration(ctx context.Context, conn *pgxpool.Conn, version string, body []byte) (retErr error) { + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + committed := false + defer func() { + if committed { + return + } + cleanupCtx, cancel := context.WithTimeout(context.Background(), migrationCleanupTimeout) + defer cancel() + if err := tx.Rollback(cleanupCtx); err != nil && !errors.Is(err, pgx.ErrTxClosed) { + retErr = errors.Join(retErr, fmt.Errorf("db: rollback migration %s: %w", version, err)) + } + }() + var present int + err = tx.QueryRow(ctx, `SELECT 1 FROM schema_migrations WHERE version = $1`, version).Scan(&present) + if err == nil { + return nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return err + } + if _, err := tx.Exec(ctx, string(body)); err != nil { + return err + } + if _, err := tx.Exec(ctx, `INSERT INTO schema_migrations (version) VALUES ($1)`, version); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + committed = true + return nil +} diff --git a/internal/db/apply_test.go b/internal/db/apply_test.go new file mode 100644 index 0000000..67fdd1d --- /dev/null +++ b/internal/db/apply_test.go @@ -0,0 +1,63 @@ +package db + +import ( + "io/fs" + "path" + "strings" + "testing" + + "evobgp/migrations" +) + +func TestPostgresEmbeddedMigrationsTransactional(t *testing.T) { + entries, err := fs.ReadDir(migrations.Postgres, "postgres") + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".up.sql") { + continue + } + body, err := fs.ReadFile(migrations.Postgres, path.Join("postgres", entry.Name())) + if err != nil { + t.Fatal(err) + } + if bad := unsupportedMigrationStatement(string(body)); bad != "" { + t.Errorf("%s contains unsupported transaction control or DDL: %q", entry.Name(), bad) + } + } +} + +func TestPostgresMigrationRejectsTransactionControl(t *testing.T) { + for _, statement := range []string{ + "COMMIT;", "END;", "ROLLBACK;", "ABORT;", "BEGIN;", + "CREATE INDEX CONCURRENTLY idx ON t (id);", "VACUUM t;", + "CREATE TABLE t(id int); /* boundary */ COMMIT; SELECT 1/0;", + "/* comment */COMMIT;", + "-- comment\nCOMMIT;", + "/* nested /* comment */ still comment */ END;", + "CREATE TABLE foo$tag$ (id int); COMMIT; -- $tag$", + "-- comment\rCOMMIT;", + `foo$E'abc\'; COMMIT;`, + `éE'abc\'; COMMIT;`, + } { + if unsupportedMigrationStatement(statement) == "" { + t.Errorf("did not reject %q", statement) + } + } +} + +func TestPostgresMigrationIgnoresQuotedControlWords(t *testing.T) { + for _, statement := range []string{ + "SELECT 'COMMIT;'::text;", + "SELECT $$COMMIT;$$;", + "SELECT $tag$END;$tag$;", + "SELECT E'ABORT;\\'';", + "SELECT \"ROLLBACK\" FROM t;", + "-- COMMIT;\nSELECT 1;", + } { + if bad := unsupportedMigrationStatement(statement); bad != "" { + t.Errorf("rejected quoted or commented SQL %q: %s", statement, bad) + } + } +} diff --git a/internal/db/open.go b/internal/db/open.go index 182dd9c..9cf2b30 100644 --- a/internal/db/open.go +++ b/internal/db/open.go @@ -25,17 +25,7 @@ func OpenPostgresPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) { if err != nil { return nil, err } - if max := os.Getenv("EVOBGP_DB_MAX_CONNS"); max != "" { - if n, err := strconv.Atoi(strings.TrimSpace(max)); err == nil && n > 0 { - cfg.MaxConns = int32(n) - } - } - if min := os.Getenv("EVOBGP_DB_MIN_CONNS"); min != "" { - if n, err := strconv.Atoi(strings.TrimSpace(min)); err == nil && n >= 0 { - cfg.MinConns = int32(n) - } - } - cfg.MaxConnLifetime = 30 * time.Minute + applyPostgresPoolConfig(cfg) pool, err := pgxpool.NewWithConfig(ctx, cfg) if err != nil { return nil, err @@ -47,6 +37,29 @@ func OpenPostgresPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) { return pool, nil } +// applyPostgresPoolConfig sets pool limits. An explicit EVOBGP_DB_MAX_CONNS is kept. +// When it is unset, the pool defaults to 8 connections so it can cover the default +// worker count instead of pgx's default of 4. +func applyPostgresPoolConfig(cfg *pgxpool.Config) { + if cfg == nil { + return + } + if max := strings.TrimSpace(os.Getenv("EVOBGP_DB_MAX_CONNS")); max != "" { + if n, err := strconv.Atoi(max); err == nil && n > 0 { + cfg.MaxConns = int32(n) + } + } else { + cfg.MaxConns = 8 + } + if min := strings.TrimSpace(os.Getenv("EVOBGP_DB_MIN_CONNS")); min != "" { + if n, err := strconv.Atoi(min); err == nil && n >= 0 { + cfg.MinConns = int32(n) + } + } + cfg.MaxConnLifetime = 30 * time.Minute + cfg.MaxConnIdleTime = 5 * time.Minute +} + // OpenSQLite applies sqlite/*.up.sql then opens the DB (foreign keys on). func OpenSQLite(ctx context.Context, filePath string) (*sql.DB, error) { dsn := fmt.Sprintf("file:%s?_pragma=foreign_keys(1)", strings.TrimPrefix(filePath, "file:")) diff --git a/internal/db/pool_config_test.go b/internal/db/pool_config_test.go new file mode 100644 index 0000000..452f3c8 --- /dev/null +++ b/internal/db/pool_config_test.go @@ -0,0 +1,34 @@ +package db + +import ( + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestApplyPostgresPoolConfigDefaultAndExplicit(t *testing.T) { + t.Setenv("EVOBGP_DB_MAX_CONNS", "") + t.Setenv("EVOBGP_DB_MIN_CONNS", "") + cfg, err := pgxpool.ParseConfig("postgres://localhost/evobgp?sslmode=disable") + if err != nil { + t.Fatal(err) + } + applyPostgresPoolConfig(cfg) + if cfg.MaxConns != 8 { + t.Fatalf("default max conns: %d", cfg.MaxConns) + } + if cfg.MaxConnIdleTime != 5*time.Minute { + t.Fatalf("idle time: %s", cfg.MaxConnIdleTime) + } + + t.Setenv("EVOBGP_DB_MAX_CONNS", "25") + cfg, err = pgxpool.ParseConfig("postgres://localhost/evobgp?sslmode=disable") + if err != nil { + t.Fatal(err) + } + applyPostgresPoolConfig(cfg) + if cfg.MaxConns != 25 { + t.Fatalf("explicit max conns overridden: %d", cfg.MaxConns) + } +} diff --git a/internal/db/postgres_integration_test.go b/internal/db/postgres_integration_test.go new file mode 100644 index 0000000..2cf2031 --- /dev/null +++ b/internal/db/postgres_integration_test.go @@ -0,0 +1,227 @@ +package db + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "io/fs" + "os" + "strings" + "sync" + "testing" + "testing/fstest" + "time" + + "evobgp/migrations" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestPostgresMigrationsIntegration(t *testing.T) { + pool := isolatedMigrationPool(t) + ctx := context.Background() + if err := ApplyPostgresMigrations(ctx, pool, migrations.Postgres, "postgres"); err != nil { + t.Fatalf("first migration pass: %v", err) + } + if err := ApplyPostgresMigrations(ctx, pool, migrations.Postgres, "postgres"); err != nil { + t.Fatalf("second migration pass: %v", err) + } + files, err := fs.ReadDir(migrations.Postgres, "postgres") + if err != nil { + t.Fatal(err) + } + want := 0 + for _, file := range files { + if strings.HasSuffix(file.Name(), ".up.sql") { + want++ + } + } + var got int + if err := pool.QueryRow(ctx, `SELECT count(*)::int FROM schema_migrations`).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("applied migrations = %d, want %d", got, want) + } + var extension string + if err := pool.QueryRow(ctx, `SELECT extname FROM pg_extension WHERE extname = 'pg_stat_statements'`).Scan(&extension); err != nil { + t.Fatalf("pg_stat_statements extension missing: %v", err) + } +} + +func TestPostgresMigrationConcurrentApplyIntegration(t *testing.T) { + pool := isolatedMigrationPool(t) + files := fstest.MapFS{"sql/000001_probe.up.sql": {Data: []byte(`CREATE TABLE migration_probe (id integer PRIMARY KEY); SELECT pg_sleep(0.1);`)}} + var wg sync.WaitGroup + errors := make(chan error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + errors <- ApplyPostgresMigrations(context.Background(), pool, files, "sql") + }() + } + wg.Wait() + close(errors) + for err := range errors { + if err != nil { + t.Fatal(err) + } + } + assertMigrationVersionCount(t, pool, 1) +} + +func TestPostgresMigrationRollbackIntegration(t *testing.T) { + pool := isolatedMigrationPool(t) + files := fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE migration_partial (id integer); SELECT 1 / 0;`)}} + if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil { + t.Fatal("expected migration error") + } + assertMigrationVersionCount(t, pool, 0) + var exists bool + if err := pool.QueryRow(context.Background(), `SELECT to_regclass('migration_partial') IS NOT NULL`).Scan(&exists); err != nil { + t.Fatal(err) + } + if exists { + t.Fatal("failed migration left a table behind") + } + // A transaction terminator hidden behind a comment must be rejected before + // the first DDL statement can run. + files = fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE migration_partial (id integer); /* boundary */ COMMIT; SELECT 1 / 0;`)}} + if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil || !strings.Contains(err.Error(), "unsupported non-transactional SQL") { + t.Fatalf("comment-hidden COMMIT error = %v", err) + } + assertMigrationVersionCount(t, pool, 0) + if err := pool.QueryRow(context.Background(), `SELECT to_regclass('migration_partial') IS NOT NULL`).Scan(&exists); err != nil { + t.Fatal(err) + } + if exists { + t.Fatal("rejected migration left a table behind") + } + files = fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE foo$tag$ (id int); COMMIT; -- $tag$`)}} + if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil || !strings.Contains(err.Error(), "unsupported non-transactional SQL") { + t.Fatalf("identifier-embedded dollar tag error = %v", err) + } + assertMigrationVersionCount(t, pool, 0) + if err := pool.QueryRow(context.Background(), `SELECT to_regclass('foo$tag$') IS NOT NULL`).Scan(&exists); err != nil { + t.Fatal(err) + } + if exists { + t.Fatal("identifier-embedded dollar tag left a table behind") + } +} + +func TestPostgresMigrationIdempotentIntegration(t *testing.T) { + pool := isolatedMigrationPool(t) + files := fstest.MapFS{"sql/000001_once.up.sql": {Data: []byte(`CREATE TABLE migration_once (id integer); INSERT INTO migration_once VALUES (1);`)}} + for i := 0; i < 2; i++ { + if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err != nil { + t.Fatal(err) + } + } + assertMigrationVersionCount(t, pool, 1) + var rows int + if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM migration_once`).Scan(&rows); err != nil { + t.Fatal(err) + } + if rows != 1 { + t.Fatalf("migration rows = %d, want 1", rows) + } +} + +func TestPostgresMigrationLockCancellationIntegration(t *testing.T) { + pool := isolatedMigrationPool(t) + locker, err := pool.Acquire(context.Background()) + if err != nil { + t.Fatal(err) + } + defer locker.Release() + if _, err := locker.Exec(context.Background(), `SELECT pg_advisory_lock($1)`, postgresMigrationLockKey); err != nil { + t.Fatal(err) + } + defer func() { + _, _ = locker.Exec(context.Background(), `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey) + }() + files := fstest.MapFS{"sql/000001_probe.up.sql": {Data: []byte(`CREATE TABLE lock_probe (id integer);`)}} + ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond) + defer cancel() + err = ApplyPostgresMigrations(ctx, pool, files, "sql") + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("lock wait error = %v, want context deadline exceeded", err) + } + var exists bool + if err := pool.QueryRow(context.Background(), `SELECT to_regclass('schema_migrations') IS NOT NULL`).Scan(&exists); err != nil { + t.Fatal(err) + } + if exists { + t.Fatal("schema_migrations was created before migration lock acquisition") + } + var unlocked bool + if err := locker.QueryRow(context.Background(), `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey).Scan(&unlocked); err != nil || !unlocked { + t.Fatalf("release held lock: unlocked=%v err=%v", unlocked, err) + } + if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err != nil { + t.Fatalf("apply after lock release: %v", err) + } +} + +func isolatedMigrationPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("EVOBGP_TEST_DATABASE_URL not set") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + var nonce [8]byte + if _, err := rand.Read(nonce[:]); err != nil { + admin.Close() + t.Fatal(err) + } + schema := "evobgp_mig_test_" + hex.EncodeToString(nonce[:]) + if _, err := admin.Exec(ctx, `CREATE SCHEMA `+schema); err != nil { + admin.Close() + t.Fatal(err) + } + var pool *pgxpool.Pool + t.Cleanup(func() { + if pool != nil { + pool.Close() + } + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cleanupCancel() + if _, err := admin.Exec(cleanupCtx, `DROP SCHEMA `+schema+` CASCADE`); err != nil { + t.Errorf("drop isolated schema %s: %v", schema, err) + } + admin.Close() + }) + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + if cfg.ConnConfig.RuntimeParams == nil { + cfg.ConnConfig.RuntimeParams = make(map[string]string) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + cfg.MaxConns = 3 + pool, err = pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + return pool +} + +func assertMigrationVersionCount(t *testing.T, pool *pgxpool.Pool, want int) { + t.Helper() + var got int + if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM schema_migrations`).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("migration versions = %d, want %d", got, want) + } +} diff --git a/internal/httpapi/backend_stage0_repro_test.go b/internal/httpapi/backend_stage0_repro_test.go new file mode 100644 index 0000000..6816e50 --- /dev/null +++ b/internal/httpapi/backend_stage0_repro_test.go @@ -0,0 +1,39 @@ +package httpapi + +import ( + "testing" + + "evobgp/internal/store" +) + +// These regressions ensure unobserved live state cannot inherit persisted health. +func TestStage0ReproPeerWithoutObservationIsNotEstablished(t *testing.T) { + peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"} + cases := []struct { + name string + views []speakerBGPLive + }{ + {name: "no_collector"}, + {name: "all_timeout", views: []speakerBGPLive{{SpeakerID: "replica", Error: "timeout"}}}, + {name: "session_absent", views: []speakerBGPLive{{SpeakerID: "replica", Label: "replica"}}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + row := map[string]any{"session_state": "Established"} + applyPeerLiveFields(row, peer, tc.views) + if row["session_state"] == "Established" { + t.Fatal("S01: persisted Established is exposed without a current session observation") + } + }) + } +} + +func TestStage0ReproUnpolledMasterIsNotHealthy(t *testing.T) { + sp := &store.Speaker{ID: "master", Role: "master"} + // collectSpeakerLiveStatus constructs this view when no poll result exists. + view := speakerBGPLive{SpeakerID: sp.ID, Label: "master"} + row := speakerLiveStatusJSON(sp, view, nil) + if row["bgp_poll_ok"] == true || row["agent_ok"] == true { + t.Fatal("S01: absent poll is exposed as successful BGP/agent health") + } +} diff --git a/internal/httpapi/bootstrap.go b/internal/httpapi/bootstrap.go index 8e3a5c7..e7c1f3d 100644 --- a/internal/httpapi/bootstrap.go +++ b/internal/httpapi/bootstrap.go @@ -7,9 +7,9 @@ import ( "time" "evobgp/internal/db" - "evobgp/internal/httpclient" "evobgp/internal/jobs" "evobgp/internal/observability" + "evobgp/internal/pipeline" "evobgp/internal/repository" "evobgp/internal/store" @@ -18,7 +18,7 @@ import ( // NewCDNHTTPClient returns the shared HTTP client for CDN and preview fetches (PERF-02 / ERR-03). func NewCDNHTTPClient() *http.Client { - return httpclient.New(httpclient.DefaultTimeout) + return pipeline.NewCDNHTTPClient() } // BootstrapWorkers opens the same store.Backend and jobs.Registry as New (without HTTP or bundle keys). @@ -50,46 +50,15 @@ func BootstrapWorkers(ctx context.Context, opts Options) (store.Backend, *jobs.R backend = mem } - cdnHTTP := NewCDNHTTPClient() - wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: cdnHTTP} + // Configured DoH profiles and RIPE service overrides retain the previous + // general HTTP/HTTPS transport, including explicitly configured private + // endpoints. CDN calls independently secure this client in UpstreamHTTPDo. + workerHTTP := pipeline.NewPipelineHTTPClient() + wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: workerHTTP} reg := jobs.NewRegistry(wk.Process) wk.Registry = reg if pool != nil { - audit := repository.NewJobAuditWriter(pool) - jobMeta := func(j *jobs.Job) map[string]any { - if j == nil { - return nil - } - st := j.Snapshot() - meta, _ := st["meta"].(map[string]any) - return meta - } - reg.SetPersistHooks( - func(j *jobs.Job) { - if j == nil { - return - } - audit.UpsertQueued(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, j.ModuleID, jobMeta(j)) - }, - func(j *jobs.Job) { - if j == nil { - return - } - audit.UpsertRunning(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, jobMeta(j)) - }, - func(j *jobs.Job) { - if j == nil { - return - } - st := j.Snapshot() - status, _ := st["status"].(string) - var errMsg *string - if e, ok := st["error"].(string); ok && e != "" { - errMsg = &e - } - audit.MarkTerminal(context.Background(), j.TenantID, j.ID, status, errMsg, time.Now().UTC()) - }, - ) + reg.UseDurableQueue(backend) } observability.RegisterStoreBackend(backend) return backend, reg, pool, nil diff --git a/internal/httpapi/bootstrap_outbound_test.go b/internal/httpapi/bootstrap_outbound_test.go new file mode 100644 index 0000000..5b5e582 --- /dev/null +++ b/internal/httpapi/bootstrap_outbound_test.go @@ -0,0 +1,110 @@ +package httpapi + +import ( + "context" + "encoding/base64" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "evobgp/internal/httpclient" + "evobgp/internal/jobs" + "evobgp/internal/pipeline" + "evobgp/internal/store" + "github.com/miekg/dns" +) + +func TestBootstrapWorkerPreservesExplicitPrivateDOH(t *testing.T) { + t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0") + t.Setenv("EVOBGP_BIRDC_SOCKET", "") + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0") + var calls atomic.Int32 + resolver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + raw, err := base64.RawURLEncoding.DecodeString(r.URL.Query().Get("dns")) + if err != nil { + w.WriteHeader(400) + return + } + query := new(dns.Msg) + if err = query.Unpack(raw); err != nil || len(query.Question) == 0 { + w.WriteHeader(400) + return + } + answer := new(dns.Msg) + answer.SetReply(query) + if query.Question[0].Qtype == dns.TypeA { + answer.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: query.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.IPv4(192, 0, 2, 20)}} + } + body, err := answer.Pack() + if err != nil { + t.Error(err) + return + } + w.Header().Set("Content-Type", "application/dns-message") + _, _ = w.Write(body) + })) + defer resolver.Close() + st, reg, _, err := BootstrapWorkers(context.Background(), Options{SeedDemo: true}) + if err != nil { + t.Fatal(err) + } + defer func() { + reg.RequestCancelAll() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if e := reg.Drain(ctx); e != nil { + t.Error(e) + } + }() + tenant, _, _, _, _ := st.DemoIDs() + profile, err := st.CreateDohProfile(tenant, &store.DohProfile{Name: "explicit private resolver", URL: resolver.URL}) + if err != nil { + t.Fatal(err) + } + mod, err := st.CreateModule(tenant, &store.Module{Name: "private-doh", Type: "DOMAINS", Enabled: true, DohProfileIDs: []string{profile.ID}}) + if err != nil { + t.Fatal(err) + } + if _, err = st.CreateDomainEntry(tenant, mod.ID, &store.DomainEntry{FQDN: "private-resolver.example.test"}); err != nil { + t.Fatal(err) + } + j, _, err := reg.Enqueue(tenant, jobs.KindModuleRefresh, nil, &mod.ID, map[string]any{"module_id": mod.ID}) + if err != nil { + t.Fatal(err) + } + deadline := time.After(5 * time.Second) + ticker := time.NewTicker(5 * time.Millisecond) + defer ticker.Stop() + for { + snap := j.Snapshot() + status := snap["status"] + if status == jobs.StatusSucceeded { + break + } + if status == jobs.StatusFailed || status == jobs.StatusCancelled { + t.Fatalf("worker refresh failed: %+v", snap) + } + select { + case <-deadline: + t.Fatal("worker did not finish") + case <-ticker.C: + } + } + if calls.Load() < 2 { + t.Fatalf("bootstrap worker blocked private DoH: %d requests", calls.Load()) + } + // The same general-client choice must not remove CDN's independent ACL. + cdn := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("private CDN reached") })) + defer cdn.Close() + req, err := http.NewRequest(http.MethodGet, cdn.URL, nil) + if err != nil { + t.Fatal(err) + } + if _, err = pipeline.UpstreamHTTPDo(context.Background(), httpclient.New(httpclient.DefaultTimeout), req); err == nil { + t.Fatal("private CDN accepted") + } +} diff --git a/internal/httpapi/cdn_preview_body_test.go b/internal/httpapi/cdn_preview_body_test.go new file mode 100644 index 0000000..995bd94 --- /dev/null +++ b/internal/httpapi/cdn_preview_body_test.go @@ -0,0 +1,81 @@ +package httpapi + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "evobgp/internal/store" +) + +func TestPreviewCDNSourceRejectsOversizedBody(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1") + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.Copy(w, strings.NewReader(strings.Repeat("x", (8<<20)+1))) + })) + defer srv.Close() + + st := store.NewMemory() + st.SeedDemo() + tenant, _, _, _, _ := st.DemoIDs() + mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "oversized-preview", Enabled: true}) + if err != nil { + t.Fatal(err) + } + s := &Server{store: st, cdnHTTP: srv.Client()} + body, err := json.Marshal(map[string]string{"url": srv.URL, "source_kind": "plain"}) + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+mod.ID+"/cdn-sources/preview", strings.NewReader(string(body))) + req.SetPathValue("module_id", mod.ID) + req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"})) + recorder := httptest.NewRecorder() + s.handlePreviewCDNSource(recorder, req) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("oversized preview status=%d body=%s", recorder.Code, recorder.Body.String()) + } +} + +func TestPreviewCDNErrorBodyBoundAndQuotaBeforeFetch(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1") + t.Setenv("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", "1") + m := store.NewMemory() + m.SeedDemo() + tenant, module, _, _, _ := m.DemoIDs() + var calls atomic.Int32 + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, strings.Repeat("x", 64<<10)) + w.(http.Flusher).Flush() + // No EOF: an unbounded drain would wait until the caller deadline. + <-r.Context().Done() + })) + defer upstream.Close() + s := &Server{store: m, cdnHTTP: upstream.Client()} + request := func() *http.Request { + r := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/cdn-sources/preview", strings.NewReader(`{"url":"`+upstream.URL+`","source_kind":"plain"}`)) + r.SetPathValue("module_id", module) + return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"})) + } + first := httptest.NewRecorder() + firstRequest := request() + ctx, cancel := context.WithTimeout(firstRequest.Context(), 5*time.Second) + defer cancel() + s.handlePreviewCDNSource(first, firstRequest.WithContext(ctx)) + if first.Code != http.StatusBadGateway || calls.Load() != 1 || ctx.Err() != nil { + t.Fatalf("unbounded drain status=%d calls=%d ctx=%v", first.Code, calls.Load(), ctx.Err()) + } + second := httptest.NewRecorder() + s.handlePreviewCDNSource(second, request()) + if second.Code != http.StatusTooManyRequests || calls.Load() != 1 || second.Header().Get("Retry-After") == "" { + t.Fatalf("quota fetch bypass status=%d calls=%d", second.Code, calls.Load()) + } +} diff --git a/internal/httpapi/cors.go b/internal/httpapi/cors.go index ccf8266..67bab50 100644 --- a/internal/httpapi/cors.go +++ b/internal/httpapi/cors.go @@ -39,6 +39,7 @@ func (s *Server) withCORS(h http.Handler) http.Handler { } w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key, Accept, X-Tenant-Id") w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PATCH, DELETE, OPTIONS") + w.Header().Set("Access-Control-Expose-Headers", "Location, X-EvoBGP-Job-Status, Retry-After") if r.Method == http.MethodOptions { w.WriteHeader(http.StatusNoContent) return diff --git a/internal/httpapi/job_errors.go b/internal/httpapi/job_errors.go new file mode 100644 index 0000000..a6d30a6 --- /dev/null +++ b/internal/httpapi/job_errors.go @@ -0,0 +1,56 @@ +package httpapi + +import ( + "errors" + "net/http" + + "evobgp/internal/jobs" + "evobgp/internal/store" +) + +func (s *Server) replayJobResponse(w http.ResponseWriter, r *http.Request, in store.JobEnqueue) bool { + j, found, err := s.jobs.ReplayContext(r.Context(), in) + if err != nil { + writeJobEnqueueError(w, err) + return true + } + if !found { + return false + } + w.Header().Set("Location", "/v1/jobs/"+j.ID) + snap := j.Snapshot() + body := map[string]any{"job_id": j.ID, "status": snap["status"]} + if j.Kind == jobs.KindTenantRefresh { + ids := snap["meta"].(map[string]any)["module_ids"] + n := 0 + switch v := ids.(type) { + case []any: + n = len(v) + case []string: + n = len(v) + } + body["module_ids"] = ids + body["modules_due"] = n + } + writeJSON(w, http.StatusAccepted, body) + return true +} + +func writeJobEnqueueError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, store.ErrJobConflict): + writeProblem(w, http.StatusConflict, "Conflict", "idempotency key belongs to a different job request") + case errors.Is(err, store.ErrJobQueueFull): + writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "job queue is full") + default: + writeInternalError(w, "job_enqueue", err) + } +} + +func writeJobReadError(w http.ResponseWriter, err error) { + if errors.Is(err, jobs.ErrNotFound) || errors.Is(err, store.ErrJobNotFound) { + writeProblem(w, http.StatusNotFound, "Not Found", "job not found") + return + } + writeInternalError(w, "job_read", err) +} diff --git a/internal/httpapi/job_queue_test.go b/internal/httpapi/job_queue_test.go new file mode 100644 index 0000000..a109028 --- /dev/null +++ b/internal/httpapi/job_queue_test.go @@ -0,0 +1,99 @@ +package httpapi + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "evobgp/internal/jobs" + "evobgp/internal/store" +) + +type unavailableJobQueue struct{ store.JobQueue } + +func (q unavailableJobQueue) EnqueueJob(context.Context, store.JobEnqueue) (store.JobRecord, bool, error) { + return store.JobRecord{}, false, errors.New("database unavailable") +} + +func TestJobEnqueueFailureNeverAccepts(t *testing.T) { + r := jobs.NewRegistry(nil) + r.UseDurableQueue(unavailableJobQueue{}) + s := &Server{jobs: r} + req := httptest.NewRequest(http.MethodPost, "/v1/bird/reload", nil) + req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "operator"})) + w := httptest.NewRecorder() + s.handleBirdReload(w, req) + if w.Code == http.StatusAccepted || w.Code < 500 { + t.Fatalf("DB failure returned %d: %s", w.Code, w.Body.String()) + } +} + +func TestAutomaticEnqueueFailureIsVisible(t *testing.T) { + r := jobs.NewRegistry(nil) + r.UseDurableQueue(unavailableJobQueue{}) + s := &Server{jobs: r} + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/v1/peers", nil) + s.enqueuePeerReconcile(w, req, "tenant", "peer_create") + if got := w.Header().Get("X-EvoBGP-Job-Status"); got != "enqueue-failed" { + t.Fatalf("scheduling failure hidden: %q", got) + } +} + +type selectorBackend struct { + store.Backend + modules []*store.Module + reads int +} + +func (b *selectorBackend) ListModules(string) []*store.Module { b.reads++; return b.modules } + +func TestTenantRefreshReplayBeforeDynamicSelection(t *testing.T) { + m := store.NewMemory() + b := &selectorBackend{Backend: m, modules: []*store.Module{{ID: "module-a", Enabled: true, RefreshIntervalSec: 1}}} + r := jobs.NewRegistry(nil) + r.UseDurableQueue(m) + s := &Server{store: b, jobs: r} + request := func() *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/v1/tenant/refresh", strings.NewReader(`{}`)) + req.Header.Set("Idempotency-Key", "same-request") + req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "editor"})) + w := httptest.NewRecorder() + s.handleTenantRefresh(w, req) + return w + } + first := request() + if first.Code != http.StatusAccepted { + t.Fatalf("first=%d %s", first.Code, first.Body.String()) + } + var accepted map[string]any + if err := json.Unmarshal(first.Body.Bytes(), &accepted); err != nil { + t.Fatal(err) + } + claimed, err := m.ClaimJobs(context.Background(), "worker", 1, time.Minute) + if err != nil || len(claimed) != 1 { + t.Fatalf("claim %+v %v", claimed, err) + } + claimed[0].Status = "succeeded" + if err = m.FinishJob(context.Background(), claimed[0]); err != nil { + t.Fatal(err) + } + b.modules = nil + reads := b.reads + replay := request() + if replay.Code != http.StatusAccepted { + t.Fatalf("replay=%d %s", replay.Code, replay.Body.String()) + } + var got map[string]any + if err = json.Unmarshal(replay.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got["job_id"] != accepted["job_id"] || got["status"] != "succeeded" || b.reads != reads { + t.Fatalf("replay re-resolved selector: %+v reads %d->%d", got, reads, b.reads) + } +} diff --git a/internal/httpapi/job_report_test.go b/internal/httpapi/job_report_test.go new file mode 100644 index 0000000..7f5a4ed --- /dev/null +++ b/internal/httpapi/job_report_test.go @@ -0,0 +1,114 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "evobgp/internal/jobs" + "evobgp/internal/store" +) + +func TestJobReportDurableProgressAndTenantScope(t *testing.T) { + ctx := context.Background() + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, revision, speaker := m.DemoIDs() + j, _, err := m.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: jobs.KindDeployApply}) + if err != nil { + t.Fatal(err) + } + claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute) + if err != nil || len(claimed) != 1 { + t.Fatalf("claim: %v %v", claimed, err) + } + f := store.JobFence{JobID: j.ID, TenantID: tenant, OwnerID: "worker", Token: claimed[0].FencingToken} + d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision) + if err != nil { + t.Fatal(err) + } + if err := m.RecordDeploymentSpeaker(ctx, f, d, "unknown", "unconfirmed", "response lost"); err != nil { + t.Fatal(err) + } + // A fresh registry has no in-process job or progress metadata. + r := jobs.NewRegistry(nil) + r.UseDurableQueue(m) + s := &Server{store: m, jobs: r} + request := func(tenantID, jobID string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/v1/jobs/"+jobID+"/report", nil) + req.SetPathValue("job_id", jobID) + req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenantID, Role: "operator"})) + w := httptest.NewRecorder() + s.handleGetJobReport(w, req) + return w + } + check := func(id string) { + t.Helper() + w := request(tenant, id) + if w.Code != http.StatusOK { + t.Fatalf("report: %d %s", w.Code, w.Body.String()) + } + var body struct { + Progress []struct { + State, Confirmation string + JobID string `json:"job_id"` + } `json:"deployment_progress"` + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if len(body.Progress) != 1 || body.Progress[0].JobID != j.ID || body.Progress[0].State != "unknown" || body.Progress[0].Confirmation != "unconfirmed" { + t.Fatalf("lost persisted progress: %s", w.Body.String()) + } + } + check(j.ID) + recovery, _, err := r.EnqueueContext(ctx, tenant, jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID}) + if err != nil { + t.Fatal(err) + } + check(recovery.ID) + // A later explicit completion releases the delivery reservation. Once recovery + // has its own persisted row, its report must not show the parent's old state. + if err := m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil { + t.Fatal(err) + } + recoveryRows, err := m.ClaimJobs(ctx, "recovery-worker", 1, time.Minute) + if err != nil || len(recoveryRows) != 1 || recoveryRows[0].ID != recovery.ID { + t.Fatalf("recovery claim: %v %v", recoveryRows, err) + } + rf := store.JobFence{JobID: recovery.ID, TenantID: tenant, OwnerID: "recovery-worker", Token: recoveryRows[0].FencingToken} + rd, err := m.BeginDeploymentSpeaker(ctx, rf, speaker, revision) + if err != nil { + t.Fatal(err) + } + if err := m.RecordDeploymentSpeaker(ctx, rf, rd, "applied", "agent_reported", ""); err != nil { + t.Fatal(err) + } + w := request(tenant, recovery.ID) + var report struct { + Progress []struct { + State string + JobID string `json:"job_id"` + } `json:"deployment_progress"` + } + if err := json.Unmarshal(w.Body.Bytes(), &report); err != nil { + t.Fatal(err) + } + if w.Code != http.StatusOK || len(report.Progress) != 1 || report.Progress[0].JobID != recovery.ID || report.Progress[0].State != "applied" { + t.Fatalf("recovery own progress hidden: %d %s", w.Code, w.Body.String()) + } + if w := request("other-tenant", j.ID); w.Code != http.StatusNotFound { + t.Fatalf("cross-tenant report: %d", w.Code) + } + foreign, _, err := r.EnqueueContext(ctx, "other-tenant", jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID}) + if err != nil { + t.Fatal(err) + } + if w := request("other-tenant", foreign.ID); w.Code != http.StatusNotFound { + t.Fatalf("cross-tenant source: %d %s", w.Code, w.Body.String()) + } +} diff --git a/internal/httpapi/legacy_node_binary_test.go b/internal/httpapi/legacy_node_binary_test.go new file mode 100644 index 0000000..245ba83 --- /dev/null +++ b/internal/httpapi/legacy_node_binary_test.go @@ -0,0 +1,89 @@ +package httpapi + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "net/http/httptest" + + "evobgp/internal/bundle" +) + +// TestLegacyNodeBinaryCurrentControlPlane exercises an independently built old +// CLI against today's HTTP handlers. It never calls apply or touches host BIRD. +func TestLegacyNodeBinaryCurrentControlPlane(t *testing.T) { + binary := os.Getenv("EVOBGP_TEST_LEGACY_NODE_BIN") + if binary == "" { + t.Skip("EVOBGP_TEST_LEGACY_NODE_BIN not set") + } + binary, err := filepath.Abs(binary) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(binary); err != nil { + t.Fatal(err) + } + srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + tenant, _, _, revision, speaker := srv.Store().DemoIDs() + mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,other-node|other-tenant|node") + if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(srv.Handler()) + defer server.Close() + run := func(wantSuccess bool, args ...string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + output, err := exec.CommandContext(ctx, binary, args...).CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("legacy CLI timed out: %v", ctx.Err()) + } + if wantSuccess && err != nil { + t.Fatalf("legacy CLI failed: %v: %s", err, output) + } + if !wantSuccess && err == nil { + t.Fatalf("legacy CLI unexpectedly accepted invalid request: %s", output) + } + } + path := filepath.Join(t.TempDir(), "current-bundle.tar.gz") + run(true, "pull-bundle", "-base-url", server.URL, "-token", "legacy-node", "-speaker-id", speaker, "-o", path) + run(true, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64()) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64()) + if err != nil { + t.Fatal(err) + } + verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub)) + if err != nil || verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker { + t.Fatalf("legacy CLI fetched the wrong revision/speaker or an invalid bundle: %v", err) + } + // The archive is valid, but verification with an unrelated key must fail. + run(false, "verify-bundle", "-f", path, "-pubkey-hex", strings.Repeat("00", ed25519.PublicKeySize)) + for _, token := range []string{"invalid-node", "other-node"} { + denied := filepath.Join(t.TempDir(), "denied.tar.gz") + run(false, "pull-bundle", "-base-url", server.URL, "-token", token, "-speaker-id", speaker, "-o", denied) + if _, err := os.Stat(denied); !os.IsNotExist(err) { + t.Fatalf("unauthorized CLI wrote a bundle: %v", err) + } + } + // A damaged payload must not pass the old verifier with the correct key. + if err := os.WriteFile(path, []byte("not a signed bundle"), 0o600); err != nil { + t.Fatal(err) + } + run(false, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64()) +} diff --git a/internal/httpapi/node_bundle_cache.go b/internal/httpapi/node_bundle_cache.go new file mode 100644 index 0000000..b315053 --- /dev/null +++ b/internal/httpapi/node_bundle_cache.go @@ -0,0 +1,310 @@ +package httpapi + +import ( + "context" + "crypto/ed25519" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "sort" + "strings" + "sync" + + "evobgp/internal/bundle" + "evobgp/internal/pipeline" + "evobgp/internal/store" +) + +const ( + // nodeBundleCacheMaxEntries is the maximum number of finished node bundles retained. + nodeBundleCacheMaxEntries = 32 + // nodeBundleCacheMaxBytes is the maximum total size of cached bundle bodies (64 MiB). + nodeBundleCacheMaxBytes = 64 << 20 +) + +// errNodeBundleRevisionMissing is returned when the revision cannot be loaded. +// It is not cached: the next pull tries again. +var errNodeBundleRevisionMissing = errors.New("httpapi: revision not found") + +// nodeBundleOpError carries the writeInternalError operation name for a failed build. +type nodeBundleOpError struct { + op string + err error +} + +func (e *nodeBundleOpError) Error() string { + if e == nil || e.err == nil { + return "httpapi: node bundle" + } + return fmt.Sprintf("httpapi: %s: %v", e.op, e.err) +} + +func (e *nodeBundleOpError) Unwrap() error { return e.err } + +// nodeBundleBuilder produces the gzip body served to evobgp-node. +// The default implementation is bundle.BuildGzippedTar, whose archive contains +// manifest.json and bundle.sig. Tests replace it to count rebuilds. +type nodeBundleBuilder func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) + +// nodeBundleCache stores finished gzip bodies. A hit returns the bytes of the +// build that filled the entry, signature included. Errors are not stored. +// Concurrent loads of one key share a single build. +type nodeBundleCache struct { + mu sync.Mutex + maxN int + maxBytes int + total int + items map[string]*nodeBundleEntry + lru []string // index 0 is the oldest idle entry + flights map[string]*nodeBundleFlight +} + +type nodeBundleEntry struct { + body []byte +} + +type nodeBundleFlight struct { + done chan struct{} + body []byte + err error + joined int +} + +func newNodeBundleCache(maxEntries, maxBytes int) *nodeBundleCache { + return &nodeBundleCache{ + maxN: maxEntries, + maxBytes: maxBytes, + items: make(map[string]*nodeBundleEntry), + flights: make(map[string]*nodeBundleFlight), + } +} + +// nodeBundleCacheKey identifies one servable bundle. +// overlayFP is a digest of the speaker overlay output (router id / source addresses, +// node IPv4 fallback, peer and discovery render). Revision content itself is immutable, +// so the revision id stands in for the shared prefix files. +func nodeBundleCacheKey(tenantID, revisionID, speakerID, overlayFP string) string { + var b strings.Builder + b.Grow(len(tenantID) + len(revisionID) + len(speakerID) + len(overlayFP) + 3) + b.WriteString(tenantID) + b.WriteByte(0) + b.WriteString(revisionID) + b.WriteByte(0) + b.WriteString(speakerID) + b.WriteByte(0) + b.WriteString(overlayFP) + return b.String() +} + +// nodeBundleOverlayFingerprint hashes the speaker overlay that handleNodeBundle +// would apply on top of an immutable revision. Empty base fragments are enough: +// bird.conf and the peers fragment are fully re-rendered from tenant settings, +// speaker source addresses, and the live peer set. Revision file bytes are not +// part of this digest. +func nodeBundleOverlayFingerprint(st store.Backend, tenantID, speakerID, revisionID string) (string, error) { + frags, err := pipeline.OverlayFragmentsForSpeaker(st, tenantID, speakerID, revisionID, nil) + if err != nil { + return "", fmt.Errorf("httpapi: bundle overlay: %w", err) + } + keys := make([]string, 0, len(frags)) + for k := range frags { + keys = append(keys, k) + } + sort.Strings(keys) + h := sha256.New() + for _, k := range keys { + if err := hashWriteString(h, k); err != nil { + return "", fmt.Errorf("httpapi: bundle overlay: %w", err) + } + if err := hashWriteString(h, frags[k]); err != nil { + return "", fmt.Errorf("httpapi: bundle overlay: %w", err) + } + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func hashWriteString(h interface{ Write([]byte) (int, error) }, s string) error { + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], uint64(len(s))) + if _, err := h.Write(buf[:]); err != nil { + return err + } + _, err := h.Write([]byte(s)) + return err +} + +// do returns the cached body or runs build once for key. +// build's bool is false when the result must not be stored (failure is also not stored). +// Callers must not mutate the returned slice. +func (c *nodeBundleCache) do(ctx context.Context, key string, build func(context.Context) ([]byte, bool, error)) (body []byte, err error) { + if c == nil { + return nil, errors.New("httpapi: nil node bundle cache") + } + if ctx == nil { + ctx = context.Background() + } + c.mu.Lock() + if e, ok := c.items[key]; ok { + c.touchLocked(key) + body = e.body + c.mu.Unlock() + return body, nil + } + if f, ok := c.flights[key]; ok { + f.joined++ + c.mu.Unlock() + return waitNodeBundleFlight(ctx, f) + } + if err = ctx.Err(); err != nil { + c.mu.Unlock() + return nil, err + } + f := &nodeBundleFlight{done: make(chan struct{})} + c.flights[key] = f + c.mu.Unlock() + + var cacheOK bool + var finished bool + finish := func() { + if finished { + return + } + finished = true + out := body + c.mu.Lock() + delete(c.flights, key) + if err == nil && cacheOK { + if stored, ok := c.storeLocked(key, out); ok { + out = stored + } + } + f.body = out + f.err = err + close(f.done) + c.mu.Unlock() + body = out + } + defer func() { + if rec := recover(); rec != nil { + body = nil + cacheOK = false + err = fmt.Errorf("httpapi: node bundle build: %v", rec) + } + finish() + }() + + // Detach from the leader's cancel so one disconnected agent does not fail the shared build. + body, cacheOK, err = build(context.WithoutCancel(ctx)) + return +} + +func waitNodeBundleFlight(ctx context.Context, f *nodeBundleFlight) ([]byte, error) { + select { + case <-f.done: + return f.body, f.err + case <-ctx.Done(): + select { + case <-f.done: + return f.body, f.err + default: + return nil, ctx.Err() + } + } +} + +// storeLocked retains body when it fits under both caps, evicting the oldest idle +// entries first. body is copied. The stored slice is returned. +func (c *nodeBundleCache) storeLocked(key string, body []byte) ([]byte, bool) { + n := len(body) + if n == 0 || c.maxN <= 0 || c.maxBytes <= 0 || n > c.maxBytes { + return nil, false + } + if e, ok := c.items[key]; ok { + c.total -= len(e.body) + delete(c.items, key) + c.removeLRULocked(key) + } + for len(c.items) > 0 && (len(c.items) >= c.maxN || c.total+n > c.maxBytes) { + if len(c.lru) == 0 { + return nil, false + } + c.deleteOldestLocked() + } + if len(c.items) >= c.maxN || c.total+n > c.maxBytes { + return nil, false + } + stored := make([]byte, n) + copy(stored, body) + c.items[key] = &nodeBundleEntry{body: stored} + c.total += n + c.lru = append(c.lru, key) + return stored, true +} + +func (c *nodeBundleCache) touchLocked(key string) { + c.removeLRULocked(key) + c.lru = append(c.lru, key) +} + +func (c *nodeBundleCache) deleteOldestLocked() { + if len(c.lru) == 0 { + return + } + key := c.lru[0] + c.lru = c.lru[1:] + if e, ok := c.items[key]; ok { + c.total -= len(e.body) + delete(c.items, key) + } +} + +func (c *nodeBundleCache) removeLRULocked(key string) { + for i, k := range c.lru { + if k != key { + continue + } + copy(c.lru[i:], c.lru[i+1:]) + c.lru = c.lru[:len(c.lru)-1] + return + } +} + +func (s *Server) buildNodeBundle(revisionID, speakerID string, fragments map[string]string) ([]byte, error) { + if s != nil && s.nodeBundleBuild != nil { + return s.nodeBundleBuild(revisionID, speakerID, fragments, s.bundlePriv) + } + return bundle.BuildGzippedTar(revisionID, speakerID, fragments, s.bundlePriv) +} + +// loadNodeBundle reads the revision, applies the speaker overlay, and signs the gzip. +// The bool is false when overlay inputs moved during the load, so the caller must not +// cache the body under the key captured before the load. Failed loads are not cacheable. +func (s *Server) loadNodeBundle(ctx context.Context, tenantID, speakerID, revisionID, overlayFP string) ([]byte, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, err + } + fpNow, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID) + if err != nil { + return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err} + } + rev, err := s.store.GetRevision(tenantID, revisionID) + if err != nil { + return nil, false, errNodeBundleRevisionMissing + } + overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, tenantID, speakerID, revisionID, rev.PreviewFragments) + if err != nil { + return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err} + } + fpAfter, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID) + if err != nil { + return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err} + } + body, err := s.buildNodeBundle(revisionID, speakerID, overlaid) + if err != nil { + return nil, false, &nodeBundleOpError{op: "internal", err: err} + } + cacheable := fpNow == overlayFP && fpAfter == fpNow && len(body) > 0 + return body, cacheable, nil +} diff --git a/internal/httpapi/node_bundle_cache_test.go b/internal/httpapi/node_bundle_cache_test.go new file mode 100644 index 0000000..219dd8a --- /dev/null +++ b/internal/httpapi/node_bundle_cache_test.go @@ -0,0 +1,791 @@ +package httpapi + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/ed25519" + "encoding/base64" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "evobgp/internal/bundle" + "evobgp/internal/signing" + "evobgp/internal/store" +) + +func TestNodeBundleCacheKeySeparatesTenantsRevisionsSpeakers(t *testing.T) { + a := nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay") + if a == nodeBundleCacheKey("tenant-b", "rev", "spk", "overlay") { + t.Fatal("tenants collided") + } + if a == nodeBundleCacheKey("tenant-a", "rev-2", "spk", "overlay") { + t.Fatal("revisions collided") + } + if a == nodeBundleCacheKey("tenant-a", "rev", "spk-2", "overlay") { + t.Fatal("speakers collided") + } + if a == nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay-2") { + t.Fatal("overlay inputs collided") + } +} + +func TestNodeBundleCacheHitSingleflightAndErrors(t *testing.T) { + c := newNodeBundleCache(8, 1<<20) + var builds atomic.Int32 + _, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) { + builds.Add(1) + return []byte("signed-body"), true, nil + }) + if err != nil { + t.Fatal(err) + } + got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) { + builds.Add(1) + return []byte("other"), true, nil + }) + if err != nil { + t.Fatal(err) + } + if builds.Load() != 1 { + t.Fatalf("second pull rebuilt: builds=%d", builds.Load()) + } + if !bytes.Equal(got, []byte("signed-body")) { + t.Fatalf("hit body = %q", got) + } + + t.Run("singleflight", func(t *testing.T) { + c := newNodeBundleCache(8, 1<<20) + var builds atomic.Int32 + release := make(chan struct{}) + const n = 8 + var wg sync.WaitGroup + bodies := make([][]byte, n) + errs := make([]error, n) + wg.Add(n) + for i := 0; i < n; i++ { + go func(i int) { + defer wg.Done() + bodies[i], errs[i] = c.do(context.Background(), "same", func(context.Context) ([]byte, bool, error) { + builds.Add(1) + <-release + return []byte("signed-bundle"), true, nil + }) + }(i) + } + deadline := time.Now().Add(2 * time.Second) + for { + c.mu.Lock() + joined := 0 + if f := c.flights["same"]; f != nil { + joined = f.joined + } + c.mu.Unlock() + if builds.Load() > 1 { + close(release) + wg.Wait() + t.Fatalf("builds = %d before followers joined", builds.Load()) + } + if builds.Load() == 1 && joined == n-1 { + break + } + if time.Now().After(deadline) { + close(release) + wg.Wait() + t.Fatalf("builds=%d joined=%d", builds.Load(), joined) + } + time.Sleep(time.Millisecond) + } + close(release) + wg.Wait() + if builds.Load() != 1 { + t.Fatalf("builds = %d, want 1", builds.Load()) + } + for i := 0; i < n; i++ { + if errs[i] != nil { + t.Fatalf("waiter %d: %v", i, errs[i]) + } + if !bytes.Equal(bodies[i], []byte("signed-bundle")) { + t.Fatalf("waiter %d body = %q", i, bodies[i]) + } + } + }) + + t.Run("canceled waiter does not drop the build", func(t *testing.T) { + c := newNodeBundleCache(4, 1024) + release := make(chan struct{}) + var builds atomic.Int32 + leaderErr := make(chan error, 1) + go func() { + _, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) { + builds.Add(1) + <-release + return []byte("body"), true, nil + }) + leaderErr <- err + }() + deadline := time.Now().Add(2 * time.Second) + for { + c.mu.Lock() + _, ok := c.flights["k"] + c.mu.Unlock() + if ok || time.Now().After(deadline) { + break + } + time.Sleep(time.Millisecond) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := c.do(ctx, "k", func(context.Context) ([]byte, bool, error) { + builds.Add(1) + return []byte("nope"), true, nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("follower err = %v", err) + } + close(release) + if err := <-leaderErr; err != nil { + t.Fatal(err) + } + if builds.Load() != 1 { + t.Fatalf("builds = %d", builds.Load()) + } + _, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) { + builds.Add(1) + return []byte("other"), true, nil + }) + if err != nil { + t.Fatal(err) + } + if builds.Load() != 1 { + t.Fatal("successful build was not cached after a canceled waiter") + } + }) + + t.Run("errors are not cached", func(t *testing.T) { + c := newNodeBundleCache(4, 1024) + calls := 0 + boom := errors.New("httpapi: bundle build failed") + _, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) { + calls++ + return nil, true, boom + }) + if !errors.Is(err, boom) { + t.Fatalf("err = %v", err) + } + _, err = c.do(context.Background(), "miss", func(context.Context) ([]byte, bool, error) { + calls++ + return nil, false, errNodeBundleRevisionMissing + }) + if !errors.Is(err, errNodeBundleRevisionMissing) { + t.Fatalf("err = %v", err) + } + got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) { + calls++ + return []byte("ok"), true, nil + }) + if err != nil { + t.Fatal(err) + } + if string(got) != "ok" || calls != 3 { + t.Fatalf("calls=%d body=%q", calls, got) + } + _, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) { + calls++ + return []byte("other"), true, nil + }) + if err != nil || calls != 3 { + t.Fatalf("cached error path rebuilt: calls=%d err=%v", calls, err) + } + }) + + t.Run("unstorable success is not cached", func(t *testing.T) { + c := newNodeBundleCache(4, 1024) + calls := 0 + for i := 0; i < 2; i++ { + got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) { + calls++ + return []byte("fresh"), false, nil + }) + if err != nil || string(got) != "fresh" { + t.Fatalf("i=%d err=%v body=%q", i, err, got) + } + } + if calls != 2 { + t.Fatalf("calls = %d, want 2", calls) + } + }) +} + +func TestNodeBundleCacheEvictsIdleAndRespectsCaps(t *testing.T) { + t.Run("entry cap evicts oldest idle", func(t *testing.T) { + c := newNodeBundleCache(2, 100) + put := func(key, body string) { + t.Helper() + got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) { + return []byte(body), true, nil + }) + if err != nil || string(got) != body { + t.Fatalf("put %s: %v %q", key, err, got) + } + } + put("a", "A") + put("b", "B") + if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) { + t.Fatal("recent entry was rebuilt") + return nil, false, nil + }); err != nil { + t.Fatal(err) + } + put("c", "C") + assertBundleCacheAccounting(t, c) + if _, ok := c.items["b"]; ok { + t.Fatal("idle entry b was kept") + } + if _, ok := c.items["a"]; !ok || c.items["c"] == nil { + t.Fatalf("kept %v", cacheKeys(c)) + } + if len(c.items) > 2 { + t.Fatalf("entry cap exceeded: %d", len(c.items)) + } + }) + + t.Run("byte cap", func(t *testing.T) { + c := newNodeBundleCache(8, 10) + put := func(key string, n int) { + t.Helper() + body := bytes.Repeat([]byte("x"), n) + got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) { + return body, true, nil + }) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("put %s: %v len=%d", key, err, len(got)) + } + assertBundleCacheAccounting(t, c) + } + put("a", 4) + put("b", 4) + if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) { + t.Fatal("rebuilt a") + return nil, false, nil + }); err != nil { + t.Fatal(err) + } + put("c", 4) + assertBundleCacheAccounting(t, c) + if _, ok := c.items["b"]; ok { + t.Fatal("byte cap kept idle entry b") + } + if len(c.items) != 2 || c.total != 8 { + t.Fatalf("n=%d total=%d", len(c.items), c.total) + } + put("d", 6) + put("e", 6) + assertBundleCacheAccounting(t, c) + if len(c.items) != 1 || c.total != 6 { + t.Fatalf("after oversized pair n=%d total=%d keys=%v", len(c.items), c.total, cacheKeys(c)) + } + }) + + t.Run("entry larger than the byte cap is not stored", func(t *testing.T) { + c := newNodeBundleCache(4, 8) + calls := 0 + big := bytes.Repeat([]byte("y"), 9) + for i := 0; i < 2; i++ { + got, err := c.do(context.Background(), "big", func(context.Context) ([]byte, bool, error) { + calls++ + return big, true, nil + }) + if err != nil || !bytes.Equal(got, big) { + t.Fatalf("i=%d err=%v", i, err) + } + } + if calls != 2 { + t.Fatalf("oversize body was cached, calls=%d", calls) + } + if len(c.items) != 0 || c.total != 0 { + t.Fatalf("stored oversize n=%d total=%d", len(c.items), c.total) + } + }) +} + +func TestNodeBundleOverlayFingerprintInputs(t *testing.T) { + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, rev, speaker := m.DemoIDs() + + base, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev) + if err != nil { + t.Fatal(err) + } + again, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev) + if err != nil { + t.Fatal(err) + } + if base != again { + t.Fatal("fingerprint changed without an overlay input change") + } + + sp, err := m.GetSpeaker(tenant, speaker) + if err != nil { + t.Fatal(err) + } + dispatch := store.MergeSpeakerMetaJSON(sp.MetaJSON, store.SpeakerMeta{ + LastDispatchAt: "2026-01-01T00:00:00Z", + LastDispatchStatus: "ok", + SyncStatus: "synced", + }) + if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &dispatch}); err != nil { + t.Fatal(err) + } + if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base { + t.Fatalf("dispatch meta changed fingerprint: %v %s", err, got) + } + + peers := m.ListPeers(tenant) + if len(peers) == 0 { + t.Fatal("expected demo peers") + } + state := "Active" + if _, err := m.UpdatePeer(tenant, peers[0].ID, &store.PeerPatch{SessionState: &state}); err != nil { + t.Fatal(err) + } + if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base { + t.Fatalf("session state changed fingerprint: %v %s", err, got) + } + + if err := m.PatchGlobalSettings(tenant, map[string]any{"bird_router_id": "203.0.113.7"}); err != nil { + t.Fatal(err) + } + routed, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev) + if err != nil { + t.Fatal(err) + } + if routed == base { + t.Fatal("router id setting did not change the overlay fingerprint") + } + + nodeMeta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"}) + if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeMeta}); err != nil { + t.Fatal(err) + } + nodeFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev) + if err != nil { + t.Fatal(err) + } + if nodeFP == routed { + t.Fatal("node ipv4 did not change the overlay fingerprint") + } + + sourceMeta := store.SpeakerMetaJSON(store.SpeakerMeta{ + NodeIPv4: "198.51.100.9", + BirdBgpSourceIPv4: "203.0.113.40", + }) + if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &sourceMeta}); err != nil { + t.Fatal(err) + } + sourceFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev) + if err != nil { + t.Fatal(err) + } + if sourceFP == nodeFP { + t.Fatal("source ipv4 did not change the overlay fingerprint") + } + + nodeOnly := store.SpeakerMetaJSON(store.SpeakerMeta{ + NodeIPv4: "198.51.100.8", + BirdBgpSourceIPv4: "203.0.113.40", + }) + if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeOnly}); err != nil { + t.Fatal(err) + } + if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != sourceFP { + t.Fatalf("unused node ipv4 changed fingerprint: %v %s vs %s", err, got, sourceFP) + } + + v6 := store.SpeakerMetaJSON(store.SpeakerMeta{ + NodeIPv4: "198.51.100.8", + BirdBgpSourceIPv4: "203.0.113.40", + BirdBgpSourceIPv6: "2001:db8::40", + }) + if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &v6}); err != nil { + t.Fatal(err) + } + v6FP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev) + if err != nil { + t.Fatal(err) + } + // Source v6 changes local v6. When no peer policy overrides it, the rendered + // peers fragment can stay the same. A difference means the overlay emitted it; + // equality means a rebuild would not change the bundle. + _ = v6FP + + if _, err := m.CreatePeer(tenant, &store.BGPPeer{ + Name: "cache-extra", + Neighbor: "203.0.113.50", + RemoteASN: 64512, + Enabled: true, + }); err != nil { + t.Fatal(err) + } + withPeer, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev) + if err != nil { + t.Fatal(err) + } + if withPeer == v6FP { + t.Fatal("added peer did not change the overlay fingerprint") + } + + sp2, err := m.CreateSpeaker(tenant, &store.Speaker{ + Role: "replica", + Endpoint: "https://203.0.113.2", + MetaJSON: v6, + }) + if err != nil { + t.Fatal(err) + } + other, err := nodeBundleOverlayFingerprint(m, tenant, sp2.ID, rev) + if err != nil { + t.Fatal(err) + } + if other == withPeer { + t.Fatal("speakers with the same addresses shared an overlay fingerprint") + } + + otherRev, err := m.CreateRollbackRevision(tenant, rev) + if err != nil { + t.Fatal(err) + } + revFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, otherRev) + if err != nil { + t.Fatal(err) + } + if revFP == withPeer { + t.Fatal("revision id did not change the overlay fingerprint") + } +} + +func TestHandleNodeBundleCache(t *testing.T) { + srv, ts, ids := startNodeBundleServer(t) + if srv.bundleCache.maxN != nodeBundleCacheMaxEntries || srv.bundleCache.maxBytes != nodeBundleCacheMaxBytes { + t.Fatalf("caps n=%d bytes=%d", srv.bundleCache.maxN, srv.bundleCache.maxBytes) + } + var builds atomic.Int32 + srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) { + builds.Add(1) + return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv) + } + client := ts.Client() + + status, raw := pullNodeBundle(t, client, ts.URL, ids.speaker, "00000000-0000-0000-0000-000000000099") + if status != http.StatusNotFound { + t.Fatalf("missing revision status %d: %s", status, raw) + } + if builds.Load() != 0 { + t.Fatal("missing revision built a bundle") + } + + body1 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev) + body2 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev) + if builds.Load() != 1 { + t.Fatalf("second pull rebuilt: builds=%d", builds.Load()) + } + if !bytes.Equal(body1, body2) { + t.Fatal("cache hit was not byte-for-byte identical") + } + verifyNodeBundle(t, srv, body2) + + sp2, err := srv.Store().CreateSpeaker(ids.tenant, &store.Speaker{ + Role: "replica", + Endpoint: "https://203.0.113.2", + MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "203.0.113.2"}), + }) + if err != nil { + t.Fatal(err) + } + bodyB := mustPullNodeBundle(t, client, ts.URL, sp2.ID, ids.rev) + if builds.Load() != 2 { + t.Fatalf("second speaker builds=%d", builds.Load()) + } + if bytes.Equal(body1, bodyB) { + t.Fatal("different speakers returned the same bundle") + } + if !strings.Contains(bundleFile(t, bodyB, "bird.conf"), "203.0.113.2") { + t.Fatal("speaker B bundle missing its router id") + } + body3 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev) + if builds.Load() != 2 || !bytes.Equal(body1, body3) { + t.Fatalf("speaker A collided with B: builds=%d equal=%v", builds.Load(), bytes.Equal(body1, body3)) + } + + meta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"}) + if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil { + t.Fatal(err) + } + body4 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev) + if builds.Load() != 3 { + t.Fatalf("changed node ipv4 builds=%d, want 3", builds.Load()) + } + if bytes.Equal(body1, body4) { + t.Fatal("changed overlay input reused the previous bundle") + } + conf := bundleFile(t, body4, "bird.conf") + if !strings.Contains(conf, "198.51.100.9") { + t.Fatalf("rebuilt bird.conf missing node ipv4: %s", conf) + } + verifyNodeBundle(t, srv, body4) + body5 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev) + if builds.Load() != 3 || !bytes.Equal(body4, body5) { + t.Fatal("rebuilt bundle was not cached") + } + + source := store.SpeakerMetaJSON(store.SpeakerMeta{ + NodeIPv4: "198.51.100.9", + BirdBgpSourceIPv4: "203.0.113.40", + }) + if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &source}); err != nil { + t.Fatal(err) + } + body6 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev) + if builds.Load() != 4 { + t.Fatalf("changed source ipv4 builds=%d, want 4", builds.Load()) + } + conf = bundleFile(t, body6, "bird.conf") + if !strings.Contains(conf, "203.0.113.40") { + t.Fatalf("source ipv4 not applied: %s", conf) + } + verifyNodeBundle(t, srv, body6) + + otherRev, err := srv.Store().CreateRollbackRevision(ids.tenant, ids.rev) + if err != nil { + t.Fatal(err) + } + bodyRev := mustPullNodeBundle(t, client, ts.URL, ids.speaker, otherRev) + if builds.Load() != 5 { + t.Fatalf("other revision builds=%d", builds.Load()) + } + if bytes.Equal(body6, bodyRev) { + t.Fatal("revisions shared a bundle body") + } + cached := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev) + if builds.Load() != 5 || !bytes.Equal(body6, cached) { + t.Fatal("original revision was evicted or rebuilt") + } +} + +func TestHandleNodeBundleSingleflight(t *testing.T) { + srv, ts, ids := startNodeBundleServer(t) + var builds atomic.Int32 + release := make(chan struct{}) + srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) { + builds.Add(1) + <-release + return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv) + } + client := ts.Client() + const n = 2 + bodies := make([][]byte, n) + errs := make([]error, n) + var wg sync.WaitGroup + wg.Add(n) + for i := 0; i < n; i++ { + go func(i int) { + defer wg.Done() + bodies[i], errs[i] = pullNodeBundleOK(client, ts.URL, ids.speaker, ids.rev) + }(i) + } + deadline := time.Now().Add(3 * time.Second) + joined := 0 + for { + if builds.Load() > 1 { + close(release) + wg.Wait() + t.Fatalf("concurrent pulls built %d times", builds.Load()) + } + srv.bundleCache.mu.Lock() + joined = 0 + for _, f := range srv.bundleCache.flights { + joined += f.joined + } + srv.bundleCache.mu.Unlock() + if builds.Load() == 1 && joined >= 1 { + break + } + if time.Now().After(deadline) { + close(release) + wg.Wait() + t.Fatalf("timed out builds=%d joined=%d", builds.Load(), joined) + } + time.Sleep(time.Millisecond) + } + close(release) + wg.Wait() + if builds.Load() != 1 { + t.Fatalf("builds = %d, want 1", builds.Load()) + } + for i := 0; i < n; i++ { + if errs[i] != nil { + t.Fatal(errs[i]) + } + } + if !bytes.Equal(bodies[0], bodies[1]) { + t.Fatal("singleflight returned different bodies") + } + verifyNodeBundle(t, srv, bodies[0]) +} + +type nodeBundleIDs struct { + tenant, rev, speaker string +} + +func startNodeBundleServer(t *testing.T) (*Server, *httptest.Server, nodeBundleIDs) { + t.Helper() + srv, err := New(Options{ + InsecureDev: true, + SeedDemo: true, + BundleSeedHex: testBundleSeed, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(srv.Close) + tenant, _, _, rev, speaker := srv.Store().DemoIDs() + mustSetTestAPIKeys(t, srv, "nodekey|"+tenant+"|node") + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(ts.Close) + return srv, ts, nodeBundleIDs{tenant: tenant, rev: rev, speaker: speaker} +} + +func mustPullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) []byte { + t.Helper() + status, body := pullNodeBundle(t, client, base, speaker, rev) + if status != http.StatusOK { + t.Fatalf("status %d: %s", status, body) + } + return body +} + +func pullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) (int, []byte) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer nodekey") + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return resp.StatusCode, body +} + +func pullNodeBundleOK(client *http.Client, base, speaker, rev string) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer nodekey") + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, errors.New(resp.Status + ": " + string(body)) + } + return body, nil +} + +func verifyNodeBundle(t *testing.T, srv *Server, raw []byte) { + t.Helper() + pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64()) + if err != nil { + t.Fatal(err) + } + if _, err := signing.VerifyGzippedTar(raw, ed25519.PublicKey(pub)); err != nil { + t.Fatal(err) + } +} + +func bundleFile(t *testing.T, gz []byte, name string) string { + t.Helper() + gr, err := gzip.NewReader(bytes.NewReader(gz)) + if err != nil { + t.Fatal(err) + } + defer func() { _ = gr.Close() }() + tr := tar.NewReader(gr) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatal(err) + } + if hdr.Name != name { + continue + } + var b strings.Builder + if _, err := io.Copy(&b, tr); err != nil { + t.Fatal(err) + } + return b.String() + } + t.Fatalf("archive missing %s", name) + return "" +} + +func assertBundleCacheAccounting(t *testing.T, c *nodeBundleCache) { + t.Helper() + if len(c.items) != len(c.lru) { + t.Fatalf("items %d lru %d", len(c.items), len(c.lru)) + } + sum := 0 + seen := make(map[string]bool, len(c.lru)) + for _, k := range c.lru { + e, ok := c.items[k] + if !ok { + t.Fatalf("lru key %q missing from items", k) + } + if seen[k] { + t.Fatalf("duplicate lru key %q", k) + } + seen[k] = true + sum += len(e.body) + } + if sum != c.total { + t.Fatalf("total=%d sum=%d", c.total, sum) + } + if c.maxN > 0 && len(c.items) > c.maxN { + t.Fatalf("entries %d over cap %d", len(c.items), c.maxN) + } + if c.maxBytes > 0 && c.total > c.maxBytes { + t.Fatalf("bytes %d over cap %d", c.total, c.maxBytes) + } +} + +func cacheKeys(c *nodeBundleCache) []string { + keys := make([]string, 0, len(c.items)) + for k := range c.items { + keys = append(keys, k) + } + return keys +} diff --git a/internal/httpapi/node_legacy_compatibility_test.go b/internal/httpapi/node_legacy_compatibility_test.go new file mode 100644 index 0000000..4f2dd08 --- /dev/null +++ b/internal/httpapi/node_legacy_compatibility_test.go @@ -0,0 +1,98 @@ +package httpapi + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "evobgp/internal/bundle" +) + +func TestLegacyCompatibilityNodePullAndAuth(t *testing.T) { + srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + tenant, _, _, revision, speaker := srv.Store().DemoIDs() + mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,operator|"+tenant+"|operator,other-node|other-tenant|node") + if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil { + t.Fatal(err) + } + handler := srv.Handler() + call := func(path, token string) (int, []byte, string) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, path, nil) + if err != nil { + t.Fatal(err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + resp := w.Result() + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return resp.StatusCode, body, resp.Header.Get("Content-Type") + } + latestPath := "/v1/speakers/" + speaker + "/revisions/latest" + bundlePath := "/v1/speakers/" + speaker + "/bundle/" + revision + for _, path := range []string{latestPath, bundlePath} { + for _, tc := range []struct { + token string + want int + }{{"", http.StatusUnauthorized}, {"operator", http.StatusForbidden}, {"other-node", http.StatusForbidden}} { + status, _, _ := call(path, tc.token) + if status != tc.want { + t.Fatalf("%s token %q: status %d, want %d", path, tc.token, status, tc.want) + } + } + } + status, raw, _ := call(latestPath, "legacy-node") + if status != http.StatusOK { + t.Fatalf("legacy latest: status %d: %s", status, raw) + } + var latest struct { + RevisionID string `json:"revision_id"` + PublishedAt string `json:"published_at"` + } + if err := json.Unmarshal(raw, &latest); err != nil { + t.Fatal(err) + } + if latest.RevisionID != revision || latest.PublishedAt == "" { + t.Fatalf("legacy latest response: %+v", latest) + } + status, raw, contentType := call("/v1/bundle/signing-public-key", "operator") + if status != http.StatusOK || contentType != "application/json" { + t.Fatalf("public key status %d, type %q: %s", status, contentType, raw) + } + var key struct { + PublicKeyBase64 string `json:"public_key_base64"` + } + if err := json.Unmarshal(raw, &key); err != nil { + t.Fatal(err) + } + pub, err := base64.StdEncoding.DecodeString(key.PublicKeyBase64) + if err != nil || len(pub) != ed25519.PublicKeySize { + t.Fatalf("legacy public key: len=%d err=%v", len(pub), err) + } + status, raw, contentType = call(bundlePath, "legacy-node") + if status != http.StatusOK || contentType != "application/gzip" { + t.Fatalf("legacy bundle: status %d, type %q: %s", status, contentType, raw) + } + verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub)) + if err != nil { + t.Fatalf("legacy signed bundle verification: %v", err) + } + if verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker || len(verified.Files) == 0 { + t.Fatalf("legacy bundle manifest: %+v", verified.Manifest) + } +} diff --git a/internal/httpapi/operation_limits.go b/internal/httpapi/operation_limits.go new file mode 100644 index 0000000..1f102a5 --- /dev/null +++ b/internal/httpapi/operation_limits.go @@ -0,0 +1,105 @@ +package httpapi + +import ( + "net/http" + "os" + "strconv" + "sync" + "time" +) + +type expensiveOperation int + +const ( + lookupOperation expensiveOperation = iota + cdnPreviewOperation + revisionPreviewOperation +) + +type tenantWork struct { + start time.Time + counts [3]int + active int +} + +// operationLimiter is deliberately local to one API process. It never queues +// waiters or evicts active tenants to admit new identities. +type operationLimiter struct { + mu sync.Mutex + tenants map[string]*tenantWork + active, maxActive, maxTenantActive, maxTenants int + rates [3]int + nextSweep time.Time +} + +func newOperationLimiter() *operationLimiter { + return &operationLimiter{ + tenants: make(map[string]*tenantWork), maxTenants: 4096, + maxActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_MAX_CONCURRENT", 4, 64), + maxTenantActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", 2, 64), + rates: [3]int{operationEnvLimit("EVOBGP_API_LOOKUP_PER_MINUTE", 120, 100000), operationEnvLimit("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", 30, 100000), operationEnvLimit("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", 30, 100000)}, + } +} + +func operationEnvLimit(name string, fallback, maximum int) int { + n, err := strconv.Atoi(os.Getenv(name)) + if err != nil || n < 1 || n > maximum { + return fallback + } + return n +} + +func (l *operationLimiter) acquire(tenant string, op expensiveOperation, now time.Time) (func(), int) { + l.mu.Lock() + defer l.mu.Unlock() + if !now.Before(l.nextSweep) { + for key, entry := range l.tenants { + if entry.active == 0 && !now.Before(entry.start.Add(time.Minute)) { + delete(l.tenants, key) + } + } + l.nextSweep = now.Add(time.Second) + } + if l.active >= l.maxActive { + return nil, 1 + } + entry := l.tenants[tenant] + if entry == nil { + if len(l.tenants) >= l.maxTenants { + return nil, 1 + } + entry = &tenantWork{start: now} + l.tenants[tenant] = entry + } + if !now.Before(entry.start.Add(time.Minute)) { + entry.start = now + entry.counts = [3]int{} + } + if entry.active >= l.maxTenantActive { + return nil, 1 + } + if entry.counts[op] >= l.rates[op] { + wait := entry.start.Add(time.Minute).Sub(now) + return nil, max(1, int((wait+time.Second-1)/time.Second)) + } + entry.counts[op]++ + entry.active++ + l.active++ + var once sync.Once + return func() { once.Do(func() { l.mu.Lock(); defer l.mu.Unlock(); entry.active--; l.active-- }) }, 0 +} + +func (s *Server) acquireOperation(w http.ResponseWriter, r *http.Request, tenant string, op expensiveOperation) (func(), bool) { + if r.Context().Err() != nil { + writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request cancelled") + return nil, false + } + s.operationOnce.Do(func() { s.operationLimiter = newOperationLimiter() }) + release, retry := s.operationLimiter.acquire(tenant, op, time.Now()) + if release == nil { + w.Header().Set("Retry-After", strconv.Itoa(retry)) + writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "preview/lookup quota exceeded; retry later") + return nil, false + } + return release, true +} diff --git a/internal/httpapi/operation_limits_test.go b/internal/httpapi/operation_limits_test.go new file mode 100644 index 0000000..b456fef --- /dev/null +++ b/internal/httpapi/operation_limits_test.go @@ -0,0 +1,143 @@ +package httpapi + +import ( + "context" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "evobgp/internal/store" +) + +func TestOperationLimitsSharedCapacityAndWindows(t *testing.T) { + l := newOperationLimiter() + l.maxActive = 2 + l.maxTenantActive = 1 + l.rates = [3]int{1, 2, 2} + now := time.Now() + a, _ := l.acquire("a", lookupOperation, now) + if a == nil { + t.Fatal("first denied") + } + if r, _ := l.acquire("a", cdnPreviewOperation, now); r != nil { + t.Fatal("tenant concurrency bypass") + } + b, _ := l.acquire("b", revisionPreviewOperation, now) + if b == nil { + t.Fatal("other tenant denied") + } + if r, _ := l.acquire("c", lookupOperation, now); r != nil { + t.Fatal("global concurrency bypass") + } + a() + a() + b() + if l.active != 0 { + t.Fatal("release not idempotent") + } + if r, retry := l.acquire("a", lookupOperation, now.Add(time.Second)); r != nil || retry != 59 { + t.Fatalf("rate bypass/retry admitted=%t seconds=%d", r != nil, retry) + } + c, _ := l.acquire("a", cdnPreviewOperation, now.Add(time.Second)) + if c == nil { + t.Fatal("independent operation denied") + } + c() + d, _ := l.acquire("a", lookupOperation, now.Add(time.Minute)) + if d == nil { + t.Fatal("expired window not reset") + } + d() +} + +func TestOperationTenantMapBoundWithoutActiveEviction(t *testing.T) { + l := newOperationLimiter() + l.maxTenants = 1 + now := time.Now() + a, _ := l.acquire("a", lookupOperation, now) + if r, _ := l.acquire("b", lookupOperation, now.Add(2*time.Minute)); r != nil { + t.Fatal("active entry evicted") + } + a() + b, _ := l.acquire("b", lookupOperation, now.Add(3*time.Minute)) + if b == nil || len(l.tenants) != 1 { + t.Fatal("idle expiry did not free bounded state") + } + b() +} + +func TestOperationConcurrentAdmissionBound(t *testing.T) { + l := newOperationLimiter() + l.maxActive = 4 + l.maxTenantActive = 4 + l.rates = [3]int{100, 100, 100} + results := make(chan func(), 32) + var wg sync.WaitGroup + for range 32 { + wg.Add(1) + go func() { defer wg.Done(); r, _ := l.acquire("tenant", lookupOperation, time.Now()); results <- r }() + } + wg.Wait() + close(results) + accepted := 0 + for r := range results { + if r != nil { + accepted++ + r() + } + } + if accepted != 4 || l.active != 0 { + t.Fatalf("accepted=%d active=%d", accepted, l.active) + } +} + +type waitingRevisionStore struct { + store.Backend + entered chan struct{} + unblock chan struct{} +} + +func (s waitingRevisionStore) GetRevision(tenant, id string) (*store.Revision, error) { + s.entered <- struct{}{} + <-s.unblock + return s.Backend.GetRevision(tenant, id) +} + +func TestRevisionQuotaHeldUntilWorkReturns(t *testing.T) { + t.Setenv("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", "1") + t.Setenv("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", "30") + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, revision, _ := m.DemoIDs() + entered := make(chan struct{}, 2) + unblock := make(chan struct{}) + s := &Server{store: waitingRevisionStore{Backend: m, entered: entered, unblock: unblock}} + request := func() *http.Request { + r := httptest.NewRequest(http.MethodGet, "/v1/revisions/"+revision+"/preview", nil) + r.SetPathValue("revision_id", revision) + return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"})) + } + ctx, cancel := context.WithCancel(request().Context()) + defer cancel() + first := httptest.NewRecorder() + done := make(chan struct{}) + go func() { defer close(done); s.handleRevisionPreview(first, request().WithContext(ctx)) }() + <-entered + cancel() // Backend deliberately ignores cancellation: capacity stays reserved. + second := httptest.NewRecorder() + s.handleRevisionPreview(second, request()) + if second.Code != http.StatusTooManyRequests || second.Header().Get("Retry-After") == "" { + close(unblock) + <-done + t.Fatalf("quota: %d %s", second.Code, second.Body.String()) + } + close(unblock) + <-done + third := httptest.NewRecorder() + s.handleRevisionPreview(third, request()) + if third.Code != http.StatusOK { + t.Fatalf("release failed: %d %s", third.Code, third.Body.String()) + } +} diff --git a/internal/httpapi/peers_discovery.go b/internal/httpapi/peers_discovery.go index 618c60a..c6a7cc3 100644 --- a/internal/httpapi/peers_discovery.go +++ b/internal/httpapi/peers_discovery.go @@ -6,7 +6,6 @@ import ( "strings" "time" - "evobgp/internal/birdfmt" "evobgp/internal/store" ) @@ -44,46 +43,6 @@ func peerDiscoveryJSON(d *store.BGPPeerDiscovery) map[string]any { return m } -// syncPeerDiscoveriesFromLive upserts pending discoveries from dynamic BGP sessions. -func (s *Server) syncPeerDiscoveriesFromLive(tenantID string, views []speakerBGPLive) { - if s.store == nil || tenantID == "" { - return - } - knownNeighbors := map[string]struct{}{} - for _, p := range s.store.ListPeers(tenantID) { - if p == nil { - continue - } - if n, ok := store.NormalizePeerNeighborString(p.Neighbor); ok { - knownNeighbors[n] = struct{}{} - } - } - now := time.Now().UTC() - for _, v := range views { - for _, sess := range v.Sessions { - if !birdfmt.IsDynamicDiscoverySession(sess.Name) { - continue - } - neighbor, ok := store.NormalizePeerNeighborString(sess.Neighbor) - if !ok { - continue - } - if _, known := knownNeighbors[neighbor]; known { - continue - } - _, _ = s.store.UpsertPeerDiscovery(tenantID, &store.PeerDiscoveryUpsert{ - SpeakerID: v.SpeakerID, - NeighborID: strings.TrimSpace(sess.NeighborID), - Neighbor: neighbor, - RemoteASN: sess.NeighborAS, - ProtocolName: sess.Name, - SessionState: sess.State, - SeenAt: now, - }) - } - } -} - func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Request) { a, ok := authFromContext(r.Context()) if !ok { @@ -97,11 +56,6 @@ func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Reques if status == "" { status = store.PeerDiscoveryPending } - // Refresh live discovery when listing pending. - if status == store.PeerDiscoveryPending || status == "all" { - views := s.collectSpeakerBGPLive(r.Context(), a.TenantID, true) - s.syncPeerDiscoveriesFromLive(a.TenantID, views) - } listStatus := status if status == "all" { listStatus = "" @@ -138,7 +92,7 @@ func (s *Server) handleApprovePeerDiscovery(w http.ResponseWriter, r *http.Reque writeStoreErr(w, err) return } - s.enqueuePeerReconcile(a.TenantID, "peer_discovery_approve") + s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_approve") s.recordCRUDAudit(r, a, "bgp.peer.discovery.approve", "Approved discovered peer "+peer.Neighbor, peer.ID, map[string]any{ "peer_id": peer.ID, "discovery_id": disc.ID, diff --git a/internal/httpapi/peers_live.go b/internal/httpapi/peers_live.go index 581f64d..2ebd5f5 100644 --- a/internal/httpapi/peers_live.go +++ b/internal/httpapi/peers_live.go @@ -1,47 +1,98 @@ package httpapi import ( - "context" "net/netip" - "os" "strings" - "sync" "time" "evobgp/internal/birdfmt" - "evobgp/internal/nodedispatch" "evobgp/internal/store" ) -const peerLiveCacheTTL = 15 * time.Second - type speakerBGPLive struct { - SpeakerID string - Label string - Sessions []birdfmt.BGPSession - Error string + SpeakerID string + Label string + Sessions []birdfmt.BGPSession + Error string + Polled bool + ObservedAt time.Time + ReceivedAt time.Time + LastSuccessAt time.Time + Source string + ObservationOverride *liveObservation } -type peerLiveCacheEntry struct { - at time.Time - views []speakerBGPLive +type liveObservation struct { + ObservedAt *string `json:"observed_at"` + ReceivedAt *string `json:"received_at"` + LastSuccessAt *string `json:"last_success_at"` + ExpiresAt *string `json:"expires_at,omitempty"` + Source string `json:"source"` + Quality string `json:"quality"` + ErrorCode string `json:"error_code,omitempty"` } -var peerLiveCache sync.Map // tenantID -> peerLiveCacheEntry +func liveTime(t time.Time) *string { + if t.IsZero() { + return nil + } + v := t.UTC().Format(time.RFC3339Nano) + return &v +} + +func (v speakerBGPLive) observed() bool { + return v.Polled || v.Error != "" || len(v.Sessions) > 0 +} + +func (v speakerBGPLive) successful() bool { + return v.observed() && v.Error == "" +} + +func (v speakerBGPLive) observation() liveObservation { + if v.ObservationOverride != nil { + return *v.ObservationOverride + } + o := liveObservation{ObservedAt: liveTime(v.ObservedAt), ReceivedAt: liveTime(v.ReceivedAt), LastSuccessAt: liveTime(v.LastSuccessAt), Source: v.Source, Quality: "unknown"} + if o.Source == "" { + o.Source = "none" + } + switch { + case v.successful(): + o.Quality = "ok" + case v.Error != "" && !v.LastSuccessAt.IsZero(): + o.Quality = "stale" + o.ErrorCode = liveErrorCode(v.Error) + case v.Error != "": + o.Quality = "unreachable" + o.ErrorCode = liveErrorCode(v.Error) + default: + o.ErrorCode = "not_polled" + } + return o +} + +func liveErrorCode(err string) string { + if strings.Contains(strings.ToLower(err), "not configured") || strings.Contains(strings.ToLower(err), "missing") { + return "not_configured" + } + return "poll_failed" +} type peerSessionOnSpeaker struct { - SpeakerID string `json:"speaker_id"` - Label string `json:"label"` - State string `json:"state"` - PollError string `json:"poll_error,omitempty"` + SpeakerID string `json:"speaker_id"` + Label string `json:"label"` + State string `json:"state"` + PollError string `json:"poll_error,omitempty"` + Observation liveObservation `json:"observation"` } type liveSpeakerPoll struct { - SpeakerID string `json:"speaker_id"` - Label string `json:"label"` - OK bool `json:"ok"` - SessionCount int `json:"session_count"` - PollError string `json:"poll_error,omitempty"` + SpeakerID string `json:"speaker_id"` + Label string `json:"label"` + OK bool `json:"ok"` + SessionCount *int `json:"session_count"` + PollError string `json:"poll_error,omitempty"` + Observation liveObservation `json:"observation"` } func speakerDisplayLabel(sp *store.Speaker) string { @@ -65,95 +116,21 @@ func speakerDisplayLabel(sp *store.Speaker) string { return sp.ID } -func masterSpeakerID(speakers []*store.Speaker) string { - for _, sp := range speakers { - if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") { - return sp.ID - } - } - return "" -} - -func (s *Server) collectSpeakerBGPLive(ctx context.Context, tenantID string, fresh bool) []speakerBGPLive { - if !fresh { - if v, ok := peerLiveCache.Load(tenantID); ok { - ent := v.(peerLiveCacheEntry) - if time.Since(ent.at) < peerLiveCacheTTL { - return ent.views - } - } - } - speakers := s.store.ListSpeakersForTenant(tenantID) - views := make([]speakerBGPLive, 0, len(speakers)+1) - - if sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")); sock != "" { - v := speakerBGPLive{Label: "CP (local BIRD)"} - if mid := masterSpeakerID(speakers); mid != "" { - v.SpeakerID = mid - for _, sp := range speakers { - if sp != nil && sp.ID == mid { - v.Label = speakerDisplayLabel(sp) - break - } - } - } - out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN"))) - if err != nil { - v.Error = err.Error() - } else { - v.Sessions = birdfmt.ParseBGPSessions(out) - } - views = append(views, v) - } - - opts := nodedispatch.Options{Timeout: 8 * time.Second} - type resWrap struct { - sp *store.Speaker - res nodedispatch.BirdProtocolsResult - } - ch := make(chan resWrap, len(speakers)) - var wg sync.WaitGroup - for _, sp := range speakers { - if sp == nil { - continue - } - meta := store.ParseSpeakerMeta(sp.MetaJSON) - if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) { - continue - } - wg.Add(1) - go func(speaker *store.Speaker) { - defer wg.Done() - ch <- resWrap{ - sp: speaker, - res: nodedispatch.FetchBirdProtocols(ctx, speaker, opts), - } - }(sp) - } - wg.Wait() - close(ch) - for rw := range ch { - views = append(views, speakerBGPLive{ - SpeakerID: rw.sp.ID, - Label: speakerDisplayLabel(rw.sp), - Sessions: rw.res.Sessions, - Error: rw.res.Error, - }) - } - - peerLiveCache.Store(tenantID, peerLiveCacheEntry{at: time.Now(), views: views}) - return views -} - func liveSpeakerPollJSON(views []speakerBGPLive) []liveSpeakerPoll { out := make([]liveSpeakerPoll, 0, len(views)) for _, v := range views { + var sessionCount *int + if v.successful() { + count := len(v.Sessions) + sessionCount = &count + } out = append(out, liveSpeakerPoll{ SpeakerID: v.SpeakerID, Label: v.Label, - OK: v.Error == "", - SessionCount: len(v.Sessions), + OK: v.successful(), + SessionCount: sessionCount, PollError: v.Error, + Observation: v.observation(), }) } return out @@ -183,27 +160,30 @@ func matchPeerOnSpeakers(peer *store.BGPPeer, views []speakerBGPLive) ( protoName := birdfmt.PeerProtocolName(peer.ID) for _, v := range views { - if v.Error != "" && len(v.Sessions) == 0 { + if !v.successful() { on = append(on, peerSessionOnSpeaker{ - SpeakerID: v.SpeakerID, - Label: v.Label, - PollError: v.Error, + SpeakerID: v.SpeakerID, + Label: v.Label, + PollError: v.Error, + Observation: v.observation(), }) continue } sess := findPeerSession(v.Sessions, protoName, neighbor, hasNeighbor) if sess == nil { on = append(on, peerSessionOnSpeaker{ - SpeakerID: v.SpeakerID, - Label: v.Label, - State: "absent", + SpeakerID: v.SpeakerID, + Label: v.Label, + State: "absent", + Observation: v.observation(), }) continue } hit := peerSessionOnSpeaker{ - SpeakerID: v.SpeakerID, - Label: v.Label, - State: sess.State, + SpeakerID: v.SpeakerID, + Label: v.Label, + State: sess.State, + Observation: v.observation(), } on = append(on, hit) if strings.EqualFold(strings.TrimSpace(sess.State), "Established") { @@ -273,20 +253,109 @@ func sessionStateRank(state string) int { } func applyPeerLiveFields(row map[string]any, peer *store.BGPPeer, views []speakerBGPLive) { - state, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views) + expected := expectedPeerViews(peer, views) + state, _, _, _, _, _ := matchPeerOnSpeakers(peer, expected) + _, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views) row["session_on_speakers"] = on row["established_on_speakers"] = establishedOn row["session_conflict"] = false row["session_mismatch"] = mismatch + row["last_known_session_state"] = row["session_state"] + row["session_state"] = nil if state != "" { row["session_state"] = state } + if state == "" && len(expected) > 0 && allSpeakerPollsSuccessful(expected) { + row["session_state"] = "absent" + } + row["live_observation"] = peerLiveObservation(expected) if connLabel != "" { row["connected_speaker_label"] = connLabel } row["connected_speaker_id"] = peerLiveSpeakerIDOrNull(connID) } +func expectedPeerViews(peer *store.BGPPeer, views []speakerBGPLive) []speakerBGPLive { + if peer == nil || peer.SpeakerID == nil || strings.TrimSpace(*peer.SpeakerID) == "" { + return views + } + want := strings.TrimSpace(*peer.SpeakerID) + expected := make([]speakerBGPLive, 0, 1) + for _, v := range views { + if strings.EqualFold(v.SpeakerID, want) { + expected = append(expected, v) + } + } + return expected +} + +func allSpeakerPollsSuccessful(views []speakerBGPLive) bool { + for _, v := range views { + if !v.successful() { + return false + } + } + return true +} + +func peerLiveObservation(views []speakerBGPLive) liveObservation { + if len(views) == 0 { + return speakerBGPLive{}.observation() + } + result := liveObservation{Source: views[0].Source, Quality: "unknown"} + if result.Source == "" { + result.Source = "none" + } + var latestObserved, latestReceived, latestSuccess time.Time + allOK, allFailed, anyStale := true, true, false + for _, v := range views { + o := v.observation() + if v.ObservedAt.After(latestObserved) { + latestObserved = v.ObservedAt + } + if v.ReceivedAt.After(latestReceived) { + latestReceived = v.ReceivedAt + } + if v.LastSuccessAt.After(latestSuccess) { + latestSuccess = v.LastSuccessAt + } + allOK = allOK && v.successful() + allFailed = allFailed && o.Quality == "unreachable" + anyStale = anyStale || o.Quality == "stale" + if result.ErrorCode == "" && o.ErrorCode != "" { + result.ErrorCode = o.ErrorCode + } + if o.ExpiresAt != nil { + expires, err := time.Parse(time.RFC3339Nano, *o.ExpiresAt) + if err == nil { + previous := time.Time{} + if result.ExpiresAt != nil { + previous, _ = time.Parse(time.RFC3339Nano, *result.ExpiresAt) + } + if previous.IsZero() || expires.Before(previous) { + result.ExpiresAt = o.ExpiresAt + } + } + } + } + result.ObservedAt = liveTime(latestObserved) + result.ReceivedAt = liveTime(latestReceived) + result.LastSuccessAt = liveTime(latestSuccess) + if len(views) > 1 { + result.Source = "mixed" + } + switch { + case allOK: + result.Quality = "ok" + result.ErrorCode = "" + case anyStale: + result.Quality = "stale" + case allFailed: + result.Quality = "unreachable" + } + return result +} + func peerLiveSpeakerIDOrNull(id string) any { if strings.TrimSpace(id) == "" { return nil diff --git a/internal/httpapi/peers_live_test.go b/internal/httpapi/peers_live_test.go index 374a820..cb63f6e 100644 --- a/internal/httpapi/peers_live_test.go +++ b/internal/httpapi/peers_live_test.go @@ -2,11 +2,84 @@ package httpapi import ( "testing" + "time" "evobgp/internal/birdfmt" "evobgp/internal/store" ) +func TestApplyPeerLiveFieldsObservationStates(t *testing.T) { + peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"} + now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC) + cases := []struct { + name string + views []speakerBGPLive + wantState any + wantQuality string + }{ + {"no collector", nil, nil, "unknown"}, + {"successful empty poll", []speakerBGPLive{{Polled: true, Source: "agent_bird_protocols", ObservedAt: now, ReceivedAt: now, LastSuccessAt: now}}, "absent", "ok"}, + {"first failure", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now}}, nil, "unreachable"}, + {"failure after success", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now, LastSuccessAt: now.Add(-time.Minute)}}, nil, "stale"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + row := map[string]any{"session_state": "Established"} + applyPeerLiveFields(row, peer, tc.views) + if row["session_state"] != tc.wantState || row["last_known_session_state"] != "Established" { + t.Fatalf("session_state=%v last_known=%v", row["session_state"], row["last_known_session_state"]) + } + observation := row["live_observation"].(liveObservation) + if observation.Quality != tc.wantQuality { + t.Fatalf("observation=%+v", observation) + } + }) + } +} + +func TestApplyPeerLiveFieldsExpectedSpeakerCoverage(t *testing.T) { + now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC) + assigned := "speaker-a" + peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2", SpeakerID: &assigned} + viewAError := speakerBGPLive{SpeakerID: assigned, Polled: true, Error: "timeout", ObservedAt: now, Source: "agent_bird_protocols"} + viewAEmpty := speakerBGPLive{SpeakerID: assigned, Polled: true, ObservedAt: now, LastSuccessAt: now, Source: "agent_bird_protocols"} + viewBEmpty := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, ObservedAt: now.Add(time.Minute), LastSuccessAt: now.Add(time.Minute), Source: "agent_bird_protocols"} + viewBError := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, Error: "timeout", ObservedAt: now.Add(time.Minute), Source: "agent_bird_protocols"} + cases := []struct { + name string + assigned bool + views []speakerBGPLive + wantState any + wantQuality string + wantObserved string + }{ + {"assigned timeout unrelated success", true, []speakerBGPLive{viewAError, viewBEmpty}, nil, "unreachable", now.Format(time.RFC3339Nano)}, + {"assigned success unrelated failure", true, []speakerBGPLive{viewAEmpty, viewBError}, "absent", "ok", now.Format(time.RFC3339Nano)}, + {"global mixed outcome", false, []speakerBGPLive{viewAEmpty, viewBError}, nil, "unknown", now.Add(time.Minute).Format(time.RFC3339Nano)}, + {"assigned speaker not observed", true, []speakerBGPLive{viewBEmpty}, nil, "unknown", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + candidate := *peer + if !tc.assigned { + candidate.SpeakerID = nil + } + row := map[string]any{"session_state": "Established"} + applyPeerLiveFields(row, &candidate, tc.views) + obs := row["live_observation"].(liveObservation) + if row["session_state"] != tc.wantState || obs.Quality != tc.wantQuality { + t.Fatalf("state=%v observation=%+v", row["session_state"], obs) + } + if tc.wantObserved == "" && obs.ObservedAt != nil { + t.Fatalf("unexpected timestamp %v", *obs.ObservedAt) + } + if tc.wantObserved != "" && (obs.ObservedAt == nil || *obs.ObservedAt != tc.wantObserved) { + t.Fatalf("observed_at=%v want=%q", obs.ObservedAt, tc.wantObserved) + } + }) + } +} + func TestMatchPeerOnSpeakers_establishedOnReplica(t *testing.T) { peer := &store.BGPPeer{ ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", diff --git a/internal/httpapi/ratelimit.go b/internal/httpapi/ratelimit.go index 6338048..29bdb2e 100644 --- a/internal/httpapi/ratelimit.go +++ b/internal/httpapi/ratelimit.go @@ -9,23 +9,33 @@ import ( "time" ) -// authRateLimiter is a simple per-IP token bucket for public auth-ish endpoints. +const maxAuthRateLimitClients = 4096 + +type authRateHit struct { + start time.Time + count int +} + +// authRateLimiter keeps a bounded fixed-window counter per client IP. type authRateLimiter struct { - mu sync.Mutex - hits map[string][]time.Time - limit int - window time.Duration + mu sync.Mutex + hits map[string]authRateHit + limit int + window time.Duration + maxClients int + nextSweep time.Time } func newAuthRateLimiter() *authRateLimiter { limit := 60 - if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 { + if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 && n <= 1000 { limit = n } return &authRateLimiter{ - hits: make(map[string][]time.Time), - limit: limit, - window: time.Minute, + hits: make(map[string]authRateHit), + limit: limit, + window: time.Minute, + maxClients: maxAuthRateLimitClients, } } @@ -33,23 +43,40 @@ func (l *authRateLimiter) allow(ip string) bool { if l == nil { return true } - now := time.Now() + return l.allowAt(ip, time.Now()) +} + +func (l *authRateLimiter) allowAt(ip string, now time.Time) bool { l.mu.Lock() defer l.mu.Unlock() - cut := now.Add(-l.window) - arr := l.hits[ip] - kept := arr[:0] - for _, t := range arr { - if t.After(cut) { - kept = append(kept, t) + if !l.nextSweep.IsZero() && !now.Before(l.nextSweep) { + l.nextSweep = time.Time{} + for key, hit := range l.hits { + expiry := hit.start.Add(l.window) + if !now.Before(expiry) { + delete(l.hits, key) + } else if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) { + l.nextSweep = expiry + } } } - if len(kept) >= l.limit { - l.hits[ip] = kept + hit, exists := l.hits[ip] + if !exists && len(l.hits) >= l.maxClients { + // Never evict an active counter: a new identity must wait for expiry. return false } - kept = append(kept, now) - l.hits[ip] = kept + if exists && hit.count >= l.limit { + return false + } + if !exists { + hit.start = now + expiry := now.Add(l.window) + if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) { + l.nextSweep = expiry + } + } + hit.count++ + l.hits[ip] = hit return true } diff --git a/internal/httpapi/request_body.go b/internal/httpapi/request_body.go new file mode 100644 index 0000000..4ba6a23 --- /dev/null +++ b/internal/httpapi/request_body.go @@ -0,0 +1,100 @@ +package httpapi + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "net/http" + "os" + "strconv" + "strings" +) + +const requestBodyLimit int64 = 8 << 20 + +// withRequestBodyLimit must run inside authMiddleware. Reading the complete +// bounded body before dispatch prevents a valid JSON prefix from committing a +// mutation before a later size/read error is discovered. Content-Type is not a +// security boundary; CSV retains its additional row/field checks in its handler. +func withRequestBodyLimit(next http.Handler) http.Handler { + return withBodyLimitAndConcurrency(requestBodyLimit, requestBodyConcurrency(), next) +} + +func withBodyLimit(limit int64, next http.Handler) http.Handler { + return withBodyLimitAndConcurrency(limit, 4, next) +} + +func requestBodyConcurrency() int { + n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_API_BODY_MAX_CONCURRENT"))) + if err == nil && n >= 1 && n <= 64 { + return n + } + return 4 +} + +func withBodyLimitAndConcurrency(limit int64, concurrency int, next http.Handler) http.Handler { + slots := make(chan struct{}, concurrency) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + original := r.Body + if original != nil { + defer func() { _ = original.Close() }() + } + if r.ContentLength > limit { + writeRequestBodyTooLarge(w) + return + } + if r.Context().Err() != nil { + writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled") + return + } + if r.Body == nil || r.Body == http.NoBody { + next.ServeHTTP(w, r) + return + } + select { + case slots <- struct{}{}: + defer func() { <-slots }() + default: + w.Header().Set("Retry-After", "1") + writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "request body processing capacity is full") + return + } + bounded := http.MaxBytesReader(w, r.Body, limit) + data, err := io.ReadAll(requestContextReader{ctx: r.Context(), reader: bounded}) + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + writeRequestBodyTooLarge(w) + return + } + var timeout net.Error + if r.Context().Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timeout) && timeout.Timeout()) { + writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled") + return + } + if err != nil { + writeProblem(w, http.StatusBadRequest, "Bad Request", "could not read request body") + return + } + r.Body = io.NopCloser(bytes.NewReader(data)) + r.ContentLength = int64(len(data)) + next.ServeHTTP(w, r) + }) +} + +func writeRequestBodyTooLarge(w http.ResponseWriter) { + writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "request body exceeds the 8 MiB limit") +} + +type requestContextReader struct { + ctx context.Context + reader io.Reader +} + +func (r requestContextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} diff --git a/internal/httpapi/request_body_test.go b/internal/httpapi/request_body_test.go new file mode 100644 index 0000000..94371f0 --- /dev/null +++ b/internal/httpapi/request_body_test.go @@ -0,0 +1,273 @@ +package httpapi + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +func TestRequestBodyKnownAndUnknownBoundsBeforeMutation(t *testing.T) { + const limit = 32 + for _, unknown := range []bool{false, true} { + for _, contentType := range []string{"", "application/json", "text/plain"} { + for _, extra := range []int{0, 1} { + label := "known" + if unknown { + label = "unknown" + } + t.Run(label+"/"+contentType+"/"+string(rune('0'+extra)), func(t *testing.T) { + mutations := 0 + h := withBodyLimit(limit, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var input map[string]any + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + t.Error(err) + } + mutations++ + w.WriteHeader(http.StatusCreated) + })) + body := `{"x":1}` + strings.Repeat(" ", limit-len(`{"x":1}`)+extra) + req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body)) + if unknown { + req.ContentLength = -1 + } + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if extra == 0 { + if w.Code != http.StatusCreated || mutations != 1 { + t.Fatalf("exact limit rejected: %d mutations=%d", w.Code, mutations) + } + } else if w.Code != http.StatusRequestEntityTooLarge || mutations != 0 { + t.Fatalf("oversized tail mutated: %d mutations=%d", w.Code, mutations) + } + }) + } + } + } +} + +type checkedRequestBody struct { + reads int + closed bool + reader io.Reader +} + +func (b *checkedRequestBody) Read(p []byte) (int, error) { b.reads++; return b.reader.Read(p) } +func (b *checkedRequestBody) Close() error { b.closed = true; return nil } + +type failingRequestReader struct{} + +func (failingRequestReader) Read([]byte) (int, error) { + return 0, errors.New("untrusted detailed read failure") +} + +func TestRequestBodyErrorsDoNotMutate(t *testing.T) { + for _, cancelled := range []bool{false, true} { + calls := 0 + b := &checkedRequestBody{reader: io.MultiReader(strings.NewReader(`{"x":1}`), failingRequestReader{})} + req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil) + req.Body = b + req.ContentLength = -1 + if cancelled { + ctx, cancel := context.WithCancel(req.Context()) + cancel() + req = req.WithContext(ctx) + } + w := httptest.NewRecorder() + withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { calls++ })).ServeHTTP(w, req) + want := http.StatusBadRequest + if cancelled { + want = http.StatusRequestTimeout + } + if w.Code != want || calls != 0 || !b.closed || strings.Contains(w.Body.String(), "untrusted detailed") { + t.Fatalf("error handling status=%d calls=%d body=%s", w.Code, calls, w.Body.String()) + } + if cancelled && b.reads != 0 { + t.Fatal("read body after cancellation") + } + } +} + +func TestRequestBodyAuthenticationRunsFirst(t *testing.T) { + b := &checkedRequestBody{reader: failingRequestReader{}} + req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil) + req.Body = b + req.ContentLength = requestBodyLimit + 1 + s := &Server{} + called := false + h := s.authMiddleware(withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true }))) + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusUnauthorized || b.reads != 0 || called { + t.Fatalf("body processed before auth: %d reads=%d", w.Code, b.reads) + } +} + +func TestRequestBodyAppliesToGETAndIgnoresClaimedZeroLength(t *testing.T) { + for _, length := range []int64{-1, 0, requestBodyLimit + 1} { + req := httptest.NewRequest(http.MethodGet, "/v1/jobs", strings.NewReader(strings.Repeat("x", int(requestBodyLimit)+1))) + req.ContentLength = length + called := false + w := httptest.NewRecorder() + withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })).ServeHTTP(w, req) + if w.Code != http.StatusRequestEntityTooLarge || called { + t.Fatalf("GET size bypass length=%d code=%d", length, w.Code) + } + } +} + +func TestRequestBodyHandlerWiringAndNoMutation(t *testing.T) { + s, err := New(Options{SeedDemo: true, InsecureDev: true, BundleSeedHex: testBundleSeed}) + if err != nil { + t.Fatal(err) + } + defer s.Close() + tenant, _, _, _, _ := s.store.DemoIDs() + before := len(s.store.ListModules(tenant)) + h := s.Handler() + prefix := `{"type":"IP_RANGES","name":"bounded-create","enabled":false}` + for _, extra := range []int{1, 0} { + body := prefix + strings.Repeat(" ", int(requestBodyLimit)-len(prefix)+extra) + req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body)) + req.ContentLength = -1 + req.Header.Set("Authorization", "Bearer dev") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if extra == 1 { + if w.Code != http.StatusRequestEntityTooLarge || len(s.store.ListModules(tenant)) != before { + t.Fatalf("oversized JSON committed: %d %s", w.Code, w.Body.String()) + } + } else if w.Code != http.StatusCreated || len(s.store.ListModules(tenant)) != before+1 { + t.Fatalf("8MiB body rejected: %d %s", w.Code, w.Body.String()) + } + } +} + +func TestRequestBodyLeavesStreamingResponseUnbuffered(t *testing.T) { + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/runtime-logs/stream", nil) + withRequestBodyLimit(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f, ok := w.(http.Flusher) + if !ok { + t.Fatal("response lost Flusher") + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, "data: ready\n\n") + f.Flush() + })).ServeHTTP(w, req) + if !w.Flushed || w.Body.String() != "data: ready\n\n" { + t.Fatal("stream response buffered") + } +} + +type blockedAdmissionBody struct { + entered chan struct{} + resume chan struct{} + sent bool +} + +func (b *blockedAdmissionBody) Read(p []byte) (int, error) { + if b.sent { + return 0, io.EOF + } + b.sent = true + close(b.entered) + <-b.resume + p[0] = 'x' + return 1, nil +} +func (*blockedAdmissionBody) Close() error { return nil } + +func TestRequestBodyAdmissionRejectsWithoutWaitingAndHoldsThroughHandler(t *testing.T) { + readEntered, readResume := make(chan struct{}), make(chan struct{}) + handlerEntered, handlerResume := make(chan struct{}), make(chan struct{}) + var once sync.Once + h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/empty" { + once.Do(func() { close(handlerEntered) }) + <-handlerResume + } + w.WriteHeader(http.StatusNoContent) + })) + first := httptest.NewRequest(http.MethodPost, "/first", nil) + first.Body = &blockedAdmissionBody{entered: readEntered, resume: readResume} + first.ContentLength = 0 // A claimed zero must not bypass body admission. + done := make(chan struct{}) + go func() { defer close(done); h.ServeHTTP(httptest.NewRecorder(), first) }() + select { + case <-readEntered: + case <-time.After(3 * time.Second): + t.Fatal("first body did not start") + } + reject := func() { + t.Helper() + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/second", strings.NewReader("x")) + h.ServeHTTP(w, req) + if w.Code != http.StatusTooManyRequests || w.Header().Get("Retry-After") != "1" { + t.Fatalf("admission did not reject: %d %s", w.Code, w.Body.String()) + } + } + reject() + // An empty legacy GET remains available while all body slots are occupied. + empty := httptest.NewRecorder() + h.ServeHTTP(empty, httptest.NewRequest(http.MethodGet, "/empty", nil)) + if empty.Code != http.StatusNoContent { + t.Fatal("empty GET consumed body slot") + } + close(readResume) + select { + case <-handlerEntered: + case <-time.After(3 * time.Second): + t.Fatal("handler did not start") + } + reject() // The fully read buffer is still retained by the running handler. + close(handlerResume) + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("first request did not finish") + } + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/third", strings.NewReader("x"))) + if w.Code != http.StatusNoContent { + t.Fatalf("body slot not released: %d", w.Code) + } +} + +func TestRequestBodyAdmissionReleasesOnReadError(t *testing.T) { + h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("read error reached handler") })) + for range 2 { + req := httptest.NewRequest(http.MethodPost, "/bad", nil) + req.Body = io.NopCloser(failingRequestReader{}) + req.ContentLength = -1 + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Fatalf("read failure retained slot: %d", w.Code) + } + } +} + +func TestRequestBodyAdmissionEnvironment(t *testing.T) { + for _, tc := range []struct { + raw string + want int + }{{"", 4}, {"1", 1}, {"64", 64}, {"65", 4}, {"0", 4}, {"-1", 4}, {"invalid", 4}} { + t.Run(tc.raw, func(t *testing.T) { + t.Setenv("EVOBGP_API_BODY_MAX_CONCURRENT", tc.raw) + if got := requestBodyConcurrency(); got != tc.want { + t.Fatalf("concurrency=%d want%d", got, tc.want) + } + }) + } +} diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 6d988a5..0b30109 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -8,14 +8,12 @@ import ( "errors" "io" "net/http" - "os" "sort" "strconv" "strings" "time" "evobgp/internal/birdfmt" - "evobgp/internal/bundle" "evobgp/internal/jobs" "evobgp/internal/observability" "evobgp/internal/pipeline" @@ -39,7 +37,7 @@ func (s *Server) Handler() http.Handler { // Firewall subsystem moved to the standalone EvoFirewall service; see docs/firewall.md. // Registered on the public mux so it wins over the "/v1/" subtree below regardless of auth. s.mux.HandleFunc("/v1/firewall/", s.handleFirewallGone) - s.mux.Handle("/v1/", s.authMiddleware(wrappedV1)) + s.mux.Handle("/v1/", s.authMiddleware(withRequestBodyLimit(wrappedV1))) return s.withCORS(observability.HTTPMiddleware(s.mux)) } @@ -60,6 +58,7 @@ func (s *Server) registerV1(m *http.ServeMux) { m.HandleFunc("GET /modules/{module_id}", s.handleGetModule) m.HandleFunc("GET /peers", s.handleListPeers) m.HandleFunc("GET /speakers", s.handleListSpeakers) + m.HandleFunc("POST /telemetry/refresh", s.handleTelemetryRefresh) m.HandleFunc("GET /bundle/signing-public-key", s.handleBundleSigningPublicKey) m.HandleFunc("POST /modules/{module_id}/refresh", s.handleModuleRefresh) m.HandleFunc("POST /tenant/refresh", s.handleTenantRefresh) @@ -79,6 +78,7 @@ func (s *Server) registerV1(m *http.ServeMux) { m.HandleFunc("GET /jobs/{job_id}", s.handleGetJob) m.HandleFunc("GET /jobs/{job_id}/report", s.handleGetJobReport) m.HandleFunc("POST /jobs/{job_id}/cancel", s.handleCancelJob) + m.HandleFunc("POST /jobs/{job_id}/retry", s.handleRetryJob) m.HandleFunc("GET /speakers/{speaker_id}/revisions/latest", s.handleNodeLatestRevision) m.HandleFunc("GET /speakers/{speaker_id}/bundle/{revision_id}", s.handleNodeBundle) m.HandleFunc("POST /nodes/enroll", s.handleNodeEnroll) @@ -295,12 +295,10 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) { allPeers := s.store.ListPeers(a.TenantID) allPeers = store.FilterOwned(allPeers, func(p *store.BGPPeer) string { return p.CreatedByUserID }, a.Kind, a.IsAdmin, a.UserID) page, next, more := store.PaginateOffset(allPeers, r.URL.Query().Get("cursor"), parseListLimit(r)) - fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1") - ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second) - defer cancel() - liveViews := s.collectSpeakerBGPLive(ctx, a.TenantID, fresh) - if fresh { - s.syncPeerDiscoveriesFromLive(a.TenantID, liveViews) + liveViews, _, err := s.loadTelemetry(r.Context(), a.TenantID, s.store.ListSpeakersForTenant(a.TenantID)) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable") + return } items := make([]map[string]any, 0, len(page)) for _, p := range page { @@ -316,30 +314,6 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) { }) } -func (s *Server) liveBGPProtocolStates(r *http.Request) map[string]string { - if r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1") { - return s.liveBGPProtocolStatesFresh(r.Context()) - } - if cached, ok := observability.CachedBirdProtocolStates(90 * time.Second); ok { - return cached - } - return s.liveBGPProtocolStatesFresh(r.Context()) -} - -func (s *Server) liveBGPProtocolStatesFresh(ctx context.Context) map[string]string { - sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")) - if sock == "" { - return map[string]string{} - } - out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN"))) - if err != nil { - return map[string]string{} - } - states := birdfmt.ParseBGPProtocolStates(out) - observability.SetBirdProtocolStates(states) - return states -} - // parseBGPProtocolStates parses `birdc show protocols all` summary rows into protocol_name -> state. func parseBGPProtocolStates(output string) map[string]string { out := make(map[string]string) @@ -403,9 +377,12 @@ func (s *Server) handleListSpeakers(w http.ResponseWriter, r *http.Request) { fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1") var liveByID map[string]map[string]any if fresh { - ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second) - defer cancel() - liveByID = s.collectSpeakerLiveStatus(ctx, a.TenantID, true, speakers) + var err error + _, liveByID, err = s.loadTelemetry(r.Context(), a.TenantID, speakers) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable") + return + } } items := make([]map[string]any, 0, len(speakers)) for _, sp := range speakers { @@ -448,12 +425,12 @@ func (s *Server) handleModuleRefresh(w http.ResponseWriter, r *http.Request) { idemPtr = &idem } mid := mod.ID - j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{ + j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{ "module_id": moduleID, "trigger": "api", }) if err != nil { - writeInternalError(w, "internal", err) + writeJobEnqueueError(w, err) return } w.Header().Set("Location", "/v1/jobs/"+j.ID) @@ -480,6 +457,17 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) { } } moduleIDs := body.ModuleIDs + sort.Strings(body.ModuleIDs) + requestIDs := append([]string{}, body.ModuleIDs...) + idem := strings.TrimSpace(r.Header.Get("Idempotency-Key")) + var idemPtr *string + if idem != "" { + idemPtr = &idem + } + request := map[string]any{"module_ids": requestIDs, "trigger": "api"} + if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindTenantRefresh, IdempotencyKey: idemPtr, Request: request}) { + return + } if len(moduleIDs) == 0 { now := time.Now().UTC() for _, mod := range s.store.ListModules(a.TenantID) { @@ -488,21 +476,16 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) { } } } - if len(moduleIDs) == 0 { + if len(moduleIDs) == 0 && idem == "" { writeJSON(w, http.StatusNoContent, map[string]any{"message": "no modules due for refresh"}) return } - idem := strings.TrimSpace(r.Header.Get("Idempotency-Key")) - var idemPtr *string - if idem != "" { - idemPtr = &idem - } - j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{ + j, _, err := s.jobs.EnqueueRequestContext(r.Context(), a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{ "module_ids": moduleIDs, "trigger": "api", - }) + }, request) if err != nil { - writeInternalError(w, "internal", err) + writeJobEnqueueError(w, err) return } w.Header().Set("Location", "/v1/jobs/"+j.ID) @@ -510,7 +493,7 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusAccepted, map[string]any{ "job_id": snap["job_id"], "status": snap["status"], - "module_ids": moduleIDs, + "module_ids": j.Snapshot()["meta"].(map[string]any)["module_ids"], "modules_due": len(moduleIDs), }) } @@ -560,8 +543,8 @@ func revisionJSON(rev *store.Revision) map[string]any { return m } -// enqueueModuleRefreshIfEnabled queues module_refresh when the module exists and is enabled (best-effort, no HTTP error). -func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger string) { +// enqueueModuleRefreshIfEnabled reports scheduling failure after a committed CRUD mutation. +func (s *Server) enqueueModuleRefreshIfEnabled(w http.ResponseWriter, r *http.Request, tenantID, moduleID, trigger string) { if s.jobs == nil { return } @@ -570,22 +553,35 @@ func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger strin return } mid := moduleID - key := "module_refresh:" + moduleID - _, _, _ = s.jobs.Enqueue(tenantID, jobs.KindModuleRefresh, &key, &mid, map[string]any{ + // Each mutation is a new logical refresh. Reusing a module-wide key would + // incorrectly replay an older terminal job after later configuration changes. + _, _, err = s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindModuleRefresh, nil, &mid, map[string]any{ "module_id": moduleID, "trigger": trigger, }) + s.reportAutomaticEnqueue(w, r, trigger, moduleID, err) } // enqueuePeerReconcile queues fast peer-only reconcile/render (best-effort, no HTTP error). -func (s *Server) enqueuePeerReconcile(tenantID, trigger string) { +func (s *Server) enqueuePeerReconcile(w http.ResponseWriter, r *http.Request, tenantID, trigger string) { if s.jobs == nil { return } - _, _, _ = s.jobs.Enqueue(tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{ + _, _, err := s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{ "trigger": trigger, "job_title": "Обновление BGP пиров", }) + s.reportAutomaticEnqueue(w, r, trigger, "", err) +} + +func (s *Server) reportAutomaticEnqueue(w http.ResponseWriter, r *http.Request, trigger, target string, err error) { + if err == nil { + return + } + w.Header().Set("X-EvoBGP-Job-Status", "enqueue-failed") + if a, ok := authFromContext(r.Context()); ok { + s.recordCRUDAudit(r, a, "bgp.job.enqueue_failed", "Configuration saved; automatic job was not queued", target, map[string]any{"trigger": trigger, "error": err.Error()}) + } } func (s *Server) handleGetRevision(w http.ResponseWriter, r *http.Request) { @@ -614,6 +610,11 @@ func (s *Server) handleRevisionPreview(w http.ResponseWriter, r *http.Request) { if !s.requirePerm(w, a, "bgp:operations:read") { return } + release, ok := s.acquireOperation(w, r, a.TenantID, revisionPreviewOperation) + if !ok { + return + } + defer release() rev, err := s.store.GetRevision(a.TenantID, r.PathValue("revision_id")) if err != nil { writeProblem(w, http.StatusNotFound, "Not Found", "revision not found") @@ -706,11 +707,11 @@ func (s *Server) handleRevisionRollback(w http.ResponseWriter, r *http.Request) if idem != "" { idemPtr = &idem } - j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{ + j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{ "source_revision_id": revID, }) if err != nil { - writeInternalError(w, "internal", err) + writeJobEnqueueError(w, err) return } w.Header().Set("Location", "/v1/jobs/"+j.ID) @@ -751,13 +752,13 @@ func (s *Server) handleApply(w http.ResponseWriter, r *http.Request) { if idem != "" { idemPtr = &idem } - j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{ + j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{ "revision_id": revID, "strategy": body.Strategy, "trigger": "api", }) if err != nil { - writeInternalError(w, "internal", err) + writeJobEnqueueError(w, err) return } w.Header().Set("Location", "/v1/jobs/"+j.ID) @@ -802,13 +803,13 @@ func (s *Server) handleSpeakerApply(w http.ResponseWriter, r *http.Request) { if idem != "" { idemPtr = &idem } - j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{ + j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{ "revision_id": revID, "speaker_id": spkID, "trigger": "api", }) if err != nil { - writeInternalError(w, "internal", err) + writeJobEnqueueError(w, err) return } w.Header().Set("Location", "/v1/jobs/"+j.ID) @@ -830,9 +831,9 @@ func (s *Server) handleBirdReload(w http.ResponseWriter, r *http.Request) { if idem != "" { idemPtr = &idem } - j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil) + j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil) if err != nil { - writeInternalError(w, "internal", err) + writeJobEnqueueError(w, err) return } w.Header().Set("Location", "/v1/jobs/"+j.ID) @@ -867,7 +868,11 @@ func (s *Server) handleListJobs(w http.ResponseWriter, r *http.Request) { status := r.URL.Query().Get("status") kind := r.URL.Query().Get("kind") cursor := r.URL.Query().Get("cursor") - list, next, more := s.jobs.List(a.TenantID, status, kind, cursor, limit) + list, next, more, err := s.jobs.ListContext(r.Context(), a.TenantID, status, kind, cursor, limit) + if err != nil { + writeInternalError(w, "jobs_list", err) + return + } items := make([]map[string]any, 0, len(list)) for _, j := range list { items = append(items, j.Snapshot()) @@ -886,9 +891,9 @@ func (s *Server) handleGetJob(w http.ResponseWriter, r *http.Request) { if !s.requirePerm(w, a, "bgp:operations:read") { return } - j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id")) + j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id")) if err != nil { - writeProblem(w, http.StatusNotFound, "Not Found", "job not found") + writeJobReadError(w, err) return } writeJSON(w, http.StatusOK, j.Snapshot()) @@ -903,9 +908,9 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) { if !s.requirePerm(w, a, "bgp:operations:read") { return } - j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id")) + j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id")) if err != nil { - writeProblem(w, http.StatusNotFound, "Not Found", "job not found") + writeJobReadError(w, err) return } snap := j.Snapshot() @@ -929,6 +934,35 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) { out["revision_id"] = v } } + sourceJobID := "" + if source, ok := meta["source_deployment_job_id"].(string); ok && source != "" { + // Resolve through the tenant-scoped registry before exposing linked progress. + if _, err := s.jobs.GetContext(r.Context(), a.TenantID, source); err != nil { + writeJobReadError(w, err) + return + } + sourceJobID = source + out["source_deployment_job_id"] = source + } + rows, err := s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, j.ID) + if err == nil && len(rows) == 0 && sourceJobID != "" { + rows, err = s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, sourceJobID) + } + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "deployment progress unavailable") + return + } + progress := make([]map[string]any, 0, len(rows)) + for _, d := range rows { + progress = append(progress, map[string]any{ + "job_id": d.JobID, "speaker_id": d.SpeakerID, "revision_id": d.RevisionID, + "generation": d.Generation, "state": d.State, "confirmation": d.Confirmation, + "error": d.Error, "dispatch_attempts": d.DispatchAttempts, "retry_round": d.RetryRound, + "health_received_at": d.HealthReceivedAt, "health_revision_id": d.HealthRevisionID, + "health_source_at": d.HealthSourceAt, "health_error": d.HealthError, + }) + } + out["deployment_progress"] = progress writeJSON(w, http.StatusOK, out) } @@ -941,9 +975,34 @@ func (s *Server) handleCancelJob(w http.ResponseWriter, r *http.Request) { if !s.requirePerm(w, a, "bgp:operations:admin") { return } - j, err := s.jobs.RequestCancel(a.TenantID, r.PathValue("job_id")) + j, err := s.jobs.RequestCancelContext(r.Context(), a.TenantID, r.PathValue("job_id")) if err != nil { - writeProblem(w, http.StatusNotFound, "Not Found", "job not found") + writeJobReadError(w, err) + return + } + writeJSON(w, http.StatusAccepted, j.Snapshot()) +} + +func (s *Server) handleRetryJob(w http.ResponseWriter, r *http.Request) { + a, ok := authFromContext(r.Context()) + if !ok { + writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth") + return + } + if !s.requirePerm(w, a, "bgp:operations:admin") { + return + } + j, err := s.jobs.RetryContext(r.Context(), a.TenantID, r.PathValue("job_id")) + if err != nil { + if errors.Is(err, store.ErrJobQueueFull) { + writeJobEnqueueError(w, err) + return + } + if errors.Is(err, store.ErrJobConflict) { + writeProblem(w, http.StatusConflict, "Conflict", "job is not failed") + return + } + writeJobReadError(w, err) return } writeJSON(w, http.StatusAccepted, j.Snapshot()) @@ -998,21 +1057,30 @@ func (s *Server) handleNodeBundle(w http.ResponseWriter, r *http.Request) { writeProblem(w, http.StatusForbidden, "Forbidden", "speaker not in tenant scope") return } - rev, err := s.store.GetRevision(a.TenantID, rid) - if err != nil { - writeProblem(w, http.StatusNotFound, "Not Found", "revision not found") - return - } - frags := rev.PreviewFragments - overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, a.TenantID, sid, rid, frags) + overlayFP, err := nodeBundleOverlayFingerprint(s.store, a.TenantID, sid, rid) if err != nil { writeInternalError(w, "bundle overlay", err) return } - frags = overlaid - tgz, err := bundle.BuildGzippedTar(rid, sid, frags, s.bundlePriv) + // Finished gzip, including manifest and bundle.sig. Revision bytes are immutable; + // the overlay fingerprint keeps speaker-specific renders from colliding. + tgz, err := s.bundleCache.do(r.Context(), nodeBundleCacheKey(a.TenantID, rid, sid, overlayFP), func(ctx context.Context) ([]byte, bool, error) { + return s.loadNodeBundle(ctx, a.TenantID, sid, rid, overlayFP) + }) if err != nil { - writeInternalError(w, "internal", err) + if errors.Is(err, errNodeBundleRevisionMissing) { + writeProblem(w, http.StatusNotFound, "Not Found", "revision not found") + return + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return + } + op := "internal" + var opErr *nodeBundleOpError + if errors.As(err, &opErr) && opErr.op != "" { + op = opErr.op + } + writeInternalError(w, op, err) return } w.Header().Set("Content-Type", "application/gzip") diff --git a/internal/httpapi/routes_audit.go b/internal/httpapi/routes_audit.go index b00ceca..36b7e5b 100644 --- a/internal/httpapi/routes_audit.go +++ b/internal/httpapi/routes_audit.go @@ -3,7 +3,6 @@ package httpapi import ( "context" "log" - "net" "net/http" "strings" "time" @@ -132,23 +131,7 @@ func (s *Server) pushAuditToPortal(entry *store.AuditEntry) { } func clientIP(r *http.Request) string { - if r == nil { - return "" - } - if xff := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); xff != "" { - parts := strings.Split(xff, ",") - if len(parts) > 0 { - return strings.TrimSpace(parts[0]) - } - } - if xrip := strings.TrimSpace(r.Header.Get("X-Real-IP")); xrip != "" { - return xrip - } - host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr)) - if err != nil { - return strings.TrimSpace(r.RemoteAddr) - } - return host + return clientIPWithTrustedProxies(r, trustedProxyPrefixes()) } // initAuditPusher wires portal push when URL and secret are configured. diff --git a/internal/httpapi/routes_crud.go b/internal/httpapi/routes_crud.go index e673b1b..674cae8 100644 --- a/internal/httpapi/routes_crud.go +++ b/internal/httpapi/routes_crud.go @@ -296,10 +296,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request) writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail) return } - if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil { - writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail) + release, ok := s.acquireOperation(w, r, a.TenantID, cdnPreviewOperation) + if !ok { return } + defer release() mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id")) if err != nil { writeStoreErr(w, err) @@ -309,6 +310,10 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request) writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "module type must be CDN_CIDRS") return } + if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail) + return + } req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, u, nil) if err != nil { writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "invalid url") @@ -321,11 +326,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - _, _ = io.Copy(io.Discard, resp.Body) + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) writeBadGateway(w, "cdn preview fetch", fmt.Errorf("upstream status: %s", resp.Status)) return } - raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + raw, err := pipeline.ReadCDNBody(resp.Body) if err != nil { writeBadGateway(w, "cdn preview read body", err) return @@ -379,7 +384,7 @@ func (s *Server) handlePostCDNSource(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.cdn_source.create", "Created CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID, "url": x.URL}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_create") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_create") writeJSON(w, http.StatusCreated, cdnSourceJSON(x)) } @@ -410,7 +415,7 @@ func (s *Server) handlePatchCDNSource(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.cdn_source.update", "Updated CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_patch") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_patch") writeJSON(w, http.StatusOK, cdnSourceJSON(x)) } @@ -426,7 +431,7 @@ func (s *Server) handleDeleteCDNSource(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.cdn_source.delete", "Deleted CDN source", sourceID, map[string]any{"module_id": mid, "source_id": sourceID}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_delete") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_delete") w.WriteHeader(http.StatusNoContent) } @@ -485,7 +490,7 @@ func (s *Server) handlePostAS(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.as_entry.create", "Created AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_create") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_create") writeJSON(w, http.StatusCreated, asEntryJSON(x)) } @@ -506,7 +511,7 @@ func (s *Server) handlePatchAS(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.as_entry.update", "Updated AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_patch") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_patch") writeJSON(w, http.StatusOK, asEntryJSON(x)) } @@ -522,7 +527,7 @@ func (s *Server) handleDeleteAS(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.as_entry.delete", "Deleted AS entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_delete") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_delete") w.WriteHeader(http.StatusNoContent) } @@ -566,7 +571,7 @@ func (s *Server) handlePostDomain(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.domain_entry.create", "Created domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_create") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_create") writeJSON(w, http.StatusCreated, domainEntryJSON(x)) } @@ -587,7 +592,7 @@ func (s *Server) handlePatchDomain(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.domain_entry.update", "Updated domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_patch") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_patch") writeJSON(w, http.StatusOK, domainEntryJSON(x)) } @@ -603,7 +608,7 @@ func (s *Server) handleDeleteDomain(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.domain_entry.delete", "Deleted domain entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_delete") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_delete") w.WriteHeader(http.StatusNoContent) } @@ -647,7 +652,7 @@ func (s *Server) handlePostIPRange(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.ip_range.create", "Created IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_create") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_create") writeJSON(w, http.StatusCreated, ipRangeJSON(x)) } @@ -668,7 +673,7 @@ func (s *Server) handlePatchIPRange(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.ip_range.update", "Updated IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_patch") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_patch") writeJSON(w, http.StatusOK, ipRangeJSON(x)) } @@ -684,7 +689,7 @@ func (s *Server) handleDeleteIPRange(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.ip_range.delete", "Deleted IP range entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID}) - s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_delete") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_delete") w.WriteHeader(http.StatusNoContent) } @@ -777,8 +782,12 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req return } moduleID := r.PathValue("module_id") - res, err := importer.ImportModuleEntriesCSV(s.store, a.TenantID, moduleID, r.Body) + res, err := importer.ImportModuleEntriesCSV(r.Context(), s.store, a.TenantID, moduleID, r.Body) if err != nil { + if errors.Is(err, importer.ErrCSVTooLarge) { + writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "csv exceeds 8 MiB, 10000 data rows or 64 fields per record; split the import into smaller requests") + return + } if errors.Is(err, store.ErrInvalidInput) { writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv is empty") return @@ -802,11 +811,11 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req if res.Imported > 0 { switch res.ModuleType { case "AS_PREFIXES": - s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "as_entry_import_csv") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "as_entry_import_csv") case "DOMAINS": - s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "domain_entry_import_csv") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "domain_entry_import_csv") case "IP_RANGES": - s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "ip_range_import_csv") + s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "ip_range_import_csv") } } writeJSON(w, http.StatusOK, map[string]any{ @@ -1055,7 +1064,7 @@ func (s *Server) handlePostPeer(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.peer.create", "Created BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor}) - s.enqueuePeerReconcile(a.TenantID, "peer_create") + s.enqueuePeerReconcile(w, r, a.TenantID, "peer_create") writeJSON(w, http.StatusCreated, peerJSON(x)) } @@ -1099,7 +1108,7 @@ func (s *Server) handlePatchPeer(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.peer.update", "Updated BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor}) - s.enqueuePeerReconcile(a.TenantID, "peer_patch") + s.enqueuePeerReconcile(w, r, a.TenantID, "peer_patch") writeJSON(w, http.StatusOK, peerJSON(x)) } @@ -1120,7 +1129,7 @@ func (s *Server) handleDeletePeer(w http.ResponseWriter, r *http.Request) { return } s.recordCRUDAudit(r, a, "bgp.peer.delete", "Deleted BGP peer", peerID, map[string]any{"peer_id": peerID}) - s.enqueuePeerReconcile(a.TenantID, "peer_delete") + s.enqueuePeerReconcile(w, r, a.TenantID, "peer_delete") w.WriteHeader(http.StatusNoContent) } @@ -1295,7 +1304,7 @@ func (s *Server) handlePatchSettings(w http.ResponseWriter, r *http.Request) { return } if peerDiscoverySettingsChanged(body) { - s.enqueuePeerReconcile(a.TenantID, "peer_discovery_settings") + s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_settings") } s.recordCRUDAudit(r, a, "bgp.settings.update", "Updated tenant settings", a.TenantID, map[string]any{"keys": settingsAuditKeys(body)}) writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) diff --git a/internal/httpapi/routes_crud_csv_test.go b/internal/httpapi/routes_crud_csv_test.go index a397a11..26f783a 100644 --- a/internal/httpapi/routes_crud_csv_test.go +++ b/internal/httpapi/routes_crud_csv_test.go @@ -1,6 +1,7 @@ package httpapi import ( + "context" "encoding/csv" "encoding/json" "io" @@ -8,8 +9,60 @@ import ( "net/http/httptest" "strings" "testing" + + "evobgp/internal/importer" + "evobgp/internal/store" ) +func TestCSVImportLimitsBeforeMutations(t *testing.T) { + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + mod, err := m.CreateModule(tenant, &store.Module{Type: "AS_PREFIXES", Name: "csv-limits"}) + if err != nil { + t.Fatal(err) + } + module := mod.ID + s := &Server{store: m} + before, err := m.ListASEntries(tenant, module) + if err != nil { + t.Fatal(err) + } + header := "asn,community\n" + validPrefix := header + "64512,\n" + oversized := validPrefix + strings.Repeat("\n", importer.MaxCSVBytes-len(validPrefix)) + "64513,\n" + for _, tc := range []struct { + name, body string + unknownLength bool + status int + }{ + {"exact", header + strings.Repeat("\n", importer.MaxCSVBytes-len(header)), false, http.StatusOK}, + {"oversize", oversized, false, http.StatusRequestEntityTooLarge}, + {"unknown_length", oversized, true, http.StatusRequestEntityTooLarge}, + {"rows", header + strings.Repeat("64512,\n", importer.MaxCSVRows+1), true, http.StatusRequestEntityTooLarge}, + {"fields", header + "64512," + strings.Repeat(",", importer.MaxCSVFields), false, http.StatusRequestEntityTooLarge}, + } { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/entries.csv", strings.NewReader(tc.body)) + if tc.unknownLength { + req.ContentLength = -1 + req.TransferEncoding = []string{"chunked"} + } + req.SetPathValue("module_id", module) + req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"})) + w := httptest.NewRecorder() + s.handleImportModuleEntriesCSV(w, req) + if w.Code != tc.status { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + after, e := m.ListASEntries(tenant, module) + if e != nil || len(after) != len(before) { + t.Fatalf("mutations on rejected/header-only request: count=%d err=%v", len(after), e) + } + }) + } +} + func TestModuleEntriesCSVImportExportIPRanges(t *testing.T) { srv, err := New(Options{ InsecureDev: true, diff --git a/internal/httpapi/routes_lookup.go b/internal/httpapi/routes_lookup.go index b3ecfce..4a83188 100644 --- a/internal/httpapi/routes_lookup.go +++ b/internal/httpapi/routes_lookup.go @@ -24,6 +24,11 @@ func (s *Server) handleLookup(w http.ResponseWriter, r *http.Request) { writeProblem(w, http.StatusBadRequest, "Bad Request", "query parameter q is required") return } + release, ok := s.acquireOperation(w, r, a.TenantID, lookupOperation) + if !ok { + return + } + defer release() res, err := lookup.Lookup(r.Context(), s.store, a.TenantID, q) if err != nil { if errors.Is(err, store.ErrInvalidInput) { diff --git a/internal/httpapi/routes_lookup_test.go b/internal/httpapi/routes_lookup_test.go index c1071a6..bee1bd9 100644 --- a/internal/httpapi/routes_lookup_test.go +++ b/internal/httpapi/routes_lookup_test.go @@ -1,6 +1,7 @@ package httpapi import ( + "context" "encoding/json" "io" "net/http" @@ -31,7 +32,7 @@ func TestLookupMembershipHTTP(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := srv.Store().SetModulePrefixSnapshot(tenant, modIP, "t", []store.PrefixRow{ + if err := srv.Store().SetModulePrefixSnapshot(context.Background(), tenant, modIP, "t", []store.PrefixRow{ {Prefix: "198.51.100.0/24", CommunityID: &cid, Source: "ip_range"}, }); err != nil { t.Fatal(err) diff --git a/internal/httpapi/routes_maintenance.go b/internal/httpapi/routes_maintenance.go index 0334a3d..b2989b6 100644 --- a/internal/httpapi/routes_maintenance.go +++ b/internal/httpapi/routes_maintenance.go @@ -242,10 +242,6 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request return } kind := "maintenance_policy_run" - if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) { - writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation") - return - } idem := strings.TrimSpace(r.Header.Get("Idempotency-Key")) var idemPtr *string if idem != "" { @@ -255,11 +251,18 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request if dryRun { title = "Maintenance policy dry-run" } - j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{ + if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindMaintenancePolicyRun, IdempotencyKey: idemPtr, Meta: map[string]any{"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title}}) { + return + } + if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) { + writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation") + return + } + j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{ "policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title, }) if err != nil { - writeInternalError(w, "maintenance_policy_enqueue", err) + writeJobEnqueueError(w, err) return } w.Header().Set("Location", "/v1/jobs/"+j.ID) diff --git a/internal/httpapi/routes_postgres_maintenance.go b/internal/httpapi/routes_postgres_maintenance.go index f0d9eea..7ee0ad5 100644 --- a/internal/httpapi/routes_postgres_maintenance.go +++ b/internal/httpapi/routes_postgres_maintenance.go @@ -11,6 +11,7 @@ import ( "evobgp/internal/jobs" "evobgp/internal/pgmonitor" + "evobgp/internal/store" ) var ( @@ -68,19 +69,22 @@ func (s *Server) enqueuePostgresMaint(w http.ResponseWriter, r *http.Request, a if !s.requirePostgres(w) || !s.requireOperatorStrict(w, a) { return } - if !s.checkPgMaintRateLimit(a.TenantID, kind) { - writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation") - return - } idem := strings.TrimSpace(r.Header.Get("Idempotency-Key")) var idemPtr *string if idem != "" { idemPtr = &idem } meta["actor_prefix"] = actorPrefix(a) - j, _, err := s.jobs.Enqueue(a.TenantID, kind, idemPtr, nil, meta) + if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: kind, IdempotencyKey: idemPtr, Meta: meta}) { + return + } + if !s.checkPgMaintRateLimit(a.TenantID, kind) { + writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation") + return + } + j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, kind, idemPtr, nil, meta) if err != nil { - writeInternalError(w, "postgres_maint_enqueue", err) + writeJobEnqueueError(w, err) return } w.Header().Set("Location", "/v1/jobs/"+j.ID) @@ -184,12 +188,12 @@ func (s *Server) handlePostgresCleanup(w http.ResponseWriter, r *http.Request) { if idem != "" { idemPtr = &idem } - j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{ + j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{ "policy_id": policyID, "dry_run": body.DryRun, "actor_prefix": actorPrefix(a), "job_title": "PostgreSQL cleanup (deprecated path)", }) if err != nil { - writeInternalError(w, "postgres_maint_enqueue", err) + writeJobEnqueueError(w, err) return } w.Header().Set("Location", "/v1/jobs/"+j.ID) diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index a724471..0c30354 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -8,15 +8,17 @@ import ( "errors" "net/http" "strings" + "sync" "time" "evobgp/internal/audit" "evobgp/internal/jobs" + "evobgp/internal/logging" "evobgp/internal/maintenance" "evobgp/internal/pgmonitor" - "evobgp/internal/repository" "evobgp/internal/runtimelogs" "evobgp/internal/store" + "evobgp/internal/telemetry" "github.com/jackc/pgx/v5/pgxpool" ) @@ -30,7 +32,11 @@ type Server struct { maintStats *maintenance.DBStatsProvider jobs *jobs.Registry authLimiter *authRateLimiter + operationOnce sync.Once + operationLimiter *operationLimiter bundlePriv ed25519.PrivateKey + bundleCache *nodeBundleCache + nodeBundleBuild nodeBundleBuilder keyResolver *apiKeyResolver firewallResolver *firewallTokenResolver bundleSeedHex string @@ -39,6 +45,11 @@ type Server struct { runtimeLogs *runtimelogs.Service runtimeLogsPolicyTenant string mux *http.ServeMux + backgroundMu sync.Mutex + backgroundStarted bool + backgroundClosed bool + backgroundCancel context.CancelFunc + telemetryCollector *telemetry.Collector // Portal / dual-auth (JWT) configuration. jwtSecret string @@ -118,6 +129,7 @@ func New(opts Options) (*Server, error) { maintStats: maintStats, jobs: reg, bundlePriv: priv, + bundleCache: newNodeBundleCache(nodeBundleCacheMaxEntries, nodeBundleCacheMaxBytes), keyResolver: resolver, firewallResolver: fwResolver, bundleSeedHex: strings.TrimSpace(opts.BundleSeedHex), @@ -143,6 +155,29 @@ func New(opts Options) (*Server, error) { // Close releases database resources. func (s *Server) Close() { + if s == nil { + return + } + s.backgroundMu.Lock() + if s.backgroundClosed { + s.backgroundMu.Unlock() + return + } + s.backgroundClosed = true + cancel := s.backgroundCancel + collector := s.telemetryCollector + s.backgroundMu.Unlock() + if cancel != nil { + cancel() + } + if collector != nil { + _ = collector.Close(context.Background()) + } + // Do not close the pool while a worker is still persisting its final state. + if s.jobs != nil { + s.jobs.RequestCancelAll() + _ = s.jobs.Drain(context.Background()) + } if s.pgPool != nil { s.pgPool.Close() } @@ -154,26 +189,44 @@ func (s *Server) Store() store.Backend { return s.store } // Jobs exposes the in-process async job registry (for scheduler / evobgp-all). func (s *Server) Jobs() *jobs.Registry { return s.jobs } -// StartBackground starts PostgreSQL monitoring and maintenance schedulers until ctx is cancelled. +// StartBackground starts shared schedulers and telemetry polling once until ctx is cancelled. func (s *Server) StartBackground(ctx context.Context) { - if s != nil && s.pgPool != nil { - pgmonitor.StartScheduler(ctx, s.pgPool) + if s == nil { + return + } + s.backgroundMu.Lock() + defer s.backgroundMu.Unlock() + if s.backgroundClosed || s.backgroundStarted { + return + } + s.backgroundStarted = true + backgroundCtx, cancel := context.WithCancel(ctx) + s.backgroundCancel = cancel + if s.store != nil { + s.telemetryCollector = telemetry.Start(backgroundCtx, s.store, telemetry.ConfigFromEnv()) + } + if s.pgPool != nil { + pgmonitor.StartScheduler(backgroundCtx, s.pgPool) if s.jobs != nil { - jobs.StartDurableQueueLoop(ctx, s.jobs, repository.NewJobAuditWriter(s.pgPool)) + jobs.StartDurableQueueLoop(backgroundCtx, s.jobs) + jobs.StartDeploymentReconciliationLoop(backgroundCtx, s.jobs, s.store) } } if s != nil && s.maintConfig != nil && s.jobs != nil { - maintenance.StartScheduler(ctx, s.maintConfig, func(policyID string, dryRun bool, idem string) { + maintenance.StartScheduler(backgroundCtx, s.maintConfig, func(policyID string, dryRun bool, idem string) { key := idem - _, _, _ = s.jobs.Enqueue("", jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{ + _, _, err := s.jobs.EnqueueContext(backgroundCtx, store.SystemJobTenantID, jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{ "policy_id": policyID, "dry_run": dryRun, "trigger": "scheduler", }) + if err != nil { + logging.With("component", "maintenance.scheduler").Error("enqueue failed", "policy_id", policyID, "err", err) + } }, 30*time.Second) } if s != nil && s.runtimeLogs != nil && s.store != nil { - runtimelogs.StartAutoCleanupScheduler(ctx, runtimelogs.SchedulerDeps{ + runtimelogs.StartAutoCleanupScheduler(backgroundCtx, runtimelogs.SchedulerDeps{ Service: s.runtimeLogs, Store: s.store, PolicyTenant: s.runtimeLogsPolicyTenant, diff --git a/internal/httpapi/speakers.go b/internal/httpapi/speakers.go index de917c5..089315d 100644 --- a/internal/httpapi/speakers.go +++ b/internal/httpapi/speakers.go @@ -6,7 +6,11 @@ import ( "encoding/hex" "encoding/json" "errors" + "net" "net/http" + "net/netip" + "net/url" + "strconv" "strings" "time" @@ -165,24 +169,65 @@ func publicControlPlaneURL(r *http.Request, override string) string { if s := strings.TrimSpace(override); s != "" { return strings.TrimRight(s, "/") } - if origin := strings.TrimSpace(r.Header.Get("Origin")); strings.HasPrefix(origin, "http://") || strings.HasPrefix(origin, "https://") { - return strings.TrimRight(origin, "/") - } - proto := strings.TrimSpace(r.Header.Get("X-Forwarded-Proto")) - if proto == "" { - proto = "https" - } - host := strings.TrimSpace(r.Header.Get("X-Forwarded-Host")) - if i := strings.Index(host, ","); i >= 0 { - host = strings.TrimSpace(host[:i]) - } - if host == "" { - host = strings.TrimSpace(r.Host) - } - if host == "" { + if r == nil { return "" } - return proto + "://" + host + if peer, ok := peerAddress(r); ok && isTrustedProxy(peer, trustedProxyPrefixes()) { + protos := r.Header.Values("X-Forwarded-Proto") + hosts := r.Header.Values("X-Forwarded-Host") + if len(protos) == 1 && len(hosts) == 1 { + proto := strings.TrimSpace(protos[0]) + host := strings.TrimSpace(hosts[0]) + if (proto == "http" || proto == "https") && validControlPlaneHost(host) { + return proto + "://" + host + } + } + } + if !validControlPlaneHost(r.Host) { + return "" + } + proto := "http" + if r.TLS != nil { + proto = "https" + } + return proto + "://" + r.Host +} + +func validControlPlaneHost(host string) bool { + if host == "" || strings.TrimSpace(host) != host || strings.HasSuffix(host, ":") || strings.ContainsAny(host, " ,/@?#\\\t\r\n") { + return false + } + u, err := url.Parse("http://" + host) + if err != nil || u.Host != host || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" { + return false + } + name := u.Hostname() + if name == "" { + return false + } + if strings.HasPrefix(host, "[") { + if _, err := netip.ParseAddr(name); err != nil { + return false + } + } else if net.ParseIP(name) == nil { + for _, label := range strings.Split(name, ".") { + if label == "" || label[0] == '-' || label[len(label)-1] == '-' { + return false + } + for _, c := range label { + if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') { + return false + } + } + } + } + if port := u.Port(); port != "" { + n, err := strconv.Atoi(port) + if err != nil || n < 1 || n > 65535 { + return false + } + } + return true } func (s *Server) attachReplicaInstall(resp map[string]any, sp *store.Speaker, nodeToken string, req speakerCreateRequest, r *http.Request) { diff --git a/internal/httpapi/speakers_live.go b/internal/httpapi/speakers_live.go index 3ecd82c..6680fdc 100644 --- a/internal/httpapi/speakers_live.go +++ b/internal/httpapi/speakers_live.go @@ -1,9 +1,7 @@ package httpapi import ( - "context" "strings" - "sync" "time" "evobgp/internal/birdfmt" @@ -11,6 +9,34 @@ import ( "evobgp/internal/store" ) +type agentHealthPoll struct { + result nodedispatch.AgentHealthResult + at time.Time + lastSuccessAt time.Time + observationOverride *liveObservation +} + +func agentHealthObservation(poll *agentHealthPoll) liveObservation { + if poll == nil { + return liveObservation{Source: "none", Quality: "unknown", ErrorCode: "not_polled"} + } + if poll.observationOverride != nil { + return *poll.observationOverride + } + o := liveObservation{ObservedAt: liveTime(poll.at), ReceivedAt: liveTime(poll.at), LastSuccessAt: liveTime(poll.lastSuccessAt), Source: "agent_health", Quality: "ok"} + if !poll.result.OK { + o.Quality = "unreachable" + if !poll.lastSuccessAt.IsZero() { + o.Quality = "stale" + } + o.ErrorCode = liveErrorCode(poll.result.Error) + if poll.result.Error == "agent reported ok=false" { + o.ErrorCode = "reported_unhealthy" + } + } + return o +} + func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) { total = len(sessions) for _, s := range sessions { @@ -22,29 +48,48 @@ func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) { } func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *nodedispatch.AgentHealthResult) map[string]any { + var poll *agentHealthPoll + if health != nil { + poll = &agentHealthPoll{result: *health} + } + return speakerLiveStatusWithPoll(sp, view, poll) +} + +func speakerLiveStatusWithPoll(sp *store.Speaker, view speakerBGPLive, health *agentHealthPoll) map[string]any { total, established := countBGPSessions(view.Sessions) + bgpObservation := view.observation() + agentObservation := agentHealthObservation(health) + if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") && health == nil { + agentObservation = bgpObservation + } m := map[string]any{ "label": view.Label, - "bgp_poll_ok": view.Error == "", - "bgp_sessions_total": total, - "bgp_established": established, + "bgp_poll_ok": view.successful(), + "bgp_sessions_total": nil, + "bgp_established": nil, + "bgp_observation": bgpObservation, + "agent_observation": agentObservation, + } + if view.successful() { + m["bgp_sessions_total"] = total + m["bgp_established"] = established } if view.Error != "" { m["bgp_poll_error"] = view.Error } if health != nil { - m["agent_ok"] = health.OK - if health.Error != "" { - m["agent_error"] = health.Error + m["agent_ok"] = health.result.OK + if health.result.Error != "" { + m["agent_error"] = health.result.Error } - if health.LastSyncAt != "" { - m["agent_last_sync_at"] = health.LastSyncAt + if health.result.LastSyncAt != "" { + m["agent_last_sync_at"] = health.result.LastSyncAt } - if health.LastAppliedRevisionID != "" { - m["agent_last_applied_revision_id"] = health.LastAppliedRevisionID + if health.result.LastAppliedRevisionID != "" { + m["agent_last_applied_revision_id"] = health.result.LastAppliedRevisionID } } else if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") { - m["agent_ok"] = view.Error == "" + m["agent_ok"] = view.successful() if view.Error != "" { m["agent_error"] = view.Error } @@ -52,7 +97,7 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded m["agent_ok"] = false m["agent_error"] = "agent health not polled" } - if len(view.Sessions) > 0 { + if view.successful() { sess := make([]map[string]any, 0, len(view.Sessions)) for _, s := range view.Sessions { row := map[string]any{ @@ -68,62 +113,3 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded } return m } - -func (s *Server) collectSpeakerLiveStatus(ctx context.Context, tenantID string, fresh bool, speakers []*store.Speaker) map[string]map[string]any { - views := s.collectSpeakerBGPLive(ctx, tenantID, fresh) - viewByID := make(map[string]speakerBGPLive, len(views)) - for _, v := range views { - if v.SpeakerID != "" { - viewByID[v.SpeakerID] = v - } - } - - opts := nodedispatch.Options{Timeout: 8 * time.Second} - type healthWrap struct { - id string - h nodedispatch.AgentHealthResult - } - healthCh := make(chan healthWrap, len(speakers)) - var wg sync.WaitGroup - for _, sp := range speakers { - if sp == nil { - continue - } - meta := store.ParseSpeakerMeta(sp.MetaJSON) - if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) { - continue - } - wg.Add(1) - go func(speaker *store.Speaker) { - defer wg.Done() - healthCh <- healthWrap{ - id: speaker.ID, - h: nodedispatch.FetchAgentHealth(ctx, speaker, opts), - } - }(sp) - } - wg.Wait() - close(healthCh) - healthByID := make(map[string]nodedispatch.AgentHealthResult, len(speakers)) - for hw := range healthCh { - healthByID[hw.id] = hw.h - } - - out := make(map[string]map[string]any, len(speakers)) - for _, sp := range speakers { - if sp == nil { - continue - } - view, ok := viewByID[sp.ID] - if !ok { - view = speakerBGPLive{SpeakerID: sp.ID, Label: speakerDisplayLabel(sp)} - } - var hp *nodedispatch.AgentHealthResult - if h, ok := healthByID[sp.ID]; ok { - hCopy := h - hp = &hCopy - } - out[sp.ID] = speakerLiveStatusJSON(sp, view, hp) - } - return out -} diff --git a/internal/httpapi/speakers_live_test.go b/internal/httpapi/speakers_live_test.go index a620acf..34a403b 100644 --- a/internal/httpapi/speakers_live_test.go +++ b/internal/httpapi/speakers_live_test.go @@ -2,12 +2,29 @@ package httpapi import ( "testing" + "time" "evobgp/internal/birdfmt" "evobgp/internal/nodedispatch" "evobgp/internal/store" ) +func TestSpeakerLiveStatusJSONEmptyAndUnpolled(t *testing.T) { + sp := &store.Speaker{ID: "master", Role: "master"} + unpolled := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID}, nil) + if unpolled["agent_ok"] != false || unpolled["bgp_poll_ok"] != false || unpolled["bgp_sessions_total"] != nil { + t.Fatalf("unpolled=%#v", unpolled) + } + if unpolled["bgp_observation"].(liveObservation).Quality != "unknown" { + t.Fatalf("unpolled observation=%#v", unpolled["bgp_observation"]) + } + now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC) + empty := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID, Polled: true, ObservedAt: now, ReceivedAt: now, LastSuccessAt: now, Source: "cp_birdc"}, nil) + if empty["agent_ok"] != true || empty["bgp_poll_ok"] != true || empty["bgp_sessions_total"] != 0 { + t.Fatalf("empty=%#v", empty) + } +} + func TestCountBGPSessions(t *testing.T) { total, est := countBGPSessions([]birdfmt.BGPSession{ {Name: "p1", State: "Established"}, diff --git a/internal/httpapi/telemetry.go b/internal/httpapi/telemetry.go new file mode 100644 index 0000000..eeb5189 --- /dev/null +++ b/internal/httpapi/telemetry.go @@ -0,0 +1,154 @@ +package httpapi + +import ( + "context" + "net/http" + "time" + + "evobgp/internal/nodedispatch" + "evobgp/internal/store" + "evobgp/internal/telemetry" +) + +func (s *Server) loadTelemetry(ctx context.Context, tenant string, speakers []*store.Speaker) ([]speakerBGPLive, map[string]map[string]any, error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + snapshots, err := s.store.ListTelemetry(ctx, tenant) + if err != nil { + return nil, nil, err + } + views, statuses := hydrateTelemetry(speakers, snapshots, time.Now(), telemetry.ConfigFromEnv().StaleAfter) + return views, statuses, nil +} + +// snapshotObservation uses control-plane receipt time, never the remote clock. +func snapshotObservation(o store.TelemetryObservation, now time.Time, staleAfter time.Duration, invalid string) liveObservation { + v := liveObservation{ObservedAt: liveTime(o.ObservedAt), ReceivedAt: liveTime(o.ReceivedAt), LastSuccessAt: liveTime(o.LastSuccessAt), Source: o.Source, Quality: "unknown"} + if !o.ReceivedAt.IsZero() { + v.ExpiresAt = liveTime(o.ReceivedAt.Add(staleAfter)) + } + if v.Source == "" { + v.Source = "none" + } + switch { + case invalid != "": + v.ErrorCode = invalid + case o.ReceivedAt.IsZero(): + v.ErrorCode = "not_polled" + case o.ReceivedAt.After(now) || now.Sub(o.ReceivedAt) > staleAfter: + v.Quality, v.ErrorCode = "stale", "snapshot_expired" + case o.Error != "": + v.Quality, v.ErrorCode = "unreachable", liveErrorCode(o.Error) + if !o.LastSuccessAt.IsZero() { + v.Quality = "stale" + } + default: + v.Quality = "ok" + } + return v +} + +func hydrateTelemetry(speakers []*store.Speaker, snapshots []store.TelemetrySnapshot, now time.Time, staleAfter time.Duration) ([]speakerBGPLive, map[string]map[string]any) { + byID := make(map[string]*store.Speaker, len(speakers)) + for _, sp := range speakers { + if sp != nil { + byID[sp.ID] = sp + } + } + grouped := make(map[string][]store.TelemetrySnapshot) + for _, snapshot := range snapshots { + if snapshot.SpeakerID != "" && byID[snapshot.SpeakerID] == nil { + continue + } + if snapshot.SpeakerID == "" && snapshot.Kind != "local_bird" { + continue + } + grouped[snapshot.SpeakerID] = append(grouped[snapshot.SpeakerID], snapshot) + } + views := make([]speakerBGPLive, 0, len(speakers)+1) + statuses := make(map[string]map[string]any, len(speakers)) + appendView := func(id string, sp *store.Speaker) { + view := speakerBGPLive{SpeakerID: id, Label: speakerDisplayLabel(sp)} + if sp == nil { + view.Label = "CP (local)" + } + var health *agentHealthPoll + rows := grouped[id] + // A restarted local collector may leave an expired authority behind. + // Only simultaneously current authorities conflict; never prefer one + // of two current BIRD nodes arbitrarily. + if len(rows) > 1 { + current := make([]store.TelemetrySnapshot, 0, len(rows)) + for _, row := range rows { + if row.Kind == "local_bird" && row.ConfigFingerprint == store.TelemetrySpeakerFingerprint(sp) && !row.BGP.ReceivedAt.IsZero() && !row.BGP.ReceivedAt.After(now) && now.Sub(row.BGP.ReceivedAt) <= staleAfter { + current = append(current, row) + } + } + if len(current) > 0 { + rows = current + } + } + if len(rows) > 1 { + view.Error = "ambiguous telemetry authorities" + view.ObservationOverride = &liveObservation{Source: "none", Quality: "unknown", ErrorCode: "ambiguous_authority"} + } else if len(rows) == 1 { + row := rows[0] + invalid := "" + if row.ConfigFingerprint != store.TelemetrySpeakerFingerprint(sp) { + invalid = "configuration_changed" + } + bgp := snapshotObservation(row.BGP.TelemetryObservation, now, staleAfter, invalid) + view.ObservationOverride = &bgp + view.Polled = !row.BGP.ReceivedAt.IsZero() + view.ObservedAt, view.ReceivedAt, view.LastSuccessAt, view.Source = row.BGP.ObservedAt, row.BGP.ReceivedAt, row.BGP.LastSuccessAt, row.BGP.Source + if bgp.Quality == "ok" { + view.Sessions = row.BGP.Sessions + } else { + view.Error = bgp.ErrorCode + } + if row.Kind == "remote_agent" { + h := snapshotObservation(row.Health.TelemetryObservation, now, staleAfter, invalid) + result := nodedispatch.AgentHealthResult{OK: row.Health.OK && h.Quality == "ok", Error: row.Health.Error, LastSyncAt: row.Health.LastSyncAt, LastAppliedRevisionID: row.Health.LastAppliedRevisionID} + if h.Quality != "ok" { + result.Error = h.ErrorCode + } + if h.Quality == "ok" && !result.OK { + h.Quality, h.ErrorCode = "unreachable", "reported_unhealthy" + } + health = &agentHealthPoll{result: result, observationOverride: &h} + } + } + views = append(views, view) + if sp != nil { + statuses[id] = speakerLiveStatusWithPoll(sp, view, health) + } + } + for _, sp := range speakers { + if sp != nil { + appendView(sp.ID, sp) + } + } + if len(grouped[""]) > 0 { + appendView("", nil) + } + return views, statuses +} + +func (s *Server) handleTelemetryRefresh(w http.ResponseWriter, r *http.Request) { + a, ok := authFromContext(r.Context()) + if !ok { + writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth") + return + } + if !s.requirePerm(w, a, "bgp:network:read") { + return + } + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + requested, err := s.store.RequestTelemetryRefresh(ctx, a.TenantID, 10*time.Second) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry refresh unavailable") + return + } + writeJSON(w, http.StatusAccepted, map[string]any{"requested": requested}) +} diff --git a/internal/httpapi/telemetry_http_test.go b/internal/httpapi/telemetry_http_test.go new file mode 100644 index 0000000..497f9a8 --- /dev/null +++ b/internal/httpapi/telemetry_http_test.go @@ -0,0 +1,126 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "evobgp/internal/nodedispatch" + "evobgp/internal/store" +) + +type telemetryHTTPStore struct { + store.Backend + discoveryWrites int + snapshotErr error + refreshCalls int +} + +func (s *telemetryHTTPStore) UpsertPeerDiscovery(tenant string, in *store.PeerDiscoveryUpsert) (*store.BGPPeerDiscovery, error) { + s.discoveryWrites++ + return s.Backend.UpsertPeerDiscovery(tenant, in) +} +func (s *telemetryHTTPStore) ListTelemetry(ctx context.Context, tenant string) ([]store.TelemetrySnapshot, error) { + if s.snapshotErr != nil { + return nil, s.snapshotErr + } + return s.Backend.ListTelemetry(ctx, tenant) +} +func (s *telemetryHTTPStore) RequestTelemetryRefresh(ctx context.Context, tenant string, cooldown time.Duration) (bool, error) { + s.refreshCalls++ + return s.Backend.RequestTelemetryRefresh(ctx, tenant, cooldown) +} +func telemetryRequest(method, path, tenant string) *http.Request { + r := httptest.NewRequest(method, path, nil) + return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "viewer"})) +} + +func TestTelemetryGETsAndRefreshNeverPollAgents(t *testing.T) { + var calls atomic.Int64 + agent := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + _, _ = w.Write([]byte(`{"ok":true,"sessions":[]}`)) + })) + defer agent.Close() + t.Setenv("EVOBGP_NODE_DISPATCH_INSECURE_TLS", "1") + mem := store.NewMemory() + mem.SeedDemo() + tenant, _, _, _, _ := mem.DemoIDs() + sp, err := mem.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: strings.TrimPrefix(agent.URL, "https://"), AgentSecret: "test"})}) + if err != nil { + t.Fatal(err) + } + // Positive control: the legacy adapter reaches the fake configured agent. + if result := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: agent.Client()}); !result.OK { + t.Fatal(result) + } + calls.Store(0) + st := &telemetryHTTPStore{Backend: mem} + server := &Server{store: st} + claims, err := mem.ClaimTelemetry(context.Background(), "collector", 1, time.Minute, "") + if err != nil || len(claims) != 1 { + t.Fatalf("claim: %v %v", claims, err) + } + for i := 0; i < 20; i++ { + for _, tc := range []struct { + path string + handler http.HandlerFunc + }{ + {"/v1/peers", server.handleListPeers}, {"/v1/peers?live=1", server.handleListPeers}, + {"/v1/speakers?live=1", server.handleListSpeakers}, {"/v1/peers/discovered?status=all", server.handleListPeerDiscoveries}, + } { + w := httptest.NewRecorder() + tc.handler(w, telemetryRequest(http.MethodGet, tc.path, tenant)) + if w.Code != http.StatusOK { + t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String()) + } + } + w := httptest.NewRecorder() + server.handleTelemetryRefresh(w, telemetryRequest(http.MethodPost, "/v1/telemetry/refresh", tenant)) + if w.Code != http.StatusAccepted { + t.Fatalf("refresh: %d %s", w.Code, w.Body.String()) + } + want := `"requested":false` + if i == 0 { + want = `"requested":true` + } + if !strings.Contains(w.Body.String(), want) { + t.Fatalf("refresh coalescing: %s", w.Body.String()) + } + } + if calls.Load() != 0 || st.discoveryWrites != 0 { + t.Fatalf("GET/refresh caused IO: agent=%d discovery=%d", calls.Load(), st.discoveryWrites) + } +} + +func TestTelemetryReadFailureAndRefreshPermission(t *testing.T) { + mem := store.NewMemory() + mem.SeedDemo() + tenant, _, _, _, _ := mem.DemoIDs() + st := &telemetryHTTPStore{Backend: mem, snapshotErr: errors.New("private database failure")} + server := &Server{store: st} + for _, handler := range []http.HandlerFunc{server.handleListPeers, server.handleListSpeakers} { + w := httptest.NewRecorder() + handler(w, telemetryRequest(http.MethodGet, "/v1/resource?live=1", tenant)) + if w.Code != http.StatusServiceUnavailable || strings.Contains(w.Body.String(), "private") { + t.Fatalf("snapshot failure: %d %s", w.Code, w.Body.String()) + } + } + r := httptest.NewRequest(http.MethodPost, "/v1/telemetry/refresh", nil) + w := httptest.NewRecorder() + server.handleTelemetryRefresh(w, r) + if w.Code != http.StatusUnauthorized || st.refreshCalls != 0 { + t.Fatal("unauthenticated refresh admitted") + } + r = r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Kind: AuthKindJWT})) + w = httptest.NewRecorder() + server.handleTelemetryRefresh(w, r) + if w.Code != http.StatusForbidden || st.refreshCalls != 0 { + t.Fatal("unprivileged refresh admitted") + } +} diff --git a/internal/httpapi/telemetry_lifecycle_test.go b/internal/httpapi/telemetry_lifecycle_test.go new file mode 100644 index 0000000..c8fe693 --- /dev/null +++ b/internal/httpapi/telemetry_lifecycle_test.go @@ -0,0 +1,30 @@ +package httpapi + +import ( + "context" + "testing" +) + +func TestTelemetryBackgroundStartsOnceAndStopsBeforeClose(t *testing.T) { + s, err := New(Options{SeedDemo: true}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s.StartBackground(ctx) + first := s.telemetryCollector + if first == nil { + t.Fatal("telemetry collector not started") + } + s.StartBackground(ctx) + if s.telemetryCollector != first { + t.Fatal("duplicate telemetry collector") + } + s.Close() + s.StartBackground(ctx) + if s.telemetryCollector != first { + t.Fatal("collector started after close") + } + s.Close() +} diff --git a/internal/httpapi/telemetry_test.go b/internal/httpapi/telemetry_test.go new file mode 100644 index 0000000..fa5b558 --- /dev/null +++ b/internal/httpapi/telemetry_test.go @@ -0,0 +1,102 @@ +package httpapi + +import ( + "evobgp/internal/birdfmt" + "evobgp/internal/store" + "testing" + "time" +) + +func TestTelemetryHydrationFreshnessAndConfiguration(t *testing.T) { + now := time.Now().UTC() + sp := &store.Speaker{ID: "replica", Role: "replica", Endpoint: "old"} + observation := store.TelemetryObservation{ObservedAt: now.Add(-time.Hour), ReceivedAt: now, LastSuccessAt: now, Source: "agent"} + base := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: observation, Sessions: []birdfmt.BGPSession{{Name: "peer", State: "Established"}}}, Health: store.TelemetryHealth{TelemetryObservation: observation, OK: true}} + cases := []struct { + name string + mutate func(*store.TelemetrySnapshot) + bgp, health bool + quality string + }{ + {"fresh receipt despite remote clock", func(*store.TelemetrySnapshot) {}, true, true, "ok"}, + {"expired BGP independent of health", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(-2 * time.Minute) }, false, true, "stale"}, + {"expired health independent of BGP", func(s *store.TelemetrySnapshot) { s.Health.ReceivedAt = now.Add(-2 * time.Minute) }, true, false, "ok"}, + {"future receipt is not fresh", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(time.Hour) }, false, true, "stale"}, + {"configuration changed", func(s *store.TelemetrySnapshot) { s.ConfigFingerprint = "old" }, false, false, "unknown"}, + {"failed BGP", func(s *store.TelemetrySnapshot) { s.BGP.Error = "timeout" }, false, true, "stale"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + snapshot := base + tc.mutate(&snapshot) + views, statuses := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{snapshot}, now, time.Minute) + status := statuses[sp.ID] + if status["bgp_poll_ok"] != tc.bgp || status["agent_ok"] != tc.health || views[0].observation().Quality != tc.quality { + t.Fatalf("status=%+v observation=%+v", status, views[0].observation()) + } + if !tc.bgp && (status["bgp_established"] != nil || len(views[0].Sessions) != 0) { + t.Fatal("stale BGP exposed live sessions") + } + }) + } +} + +func TestTelemetryHydrationMissingAndAmbiguousLocal(t *testing.T) { + now := time.Now() + sp := &store.Speaker{ID: "master", Role: "master"} + for _, rows := range [][]store.TelemetrySnapshot{nil, {{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "a"}, {SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "b"}}} { + views, statuses := hydrateTelemetry([]*store.Speaker{sp}, rows, now, time.Minute) + if views[0].observation().Quality != "unknown" || statuses[sp.ID]["agent_ok"] == true || statuses[sp.ID]["bgp_poll_ok"] == true { + t.Fatalf("unexpected healthy master: %+v", statuses) + } + } +} + +func TestTelemetryHydrationLocalAuthorityRestart(t *testing.T) { + now := time.Now() + sp := &store.Speaker{ID: "master", Role: "master"} + fresh := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "new", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now}}} + old := fresh + old.LocalAuthority = "old" + old.BGP.ReceivedAt = now.Add(-2 * time.Minute) + _, status := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute) + if status[sp.ID]["bgp_poll_ok"] != true { + t.Fatal("expired authority hides current local snapshot") + } + old.BGP.ReceivedAt = now + _, status = hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute) + if status[sp.ID]["bgp_poll_ok"] != false { + t.Fatal("two current local authorities are ambiguous") + } +} + +func TestTelemetryPeerObservationPreservesUnknownConfiguration(t *testing.T) { + now := time.Now() + sp := &store.Speaker{ID: "replica", Role: "replica"} + row := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: "previous", BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now, LastSuccessAt: now}}} + views, _ := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{row}, now, time.Minute) + observation := peerLiveObservation(views) + if observation.Quality != "unknown" || observation.ErrorCode != "configuration_changed" { + t.Fatalf("changed config is not a failed current poll: %+v", observation) + } +} + +func TestTelemetryExpiryUsesConfiguredIntervalAndEarliestSource(t *testing.T) { + now := time.Now().UTC() + old := now.Add(-30 * time.Second) + a := snapshotObservation(store.TelemetryObservation{ReceivedAt: old}, now, 5*time.Minute, "") + b := snapshotObservation(store.TelemetryObservation{ReceivedAt: now}, now, 5*time.Minute, "") + want := old.Add(5 * time.Minute).Format(time.RFC3339Nano) + if a.ExpiresAt == nil || *a.ExpiresAt != want { + t.Fatalf("expiry ignores configured interval: %+v", a) + } + views := []speakerBGPLive{{Polled: true, ObservationOverride: &b}, {Polled: true, ObservationOverride: &a}} + combined := peerLiveObservation(views) + if combined.ExpiresAt == nil || *combined.ExpiresAt != want { + t.Fatalf("aggregate extends oldest source: %+v", combined) + } + missing := snapshotObservation(store.TelemetryObservation{}, now, time.Minute, "") + if missing.ExpiresAt != nil { + t.Fatal("unobserved source has expiry") + } +} diff --git a/internal/httpapi/trusted_proxy.go b/internal/httpapi/trusted_proxy.go new file mode 100644 index 0000000..e06c9d5 --- /dev/null +++ b/internal/httpapi/trusted_proxy.go @@ -0,0 +1,91 @@ +package httpapi + +import ( + "net" + "net/http" + "net/netip" + "os" + "strings" +) + +// trustedProxyPrefixes returns no trusted peers when any configured CIDR is invalid. +// An empty configuration deliberately treats every connection as direct. +func trustedProxyPrefixes() []netip.Prefix { + raw := strings.TrimSpace(os.Getenv("EVOBGP_TRUSTED_PROXY_CIDRS")) + if raw == "" { + return nil + } + parts := strings.Split(raw, ",") + prefixes := make([]netip.Prefix, 0, len(parts)) + for _, part := range parts { + prefix, err := netip.ParsePrefix(strings.TrimSpace(part)) + if err != nil { + return nil + } + if prefix.Addr().Is4In6() { + if prefix.Bits() < 96 { + return nil + } + prefix = netip.PrefixFrom(prefix.Addr().Unmap(), prefix.Bits()-96) + } + prefixes = append(prefixes, prefix.Masked()) + } + return prefixes +} + +func peerAddress(r *http.Request) (netip.Addr, bool) { + if r == nil { + return netip.Addr{}, false + } + host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr)) + if err != nil { + host = strings.TrimSpace(r.RemoteAddr) + } + ip, err := netip.ParseAddr(host) + if err != nil { + return netip.Addr{}, false + } + return ip.WithZone("").Unmap(), true +} + +func isTrustedProxy(ip netip.Addr, prefixes []netip.Prefix) bool { + for _, prefix := range prefixes { + if prefix.Contains(ip) || prefix.Contains(ip.Unmap()) { + return true + } + } + return false +} + +func clientIPWithTrustedProxies(r *http.Request, prefixes []netip.Prefix) string { + peer, ok := peerAddress(r) + if !ok { + if r == nil { + return "" + } + return strings.TrimSpace(r.RemoteAddr) + } + if !isTrustedProxy(peer, prefixes) { + return peer.String() + } + values := r.Header.Values("X-Forwarded-For") + if len(values) != 1 || strings.TrimSpace(values[0]) == "" { + return peer.String() + } + hops := strings.Split(values[0], ",") + current := peer + for i := len(hops) - 1; i >= 0; i-- { + if !isTrustedProxy(current, prefixes) { + return current.String() + } + next, err := netip.ParseAddr(strings.TrimSpace(hops[i])) + if err != nil { + return peer.String() + } + if next.Zone() != "" { + return peer.String() + } + current = next.Unmap() + } + return current.String() +} diff --git a/internal/httpapi/trusted_proxy_test.go b/internal/httpapi/trusted_proxy_test.go new file mode 100644 index 0000000..35ac8f6 --- /dev/null +++ b/internal/httpapi/trusted_proxy_test.go @@ -0,0 +1,129 @@ +package httpapi + +import ( + "fmt" + "net/http/httptest" + "net/netip" + "testing" + "time" +) + +func TestClientIPTrustedProxyChain(t *testing.T) { + prefixes := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")} + tests := []struct { + name string + remote string + xff string + want string + }{ + {"untrusted peer ignores spoof", "198.51.100.10:1234", "203.0.113.9", "198.51.100.10"}, + {"rightmost untrusted hop", "10.0.0.2:1234", "203.0.113.9, 198.51.100.8", "198.51.100.8"}, + {"trusted chain", "10.0.0.2:1234", "203.0.113.9, 10.0.0.3", "203.0.113.9"}, + {"mapped peer", "[::ffff:10.0.0.2]:1234", "203.0.113.9", "203.0.113.9"}, + {"mapped forwarded client", "10.0.0.2:1234", "::ffff:203.0.113.9", "203.0.113.9"}, + {"malformed nearest hop", "10.0.0.2:1234", "203.0.113.9, bogus", "10.0.0.2"}, + {"malformed farther hop ignored", "10.0.0.2:1234", "bogus, 198.51.100.8", "198.51.100.8"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := httptest.NewRequest("GET", "http://example.test/", nil) + r.RemoteAddr = tt.remote + r.Header.Set("X-Forwarded-For", tt.xff) + if got := clientIPWithTrustedProxies(r, prefixes); got != tt.want { + t.Fatalf("client IP = %q, want %q", got, tt.want) + } + }) + } +} + +func TestClientIPConfigurationFailsClosed(t *testing.T) { + r := httptest.NewRequest("GET", "http://example.test/", nil) + r.RemoteAddr = "10.0.0.2:1234" + r.Header.Set("X-Forwarded-For", "203.0.113.9") + r.Header.Set("X-Real-IP", "203.0.113.8") + if got := clientIPWithTrustedProxies(r, nil); got != "10.0.0.2" { + t.Fatalf("direct client IP = %q", got) + } + t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8, invalid") + if got := clientIP(r); got != "10.0.0.2" { + t.Fatalf("invalid CIDR client IP = %q", got) + } + t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "::ffff:10.0.0.2/128") + if got := clientIP(r); got != "203.0.113.9" { + t.Fatalf("mapped CIDR client IP = %q", got) + } +} + +func TestPublicControlPlaneURLTrust(t *testing.T) { + t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8") + r := httptest.NewRequest("POST", "http://internal.local:8080/v1/speakers", nil) + r.Header.Set("Origin", "https://evil.example") + r.Header.Set("X-Forwarded-Proto", "https") + r.Header.Set("X-Forwarded-Host", "bgp.example.test") + r.RemoteAddr = "198.51.100.8:1234" + if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" { + t.Fatalf("direct URL = %q", got) + } + r.RemoteAddr = "10.0.0.2:1234" + if got := publicControlPlaneURL(r, ""); got != "https://bgp.example.test" { + t.Fatalf("proxied URL = %q", got) + } + if got := publicControlPlaneURL(r, "https://explicit.example/"); got != "https://explicit.example" { + t.Fatalf("override URL = %q", got) + } + for _, bad := range []struct{ proto, host string }{ + {"javascript", "bgp.example.test"}, + {"https,http", "bgp.example.test"}, + {"https", "evil.example/path"}, + {"https", "evil.example@bgp.example.test"}, + {"https", "evil.example,bgp.example.test"}, + {"https", "bgp.example.test:99999"}, + {"https", "bgp.example.test:"}, + } { + r.Header.Set("X-Forwarded-Proto", bad.proto) + r.Header.Set("X-Forwarded-Host", bad.host) + if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" { + t.Errorf("bad forwarded headers (%q, %q): %q", bad.proto, bad.host, got) + } + } + r.Header.Set("X-Forwarded-Proto", "https") + r.Header.Set("X-Forwarded-Host", "bgp.example.test") + r.Header.Add("X-Forwarded-Host", "evil.example") + if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" { + t.Fatalf("duplicate forwarded host = %q", got) + } +} + +func TestAuthRateLimiterManyIdentities(t *testing.T) { + l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 1, window: time.Minute, maxClients: 32} + now := time.Unix(1000, 0) + for i := 0; i < 1000; i++ { + allowed := l.allowAt(fmt.Sprintf("ip-%d", i), now) + if allowed != (i < l.maxClients) { + t.Fatalf("identity %d allowed = %t", i, allowed) + } + } + if len(l.hits) != l.maxClients || l.allowAt("ip-0", now) { + t.Fatalf("active counters changed under cardinality pressure: %d", len(l.hits)) + } +} + +func TestAuthRateLimiterCapacityAndExpiry(t *testing.T) { + l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 2, window: time.Minute, maxClients: 2} + now := time.Unix(1000, 0) + if !l.allowAt("a", now) || !l.allowAt("b", now) || !l.allowAt("a", now) { + t.Fatal("initial requests rejected") + } + if l.allowAt("a", now) || l.allowAt("c", now) { + t.Fatal("active limit or capacity bypassed") + } + if len(l.hits) != 2 { + t.Fatalf("map size = %d", len(l.hits)) + } + if !l.allowAt("c", now.Add(time.Minute)) { + t.Fatal("expired entries did not release capacity") + } + if len(l.hits) != 1 { + t.Fatalf("expired entries remained: %d", len(l.hits)) + } +} diff --git a/internal/httpclient/body.go b/internal/httpclient/body.go new file mode 100644 index 0000000..67b920e --- /dev/null +++ b/internal/httpclient/body.go @@ -0,0 +1,31 @@ +package httpclient + +import ( + "errors" + "fmt" + "io" + "net/http" +) + +// ErrResponseTooLarge means the complete decoded HTTP response exceeded its budget. +var ErrResponseTooLarge = errors.New("httpclient: response body exceeds limit") + +// ReadResponseBody reads a complete bounded body, including one extra byte to +// distinguish an exact limit from truncation. Standard Transport gzip decoding +// occurs before this reader, so the limit covers decompressed data as well. +func ReadResponseBody(resp *http.Response, limit int64) ([]byte, error) { + if limit <= 0 || limit == int64(^uint64(0)>>1) { + return nil, fmt.Errorf("httpclient: invalid response limit") + } + if resp.ContentLength > limit { + return nil, ErrResponseTooLarge + } + body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if err != nil { + return nil, err + } + if int64(len(body)) > limit { + return nil, ErrResponseTooLarge + } + return body, nil +} diff --git a/internal/httpclient/body_test.go b/internal/httpclient/body_test.go new file mode 100644 index 0000000..fa1c23f --- /dev/null +++ b/internal/httpclient/body_test.go @@ -0,0 +1,22 @@ +package httpclient + +import ( + "errors" + "io" + "net/http" + "strings" + "testing" +) + +func TestReadResponseBodyDoesNotAcceptValidPrefixTruncation(t *testing.T) { + for _, length := range []int64{-1, 9} { + resp := &http.Response{ContentLength: length, Body: io.NopCloser(strings.NewReader("{} "))} + if _, err := ReadResponseBody(resp, 8); !errors.Is(err, ErrResponseTooLarge) { + t.Fatalf("length=%d got %v", length, err) + } + } + resp := &http.Response{ContentLength: -1, Body: io.NopCloser(strings.NewReader("{} "))} + if b, err := ReadResponseBody(resp, 8); err != nil || len(b) != 8 { + t.Fatalf("exact limit failed: %q %v", b, err) + } +} diff --git a/internal/httpclient/circuit.go b/internal/httpclient/circuit.go index 89dac5b..e0fca0b 100644 --- a/internal/httpclient/circuit.go +++ b/internal/httpclient/circuit.go @@ -1,8 +1,13 @@ package httpclient import ( + "errors" + "fmt" + "strings" "sync" "time" + + "evobgp/internal/observability" ) const ( @@ -16,14 +21,77 @@ type hostBreaker struct { openUntil time.Time } -var hostBreakers sync.Map // string -> *hostBreaker +// ErrBreakerCapacity rejects new upstream identities when the bounded cache is full. +var ErrBreakerCapacity = errors.New("httpclient: upstream circuit capacity exhausted") -func breakerForHost(host string) *hostBreaker { +type breakerEntry struct { + breaker *hostBreaker + lastUsed time.Time + active int +} + +type breakerCache struct { + mu sync.Mutex + entries map[string]*breakerEntry + maximum int + idle time.Duration + nextSweep time.Time +} + +var hostBreakers = breakerCache{entries: make(map[string]*breakerEntry), maximum: 1024, idle: 10 * time.Minute} + +func (c *breakerCache) sweep(now time.Time) { + for host, e := range c.entries { + if e.active != 0 || now.Before(e.lastUsed.Add(c.idle)) { + continue + } + e.breaker.mu.Lock() + open := now.Before(e.breaker.openUntil) + e.breaker.mu.Unlock() + if !open { + delete(c.entries, host) + observability.RemoveUpstreamBreaker(host) + } + } + c.nextSweep = now.Add(time.Second) +} + +func (c *breakerCache) acquire(host string, now time.Time) (*hostBreaker, string, func(), error) { + if len(host) > 254 { + return nil, "", nil, fmt.Errorf("httpclient: upstream hostname too long") + } + host = strings.TrimSuffix(strings.ToLower(host), ".") + if len(host) > 253 { + return nil, "", nil, fmt.Errorf("httpclient: upstream hostname too long") + } if host == "" { host = "_" } - v, _ := hostBreakers.LoadOrStore(host, &hostBreaker{}) - return v.(*hostBreaker) + c.mu.Lock() + defer c.mu.Unlock() + if !now.Before(c.nextSweep) { + c.sweep(now) + } + e := c.entries[host] + if e == nil { + if len(c.entries) >= c.maximum { + return nil, host, nil, ErrBreakerCapacity + } + e = &breakerEntry{breaker: &hostBreaker{}} + c.entries[host] = e + } + e.active++ + e.lastUsed = now + var once sync.Once + release := func() { + once.Do(func() { + c.mu.Lock() + defer c.mu.Unlock() + e.active-- + e.lastUsed = time.Now() + }) + } + return e.breaker, host, release, nil } func (b *hostBreaker) allow() bool { @@ -51,23 +119,31 @@ func (b *hostBreaker) recordFailure() { // SnapshotBreakers returns whether each known host breaker is currently open. func SnapshotBreakers() map[string]bool { + hostBreakers.mu.Lock() + defer hostBreakers.mu.Unlock() + now := time.Now() + hostBreakers.sweep(now) out := map[string]bool{} - hostBreakers.Range(func(key, value any) bool { - host, _ := key.(string) - b, _ := value.(*hostBreaker) - if b == nil { - return true - } + for host, e := range hostBreakers.entries { + b := e.breaker b.mu.Lock() open := time.Now().Before(b.openUntil) b.mu.Unlock() out[host] = open - return true - }) + } return out } -// ResetHostBreakers clears all circuit breakers (tests only). +// ResetHostBreakers clears idle circuit breakers (tests only). In-flight +// references are retained so their completions cannot recreate orphan metrics. func ResetHostBreakers() { - hostBreakers = sync.Map{} + hostBreakers.mu.Lock() + defer hostBreakers.mu.Unlock() + for host, e := range hostBreakers.entries { + if e.active == 0 { + delete(hostBreakers.entries, host) + observability.RemoveUpstreamBreaker(host) + } + } + hostBreakers.nextSweep = time.Time{} } diff --git a/internal/httpclient/circuit_cache_test.go b/internal/httpclient/circuit_cache_test.go new file mode 100644 index 0000000..c1cee2b --- /dev/null +++ b/internal/httpclient/circuit_cache_test.go @@ -0,0 +1,118 @@ +package httpclient + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "evobgp/internal/observability" + "github.com/prometheus/client_golang/prometheus" +) + +func TestBreakerCanonicalHostAndActiveRetention(t *testing.T) { + c := breakerCache{entries: make(map[string]*breakerEntry), maximum: 1, idle: time.Minute} + now := time.Now() + first, key, release, err := c.acquire("Cache-Test.Example.", now) + if err != nil || key != "cache-test.example" { + t.Fatalf("canonical key=%s err=%v", key, err) + } + second, _, release2, err := c.acquire("cache-test.example", now) + if err != nil || first != second || len(c.entries) != 1 { + t.Fatal("alias bypassed shared breaker") + } + observability.SetUpstreamBreakerOpen(key, true) + first.openUntil = now.Add(3 * time.Minute) + if _, _, _, err := c.acquire("other.example", now.Add(2*time.Minute)); !errors.Is(err, ErrBreakerCapacity) { + t.Fatalf("active entry evicted: %v", err) + } + release() + release() + release2() + if _, _, _, err := c.acquire("other.example", now.Add(2*time.Minute+2*time.Second)); !errors.Is(err, ErrBreakerCapacity) { + t.Fatalf("open breaker evicted: %v", err) + } + _, _, done, err := c.acquire("other.example", now.Add(4*time.Minute)) + if err != nil { + t.Fatal(err) + } + defer done() + if len(c.entries) != 1 || c.entries[key] != nil { + t.Fatal("idle closed entry retained") + } + families, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + if strings.HasSuffix(family.GetName(), "upstream_breaker_open") { + for _, metric := range family.Metric { + for _, label := range metric.Label { + if label.GetName() == "host" && label.GetValue() == key { + t.Fatal("evicted host metric retained") + } + } + } + } + } +} + +type breakerTransport func(*http.Request) (*http.Response, error) + +func (f breakerTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestBreakerCapacityRejectsBeforeDial(t *testing.T) { + ResetHostBreakers() + var releases []func() + defer func() { + for _, release := range releases { + release() + } + ResetHostBreakers() + }() + for i := 0; i < hostBreakers.maximum; i++ { + _, _, release, err := hostBreakers.acquire(fmt.Sprintf("capacity-%d.example", i), time.Now()) + if err != nil { + t.Fatal(err) + } + releases = append(releases, release) + } + called := false + hc := &http.Client{Transport: breakerTransport(func(*http.Request) (*http.Response, error) { called = true; return nil, errors.New("unexpected dial") })} + req, _ := http.NewRequest(http.MethodGet, "https://overflow.example/", nil) + _, err := DoWithBreaker(context.Background(), hc, req, 1) + if !errors.Is(err, ErrBreakerCapacity) || called { + t.Fatalf("capacity err=%v dial=%t", err, called) + } + if len(SnapshotBreakers()) != hostBreakers.maximum { + t.Fatal("cache capacity changed") + } + ResetHostBreakers() + if len(SnapshotBreakers()) != hostBreakers.maximum { + t.Fatal("reset discarded in-flight references") + } +} + +func TestBreakerRejectsOversizedIdentity(t *testing.T) { + c := breakerCache{entries: make(map[string]*breakerEntry), maximum: 1, idle: time.Minute} + if _, _, _, err := c.acquire(strings.Repeat("x", 254), time.Now()); err == nil || len(c.entries) != 0 { + t.Fatal("oversized identity retained") + } + name := strings.Repeat("a", 253) + first, key, release, err := c.acquire(name, time.Now()) + if err != nil { + t.Fatal(err) + } + defer release() + second, alias, release2, err := c.acquire(strings.ToUpper(name)+".", time.Now()) + if err != nil { + t.Fatal(err) + } + defer release2() + if first != second || key != alias { + t.Fatal("maximum length trailing-dot alias differs") + } +} diff --git a/internal/httpclient/httpclient.go b/internal/httpclient/httpclient.go index 1c1a9fb..8c2b8bb 100644 --- a/internal/httpclient/httpclient.go +++ b/internal/httpclient/httpclient.go @@ -54,7 +54,9 @@ func DoWithRetry(ctx context.Context, hc *http.Client, req *http.Request, maxAtt continue } if resp.StatusCode >= 500 { - _, _ = io.Copy(io.Discard, resp.Body) + // Draining is only an optimization for connection reuse, never an + // invitation to consume an unbounded error stream before retrying. + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) _ = resp.Body.Close() lastErr = fmt.Errorf("httpclient: upstream %s", resp.Status) continue @@ -72,8 +74,11 @@ func DoWithBreaker(ctx context.Context, hc *http.Client, req *http.Request, maxA if req == nil || req.URL == nil { return nil, fmt.Errorf("httpclient: nil request") } - host := req.URL.Hostname() - br := breakerForHost(host) + br, host, release, err := hostBreakers.acquire(req.URL.Hostname(), time.Now()) + if err != nil { + return nil, err + } + defer release() if !br.allow() { observability.SetUpstreamBreakerOpen(host, true) return nil, fmt.Errorf("httpclient: circuit open for %s", host) diff --git a/internal/httpclient/retry_body_test.go b/internal/httpclient/retry_body_test.go new file mode 100644 index 0000000..ad60f24 --- /dev/null +++ b/internal/httpclient/retry_body_test.go @@ -0,0 +1,44 @@ +package httpclient + +import ( + "context" + "io" + "net/http" + "testing" +) + +type endlessRetryBody struct { + read int + closed bool +} + +func (b *endlessRetryBody) Read(p []byte) (int, error) { + for i := range p { + p[i] = 'x' + } + b.read += len(p) + return len(p), nil +} +func (b *endlessRetryBody) Close() error { b.closed = true; return nil } + +type retryTransport func(*http.Request) (*http.Response, error) + +func (f retryTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestRetryDrainBoundsEndlessErrorBody(t *testing.T) { + b := &endlessRetryBody{} + c := &http.Client{Transport: retryTransport(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 503, Status: "503 Service Unavailable", Body: b, Header: make(http.Header)}, nil + })} + req, err := http.NewRequest(http.MethodGet, "http://agent.example/bundle", nil) + if err != nil { + t.Fatal(err) + } + if _, err = DoWithRetry(context.Background(), c, req, 1); err == nil { + t.Fatal("upstream failure accepted") + } + if b.read != 64<<10 || !b.closed { + t.Fatalf("retry consumed %d bytes, closed=%v", b.read, b.closed) + } + var _ io.ReadCloser = b +} diff --git a/internal/importer/csv.go b/internal/importer/csv.go index 792c854..7d1b37b 100644 --- a/internal/importer/csv.go +++ b/internal/importer/csv.go @@ -1,7 +1,10 @@ package importer import ( + "bytes" + "context" "encoding/csv" + "errors" "fmt" "io" "strconv" @@ -10,6 +13,18 @@ import ( "evobgp/internal/store" ) +// MaxCSVBytes is the complete request limit, including the optional header. +const MaxCSVBytes = 8 << 20 + +// MaxCSVRows bounds database work for a single import, excluding its header. +const MaxCSVRows = 10000 + +// MaxCSVFields bounds CSV parser allocations for delimiter-heavy records. +const MaxCSVFields = 64 + +// ErrCSVTooLarge means the entire import was rejected before creating entries. +var ErrCSVTooLarge = errors.New("importer: csv exceeds byte, row or field limit") + // Result of importing module entries from CSV. type Result struct { Imported int @@ -17,18 +32,50 @@ type Result struct { } // ImportModuleEntriesCSV parses and creates entries for AS_PREFIXES, DOMAINS, or IP_RANGES modules. -func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io.Reader) (*Result, error) { +func ImportModuleEntriesCSV(ctx context.Context, st store.Backend, tenantID, moduleID string, body io.Reader) (*Result, error) { + if err := ctx.Err(); err != nil { + return nil, err + } mod, err := st.GetModule(tenantID, moduleID) if err != nil { return nil, err } - cr := csv.NewReader(io.LimitReader(body, 8<<20)) + // Read one byte beyond the cap: a valid prefix must never be imported as if + // it were the complete request. Check all resource limits before mutations. + raw, err := io.ReadAll(io.LimitReader(body, MaxCSVBytes+1)) + if err != nil { + return nil, fmt.Errorf("importer: read csv: %w", err) + } + if len(raw) > MaxCSVBytes { + return nil, ErrCSVTooLarge + } + if err := checkCSVFields(ctx, raw); err != nil { + return nil, err + } + cr := csv.NewReader(bytes.NewReader(raw)) cr.TrimLeadingSpace = true cr.FieldsPerRecord = -1 - rows, err := cr.ReadAll() - if err != nil { - return nil, fmt.Errorf("importer: invalid csv: %w", err) + rows := make([][]string, 0) + headerRows := 0 + for { + if err := ctx.Err(); err != nil { + return nil, err + } + record, err := cr.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("importer: invalid csv: %w", err) + } + if len(rows) == 0 && isCSVHeader(record) { + headerRows = 1 + } + if len(rows)-headerRows >= MaxCSVRows { + return nil, ErrCSVTooLarge + } + rows = append(rows, record) } if len(rows) == 0 { return nil, fmt.Errorf("importer: %w", store.ErrInvalidInput) @@ -63,18 +110,17 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io } start := 0 - if len(rows[0]) >= 2 { - key := strings.ToLower(strings.TrimSpace(rows[0][0])) - switch key { - case "asn", "domain", "iprange": - start = 1 - } + if isCSVHeader(rows[0]) { + start = 1 } imported := 0 switch mod.Type { case "AS_PREFIXES": for i := start; i < len(rows); i++ { + if err := ctx.Err(); err != nil { + return nil, err + } rec := rows[i] if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) { continue @@ -97,6 +143,9 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io } case "DOMAINS": for i := start; i < len(rows); i++ { + if err := ctx.Err(); err != nil { + return nil, err + } rec := rows[i] if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) { continue @@ -119,6 +168,9 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io } case "IP_RANGES": for i := start; i < len(rows); i++ { + if err := ctx.Err(); err != nil { + return nil, err + } rec := rows[i] if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) { continue @@ -145,3 +197,49 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io return &Result{Imported: imported, ModuleType: mod.Type}, nil } + +func isCSVHeader(record []string) bool { + if len(record) < 2 { + return false + } + switch strings.ToLower(strings.TrimSpace(record[0])) { + case "asn", "domain", "iprange": + return true + default: + return false + } +} + +// checkCSVFields runs before encoding/csv allocates a string/index for each field. +// Quoted commas, escaped quotes and multiline fields don't start new fields. +// Syntax validation remains the responsibility of encoding/csv below. +func checkCSVFields(ctx context.Context, raw []byte) error { + quoted, fields := false, 1 + for i := 0; i < len(raw); i++ { + if i%4096 == 0 { + if err := ctx.Err(); err != nil { + return err + } + } + switch raw[i] { + case '"': + if quoted && i+1 < len(raw) && raw[i+1] == '"' { + i++ + continue + } + quoted = !quoted + case ',': + if !quoted { + fields++ + if fields > MaxCSVFields { + return ErrCSVTooLarge + } + } + case '\n': + if !quoted { + fields = 1 + } + } + } + return nil +} diff --git a/internal/importer/csv_test.go b/internal/importer/csv_test.go new file mode 100644 index 0000000..454b800 --- /dev/null +++ b/internal/importer/csv_test.go @@ -0,0 +1,95 @@ +package importer + +import ( + "context" + "errors" + "strings" + "testing" + + "evobgp/internal/store" +) + +type importBackend struct { + store.Backend + created int + onCreate func() +} + +func (s *importBackend) GetModule(string, string) (*store.Module, error) { + return &store.Module{Type: "AS_PREFIXES"}, nil +} +func (s *importBackend) ListCommunities(string) ([]*store.Community, error) { return nil, nil } +func (s *importBackend) CreateASEntry(_, _ string, e *store.ASEntry) (*store.ASEntry, error) { + s.created++ + if s.onCreate != nil { + s.onCreate() + } + return e, nil +} + +func TestCSVByteLimitRejectsValidTruncatedPrefix(t *testing.T) { + header := "asn,community\n" + for _, extra := range []int{0, 1} { + s := &importBackend{} + body := header + strings.Repeat("\n", MaxCSVBytes-len(header)) + strings.Repeat("\n", extra) + _, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(body)) + if extra == 0 && err != nil { + t.Fatalf("exact limit: %v", err) + } + if extra == 1 && !errors.Is(err, ErrCSVTooLarge) { + t.Fatalf("limit+1: %v", err) + } + if s.created != 0 { + t.Fatal("oversized import created entries") + } + } +} + +func TestCSVRowLimitCheckedBeforeWrites(t *testing.T) { + for _, header := range []string{"", "asn,community\n"} { + for _, extra := range []int{0, 1} { + s := &importBackend{} + _, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(header+strings.Repeat("64512,\n", MaxCSVRows+extra))) + if extra == 0 && (err != nil || s.created != MaxCSVRows) { + t.Fatalf("exact rows: created=%d err=%v", s.created, err) + } + if extra == 1 && (!errors.Is(err, ErrCSVTooLarge) || s.created != 0) { + t.Fatalf("rows+1: created=%d err=%v", s.created, err) + } + } + } +} + +func TestCSVCancellationStopsFurtherWrites(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := &importBackend{onCreate: cancel} + _, err := ImportModuleEntriesCSV(ctx, s, "tenant", "module", strings.NewReader("asn,community\n64512,\n64513,\n")) + if !errors.Is(err, context.Canceled) || s.created != 1 { + t.Fatalf("created=%d err=%v", s.created, err) + } +} + +func TestCSVFieldLimitAndQuotedFields(t *testing.T) { + for _, tc := range []struct { + name, input string + oversized bool + }{ + {"exact", "64512," + strings.Repeat(",", MaxCSVFields-2) + "\n", false}, + {"over", "64512," + strings.Repeat(",", MaxCSVFields-1) + "\n", true}, + {"many_fields", strings.Repeat(",", 1<<20) + "\n", true}, + {"quoted_extra", "64512,,\"" + strings.Repeat(",", 128) + "\nquoted \"\"text\"\"\"\n", false}, + } { + t.Run(tc.name, func(t *testing.T) { + s := &importBackend{} + _, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(tc.input)) + if tc.oversized { + if !errors.Is(err, ErrCSVTooLarge) || s.created != 0 { + t.Fatalf("created=%d err=%v", s.created, err) + } + } else if err != nil || s.created != 1 { + t.Fatalf("created=%d err=%v", s.created, err) + } + }) + } +} diff --git a/internal/jobs/deploy_confirmation_test.go b/internal/jobs/deploy_confirmation_test.go new file mode 100644 index 0000000..2f8ca96 --- /dev/null +++ b/internal/jobs/deploy_confirmation_test.go @@ -0,0 +1,231 @@ +package jobs + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + + "evobgp/internal/nodedispatch" + "evobgp/internal/store" + + "github.com/google/uuid" +) + +type deployRoundTrip func(*http.Request) (*http.Response, error) + +func (f deployRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestDeployApplyConfirmation(t *testing.T) { + for _, tc := range []struct { + name, response, confirmation, status string + dispatch bool + stale bool + wantApplied bool + }{ + {name: "dispatch disabled", confirmation: "unconfirmed", status: StatusSucceeded}, + {name: "bare success", dispatch: true, response: `{"ok":true}`, confirmation: "unconfirmed", status: StatusSucceeded}, + {name: "matching ACK", dispatch: true, response: "matching", confirmation: "agent_reported", status: StatusSucceeded, wantApplied: true}, + {name: "mismatched ACK", dispatch: true, response: `{"ok":true,"applied_revision_id":"other"}`, confirmation: "unconfirmed", status: StatusFailed}, + {name: "stale desired", dispatch: true, response: "matching", confirmation: "unconfirmed", status: StatusFailed, stale: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "") + if tc.dispatch { + // A remote-only job must not touch the control plane BIRD tree. + t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", t.TempDir()) + t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1") + } else { + t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0") + } + m := store.NewMemory() + m.SeedDemo() + tenant, _, module, previous, speakerID := m.DemoIDs() + if err := m.SetLastAppliedRevision(tenant, speakerID, previous); err != nil { + t.Fatal(err) + } + revision := uuid.NewString() + if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil { + t.Fatal(err) + } + newer := uuid.NewString() + if err := m.CreateRenderRevision(context.Background(), newer, tenant, module, &revision, "newer-hash", map[string]string{"bird.conf": "# newer\n"}, nil); err != nil { + t.Fatal(err) + } + meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"}) + if _, err := m.UpdateSpeaker(tenant, speakerID, &store.SpeakerPatch{MetaJSON: &meta}); err != nil { + t.Fatal(err) + } + client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/v1/agent/sync" || r.Header.Get("Authorization") != "Bearer secret" { + t.Errorf("unexpected sync request: %s %s", r.Method, r.URL) + } + var body map[string]string + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || len(body) != 1 || body["revision_id"] != revision { + t.Errorf("legacy sync body: %#v, %v", body, err) + } + if tc.stale { + if err := m.PublishRevisionForSpeaker(speakerID, newer); err != nil { + t.Error(err) + } + } + response := tc.response + if response == "matching" { + response = `{"ok":true,"applied_revision_id":"` + revision + `"}` + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil + })} + j := &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision, "speaker_id": speakerID}} + (&Worker{Store: m, NodeHTTPClient: client}).Process(j) + if j.Status != tc.status { + t.Fatalf("job status = %s, want %s; error=%v", j.Status, tc.status, j.Error) + } + sp, err := m.GetSpeaker(tenant, speakerID) + if err != nil { + t.Fatal(err) + } + want := previous + if tc.wantApplied { + want = revision + } + if sp.LastAppliedRevisionID == nil || *sp.LastAppliedRevisionID != want { + t.Fatalf("applied = %v, want %s", sp.LastAppliedRevisionID, want) + } + published, _, err := m.LatestPublishedRevision(speakerID) + if err != nil { + t.Fatal(err) + } + wantPublished := revision + if tc.stale { + wantPublished = newer + } + if published != wantPublished { + t.Fatalf("published = %s, want %s", published, wantPublished) + } + results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result) + if len(results) != 1 || results[0].Confirmation != tc.confirmation { + t.Fatalf("results = %+v", results) + } + }) + } +} + +func TestDeployApplyPartialResult(t *testing.T) { + t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "") + t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1") + m := store.NewMemory() + m.SeedDemo() + tenant, _, module, previous, first := m.DemoIDs() + revision := uuid.NewString() + if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil { + t.Fatal(err) + } + goodMeta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "good.example.test", AgentSecret: "secret"}) + if _, err := m.UpdateSpeaker(tenant, first, &store.SpeakerPatch{MetaJSON: &goodMeta}); err != nil { + t.Fatal(err) + } + badMeta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "bad.example.test", AgentSecret: "secret"}) + second, err := m.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: badMeta}) + if err != nil { + t.Fatal(err) + } + client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) { + response := `{"ok":true,"applied_revision_id":"` + revision + `"}` + if strings.HasPrefix(r.URL.Host, "bad.") { + response = `{"ok":false,"error":"apply failed"}` + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil + })} + j := &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision}} + (&Worker{Store: m, NodeHTTPClient: client}).Process(j) + if j.Status != StatusFailed { + t.Fatalf("job status = %s, want failed", j.Status) + } + summary := j.Meta["apply_summary"].(map[string]any) + if summary["applied_count"] != 1 || summary["failed_count"] != 1 || summary["published_count"] != 2 { + t.Fatalf("partial summary = %#v", summary) + } + firstSp, _ := m.GetSpeaker(tenant, first) + secondSp, _ := m.GetSpeaker(tenant, second.ID) + if firstSp.LastAppliedRevisionID == nil || *firstSp.LastAppliedRevisionID != revision || secondSp.LastAppliedRevisionID != nil { + t.Fatalf("partial applied state: first=%v second=%v", firstSp.LastAppliedRevisionID, secondSp.LastAppliedRevisionID) + } + results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result) + got := map[string]bool{} + for _, res := range results { + got[res.SpeakerID] = true + } + if len(results) != 2 || !got[first] || !got[second.ID] { + t.Fatalf("partial results=%+v", results) + } +} + +func TestDeployApplyCancellationStopsFurtherPublication(t *testing.T) { + t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "") + t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1") + m := store.NewMemory() + m.SeedDemo() + tenant, _, module, previous, seeded := m.DemoIDs() + if err := m.DeleteSpeaker(tenant, seeded); err != nil { + t.Fatal(err) + } + revision := uuid.NewString() + if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil { + t.Fatal(err) + } + meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"}) + speakerCount := speakerWakeLimit + 2 + for range speakerCount { + if _, err := m.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: meta}); err != nil { + t.Fatal(err) + } + } + speakers := m.ListSpeakersForTenant(tenant) + if len(speakers) != speakerCount { + t.Fatalf("speakers=%d, want %d", len(speakers), speakerCount) + } + var j *Job + var calls atomic.Int32 + client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) { + // Cancel only once the in-flight wake limit is busy, so later speakers stay unpublished. + if calls.Add(1) == int32(speakerWakeLimit) { + j.RequestCancel() + } + <-r.Context().Done() + return nil, r.Context().Err() + })} + j = &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision}} + (&Worker{Store: m, NodeHTTPClient: client}).Process(j) + if j.Status != StatusCancelled || int(calls.Load()) != speakerWakeLimit { + t.Fatalf("cancelled job status=%s dispatch calls=%d", j.Status, calls.Load()) + } + results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result) + gotResult := map[string]bool{} + for _, res := range results { + gotResult[res.SpeakerID] = res.Status == "cancelled" + } + if len(results) != speakerWakeLimit { + t.Fatalf("cancelled results=%+v", results) + } + for i, sp := range speakers { + got, _, err := m.LatestPublishedRevision(sp.ID) + if i < speakerWakeLimit { + if err != nil || got != revision || !gotResult[sp.ID] { + t.Fatalf("speaker %s published=%s err=%v result=%v", sp.ID, got, err, gotResult[sp.ID]) + } + if sp.LastAppliedRevisionID != nil { + t.Fatalf("cancelled speaker %s marked applied", sp.ID) + } + continue + } + if err == nil { + t.Fatalf("speaker %s published %s after cancellation", sp.ID, got) + } + } + if j.Meta["apply_summary"].(map[string]any)["published_count"] != speakerWakeLimit { + t.Fatalf("cancelled summary = %#v", j.Meta["apply_summary"]) + } +} diff --git a/internal/jobs/deployment_reconcile.go b/internal/jobs/deployment_reconcile.go new file mode 100644 index 0000000..ba113f7 --- /dev/null +++ b/internal/jobs/deployment_reconcile.go @@ -0,0 +1,126 @@ +package jobs + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + "time" + + "evobgp/internal/logging" + "evobgp/internal/nodedispatch" + "evobgp/internal/store" +) + +func deploymentFenceForJob(j *Job) store.JobFence { + generation, _ := strconv.ParseInt(j.metaString("reconcile_generation"), 10, 64) + round, _ := strconv.Atoi(j.metaString("reconcile_round")) + return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken, ExpectedGeneration: generation, RetryRound: round} +} + +func beginDeployment(ctx context.Context, st store.Deployments, f store.JobFence, speaker, revision string) (store.DeploymentSpeaker, error) { + for { + d, err := st.BeginDeploymentSpeaker(ctx, f, speaker, revision) + if !errors.Is(err, store.ErrDeploymentBusy) { + return d, err + } + timer := time.NewTimer(time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return store.DeploymentSpeaker{}, ctx.Err() + case <-timer.C: + } + } +} + +// ReconcileDeployments schedules at most three recovery rounds for current desired +// generations. Stable keys coalesce API replicas; the worker CAS checks generation +// again before publishing, and preserves the original desired request sequence. +func ReconcileDeployments(ctx context.Context, r *Registry, st store.Deployments) error { + rows, err := st.ListPendingDeployments(ctx, 100) + if err != nil { + return err + } + for _, d := range rows { + key := fmt.Sprintf("deployment_reconcile:%s:%s:%d", d.JobID, d.SpeakerID, d.RetryRound+1) + _, _, err = r.EnqueueContext(ctx, d.TenantID, KindDeployApply, &key, nil, map[string]any{"revision_id": d.RevisionID, "speaker_id": d.SpeakerID, "reconcile_generation": strconv.FormatInt(d.Generation, 10), "reconcile_round": strconv.Itoa(d.RetryRound + 1), "source_deployment_job_id": d.JobID}) + if err != nil { + return err + } + } + return nil +} + +// StartDeploymentReconciliationLoop belongs to the API server lifecycle. Static +// ingest/deploy/scheduler processes must not start this job-producing loop. +func StartDeploymentReconciliationLoop(ctx context.Context, r *Registry, st store.Backend) { + if r == nil || r.queue == nil || st == nil { + return + } + r.durableMu.Lock() + if r.durableStop { + r.durableMu.Unlock() + return + } + r.durableLoopWG.Add(1) + r.durableMu.Unlock() + go func() { + defer r.durableLoopWG.Done() + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + r.durableMu.Lock() + stopped := r.durableStop + r.durableMu.Unlock() + if stopped { + return + } + qctx, cancel := context.WithTimeout(ctx, 40*time.Second) + err := ObserveDeployments(qctx, st, nil) + if err == nil { + err = ReconcileDeployments(qctx, r, st) + } + cancel() + if err != nil && ctx.Err() == nil { + logging.With("component", "jobs.deployment_reconcile").Warn("reconciliation failed", "err", err) + } + } + }() +} + +// ObserveDeployments samples compatible health before reconciling drift. The CP +// receipt time and optional agent source time are stored separately; missing +// legacy revision fields never manufacture an applied/verified confirmation. +func ObserveDeployments(ctx context.Context, st store.Backend, client *http.Client) error { + rows, err := st.ListDeploymentObservations(ctx, 10) + if err != nil { + return err + } + for _, d := range rows { + sp, e := st.GetSpeaker(d.TenantID, d.SpeakerID) + if e != nil { + return e + } + observation := store.DeploymentObservation{Error: "remote health unavailable for this speaker"} + if nodedispatch.Enabled() && store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON)) { + hctx, cancel := context.WithTimeout(ctx, 3*time.Second) + h := nodedispatch.FetchAgentHealth(hctx, sp, nodedispatch.Options{HTTPClient: client, Timeout: 3 * time.Second}) + cancel() + observation = store.DeploymentObservation{RevisionID: h.LastAppliedRevisionID, SourceAt: h.LastSyncAt, Error: h.Error} + if h.OK && observation.RevisionID == "" { + observation.Error = "legacy health did not report an applied revision" + } + } + if err = st.ObserveDeployment(ctx, d, observation); err != nil && !errors.Is(err, store.ErrJobLeaseLost) { + return err + } + } + return nil +} diff --git a/internal/jobs/deployment_reconcile_test.go b/internal/jobs/deployment_reconcile_test.go new file mode 100644 index 0000000..c4f13c2 --- /dev/null +++ b/internal/jobs/deployment_reconcile_test.go @@ -0,0 +1,140 @@ +package jobs + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "evobgp/internal/store" +) + +func TestDeploymentReconcileDedupAndDesiredOrdering(t *testing.T) { + t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0") + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, revision, speaker := m.DemoIDs() + ctx := context.Background() + r := NewRegistry(nil) + r.UseDurableQueue(m) + initial, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker}) + if err != nil { + t.Fatal(err) + } + claim := func(owner string) *Job { + t.Helper() + rows, e := m.ClaimJobs(ctx, owner, 1, time.Minute) + if e != nil || len(rows) != 1 { + t.Fatalf("claim: %+v %v", rows, e) + } + return jobFromRecord(rows[0]) + } + first := claim("initial") + if first.ID != initial.ID { + t.Fatal("wrong initial job") + } + (&Worker{Store: m}).Process(first) + if err = m.FinishJob(ctx, recordFromJob(first)); err != nil { + t.Fatal(err) + } + // Enqueue an operator request before the synthetic recovery request. The latter + // must retain the earlier desired order despite its own larger queue sequence. + operator, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker}) + if err != nil { + t.Fatal(err) + } + for range 2 { + if err = ReconcileDeployments(ctx, r, m); err != nil { + t.Fatal(err) + } + } + all, _, _, err := m.ListJobs(ctx, tenant, "", "", "", 100) + if err != nil || len(all) != 3 { + t.Fatalf("reconcile dedupe: %d %v", len(all), err) + } + newer := claim("operator") + if newer.ID != operator.ID { + t.Fatal("wrong operator job") + } + recovery := claim("recovery") + f := deploymentFenceForJob(recovery) + d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision) + if err != nil { + t.Fatal(err) + } + if d.RequestSequence != first.RequestSequence { + t.Fatal("recovery lost original desired order") + } + if err = m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil { + t.Fatal(err) + } + newD, err := m.BeginDeploymentSpeaker(ctx, deploymentFenceForJob(newer), speaker, revision) + if err != nil { + t.Fatalf("newer operator should supersede recovery despite smaller queue sequence: %v", err) + } + if newD.RequestSequence != newer.RequestSequence { + t.Fatal("operator desired sequence changed") + } + if _, err = m.BeginDeploymentSpeaker(ctx, f, speaker, revision); err == nil { + t.Fatal("stale recovery generation reused") + } +} + +func TestDeploymentHealthDetectsDriftAndKeepsLegacyQuality(t *testing.T) { + t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0") + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, revision, speaker := m.DemoIDs() + ctx := context.Background() + r := NewRegistry(nil) + r.UseDurableQueue(m) + _, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker}) + if err != nil { + t.Fatal(err) + } + rows, err := m.ClaimJobs(ctx, "owner", 1, time.Minute) + if err != nil || len(rows) != 1 { + t.Fatal(err) + } + j := jobFromRecord(rows[0]) + (&Worker{Store: m}).Process(j) + if err = m.FinishJob(ctx, recordFromJob(j)); err != nil { + t.Fatal(err) + } + meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"}) + if _, err = m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil { + t.Fatal(err) + } + t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1") + response := `{"ok":true,"last_applied_revision_id":"` + revision + `"}` + client := &http.Client{Transport: deployRoundTrip(func(req *http.Request) (*http.Response, error) { + if req.Header.Get("Authorization") != "Bearer secret" || req.Method != "GET" { + t.Error("health must be authenticated GET") + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil + })} + if err = ObserveDeployments(ctx, m, client); err != nil { + t.Fatal(err) + } + progress, err := m.ListDeploymentSpeakers(ctx, tenant, j.ID) + if err != nil || len(progress) != 1 || progress[0].State != "applied" || progress[0].Confirmation != "agent_reported" || progress[0].HealthReceivedAt == nil || progress[0].HealthSourceAt != "" { + t.Fatalf("legacy health confirmation: %+v %v", progress, err) + } + response = `{"ok":true,"last_applied_revision_id":"older-late-operation"}` + if err = ObserveDeployments(ctx, m, client); err != nil { + t.Fatal(err) + } + progress, _ = m.ListDeploymentSpeakers(ctx, tenant, j.ID) + if progress[0].State != "unknown" || progress[0].Confirmation != "unconfirmed" { + t.Fatalf("late drift hidden: %+v", progress) + } + if err = ReconcileDeployments(ctx, r, m); err != nil { + t.Fatal(err) + } + jobs, _, _, _ := m.ListJobs(ctx, tenant, "queued", "", "", 100) + if len(jobs) != 1 { + t.Fatalf("drift recovery jobs: %d", len(jobs)) + } +} diff --git a/internal/jobs/durable.go b/internal/jobs/durable.go index c2ea9c3..c7fed11 100644 --- a/internal/jobs/durable.go +++ b/internal/jobs/durable.go @@ -2,136 +2,216 @@ package jobs import ( "context" - "os" - "strconv" - "strings" - "time" - + "errors" "evobgp/internal/logging" "evobgp/internal/observability" - "evobgp/internal/repository" + "evobgp/internal/store" + "sync/atomic" + "time" ) -// Adopt registers a durable job_audit claim into the in-process registry and starts the worker. -// Used after SKIP LOCKED claim so two API processes can share the queue without losing work on restart. -func (r *Registry) Adopt(j *Job) bool { - if r == nil || j == nil || j.ID == "" { - return false - } - r.mu.Lock() - if _, exists := r.byID[j.ID]; exists { - r.mu.Unlock() - return false - } - if j.Status == "" { - j.Status = StatusQueued - } - if j.Meta == nil { - j.Meta = map[string]any{} - } - r.byID[j.ID] = j - if j.IdempotencyKey != nil && *j.IdempotencyKey != "" { - r.byIdempo[idempoKey{tenant: j.TenantID, key: *j.IdempotencyKey}] = j - } - if isRefreshKind(j.Kind) { - r.inflightRefresh[j.TenantID]++ - } - workerStart := r.workerStart - r.mu.Unlock() +const durableLease = time.Minute - if workerStart != nil { - go func() { - r.workerSem <- struct{}{} - active := len(r.workerSem) - capacity := cap(r.workerSem) - observability.RecordJobQueueDepth(active, capacity) - defer func() { - <-r.workerSem - observability.RecordJobQueueDepth(len(r.workerSem), capacity) - }() - workerStart(j) - }() - } - return true -} - -// StartDurableQueueLoop periodically reclaims stale running rows and claims queued job_audit work (PG SKIP LOCKED). -// No-op when audit is nil. Interval from EVOBGP_JOB_RECLAIM_INTERVAL (default 20s); stale from EVOBGP_JOB_STALE_AFTER (default 15m). -func StartDurableQueueLoop(ctx context.Context, reg *Registry, audit *repository.JobAuditWriter) { - if ctx == nil || reg == nil || audit == nil { +// StartDurableQueueLoop claims work only after reserving local capacity. +func StartDurableQueueLoop(ctx context.Context, r *Registry) { + if r == nil || r.queue == nil { return } - interval := 20 * time.Second - if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_RECLAIM_INTERVAL")); s != "" { - if d, err := time.ParseDuration(s); err == nil && d > 0 { - interval = d - } - } - staleAfter := 15 * time.Minute - if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_STALE_AFTER")); s != "" { - if d, err := time.ParseDuration(s); err == nil && d > 0 { - staleAfter = d - } - } - limit := 8 - if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_LIMIT"))); err == nil && n > 0 { - limit = n - } - grace := 30 * time.Second - if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_GRACE")); s != "" { - if d, err := time.ParseDuration(s); err == nil && d >= 0 { - grace = d - } - } log := logging.With("component", "jobs.durable") - run := func() { - cctx, cancel := context.WithTimeout(ctx, 15*time.Second) - defer cancel() - n, err := audit.ReclaimStaleRunning(cctx, staleAfter) - if err != nil { - log.Warn("reclaim stale running failed", "err", err) - } else if n > 0 { - log.Info("reclaimed stale running jobs", "count", n) - } - claimed, err := audit.ClaimQueued(cctx, limit, grace) - if err != nil { - log.Warn("claim queued failed", "err", err) - return - } - for i := range claimed { - c := claimed[i] - j := &Job{ - ID: c.ID, - TenantID: c.TenantID, - Kind: c.Kind, - Status: StatusQueued, - IdempotencyKey: c.IdempotencyKey, - ModuleID: c.ModuleID, - CreatedAt: c.CreatedAt, - Meta: c.Meta, - } - if !reg.Adopt(j) { - // Already local — leave DB running; local worker owns it. - continue - } - log.Info("adopted durable job", "job_id", j.ID, "kind", j.Kind, "tenant_id", j.TenantID) - } - counts, err := audit.CountByStatus(cctx) - if err == nil { - observability.RecordJobAuditDepth(counts) - } + r.durableMu.Lock() + if r.durableStop { + r.durableMu.Unlock() + return } + r.durableLoopWG.Add(2) + r.durableMu.Unlock() + go r.reapExpiredJobsLoop(ctx) go func() { - run() - t := time.NewTicker(interval) - defer t.Stop() + defer r.durableLoopWG.Done() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() for { + if ctx.Err() != nil { + return + } + r.durableMu.Lock() + stopped := r.durableStop + r.durableMu.Unlock() + if stopped { + return + } + metricCtx, metricCancel := context.WithTimeout(ctx, 5*time.Second) + if counts, err := r.queue.CountJobsByStatus(metricCtx); err == nil { + observability.RecordJobAuditDepth(counts) + } + metricCancel() + free := 0 + reserve: + for free < cap(r.workerSem) { + select { + case r.workerSem <- struct{}{}: + free++ + default: + break reserve + } + } + if free > 0 { + qctx, cancel := context.WithTimeout(ctx, 10*time.Second) + claimed, err := r.queue.ClaimJobs(qctx, r.ownerID, free, durableLease) + cancel() + if err != nil { + log.Warn("claim failed", "err", err) + } + for i := len(claimed); i < free; i++ { + <-r.workerSem + } + observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem)) + for _, c := range claimed { + r.durableMu.Lock() + stopped = r.durableStop + if stopped || ctx.Err() != nil { + r.durableMu.Unlock() + releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := r.queue.ReleaseJob(releaseCtx, c.ID, c.OwnerID, c.FencingToken); err != nil { + log.Warn("release during shutdown failed", "job_id", c.ID, "err", err) + } + releaseCancel() + <-r.workerSem + continue + } + j := jobFromRecord(c) + r.mu.Lock() + r.byID[j.ID] = j + if isRefreshKind(j.Kind) { + r.inflightRefresh[j.TenantID]++ + } + r.mu.Unlock() + r.durableWG.Add(1) + go r.runClaimed(j) + r.durableMu.Unlock() + } + } select { case <-ctx.Done(): return - case <-t.C: - run() + case <-ticker.C: } } }() } + +// reapExpiredJobsLoop runs the job_audit reaper every 15s, not on each claim. +func (r *Registry) reapExpiredJobsLoop(ctx context.Context) { + defer r.durableLoopWG.Done() + r.reapExpiredJobs(ctx) + reap := time.NewTicker(15 * time.Second) + stop := time.NewTicker(time.Second) + defer reap.Stop() + defer stop.Stop() + for { + if r.durableStopped() || ctx.Err() != nil { + return + } + select { + case <-ctx.Done(): + return + case <-stop.C: + if r.durableStopped() { + return + } + case <-reap.C: + r.reapExpiredJobs(ctx) + } + } +} + +func (r *Registry) durableStopped() bool { + if r == nil { + return true + } + r.durableMu.Lock() + defer r.durableMu.Unlock() + return r.durableStop +} + +func (r *Registry) reapExpiredJobs(ctx context.Context) { + if r == nil || r.queue == nil { + return + } + cctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if err := r.queue.ReapExpiredJobs(cctx); err != nil && ctx.Err() == nil { + logging.With("component", "jobs.durable").Warn("reap expired jobs", "err", err) + } +} + +func (r *Registry) runClaimed(j *Job) { + log := logging.With("component", "jobs.durable") + var leaseHealthy atomic.Bool + leaseHealthy.Store(true) + defer func() { + r.mu.Lock() + delete(r.byID, j.ID) + r.mu.Unlock() + <-r.workerSem + observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem)) + r.durableWG.Done() + }() + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + t := time.NewTicker(durableLease / 4) + defer t.Stop() + for { + select { + case <-stop: + return + case <-t.C: + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + requested, err := r.queue.HeartbeatJob(ctx, j.ID, j.OwnerID, j.FencingToken, durableLease) + if err == nil { + err = r.queue.CheckpointJob(ctx, recordFromJob(j)) + } + cancel() + if err != nil { + leaseHealthy.Store(false) + j.RequestCancel() + log.Warn("heartbeat failed", "job_id", j.ID, "err", err) + return + } + if requested { + j.RequestCancel() + } + } + } + }() + r.workerStart(j) + close(stop) + <-done + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + r.durableMu.Lock() + stopped := r.durableStop + r.durableMu.Unlock() + c := recordFromJob(j) + if stopped && c.Status != StatusSucceeded && c.Status != StatusFailed { + if err := r.queue.ReleaseJob(ctx, j.ID, j.OwnerID, j.FencingToken); err != nil && !errors.Is(err, store.ErrJobLeaseLost) { + log.Warn("release failed", "err", err) + } + return + } + if !leaseHealthy.Load() { + // Leave an uncertain lease for expiry/reclaim. This consumes an attempt, + // whereas a deliberate graceful shutdown releases without using the budget. + return + } + if c.Status != StatusSucceeded && c.Status != StatusFailed && c.Status != StatusCancelled { + c.Status = StatusFailed + msg := "worker returned without terminal status" + c.Error = &msg + } + if err := r.queue.FinishJob(ctx, c); err != nil { + log.Warn("finish failed; lease will retry", "job_id", j.ID, "err", err) + } +} diff --git a/internal/jobs/durable_queue_test.go b/internal/jobs/durable_queue_test.go new file mode 100644 index 0000000..cbf078e --- /dev/null +++ b/internal/jobs/durable_queue_test.go @@ -0,0 +1,132 @@ +package jobs + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "evobgp/internal/store" +) + +type blockingClaimQueue struct { + store.JobQueue + entered chan int + resume chan struct{} +} + +func (q *blockingClaimQueue) ClaimJobs(ctx context.Context, owner string, limit int, lease time.Duration) ([]store.JobRecord, error) { + q.entered <- limit + <-q.resume + return q.JobQueue.ClaimJobs(context.Background(), owner, limit, lease) +} + +func TestDurableShutdownWaitsInflightClaimAndReleases(t *testing.T) { + t.Setenv("EVOBGP_JOB_MAX_CONCURRENT", "1") + m := store.NewMemory() + queued, _, err := m.EnqueueJob(context.Background(), store.JobEnqueue{TenantID: "tenant", Kind: "test"}) + if err != nil { + t.Fatal(err) + } + q := &blockingClaimQueue{JobQueue: m, entered: make(chan int, 1), resume: make(chan struct{})} + var runs atomic.Int32 + r := NewRegistry(func(j *Job) { runs.Add(1); j.Succeed() }) + r.UseDurableQueue(q) + StartDurableQueueLoop(context.Background(), r) + select { + case limit := <-q.entered: + if limit != cap(r.workerSem) { + t.Fatalf("claim capacity=%d reserved=%d", limit, cap(r.workerSem)) + } + case <-time.After(3 * time.Second): + t.Fatal("no claim") + } + r.RequestCancelAll() + drained := make(chan error, 1) + go func() { drained <- r.Drain(context.Background()) }() + select { + case <-drained: + t.Fatal("drained before in-flight claim returned") + case <-time.After(20 * time.Millisecond): + } + close(q.resume) + select { + case err := <-drained: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("shutdown did not drain") + } + if runs.Load() != 0 { + t.Fatal("started worker after stop") + } + got, err := m.GetJob(context.Background(), "tenant", queued.ID) + if err != nil || got.Status != "queued" || got.OwnerID != "" || got.Attempt != 0 { + t.Fatalf("not released: %+v %v", got, err) + } +} + +func TestDurableSharedTerminalAndRequestReplay(t *testing.T) { + ctx := context.Background() + m := store.NewMemory() + key := "request" + a := NewRegistry(nil) + a.UseDurableQueue(m) + j, _, err := a.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}}) + if err != nil { + t.Fatal(err) + } + claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute) + if err != nil || len(claimed) != 1 { + t.Fatalf("claim %+v %v", claimed, err) + } + claimed[0].Status = StatusSucceeded + claimed[0].Meta = map[string]any{"result": "ok"} + if err = m.FinishJob(ctx, claimed[0]); err != nil { + t.Fatal(err) + } + b := NewRegistry(nil) + b.UseDurableQueue(m) + got, err := b.GetContext(ctx, "tenant", j.ID) + if err != nil || got.Status != StatusSucceeded { + t.Fatalf("shared get %+v %v", got, err) + } + list, _, _, err := b.ListContext(ctx, "tenant", StatusSucceeded, "", "", 10) + if err != nil || len(list) != 1 { + t.Fatalf("shared list %+v %v", list, err) + } + replay, created, err := b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}}) + if err != nil || created || replay.ID != j.ID || replay.Status != StatusSucceeded { + t.Fatalf("terminal replay %+v %v %v", replay, created, err) + } + if _, _, err = b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"input": "different"}); !errors.Is(err, store.ErrJobConflict) { + t.Fatalf("changed payload accepted: %v", err) + } +} + +func TestParentRetryRecoversCommittedDeployChild(t *testing.T) { + ctx := context.Background() + m := store.NewMemory() + r := NewRegistry(nil) + r.UseDurableQueue(m) + parent, _, err := r.EnqueueContext(ctx, "tenant", KindRevisionRollback, nil, nil, map[string]any{"revision_id": "original"}) + if err != nil { + t.Fatal(err) + } + key := "deploy_from:" + parent.ID + child, _, err := r.EnqueueContext(ctx, "tenant", KindDeployApply, &key, nil, map[string]any{"revision_id": "already-rendered"}) + if err != nil { + t.Fatal(err) + } + worker := &Worker{Store: m, Registry: r} + worker.Process(parent) + if parent.statusLocked() != StatusSucceeded || parent.metaString("revision_id") != "already-rendered" || parent.metaString("deploy_apply_job_id") != child.ID { + t.Fatalf("did not recover child: %+v", parent.Snapshot()) + } + items, _, _, err := m.ListJobs(ctx, "tenant", "", KindDeployApply, "", 10) + if err != nil || len(items) != 1 { + t.Fatalf("duplicate child: %+v %v", items, err) + } +} diff --git a/internal/jobs/job.go b/internal/jobs/job.go index 5e69d25..6b69210 100644 --- a/internal/jobs/job.go +++ b/internal/jobs/job.go @@ -11,6 +11,7 @@ import ( "time" "evobgp/internal/observability" + "evobgp/internal/store" "github.com/google/uuid" ) @@ -38,6 +39,11 @@ type Job struct { Error *string ProgressPct *int16 Meta map[string]any + OwnerID string + FencingToken int64 + RequestSequence int64 + Attempt int + requestHash string cancelRequested bool mu sync.Mutex } @@ -225,6 +231,30 @@ type Registry struct { // have been enqueued but not yet finalized in finishModuleRefreshSuccess. Used for deterministic // deploy coalescing under tenantRefreshMu (instead of polling job statuses). inflightRefresh map[string]int + queue store.JobQueue + ownerID string + durableMu sync.Mutex + durableWG sync.WaitGroup + durableLoopWG sync.WaitGroup + durableStop bool +} + +// UseDurableQueue makes the shared store authoritative for new work and job reads. +func (r *Registry) UseDurableQueue(q store.JobQueue) { + r.mu.Lock() + defer r.mu.Unlock() + r.queue = q + r.ownerID = uuid.NewString() +} + +func jobFromRecord(c store.JobRecord) *Job { + return &Job{ID: c.ID, TenantID: c.TenantID, Kind: c.Kind, Status: c.Status, IdempotencyKey: c.IdempotencyKey, ModuleID: c.ModuleID, Meta: c.Meta, CreatedAt: c.CreatedAt, StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, Error: c.Error, OwnerID: c.OwnerID, FencingToken: c.FencingToken, RequestSequence: c.RequestSequence, Attempt: c.Attempt, cancelRequested: c.CancelRequested} +} + +func recordFromJob(j *Job) store.JobRecord { + j.mu.Lock() + defer j.mu.Unlock() + return store.JobRecord{ID: j.ID, TenantID: j.TenantID, Kind: j.Kind, Status: j.Status, IdempotencyKey: j.IdempotencyKey, ModuleID: j.ModuleID, Meta: cloneMeta(j.Meta), CreatedAt: j.CreatedAt, StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, Error: j.Error, OwnerID: j.OwnerID, FencingToken: j.FencingToken, RequestSequence: j.RequestSequence, Attempt: j.Attempt, CancelRequested: j.cancelRequested} } type idempoKey struct { @@ -310,6 +340,24 @@ func registryMaxConcurrentJobs() int { return 8 } +func jobRegistryPendingLimit() int { + if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_MAX_PENDING"))); err == nil && n > 0 { + return n + } + return 10000 +} + +func (r *Registry) ActiveCountUnsafe() int { + n := 0 + for _, j := range r.byID { + st := j.statusLocked() + if st == StatusQueued || st == StatusRunning { + n++ + } + } + return n +} + // pruneTerminalIfOver удаляет самые старые завершённые джобы (succeeded/failed/cancelled), пока len(byID) > maxJobs. func (r *Registry) pruneTerminalIfOver(maxJobs int) { if r == nil || maxJobs <= 0 || len(r.byID) <= maxJobs { @@ -352,6 +400,33 @@ func (r *Registry) pruneTerminalIfOver(maxJobs int) { // Enqueue creates a job or returns an existing one for the same idempotency key. func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) { + return r.EnqueueContext(context.Background(), tenantID, kind, idempotencyKey, moduleID, meta) +} + +// EnqueueContext commits a durable job before returning it to an HTTP caller. +func (r *Registry) EnqueueContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) { + return r.EnqueueRequestContext(ctx, tenantID, kind, idempotencyKey, moduleID, meta, nil) +} + +// EnqueueRequestContext preserves idempotency of dynamically resolved selectors. +func (r *Registry) EnqueueRequestContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta, request map[string]any) (*Job, bool, error) { + if r.queue != nil { + r.durableMu.Lock() + stopped := r.durableStop + r.durableMu.Unlock() + if stopped { + return nil, false, fmt.Errorf("jobs: shutting down") + } + c, created, err := r.queue.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request}) + if err != nil { + return nil, false, err + } + return jobFromRecord(c), created, nil + } + hash, err := store.JobRequestHash(store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request}) + if err != nil { + return nil, false, err + } r.mu.Lock() maxJobs := registryMaxJobsFromEnv() r.pruneTerminalIfOver(maxJobs) @@ -359,14 +434,17 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module if idempotencyKey != nil && *idempotencyKey != "" { k := idempoKey{tenant: tenantID, key: *idempotencyKey} if existing, ok := r.byIdempo[k]; ok { - st := existing.statusLocked() - if st == StatusQueued || st == StatusRunning { - r.mu.Unlock() - return existing, false, nil + r.mu.Unlock() + if existing.requestHash != hash { + return nil, false, store.ErrJobConflict } - delete(r.byIdempo, k) + return existing, false, nil } } + if r.ActiveCountUnsafe() >= jobRegistryPendingLimit() { + r.mu.Unlock() + return nil, false, store.ErrJobQueueFull + } j := &Job{ ID: uuid.NewString(), @@ -377,6 +455,7 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module ModuleID: moduleID, CreatedAt: time.Now().UTC(), Meta: cloneMeta(meta), + requestHash: hash, } if idempotencyKey != nil && *idempotencyKey != "" { r.byIdempo[idempoKey{tenant: tenantID, key: *idempotencyKey}] = j @@ -571,6 +650,11 @@ func (r *Registry) RequestCancelAll() int { if r == nil { return 0 } + if r.queue != nil { + r.durableMu.Lock() + r.durableStop = true + r.durableMu.Unlock() + } r.mu.RLock() defer r.mu.RUnlock() n := 0 @@ -585,6 +669,9 @@ func (r *Registry) RequestCancelAll() int { j.RequestCancel() n++ } + if r.queue != nil && len(r.workerSem) > n { + n = len(r.workerSem) + } return n } @@ -614,6 +701,16 @@ func (r *Registry) Drain(ctx context.Context) error { if r == nil { return nil } + if r.queue != nil { + done := make(chan struct{}) + go func() { r.durableLoopWG.Wait(); r.durableWG.Wait(); close(done) }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } ticker := time.NewTicker(50 * time.Millisecond) defer ticker.Stop() for { diff --git a/internal/jobs/registry_shared.go b/internal/jobs/registry_shared.go new file mode 100644 index 0000000..88b1f23 --- /dev/null +++ b/internal/jobs/registry_shared.go @@ -0,0 +1,115 @@ +package jobs + +import ( + "context" + "errors" + + "evobgp/internal/store" +) + +// FindIdempotentContext looks up a retained logical operation without inserting. +func (r *Registry) FindIdempotentContext(ctx context.Context, tenant, key string) (*Job, error) { + if r.queue != nil { + c, err := r.queue.FindJobByIdempotencyKey(ctx, tenant, key) + if err != nil { + return nil, err + } + j := jobFromRecord(c) + j.requestHash = c.RequestHash + return j, nil + } + r.mu.RLock() + defer r.mu.RUnlock() + j := r.byIdempo[idempoKey{tenant: tenant, key: key}] + if j == nil { + return nil, store.ErrJobNotFound + } + return j, nil +} + +// ReplayContext validates an immutable request before expensive live resolution. +func (r *Registry) ReplayContext(ctx context.Context, in store.JobEnqueue) (*Job, bool, error) { + if in.IdempotencyKey == nil || *in.IdempotencyKey == "" { + return nil, false, nil + } + j, err := r.FindIdempotentContext(ctx, in.TenantID, *in.IdempotencyKey) + if errors.Is(err, store.ErrJobNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + h, err := store.JobRequestHash(in) + if err != nil { + return nil, false, err + } + if j.requestHash != h { + return nil, false, store.ErrJobConflict + } + return j, true, nil +} + +// GetContext reads a job from the shared store when durability is enabled. +func (r *Registry) GetContext(ctx context.Context, tenant, id string) (*Job, error) { + if r.queue == nil { + return r.Get(tenant, id) + } + c, err := r.queue.GetJob(ctx, tenant, id) + if errors.Is(err, store.ErrJobNotFound) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + return jobFromRecord(c), nil +} + +// ListContext returns a bounded tenant page from the authoritative queue. +func (r *Registry) ListContext(ctx context.Context, tenant, status, kind, cursor string, limit int) ([]*Job, string, bool, error) { + if r.queue == nil { + j, n, m := r.List(tenant, status, kind, cursor, limit) + return j, n, m, nil + } + c, next, more, err := r.queue.ListJobs(ctx, tenant, status, kind, cursor, limit) + if err != nil { + return nil, "", false, err + } + out := make([]*Job, 0, len(c)) + for _, v := range c { + out = append(out, jobFromRecord(v)) + } + return out, next, more, nil +} + +// RequestCancelContext persists a cancel request visible to any worker replica. +func (r *Registry) RequestCancelContext(ctx context.Context, tenant, id string) (*Job, error) { + if r.queue == nil { + return r.RequestCancel(tenant, id) + } + c, err := r.queue.CancelJob(ctx, tenant, id) + if errors.Is(err, store.ErrJobNotFound) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + r.mu.RLock() + local := r.byID[id] + r.mu.RUnlock() + if local != nil { + local.RequestCancel() + } + return jobFromRecord(c), nil +} + +// RetryContext schedules an explicit retry of a failed shared operation. +func (r *Registry) RetryContext(ctx context.Context, tenant, id string) (*Job, error) { + if r.queue == nil { + return nil, store.ErrJobConflict + } + c, err := r.queue.RetryJob(ctx, tenant, id) + if err != nil { + return nil, err + } + return jobFromRecord(c), nil +} diff --git a/internal/jobs/worker.go b/internal/jobs/worker.go index 4b80d44..f0f17d3 100644 --- a/internal/jobs/worker.go +++ b/internal/jobs/worker.go @@ -2,6 +2,7 @@ package jobs import ( "context" + "errors" "fmt" "net/http" "net/netip" @@ -66,13 +67,18 @@ const ( // Worker executes queued jobs against store.Backend (memory or SQL). type Worker struct { - Store store.Backend - PgPool *pgxpool.Pool - HTTPClient *http.Client // optional; CDN refresh uses this (default 45s timeout). + Store store.Backend + PgPool *pgxpool.Pool + HTTPClient *http.Client // optional; CDN refresh uses this (default 45s timeout). + NodeHTTPClient *http.Client // optional; agent dispatch only, never shared with CDN transport. // Registry is set after BootstrapWorkers creates the job queue; used to chain deploy_apply after refresh/rollback. Registry *Registry - // refreshGate serializes deploy_apply gating after module_refresh per tenant (see finishModuleRefreshSuccess). - refreshGate sync.Map // map[string]*sync.Mutex + // refreshGate decides which module refresh renders for a tenant. + // The mutex covers only that decision; the render itself runs outside the lock. + refreshGate sync.Map // map[string]*tenantRefreshGate + // dispatchGate serializes publish and deployment state for one speaker in this process. + // Agent wake runs outside the mutex. + dispatchGate sync.Map // map[string]*sync.Mutex } type revisionLogEntry struct { @@ -116,6 +122,23 @@ func (w *Worker) Process(j *Job) { j.MarkCancelled() return } + // A committed child is the durable checkpoint for a parent that crashed + // before its own terminal write. Reuse it before rendering another revision. + if w.Registry != nil && (isRefreshKind(j.Kind) || j.Kind == KindRevisionRollback || j.Kind == KindPeerReconcile) { + child, err := w.Registry.FindIdempotentContext(context.Background(), j.TenantID, "deploy_from:"+j.ID) + if err == nil { + j.mergeMeta(map[string]any{"revision_id": child.metaString("revision_id"), "deploy_apply_job_id": child.ID, "recovered_child": true}) + if isRefreshKind(j.Kind) { + w.Registry.finalizeRefreshCoalesce(j.TenantID) + } + j.Succeed() + return + } + if !errors.Is(err, store.ErrJobNotFound) { + j.Fail(err.Error()) + return + } + } switch j.Kind { case KindModuleRefresh: @@ -185,50 +208,24 @@ func (w *Worker) runPeerReconcile(j *Job) { const peerJobTitle = "Обновление BGP пиров" j.mergeMeta(map[string]any{"job_title": peerJobTitle}) - var revID string latest, _, _ := w.Store.ListRevisions(j.TenantID, "", "", 1) triggerModuleID, err := w.peerTriggerModuleID(j.TenantID, latest) if err != nil { j.Fail(err.Error()) return } - if len(latest) == 0 { - ctx, cancel := j.workContext() - defer cancel() - rid, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID) - if err != nil { - if ctx.Err() != nil { - j.MarkCancelled() - return - } - j.Fail(err.Error()) + // Peer edits bump render generation. Render from module snapshots so the + // new revision includes current prefixes without paging the previous one. + ctx, cancel := j.workContext() + defer cancel() + revID, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID) + if err != nil { + if ctx.Err() != nil { + j.MarkCancelled() return } - revID = rid - } else { - baseRevID := latest[0].ID - rows := make([]store.PrefixRow, 0, 1024) - cursor := "" - for { - page, next, more := w.Store.ListRevisionPrefixes(j.TenantID, baseRevID, cursor, 2000) - rows = append(rows, page...) - if !more || strings.TrimSpace(next) == "" { - break - } - cursor = next - } - ctx, cancel := j.workContext() - defer cancel() - rid, err := pipeline.RenderTenantRevisionFromPrefixes(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID, rows) - if err != nil { - if ctx.Err() != nil { - j.MarkCancelled() - return - } - j.Fail(err.Error()) - return - } - revID = rid + j.Fail(err.Error()) + return } j.mergeMeta(map[string]any{"revision_id": revID}) if entries, total, err := w.buildRevisionLogEntries(j.TenantID, revID); err == nil { @@ -240,7 +237,10 @@ func (w *Worker) runPeerReconcile(j *Job) { } else { j.mergeMeta(map[string]any{"log_build_error": err.Error()}) } - w.enqueueDeployAllSpeakers(j, j.TenantID, revID) + if err := w.enqueueDeployAllSpeakers(j, j.TenantID, revID); err != nil { + j.Fail(err.Error()) + return + } j.Succeed() } @@ -269,8 +269,23 @@ func (w *Worker) peerTriggerModuleID(tenantID string, latest []*store.Revision) return "", fmt.Errorf("missing module_id for peer reconcile") } +type tenantRefreshGate struct { + mu sync.Mutex + rendering bool + dirty bool +} + +func (w *Worker) tenantRefreshGate(tenantID string) *tenantRefreshGate { + v, _ := w.refreshGate.LoadOrStore(tenantID, &tenantRefreshGate{}) + return v.(*tenantRefreshGate) +} + func (w *Worker) tenantRefreshMu(tenantID string) *sync.Mutex { - v, _ := w.refreshGate.LoadOrStore(tenantID, &sync.Mutex{}) + return &w.tenantRefreshGate(tenantID).mu +} + +func (w *Worker) speakerDispatchMu(speakerID string) *sync.Mutex { + v, _ := w.dispatchGate.LoadOrStore(speakerID, &sync.Mutex{}) return v.(*sync.Mutex) } @@ -327,7 +342,8 @@ func (w *Worker) runTenantRefresh(j *Job) { moduleIDs := moduleIDsFromJobMeta(j.metaCopy()) if len(moduleIDs) == 0 { - j.Fail("missing module_ids in job meta") + j.mergeMeta(map[string]any{"modules_refreshed": 0}) + j.Succeed() return } ctx, cancel := j.workContext() @@ -382,20 +398,19 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) { j.Succeed() return } - mu := w.tenantRefreshMu(j.TenantID) - mu.Lock() - defer mu.Unlock() - ctx, cancel := j.workContext() - defer cancel() - // Determine whether this is the last outstanding refresh for the tenant. The counter is - // incremented in Enqueue (under r.mu) and decremented here, so the "last one" decision is - // deterministic regardless of ingest timing — unlike the previous status-polling approach - // (CountOtherActiveRefresh) which could race under -race. + gate := w.tenantRefreshGate(j.TenantID) + gate.mu.Lock() + // The counter is incremented in Enqueue and decremented here, so the "last one" + // decision does not depend on job status polling. Render runs after the unlock. isLastRefresh := true if w.Registry != nil { isLastRefresh = w.Registry.finalizeRefreshCoalesce(j.TenantID) } - if !isLastRefresh { + if !isLastRefresh || gate.rendering { + if gate.rendering { + gate.dirty = true + } + gate.mu.Unlock() j.mergeMeta(map[string]any{ "deploy_apply_deferred": true, "deploy_apply_defer_reason": "parallel_module_refresh", @@ -403,16 +418,33 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) { j.Succeed() return } + gate.rendering = true + gate.mu.Unlock() + defer func() { + gate.mu.Lock() + gate.rendering = false + gate.mu.Unlock() + }() - rev, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID) + ctx, cancel := j.workContext() + defer cancel() + rev, err := w.renderTenantUntilQuiet(ctx, j, triggerModuleID) if err != nil { - if ctx.Err() != nil { + if errors.Is(err, errRenderSuperseded) { + j.Fail(err.Error()) + return + } + if ctx.Err() != nil || j.IsCancelRequested() { j.MarkCancelled() return } j.Fail(err.Error()) return } + if ctx.Err() != nil || j.IsCancelRequested() { + j.MarkCancelled() + return + } j.mergeMeta(map[string]any{"revision_id": rev}) if entries, total, err := w.buildRevisionLogEntries(j.TenantID, rev); err == nil { j.mergeMeta(map[string]any{ @@ -423,50 +455,240 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) { } else { j.mergeMeta(map[string]any{"log_build_error": err.Error()}) } - w.enqueueDeployAllSpeakers(j, j.TenantID, rev) + if err := w.enqueueDeployAllSpeakers(j, j.TenantID, rev); err != nil { + j.Fail(err.Error()) + return + } j.Succeed() } +const ( + // renderQuietPassLimit stops a render loop that never observes a quiet input set. + // Hitting the cap fails the job; the last revision must not be published. + renderQuietPassLimit = 32 + // speakerWakeLimit is the maximum number of concurrent agent wakes in one deploy. + speakerWakeLimit = 4 +) + +// errRenderSuperseded marks a render that must not be deployed. A newer snapshot +// landed, the job was cancelled while inputs were still changing, or the quiet +// pass cap was exhausted. +var errRenderSuperseded = errors.New("newer snapshot still pending") + +// renderTenantUntilQuiet builds a revision outside the tenant gate. +// It keeps rendering while another ingest marks the gate dirty or the snapshot +// generation moves, and returns a revision only after a fresh check shows the +// inputs are still the ones just rendered. Cancellation or the safety cap while +// inputs are still changing fails the render instead of publishing a stale revision. +func (w *Worker) renderTenantUntilQuiet(ctx context.Context, j *Job, triggerModuleID string) (string, error) { + gate := w.tenantRefreshGate(j.TenantID) + var rev string + for pass := 0; pass < renderQuietPassLimit; pass++ { + if abort := renderJobAborted(ctx, j); abort != nil { + if takeRefreshDirty(gate) { + return "", errRenderWhileDirty(abort) + } + return "", abort + } + before, err := w.captureRenderInputs(ctx, j.TenantID) + if err != nil { + if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) { + return "", errRenderWhileDirty(abort) + } + return "", err + } + rev, err = pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID) + if err != nil { + dirty := takeRefreshDirty(gate) + if abort := renderJobAborted(ctx, j); abort != nil && dirty { + return "", errRenderWhileDirty(abort) + } + if abort := renderJobAborted(ctx, j); abort != nil { + return "", abort + } + if dirty { + continue + } + return "", err + } + after, err := w.captureRenderInputs(ctx, j.TenantID) + if err != nil { + if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) { + return "", errRenderWhileDirty(abort) + } + return "", err + } + dirty := takeRefreshDirty(gate) + if dirty || renderInputsChanged(before, after) { + if abort := renderJobAborted(ctx, j); abort != nil { + return "", errRenderWhileDirty(abort) + } + continue + } + confirm, err := w.captureRenderInputs(ctx, j.TenantID) + if err != nil { + if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) { + return "", errRenderWhileDirty(abort) + } + return "", err + } + if takeRefreshDirty(gate) || renderInputsChanged(after, confirm) { + if abort := renderJobAborted(ctx, j); abort != nil { + return "", errRenderWhileDirty(abort) + } + continue + } + if abort := renderJobAborted(ctx, j); abort != nil { + return "", abort + } + return rev, nil + } + if abort := renderJobAborted(ctx, j); abort != nil { + return "", errRenderWhileDirty(abort) + } + return "", errRenderWhileDirty(nil) +} + +func errRenderWhileDirty(cause error) error { + if cause == nil { + return fmt.Errorf("jobs: tenant render did not quiesce after %d passes: %w", renderQuietPassLimit, errRenderSuperseded) + } + return fmt.Errorf("jobs: tenant render cancelled while a newer snapshot was still pending: %w", errRenderSuperseded) +} + +func renderJobAborted(ctx context.Context, j *Job) error { + if err := ctx.Err(); err != nil { + return err + } + if j != nil && j.IsCancelRequested() { + return context.Canceled + } + return nil +} + +func takeRefreshDirty(gate *tenantRefreshGate) bool { + if gate == nil { + return false + } + gate.mu.Lock() + defer gate.mu.Unlock() + dirty := gate.dirty + gate.dirty = false + return dirty +} + +type renderInputView struct { + generation int64 + snapshots []store.ModuleSnapshotUse +} + +func (w *Worker) captureRenderInputs(ctx context.Context, tenantID string) (renderInputView, error) { + if w == nil || w.Store == nil { + return renderInputView{}, fmt.Errorf("jobs: render store unavailable") + } + gen, err := w.Store.GetRenderGeneration(ctx, tenantID) + if err != nil { + if !errors.Is(err, store.ErrNotFound) { + return renderInputView{}, fmt.Errorf("jobs: read render generation: %w", err) + } + gen = 0 + } + headers, err := w.Store.ListModuleSnapshotHeaders(ctx, tenantID) + if err != nil { + return renderInputView{}, fmt.Errorf("jobs: read snapshot headers: %w", err) + } + return renderInputView{generation: gen, snapshots: headers}, nil +} + +func renderInputsChanged(before, after renderInputView) bool { + if before.generation != after.generation { + return true + } + return !sameSnapshotHeaders(before.snapshots, after.snapshots) +} + +func sameSnapshotHeaders(a, b []store.ModuleSnapshotUse) bool { + if len(a) != len(b) { + return false + } + aa := append([]store.ModuleSnapshotUse(nil), a...) + bb := append([]store.ModuleSnapshotUse(nil), b...) + sort.Slice(aa, func(i, j int) bool { return aa[i].ModuleID < aa[j].ModuleID }) + sort.Slice(bb, func(i, j int) bool { return bb[i].ModuleID < bb[j].ModuleID }) + for i := range aa { + if aa[i].ModuleID != bb[i].ModuleID || aa[i].SnapshotVersion != bb[i].SnapshotVersion || aa[i].InputHash != bb[i].InputHash { + return false + } + } + return true +} + // enqueueDeployAllSpeakers queues the same work as POST /v1/apply (all speakers, no speaker_id). -func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) { +func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) error { if w == nil || w.Registry == nil { - return + return fmt.Errorf("jobs: deploy queue unavailable") } revID = strings.TrimSpace(revID) if revID == "" { - return + return fmt.Errorf("jobs: missing revision for deploy") } meta := map[string]any{"revision_id": revID} if tr := strings.TrimSpace(j.metaString("trigger")); tr != "" { meta["trigger"] = tr } - applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, nil, nil, meta) + key := "deploy_from:" + j.ID + applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, &key, nil, meta) if err != nil { j.mergeMeta(map[string]any{"deploy_apply_enqueue_error": err.Error()}) - return + return fmt.Errorf("jobs: enqueue deploy: %w", err) } if applyJob != nil { j.mergeMeta(map[string]any{"deploy_apply_job_id": applyJob.ID}) } + return nil } func (w *Worker) runDeployApply(j *Job) { revID := strings.TrimSpace(j.metaString("revision_id")) spk := strings.TrimSpace(j.metaString("speaker_id")) - hasSpeaker := spk != "" if revID == "" { j.Fail("missing revision_id in job meta") return } + if j.OwnerID == "" { + if _, developmentMemory := w.Store.(*store.Memory); !developmentMemory { + j.Fail("deployment requires a current durable job lease") + return + } + } + ctx, cancel := j.workContext() defer cancel() - activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR")) - if activeDir != "" { - revObj, err := w.Store.GetRevision(j.TenantID, revID) + cancelled := func() bool { return ctx.Err() != nil || j.IsCancelRequested() } + var speakers []*store.Speaker + if spk != "" { + sp, err := w.Store.GetSpeaker(j.TenantID, spk) if err != nil { j.Fail(err.Error()) return } + speakers = []*store.Speaker{sp} + } else { + speakers = w.Store.ListSpeakersForTenant(j.TenantID) + } + activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR")) + localAttempted := false + var localErr error + applyLocal := func() error { + if localAttempted { + return localErr + } + localAttempted = true + revObj, err := w.Store.GetRevision(j.TenantID, revID) + if err != nil { + localErr = err + return err + } staging := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_STAGING_DIR")) if staging == "" { staging = os.TempDir() + "/evobgp-bird-staging" @@ -480,99 +702,322 @@ func (w *Worker) runDeployApply(j *Job) { } ctl := &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket} if err := birddeploy.ApplyRevision(ctx, ctl, revObj, cfg); err != nil { - if ctx.Err() != nil { - j.MarkCancelled() - return - } - j.Fail(err.Error()) - return - } - } - applied := make([]string, 0, 8) - var dispatchResults []nodedispatch.Result - applyOne := func(speakerID string) error { - if err := w.Store.SetLastAppliedRevision(j.TenantID, speakerID, revID); err != nil { + localErr = err return err } - // Replica / node pulls use LatestPublishedRevision; keep pointer in sync with successful deploy. - if err := w.Store.PublishRevisionForSpeaker(speakerID, revID); err != nil { - return err - } - applied = append(applied, speakerID) return nil } - dispatchSpeaker := func(sp *store.Speaker) { - if !nodedispatch.Enabled() || sp == nil { - return + tally := &deployTally{applied: make([]string, 0, len(speakers))} + slots := make(chan struct{}, speakerWakeLimit) + var wakeGroup sync.WaitGroup + acquireWake := func() bool { + select { + case slots <- struct{}{}: + case <-ctx.Done(): + return false } - meta := store.ParseSpeakerMeta(sp.MetaJSON) - if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) { - return - } - ctx2, cancel := context.WithTimeout(ctx, 35*time.Second) - defer cancel() - res := nodedispatch.WakeSpeaker(ctx2, sp, nodedispatch.Options{RevisionID: revID}) - dispatchResults = append(dispatchResults, res) - patch := store.SpeakerMeta{ - LastDispatchAt: time.Now().UTC().Format(time.RFC3339Nano), - LastDispatchStatus: res.Status, - } - if res.Error != "" { - patch.LastDispatchError = res.Error - patch.SyncStatus = "error" - } else if res.Status == "ok" { - patch.LastDispatchError = "" - patch.SyncStatus = "synced" - } - merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch) - _, _ = w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged}) - } - if hasSpeaker && spk != "" { - if err := applyOne(spk); err != nil { - j.Fail(err.Error()) - return - } - if sp, err := w.Store.GetSpeaker(j.TenantID, spk); err == nil { - dispatchSpeaker(sp) - } - if len(dispatchResults) > 0 { - j.mergeMeta(map[string]any{"node_dispatch": map[string]any{ - "revision_id": revID, - "results": dispatchResults, - }}) - } - mergeBirdPostApplyMeta(j) - j.Succeed() - return - } - speakers := w.Store.ListSpeakersForTenant(j.TenantID) - for _, sp := range speakers { - if err := applyOne(sp.ID); err != nil { - j.Fail(err.Error()) - return + if cancelled() { + <-slots + return false } + return true } for _, sp := range speakers { - dispatchSpeaker(sp) + if cancelled() { + break + } + if sp == nil { + continue + } + held := false + if speakerRemoteWake(sp) { + if !acquireWake() { + break + } + held = true + } + staged := w.stageSpeakerDeploy(ctx, j, sp, revID, activeDir, cancelled, applyLocal) + if staged.wake && !staged.stop { + wakeGroup.Add(1) + go func(sp *store.Speaker, deployment store.DeploymentSpeaker) { + defer wakeGroup.Done() + defer func() { <-slots }() + wakeCtx, cancelWake := context.WithTimeout(ctx, 35*time.Second) + res := nodedispatch.WakeSpeaker(wakeCtx, sp, nodedispatch.Options{RevisionID: revID, HTTPClient: w.NodeHTTPClient, MaxRetries: 1}) + cancelWake() + out, include, failedN, unconfirmedN, appliedID := w.finishSpeakerWake(ctx, j, sp, revID, deployment, res, cancelled) + tally.add(out, include, 0, failedN, unconfirmedN, appliedID) + }(sp, staged.deployment) + } else if held { + <-slots + } + tally.add(staged.result, staged.hasResult, staged.published, staged.failed, staged.unconfirmed, staged.appliedID) + if staged.stop { + break + } } + wakeGroup.Wait() + results, applied, published, failed, unconfirmed := tally.snapshot() j.mergeMeta(map[string]any{ "apply_summary": map[string]any{ - "revision_id": revID, - "speakers_count": len(applied), - "speaker_ids": applied, - "message": fmt.Sprintf("Ревизия %s применена на %d спикерах", shortID(revID), len(applied)), + "revision_id": revID, + "speakers_count": len(speakers), + "published_count": published, + "applied_count": len(applied), + "unconfirmed_count": unconfirmed, + "failed_count": failed, + "speaker_ids": applied, + "message": fmt.Sprintf("Ревизия %s: подтверждено %d, без подтверждения %d, ошибок %d", shortID(revID), len(applied), unconfirmed, failed), }, + "node_dispatch": map[string]any{"revision_id": revID, "results": results}, }) - if len(dispatchResults) > 0 { - j.mergeMeta(map[string]any{"node_dispatch": map[string]any{ - "revision_id": revID, - "results": dispatchResults, - }}) + if cancelled() { + j.MarkCancelled() + return } mergeBirdPostApplyMeta(j) + if cancelled() { + j.MarkCancelled() + return + } + if failed > 0 { + j.Fail(fmt.Sprintf("deploy delivery failed for %d of %d speakers", failed, len(speakers))) + return + } j.Succeed() } +type speakerStage struct { + stop bool + wake bool + deployment store.DeploymentSpeaker + result nodedispatch.Result + hasResult bool + published int + failed int + unconfirmed int + appliedID string +} + +type deployTally struct { + mu sync.Mutex + results []nodedispatch.Result + applied []string + published int + failed int + unconfirmed int +} + +func (t *deployTally) add(res nodedispatch.Result, include bool, published, failed, unconfirmed int, appliedID string) { + t.mu.Lock() + defer t.mu.Unlock() + if include { + t.results = append(t.results, res) + } + t.published += published + t.failed += failed + t.unconfirmed += unconfirmed + if appliedID != "" { + t.applied = append(t.applied, appliedID) + } +} + +func (t *deployTally) snapshot() (results []nodedispatch.Result, applied []string, published, failed, unconfirmed int) { + t.mu.Lock() + defer t.mu.Unlock() + results = append([]nodedispatch.Result(nil), t.results...) + applied = append([]string(nil), t.applied...) + if applied == nil { + applied = []string{} + } + sort.Slice(results, func(i, j int) bool { return results[i].SpeakerID < results[j].SpeakerID }) + sort.Strings(applied) + return results, applied, t.published, t.failed, t.unconfirmed +} + +func speakerRemoteWake(sp *store.Speaker) bool { + if sp == nil || !nodedispatch.Enabled() { + return false + } + return store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON)) +} + +func (w *Worker) bindDeploymentWriters(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker) (func(string, string, string) error, func(string) error) { + fence := deploymentFenceForJob(j) + record := func(state, confirmation, message string) error { + if j.OwnerID == "" { + return nil + } + return w.Store.RecordDeploymentSpeaker(ctx, fence, deployment, state, confirmation, message) + } + markApplied := func(confirmation string) error { + if j.OwnerID == "" { + return w.Store.SetLastAppliedRevision(j.TenantID, sp.ID, revID) + } + return record("applied", confirmation, "") + } + return record, markApplied +} + +// stageSpeakerDeploy publishes one speaker and records terminal states. +// Remote wakes are left to the caller so the speaker lock is not held across the network. +func (w *Worker) stageSpeakerDeploy(ctx context.Context, j *Job, sp *store.Speaker, revID, activeDir string, cancelled func() bool, applyLocal func() error) speakerStage { + var staged speakerStage + mu := w.speakerDispatchMu(sp.ID) + mu.Lock() + defer mu.Unlock() + if cancelled() { + staged.stop = true + return staged + } + var deployment store.DeploymentSpeaker + var publishErr error + if j.OwnerID != "" { + deployment, publishErr = beginDeployment(ctx, w.Store, deploymentFenceForJob(j), sp.ID, revID) + } else { + publishErr = w.Store.PublishRevisionForSpeaker(sp.ID, revID) + } + staged.deployment = deployment + record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment) + if publishErr != nil { + staged.failed = 1 + staged.hasResult = true + staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: publishErr.Error()} + return staged + } + staged.published = 1 + if deployment.State == "applied" { + staged.appliedID = sp.ID + staged.hasResult = true + staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: deployment.Confirmation, AppliedRevisionID: revID} + return staged + } + if cancelled() { + staged.stop = true + return staged + } + if !speakerRemoteWake(sp) { + if activeDir != "" && strings.EqualFold(sp.Role, "master") { + err := applyLocal() + if cancelled() { + staged.stop = true + return staged + } + if err != nil { + _ = record("failed", "unconfirmed", err.Error()) + staged.failed = 1 + staged.hasResult = true + staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()} + return staged + } + if err := markApplied("local_applied"); err != nil { + staged.failed = 1 + staged.hasResult = true + staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()} + return staged + } + staged.appliedID = sp.ID + staged.hasResult = true + staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: "local_applied", AppliedRevisionID: revID} + return staged + } + if err := record("unconfirmed", "unconfirmed", "dispatch disabled or speaker has no remote agent"); err != nil { + staged.failed = 1 + } + staged.unconfirmed = 1 + staged.hasResult = true + staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "skipped", Confirmation: "unconfirmed"} + return staged + } + if err := record("dispatched", "unconfirmed", "awaiting agent acknowledgement"); err != nil { + staged.failed = 1 + staged.hasResult = true + staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()} + return staged + } + if deployment.DispatchAttempts >= 3 { + _ = record("unknown", "unconfirmed", "delivery attempt budget exhausted") + staged.failed = 1 + staged.hasResult = true + staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Confirmation: "unconfirmed", Error: "delivery attempt budget exhausted"} + return staged + } + staged.wake = true + return staged +} + +// finishSpeakerWake records the agent acknowledgement. The speaker lock covers the +// state update and is not held while WakeSpeaker is in flight. +func (w *Worker) finishSpeakerWake(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker, res nodedispatch.Result, cancelled func() bool) (nodedispatch.Result, bool, int, int, string) { + mu := w.speakerDispatchMu(sp.ID) + mu.Lock() + defer mu.Unlock() + if cancelled != nil && cancelled() { + return nodedispatch.Result{SpeakerID: sp.ID, Endpoint: res.Endpoint, Status: "cancelled", Confirmation: "unconfirmed"}, true, 0, 0, "" + } + record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment) + appliedID := "" + if res.Status == "ok" && res.AppliedRevisionID != "" { + if res.AppliedRevisionID != revID { + res.Status = "error" + res.Confirmation = "unconfirmed" + res.Error = "agent reported a different applied revision" + } else if latest, _, err := w.Store.LatestPublishedRevision(sp.ID); err != nil { + res.Status = "error" + res.Confirmation = "unconfirmed" + res.Error = err.Error() + } else if latest != revID { + res.Status = "error" + res.Confirmation = "unconfirmed" + res.Error = "stale agent acknowledgement: desired revision changed" + } else if err := markApplied("agent_reported"); err != nil { + res.Status = "error" + res.Confirmation = "unconfirmed" + res.Error = err.Error() + } else { + appliedID = sp.ID + } + } + if res.Status != "ok" || res.Confirmation != "agent_reported" { + state := "unconfirmed" + if res.Status != "ok" { + state = "unknown" + } + if err := record(state, "unconfirmed", res.Error); err != nil { + res.Status = "error" + res.Error = err.Error() + } + } + patch := store.SpeakerMeta{ + LastDispatchAt: time.Now().UTC().Format(time.RFC3339Nano), + LastDispatchStatus: res.Status, + } + if res.Error != "" { + patch.LastDispatchError = res.Error + patch.SyncStatus = "error" + } else if res.Status == "ok" && res.Confirmation == "agent_reported" { + patch.LastDispatchError = "" + patch.SyncStatus = "synced" + } else if res.Status == "ok" { + patch.SyncStatus = "unconfirmed" + } + merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch) + if j.OwnerID == "" { + if _, err := w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged}); err != nil { + res.Status = "error" + res.Error = "dispatch metadata update: " + err.Error() + } + } + failed := 0 + unconfirmed := 0 + if res.Status == "ok" && res.Confirmation == "unconfirmed" { + unconfirmed = 1 + } else if res.Status != "ok" { + failed = 1 + } + return res, true, failed, unconfirmed, appliedID +} + func (w *Worker) runRollback(j *Job) { src := strings.TrimSpace(j.metaString("source_revision_id")) if src == "" { @@ -592,7 +1037,10 @@ func (w *Worker) runRollback(j *Job) { "message": fmt.Sprintf("Откат %s → %s", shortID(src), shortID(newID)), }, }) - w.enqueueDeployAllSpeakers(j, j.TenantID, newID) + if err := w.enqueueDeployAllSpeakers(j, j.TenantID, newID); err != nil { + j.Fail(err.Error()) + return + } j.Succeed() } diff --git a/internal/jobs/worker_test.go b/internal/jobs/worker_test.go index c91c7a8..b4d8c24 100644 --- a/internal/jobs/worker_test.go +++ b/internal/jobs/worker_test.go @@ -1,6 +1,9 @@ package jobs import ( + "context" + "fmt" + "strings" "testing" "time" @@ -185,3 +188,187 @@ func TestPeerReconcile_RendersFromLatestRevisionAndQueuesDeploy(t *testing.T) { t.Fatalf("expected deploy_apply_job_id in peer_reconcile meta, got %#v", meta) } } + +type renderHookStore struct { + *store.Memory + onCreate func(prefixes []store.PrefixRow) +} + +func (s *renderHookStore) CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parent *string, contentHash string, preview map[string]string, prefixes []store.PrefixRow) error { + if err := s.Memory.CreateRenderRevision(ctx, revisionID, tenantID, moduleID, parent, contentHash, preview, prefixes); err != nil { + return err + } + if s.onCreate != nil { + copied := append([]store.PrefixRow(nil), prefixes...) + s.onCreate(copied) + } + return nil +} + +func newIPRenderFixture(t *testing.T) (tenant, module string, mem *store.Memory) { + t.Helper() + t.Setenv("EVOBGP_ASN_RESOLVE", "0") + t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "") + mem = store.NewMemory() + mem.SeedDemo() + var modIP string + tenant, _, modIP, _, _ = mem.DemoIDs() + module = modIP + for _, mod := range mem.ListModules(tenant) { + if mod == nil || mod.ID == module { + continue + } + disabled := false + if _, err := mem.UpdateModule(tenant, mod.ID, &store.ModulePatch{Enabled: &disabled}); err != nil { + t.Fatal(err) + } + } + return tenant, module, mem +} + +func installModuleSnapshot(t *testing.T, mem *store.Memory, tenant, module, prefix string) { + t.Helper() + err := mem.SetModulePrefixSnapshot(context.Background(), tenant, module, "snap-"+prefix, []store.PrefixRow{{ + Prefix: prefix, + Source: "ip_range", + }}) + if err != nil { + t.Fatalf("snapshot %s: %v", prefix, err) + } +} + +func revisionHasPrefix(t *testing.T, mem *store.Memory, tenant, revID, prefix string) bool { + t.Helper() + rows, _, _ := mem.ListRevisionPrefixes(tenant, revID, "", 100) + for _, row := range rows { + if row.Prefix == prefix { + return true + } + } + return false +} + +func TestRenderTenantUntilQuietOverlappingIngests(t *testing.T) { + t.Run("latest snapshot", func(t *testing.T) { + tenant, module, mem := newIPRenderFixture(t) + prefixes := []string{"10.8.0.0/24", "10.9.0.0/24", "10.10.0.0/24", "10.11.0.0/24"} + installModuleSnapshot(t, mem, tenant, module, prefixes[0]) + hook := &renderHookStore{Memory: mem} + var wk *Worker + step := 0 + hook.onCreate = func([]store.PrefixRow) { + step++ + if step >= len(prefixes) { + return + } + installModuleSnapshot(t, mem, tenant, module, prefixes[step]) + gate := wk.tenantRefreshGate(tenant) + gate.mu.Lock() + gate.dirty = true + gate.mu.Unlock() + } + wk = &Worker{Store: hook, Registry: NewRegistry(nil)} + j := &Job{ID: "refresh-overlap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning} + wk.finishModuleRefreshSuccess(j, module) + if j.Status != StatusSucceeded { + errText := "" + if j.Error != nil { + errText = *j.Error + } + t.Fatalf("status=%s err=%s", j.Status, errText) + } + if step != len(prefixes) { + t.Fatalf("render commits=%d, want %d so three overlapping ingests are included", step, len(prefixes)) + } + rev := j.metaString("revision_id") + if !revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-1]) { + t.Fatalf("deployed revision %s missing latest prefix %s", rev, prefixes[len(prefixes)-1]) + } + if revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-2]) { + t.Fatalf("deployed revision %s still has stale prefix %s", rev, prefixes[len(prefixes)-2]) + } + deployID := j.metaString("deploy_apply_job_id") + deploy, err := wk.Registry.Get(tenant, deployID) + if err != nil { + t.Fatal(err) + } + if deploy.metaString("revision_id") != rev { + t.Fatalf("deploy revision=%s, want %s", deploy.metaString("revision_id"), rev) + } + }) + + t.Run("cancel does not deploy", func(t *testing.T) { + tenant, module, mem := newIPRenderFixture(t) + installModuleSnapshot(t, mem, tenant, module, "10.12.0.0/24") + hook := &renderHookStore{Memory: mem} + var wk *Worker + var j *Job + commits := 0 + hook.onCreate = func([]store.PrefixRow) { + commits++ + installModuleSnapshot(t, mem, tenant, module, "10.13.0.0/24") + gate := wk.tenantRefreshGate(tenant) + gate.mu.Lock() + gate.dirty = true + gate.mu.Unlock() + j.RequestCancel() + } + wk = &Worker{Store: hook, Registry: NewRegistry(nil)} + j = &Job{ID: "refresh-cancel", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning} + wk.finishModuleRefreshSuccess(j, module) + if j.Status != StatusFailed { + t.Fatalf("status=%s, want failed", j.Status) + } + if j.Error == nil || !strings.Contains(*j.Error, "newer snapshot") { + t.Fatalf("error=%v, want superseded render", j.Error) + } + if commits != 1 { + t.Fatalf("render commits=%d, want 1", commits) + } + if j.metaString("deploy_apply_job_id") != "" || j.metaString("revision_id") != "" { + t.Fatalf("cancelled dirty render published meta %#v", j.metaCopy()) + } + deploys, _, _ := wk.Registry.List(tenant, "", KindDeployApply, "", 10) + if len(deploys) != 0 { + t.Fatalf("deploy jobs=%d, want 0", len(deploys)) + } + revs, _, _ := mem.ListRevisions(tenant, "", "", 50) + for _, rev := range revs { + if rev != nil && revisionHasPrefix(t, mem, tenant, rev.ID, "10.13.0.0/24") { + t.Fatalf("revision %s contains the snapshot that landed during cancel", rev.ID) + } + } + }) + + t.Run("cap does not publish", func(t *testing.T) { + tenant, module, mem := newIPRenderFixture(t) + installModuleSnapshot(t, mem, tenant, module, "10.20.0.0/24") + hook := &renderHookStore{Memory: mem} + var wk *Worker + commits := 0 + hook.onCreate = func([]store.PrefixRow) { + commits++ + installModuleSnapshot(t, mem, tenant, module, fmt.Sprintf("10.21.%d.0/24", commits)) + gate := wk.tenantRefreshGate(tenant) + gate.mu.Lock() + gate.dirty = true + gate.mu.Unlock() + } + wk = &Worker{Store: hook, Registry: NewRegistry(nil)} + j := &Job{ID: "refresh-cap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning} + wk.finishModuleRefreshSuccess(j, module) + if j.Status != StatusFailed || j.Error == nil || !strings.Contains(*j.Error, "did not quiesce") { + errText := "" + if j.Error != nil { + errText = *j.Error + } + t.Fatalf("status=%s err=%s", j.Status, errText) + } + if commits != renderQuietPassLimit { + t.Fatalf("render commits=%d, want cap %d", commits, renderQuietPassLimit) + } + if j.metaString("deploy_apply_job_id") != "" { + t.Fatal("safety cap published a deploy") + } + }) +} diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index 8ad271b..3bfbb82 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -107,14 +107,14 @@ func LookupWithResolver( if addr, err := netip.ParseAddr(raw); err == nil { out.QueryKind = KindIP out.Normalized = addr.String() - if err := lookupIP(st, tenantID, addr, out, commByID, ""); err != nil { + if err := lookupIP(ctx, st, tenantID, addr, out, commByID, ""); err != nil { return nil, err } } else if pfx, err := netip.ParsePrefix(raw); err == nil { masked := pfx.Masked() out.QueryKind = KindCIDR out.Normalized = masked.String() - if err := lookupCIDR(st, tenantID, masked, out, commByID); err != nil { + if err := lookupCIDR(ctx, st, tenantID, masked, out, commByID); err != nil { return nil, err } } else { @@ -124,7 +124,7 @@ func LookupWithResolver( } out.QueryKind = KindDomain out.Normalized = fqdn - if err := lookupDomain(st, tenantID, fqdn, out, commByID); err != nil { + if err := lookupDomain(ctx, st, tenantID, fqdn, out, commByID); err != nil { return nil, err } if resolve == nil { @@ -178,7 +178,7 @@ func lookupResolvedIPs( out.ResolvedIPs = make([]string, 0, len(ips)) for _, ip := range ips { out.ResolvedIPs = append(out.ResolvedIPs, ip.String()) - if err := lookupIP(st, tenantID, ip, out, commByID, ip.String()); err != nil { + if err := lookupIP(ctx, st, tenantID, ip, out, commByID, ip.String()); err != nil { return err } } @@ -186,6 +186,7 @@ func lookupResolvedIPs( } func lookupIP( + ctx context.Context, st store.Backend, tenantID string, addr netip.Addr, @@ -227,7 +228,7 @@ func lookupIP( } } - snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID) + snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID) if err != nil { return err } @@ -269,6 +270,7 @@ func prefixCoversQuery(listPfx, query netip.Prefix) bool { } func lookupCIDR( + ctx context.Context, st store.Backend, tenantID string, query netip.Prefix, @@ -308,7 +310,7 @@ func lookupCIDR( } } - snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID) + snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID) if err != nil { return err } @@ -338,7 +340,7 @@ func lookupCIDR( return nil } -func lookupDomain(st store.Backend, tenantID, fqdn string, out *Result, commByID map[string]*store.Community) error { +func lookupDomain(ctx context.Context, st store.Backend, tenantID, fqdn string, out *Result, commByID map[string]*store.Community) error { matchedModuleIDs := make(map[string]*store.Module) for _, mod := range st.ListModules(tenantID) { @@ -372,7 +374,7 @@ func lookupDomain(st store.Backend, tenantID, fqdn string, out *Result, commByID } for mid, mod := range matchedModuleIDs { - snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mid) + snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mid) if err != nil { return err } diff --git a/internal/lookup/lookup_test.go b/internal/lookup/lookup_test.go index 4269a34..8c0acf4 100644 --- a/internal/lookup/lookup_test.go +++ b/internal/lookup/lookup_test.go @@ -35,7 +35,7 @@ func TestLookupIPEntryAndSnapshot(t *testing.T) { t.Fatal(err) } - if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash1", []store.PrefixRow{ + if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash1", []store.PrefixRow{ {Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"}, }); err != nil { t.Fatal(err) @@ -129,7 +129,7 @@ func TestLookupDomainEntryAndSnapshot(t *testing.T) { if err != nil { t.Fatal(err) } - if err := m.SetModulePrefixSnapshot(tenant, mod.ID, "hash-d", []store.PrefixRow{ + if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash-d", []store.PrefixRow{ {Prefix: "198.51.100.1/32", CommunityID: &cid, Source: "domain"}, {Prefix: "203.0.113.9/32", CommunityID: &cid, Source: "other"}, }); err != nil { @@ -183,7 +183,7 @@ func TestLookupDomainResolvedIPAgainstRanges(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash-r", []store.PrefixRow{ + if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash-r", []store.PrefixRow{ {Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"}, }); err != nil { t.Fatal(err) @@ -234,7 +234,7 @@ func TestLookupCIDREntryAndSnapshot(t *testing.T) { if err != nil { t.Fatal(err) } - if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash-cidr", []store.PrefixRow{ + if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash-cidr", []store.PrefixRow{ {Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"}, }); err != nil { t.Fatal(err) diff --git a/internal/nodecli/commands.go b/internal/nodecli/commands.go index 7f819d2..36dba48 100644 --- a/internal/nodecli/commands.go +++ b/internal/nodecli/commands.go @@ -61,13 +61,17 @@ func nodeHTTPClient() *http.Client { } func fetchLatestRevision(base, token, speaker string) (string, error) { + return fetchLatestRevisionContext(context.Background(), base, token, speaker) +} + +func fetchLatestRevisionContext(parent context.Context, base, token, speaker string) (string, error) { u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/revisions/latest" req, err := http.NewRequest(http.MethodGet, u, nil) if err != nil { return "", err } req.Header.Set("Authorization", "Bearer "+token) - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + ctx, cancel := context.WithTimeout(parent, 60*time.Second) defer cancel() resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3) if err != nil { @@ -75,13 +79,17 @@ func fetchLatestRevision(base, token, speaker string) (string, error) { } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) return "", fmt.Errorf("latest revision: %s: %s", resp.Status, strings.TrimSpace(string(b))) } var out struct { RevisionID string `json:"revision_id"` } - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + body, err := readHTTPBytes(resp, 64<<10) + if err != nil { + return "", err + } + if err := json.Unmarshal(body, &out); err != nil { return "", err } if out.RevisionID == "" { @@ -91,13 +99,21 @@ func fetchLatestRevision(base, token, speaker string) (string, error) { } func fetchBundle(base, token, speaker, revision string) ([]byte, error) { + return fetchBundleContext(context.Background(), base, token, speaker, revision, bundle.Limits{}) +} + +func fetchBundleContext(parent context.Context, base, token, speaker, revision string, limits bundle.Limits) ([]byte, error) { + l, err := limits.Normalized() + if err != nil { + return nil, err + } u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/bundle/" + revision req, err := http.NewRequest(http.MethodGet, u, nil) if err != nil { return nil, err } req.Header.Set("Authorization", "Bearer "+token) - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + ctx, cancel := context.WithTimeout(parent, 120*time.Second) defer cancel() resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3) if err != nil { @@ -105,10 +121,37 @@ func fetchBundle(base, token, speaker, revision string) ([]byte, error) { } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) return nil, fmt.Errorf("bundle: %s: %s", resp.Status, strings.TrimSpace(string(b))) } - return io.ReadAll(resp.Body) + return readHTTPBytes(resp, l.MaxCompressedBytes) +} + +func readHTTPBytes(resp *http.Response, limit int64) ([]byte, error) { + if resp.ContentLength > limit { + return nil, bundle.ErrLimit + } + return bundle.ReadCompressed(resp.Body, bundle.Limits{MaxCompressedBytes: limit}) +} + +func readBundleFile(path string, limits bundle.Limits) ([]byte, error) { + l, err := limits.Normalized() + if err != nil { + return nil, err + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + info, err := f.Stat() + if err != nil { + return nil, err + } + if info.Size() > l.MaxCompressedBytes { + return nil, bundle.ErrLimit + } + return bundle.ReadCompressed(f, l) } func loadPubKey(pubB64, pubHex string) (ed25519.PublicKey, error) { @@ -146,7 +189,7 @@ func CmdVerifyBundle(args []string) int { fmt.Fprintln(os.Stderr, err) return 2 } - raw, err := os.ReadFile(*path) + raw, err := readBundleFile(*path, bundle.Limits{}) if err != nil { fmt.Fprintln(os.Stderr, err) return 1 @@ -182,7 +225,7 @@ func CmdApplyBundle(args []string) int { fmt.Fprintln(os.Stderr, err) return 2 } - raw, err := os.ReadFile(*path) + raw, err := readBundleFile(*path, bundle.Limits{}) if err != nil { fmt.Fprintln(os.Stderr, err) return 1 diff --git a/internal/nodecli/limits_test.go b/internal/nodecli/limits_test.go new file mode 100644 index 0000000..8404bd0 --- /dev/null +++ b/internal/nodecli/limits_test.go @@ -0,0 +1,81 @@ +package nodecli + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "evobgp/internal/bundle" +) + +type countingBody struct { + io.Reader + read int +} + +func (b *countingBody) Read(p []byte) (int, error) { + n, e := b.Reader.Read(p) + b.read += n + return n, e +} +func (b *countingBody) Close() error { return nil } + +func TestHTTPBundleBoundKnownAndChunked(t *testing.T) { + for _, length := range []int64{-1, 2048} { + body := &countingBody{Reader: bytes.NewReader(bytes.Repeat([]byte{'x'}, 2048))} + _, err := readHTTPBytes(&http.Response{Body: body, ContentLength: length}, 1024) + if !errors.Is(err, bundle.ErrLimit) { + t.Fatalf("length=%d limit=%v", length, err) + } + if body.read > 1025 || (length > 1024 && body.read != 0) { + t.Fatalf("unbounded read %d for length %d", body.read, length) + } + } +} + +func TestLocalBundleSizeBound(t *testing.T) { + path := filepath.Join(t.TempDir(), "bundle.tar.gz") + if err := os.WriteFile(path, bytes.Repeat([]byte{'x'}, 2048), 0o600); err != nil { + t.Fatal(err) + } + if _, err := readBundleFile(path, bundle.Limits{MaxCompressedBytes: 1024}); !errors.Is(err, bundle.ErrLimit) { + t.Fatalf("local input cap: %v", err) + } +} + +func TestBundleHTTPMetadataAndDiagnosticsBound(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/revisions/latest"): + _, _ = io.WriteString(w, strings.Repeat(" ", 65<<10)) + default: + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, strings.Repeat("x", 64<<10)) + } + })) + defer server.Close() + if _, err := fetchLatestRevision(server.URL, "key", "speaker"); !errors.Is(err, bundle.ErrLimit) { + t.Fatalf("metadata unbounded: %v", err) + } + if _, err := fetchBundle(server.URL, "key", "speaker", "revision"); err == nil || len(err.Error()) > 5<<10 { + t.Fatalf("diagnostic unbounded: %v", err) + } +} + +func TestSyncFetchRespectsCallerCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := fetchBundleContext(ctx, "http://127.0.0.1:1", "key", "speaker", "revision", bundle.Limits{}); !errors.Is(err, context.Canceled) { + t.Fatalf("download ignored cancel: %v", err) + } + if _, err := fetchLatestRevisionContext(ctx, "http://127.0.0.1:1", "key", "speaker"); !errors.Is(err, context.Canceled) { + t.Fatalf("metadata ignored cancel: %v", err) + } +} diff --git a/internal/nodecli/sync.go b/internal/nodecli/sync.go index df8fdd8..7545695 100644 --- a/internal/nodecli/sync.go +++ b/internal/nodecli/sync.go @@ -12,23 +12,23 @@ import ( "evobgp/internal/birdfmt" "evobgp/internal/bundle" - "evobgp/internal/signing" ) // SyncConfig drives pull → verify → apply on a replica node. type SyncConfig struct { - BaseURL string - Token string - SpeakerID string - RevisionID string // empty = latest published on CP - PubKeyB64 string - PubKeyHex string - ExtractDir string - BundlePath string // temp file; default os.TempDir()/evobgp-bundle.tar.gz - BirdBin string - BirdcBin string - Socket string - HTTPClient interface { + BundleLimits bundle.Limits // zero selects bounded large-config defaults + BaseURL string + Token string + SpeakerID string + RevisionID string // empty = latest published on CP + PubKeyB64 string + PubKeyHex string + ExtractDir string + BundlePath string // temp file; default os.TempDir()/evobgp-bundle.tar.gz + BirdBin string + BirdcBin string + Socket string + HTTPClient interface { Do(req interface{}) (interface{}, error) } Timeout time.Duration @@ -59,12 +59,12 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) { rev := strings.TrimSpace(cfg.RevisionID) if rev == "" { var err error - rev, err = fetchLatestRevision(cfg.BaseURL, cfg.Token, cfg.SpeakerID) + rev, err = fetchLatestRevisionContext(ctx, cfg.BaseURL, cfg.Token, cfg.SpeakerID) if err != nil { return SyncResult{}, err } } - raw, err := fetchBundle(cfg.BaseURL, cfg.Token, cfg.SpeakerID, rev) + raw, err := fetchBundleContext(ctx, cfg.BaseURL, cfg.Token, cfg.SpeakerID, rev, cfg.BundleLimits) if err != nil { return SyncResult{}, err } @@ -75,7 +75,7 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) { if err := os.WriteFile(bundlePath, raw, 0o644); err != nil { return SyncResult{}, err } - v, err := signing.VerifyGzippedTar(raw, pub) + v, err := bundle.VerifyGzippedTarWithLimits(raw, pub, cfg.BundleLimits) if err != nil { return SyncResult{}, err } @@ -83,7 +83,7 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) { if err := os.MkdirAll(root, 0o755); err != nil { return SyncResult{}, err } - if err := bundle.WriteExtractedFiles(root, v); err != nil { + if err := bundle.WriteExtractedFilesWithLimits(root, v, cfg.BundleLimits); err != nil { return SyncResult{}, err } mainRel := v.FindMainBirdConf() diff --git a/internal/nodedispatch/bird.go b/internal/nodedispatch/bird.go index a2271c1..a238992 100644 --- a/internal/nodedispatch/bird.go +++ b/internal/nodedispatch/bird.go @@ -9,6 +9,7 @@ import ( "strings" "evobgp/internal/birdfmt" + "evobgp/internal/httpclient" "evobgp/internal/store" ) @@ -52,11 +53,16 @@ func FetchBirdProtocols(ctx context.Context, sp *store.Speaker, opts Options) Bi return res } defer func() { _ = resp.Body.Close() }() - b, _ := io.ReadAll(resp.Body) if resp.StatusCode < 200 || resp.StatusCode >= 300 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) res.Error = fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b))) return res } + b, err := httpclient.ReadResponseBody(resp, 8<<20) + if err != nil { + res.Error = err.Error() + return res + } var out struct { Sessions []birdfmt.BGPSession `json:"sessions"` } diff --git a/internal/nodedispatch/dispatch.go b/internal/nodedispatch/dispatch.go index ba44bd5..d9b6b67 100644 --- a/internal/nodedispatch/dispatch.go +++ b/internal/nodedispatch/dispatch.go @@ -20,6 +20,7 @@ type Result struct { SpeakerID string `json:"speaker_id"` Endpoint string `json:"endpoint,omitempty"` Status string `json:"status"` + Confirmation string `json:"confirmation,omitempty"` AppliedRevisionID string `json:"applied_revision_id,omitempty"` Error string `json:"error,omitempty"` } @@ -62,12 +63,13 @@ func Enabled() bool { // WakeSpeaker POSTs /v1/agent/sync to a replica agent (HTTPS via Traefik). func WakeSpeaker(ctx context.Context, sp *store.Speaker, opts Options) Result { - res := Result{SpeakerID: sp.ID} + res := Result{} if sp == nil { res.Status = "error" res.Error = "nil speaker" return res } + res.SpeakerID = sp.ID meta := store.ParseSpeakerMeta(sp.MetaJSON) url := store.AgentSyncURL(meta) if url == "" { @@ -117,13 +119,31 @@ func WakeSpeaker(ctx context.Context, sp *store.Speaker, opts Options) Result { _ = resp.Body.Close() if resp.StatusCode >= 200 && resp.StatusCode < 300 { var out struct { + OK *bool `json:"ok"` AppliedRevisionID string `json:"applied_revision_id"` + Error string `json:"error"` + } + if len(bytes.TrimSpace(b)) > 0 { + if err := json.Unmarshal(b, &out); err != nil { + res.Status = "error" + res.Error = "invalid agent sync response" + return res + } + } + if (out.OK != nil && !*out.OK) || strings.TrimSpace(out.Error) != "" { + res.Status = "error" + res.Error = strings.TrimSpace(out.Error) + if res.Error == "" { + res.Error = "agent sync reported failure" + } + return res } - _ = json.Unmarshal(b, &out) res.Status = "ok" res.AppliedRevisionID = strings.TrimSpace(out.AppliedRevisionID) - if res.AppliedRevisionID == "" { - res.AppliedRevisionID = strings.TrimSpace(opts.RevisionID) + if res.AppliedRevisionID != "" { + res.Confirmation = "agent_reported" + } else { + res.Confirmation = "unconfirmed" } return res } diff --git a/internal/nodedispatch/health.go b/internal/nodedispatch/health.go index a1eeba4..3bb3bdf 100644 --- a/internal/nodedispatch/health.go +++ b/internal/nodedispatch/health.go @@ -8,6 +8,7 @@ import ( "net/http" "strings" + "evobgp/internal/httpclient" "evobgp/internal/store" ) @@ -43,12 +44,16 @@ func FetchAgentHealth(ctx context.Context, sp *store.Speaker, opts Options) Agen return AgentHealthResult{Error: err.Error()} } defer func() { _ = resp.Body.Close() }() - b, _ := io.ReadAll(resp.Body) if resp.StatusCode < 200 || resp.StatusCode >= 300 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) return AgentHealthResult{ Error: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b))), } } + b, err := httpclient.ReadResponseBody(resp, 64<<10) + if err != nil { + return AgentHealthResult{Error: err.Error()} + } var out struct { OK bool `json:"ok"` LastAppliedRevisionID string `json:"last_applied_revision_id"` diff --git a/internal/nodedispatch/legacy_agent_binary_test.go b/internal/nodedispatch/legacy_agent_binary_test.go new file mode 100644 index 0000000..ca3d75c --- /dev/null +++ b/internal/nodedispatch/legacy_agent_binary_test.go @@ -0,0 +1,223 @@ +package nodedispatch_test + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "evobgp/internal/bundle" + "evobgp/internal/nodedispatch" + "evobgp/internal/store" +) + +func TestMain(m *testing.M) { + if os.Getenv("EVOBGP_TEST_LEGACY_BIRD_HELPER") != "1" { + os.Exit(m.Run()) + } + args := os.Args[1:] + log, err := os.OpenFile(os.Getenv("EVOBGP_TEST_BIRD_LOG"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + os.Exit(2) + } + _, _ = fmt.Fprintln(log, strings.Join(args, " ")) + _ = log.Close() + switch { + case len(args) == 3 && args[0] == "-c" && args[2] == "-p": + if _, err := os.Stat(args[1]); err == nil { + os.Exit(0) + } + case len(args) == 3 && args[0] == "-s" && args[2] == "configure": + os.Exit(0) + case len(args) == 5 && args[0] == "-s" && strings.Join(args[2:], " ") == "show protocols all": + fmt.Println("peer_test BGP master up 2026-01-01 Established") + os.Exit(0) + } + os.Exit(2) +} + +// TestLegacyAgentBinaryCurrentDispatch runs the historical agent as a separate +// process. All network traffic stays on loopback and BIRD executables are fakes. +func TestLegacyAgentBinaryCurrentDispatch(t *testing.T) { + binary := os.Getenv("EVOBGP_TEST_LEGACY_AGENT_BIN") + if binary == "" { + t.Skip("EVOBGP_TEST_LEGACY_AGENT_BIN not set") + } + binary, err := filepath.Abs(binary) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(binary); err != nil { + t.Fatal(err) + } + const speakerID = "legacy-speaker" + const revisionID = "legacy-revision" + const nodeToken = "local-node-token" + const agentSecret = "local-agent-secret" + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + archive, err := bundle.BuildGzippedTar(revisionID, speakerID, map[string]string{ + "bird.conf": "router id 192.0.2.1;\n", + }, priv) + if err != nil { + t.Fatal(err) + } + var bundleGets atomic.Int32 + controlPlane := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer "+nodeToken { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + switch r.URL.Path { + case "/v1/speakers/" + speakerID + "/bundle/" + revisionID: + bundleGets.Add(1) + w.Header().Set("Content-Type", "application/gzip") + _, _ = w.Write(archive) + default: + http.NotFound(w, r) + } + })) + defer controlPlane.Close() + + root := t.TempDir() + logPath := filepath.Join(root, "bird-invocations") + helper, err := os.Executable() + if err != nil { + t.Fatal(err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + agentAddr := listener.Addr().String() + _ = listener.Close() + outputPath := filepath.Join(root, "agent.log") + output, err := os.Create(outputPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = output.Close() }() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cmd := exec.CommandContext(ctx, binary, "-listen", agentAddr, "serve") + cmd.Stdout, cmd.Stderr = output, output + cmd.Env = append(os.Environ(), + "EVOBGP_AGENT_SECRET="+agentSecret, + "EVOBGP_CONTROL_PLANE_URL="+controlPlane.URL, + "EVOBGP_NODE_TOKEN="+nodeToken, + "EVOBGP_SPEAKER_ID="+speakerID, + "EVOBGP_BUNDLE_PUBKEY_BASE64="+base64.StdEncoding.EncodeToString(pub), + "EVOBGP_BIRD_EXTRACT_DIR="+filepath.Join(root, "extract"), + "EVOBGP_BIRD_BIN="+helper, + "EVOBGP_BIRDC_BIN="+helper, + "EVOBGP_BIRDC_SOCKET="+filepath.Join(root, "fake.ctl"), + "EVOBGP_TEST_LEGACY_BIRD_HELPER=1", + "EVOBGP_TEST_BIRD_LOG="+logPath, + "TMPDIR="+root, + "TMP="+root, + "TEMP="+root, + ) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + cancel() + _ = cmd.Wait() + }() + + transport := http.DefaultTransport.(*http.Transport).Clone() + // Under -race each helper process can spend a second flushing race output at + // exit. Sync executes parse-check and configure before returning its ACK. + client := &http.Client{Timeout: 10 * time.Second, Transport: legacyAgentTransport{addr: agentAddr, base: transport}} + defer transport.CloseIdleConnections() + sp := &store.Speaker{ID: speakerID, Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{ + AgentDomain: "legacy-agent.example.test", AgentSecret: agentSecret, + })} + deadline := time.Now().Add(10 * time.Second) + for { + health := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client}) + if health.OK { + if health.LastAppliedRevisionID != "" { + t.Fatalf("initial health has applied revision: %+v", health) + } + break + } + if time.Now().After(deadline) { + log, _ := os.ReadFile(outputPath) + t.Fatalf("legacy agent did not become healthy: %+v; output: %s", health, log) + } + time.Sleep(50 * time.Millisecond) + } + + // The historical agent must enforce the bearer token before reaching sync. + badReq, err := http.NewRequest(http.MethodPost, "https://legacy-agent.example.test/v1/agent/sync", strings.NewReader(`{"revision_id":"legacy-revision"}`)) + if err != nil { + t.Fatal(err) + } + badReq.Header.Set("Authorization", "Bearer wrong-secret") + badResp, err := client.Do(badReq) + if err != nil { + t.Fatal(err) + } + _ = badResp.Body.Close() + if badResp.StatusCode != http.StatusUnauthorized || bundleGets.Load() != 0 { + t.Fatalf("unauthorized sync: status=%d bundle_gets=%d", badResp.StatusCode, bundleGets.Load()) + } + + result := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{ + HTTPClient: client, MaxRetries: 1, RevisionID: revisionID, + }) + if result.Status != "ok" || result.Confirmation != "agent_reported" || result.AppliedRevisionID != revisionID { + log, _ := os.ReadFile(outputPath) + t.Fatalf("current dispatch to legacy agent: %+v; output: %s", result, log) + } + if bundleGets.Load() != 1 { + t.Fatalf("signed bundle gets = %d, want 1", bundleGets.Load()) + } + health := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client}) + if !health.OK || health.LastAppliedRevisionID != revisionID || health.LastSyncAt == "" { + t.Fatalf("post-sync legacy health: %+v", health) + } + protocols := nodedispatch.FetchBirdProtocols(context.Background(), sp, nodedispatch.Options{HTTPClient: client}) + if protocols.Error != "" || len(protocols.Sessions) != 1 || protocols.Sessions[0].State != "Established" { + t.Fatalf("legacy bird protocols: %+v", protocols) + } + mainConfig, err := os.ReadFile(filepath.Join(root, "extract", "bird.conf")) + if err != nil || string(mainConfig) != "router id 192.0.2.1;\n" { + t.Fatalf("extracted signed bundle: content=%q err=%v", mainConfig, err) + } + invocations, err := os.ReadFile(logPath) + if err != nil { + t.Fatal(err) + } + wantParse := fmt.Sprintf("-c %s -p", filepath.Join(root, "extract", "bird.conf")) + if !strings.Contains(string(invocations), wantParse) || !strings.Contains(string(invocations), "-s "+filepath.Join(root, "fake.ctl")+" configure") { + t.Fatalf("legacy agent did not parse-check and configure via stand-ins: %s", invocations) + } +} + +type legacyAgentTransport struct { + addr string + base http.RoundTripper +} + +func (tr legacyAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) { + copyReq := req.Clone(req.Context()) + copyReq.URL.Scheme = "http" + copyReq.URL.Host = tr.addr + copyReq.Host = tr.addr + return tr.base.RoundTrip(copyReq) +} diff --git a/internal/nodedispatch/legacy_compatibility_test.go b/internal/nodedispatch/legacy_compatibility_test.go new file mode 100644 index 0000000..f40a0ed --- /dev/null +++ b/internal/nodedispatch/legacy_compatibility_test.go @@ -0,0 +1,106 @@ +package nodedispatch_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "reflect" + "strings" + "testing" + + "evobgp/internal/nodedispatch" + "evobgp/internal/store" +) + +type legacyRoundTripFunc func(*http.Request) (*http.Response, error) + +func (f legacyRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestLegacyCompatibilityAgentSync(t *testing.T) { + for _, tc := range []struct { + name, response, wantApplied string + status int + }{ + {"legacy bare success", `{"ok":true}`, "", http.StatusOK}, + {"legacy empty no-content", "", "", http.StatusNoContent}, + {"legacy explicit revision", `{"ok":true,"applied_revision_id":"revision-1"}`, "revision-1", http.StatusOK}, + } { + t.Run(tc.name, func(t *testing.T) { + var gotAuth, gotType string + var gotBody map[string]json.RawMessage + client := &http.Client{Transport: legacyRoundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/v1/agent/sync" { + t.Errorf("unexpected sync request: %s %s", r.Method, r.URL) + } + gotAuth, gotType = r.Header.Get("Authorization"), r.Header.Get("Content-Type") + body, _ := io.ReadAll(r.Body) + if err := json.Unmarshal(body, &gotBody); err != nil { + t.Errorf("sync JSON: %v", err) + } + return &http.Response{StatusCode: tc.status, Body: io.NopCloser(strings.NewReader(tc.response)), Header: make(http.Header)}, nil + })} + sp := &store.Speaker{ID: "speaker-1", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{ + AgentDomain: "agent.example.test", AgentSecret: "legacy-secret", + })} + got := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{HTTPClient: client, MaxRetries: 1, RevisionID: "revision-1"}) + if got.Status != "ok" || got.AppliedRevisionID != tc.wantApplied { + t.Fatalf("sync result: %+v", got) + } + wantConfirmation := "unconfirmed" + if tc.wantApplied != "" { + wantConfirmation = "agent_reported" + } + if got.Confirmation != wantConfirmation { + t.Fatalf("confirmation = %q, want %q", got.Confirmation, wantConfirmation) + } + if gotAuth != "Bearer legacy-secret" || gotType != "application/json" { + t.Fatalf("legacy headers: authorization=%q content-type=%q", gotAuth, gotType) + } + if !reflect.DeepEqual(gotBody, map[string]json.RawMessage{"revision_id": json.RawMessage(`"revision-1"`)}) { + t.Fatalf("unexpected fields sent to strict legacy decoder: %s", gotBody) + } + }) + } +} + +func TestAgentSyncRejectsMalformedSuccess(t *testing.T) { + for _, response := range []string{`{"ok":false}`, `{"ok":true`, `{"ok":false,"applied_revision_id":"revision-1"}`} { + t.Run(response, func(t *testing.T) { + client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil + })} + sp := &store.Speaker{ID: "speaker-1", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})} + got := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{HTTPClient: client, MaxRetries: 1, RevisionID: "revision-1"}) + if got.Status != "error" || got.AppliedRevisionID != "" { + t.Fatalf("invalid 2xx response accepted: %+v", got) + } + }) + } +} + +func TestLegacyCompatibilityAgentHealthMissingNewFields(t *testing.T) { + for _, tc := range []struct { + name, body, wantRevision, wantSync string + }{ + {"bare legacy health", `{"ok":true}`, "", ""}, + {"revision without time", `{"ok":true,"last_applied_revision_id":"revision-1"}`, "revision-1", ""}, + {"revision with time", `{"ok":true,"last_applied_revision_id":"revision-1","last_sync_at":"2026-10-26T00:00:00Z"}`, "revision-1", "2026-10-26T00:00:00Z"}, + } { + t.Run(tc.name, func(t *testing.T) { + client := &http.Client{Transport: legacyRoundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodGet || r.URL.Path != "/v1/agent/health" || r.Header.Get("Authorization") != "Bearer legacy-secret" { + t.Errorf("unexpected health request: %s %s", r.Method, r.URL) + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(tc.body)), Header: make(http.Header)}, nil + })} + sp := &store.Speaker{ID: "speaker-1", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{ + AgentDomain: "agent.example.test", AgentSecret: "legacy-secret", + })} + got := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client}) + if !got.OK || got.Error != "" || got.LastAppliedRevisionID != tc.wantRevision || got.LastSyncAt != tc.wantSync { + t.Fatalf("legacy health: %+v", got) + } + }) + } +} diff --git a/internal/nodedispatch/telemetry_limits_test.go b/internal/nodedispatch/telemetry_limits_test.go new file mode 100644 index 0000000..e8789fe --- /dev/null +++ b/internal/nodedispatch/telemetry_limits_test.go @@ -0,0 +1,80 @@ +package nodedispatch_test + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + "evobgp/internal/nodedispatch" + "evobgp/internal/store" +) + +type telemetryTestBody struct { + io.Reader + read int + closed bool +} + +func (b *telemetryTestBody) Read(p []byte) (int, error) { + n, err := b.Reader.Read(p) + b.read += n + return n, err +} +func (b *telemetryTestBody) Close() error { b.closed = true; return nil } + +type telemetryErrorReader struct{} + +func (telemetryErrorReader) Read([]byte) (int, error) { return 0, errors.New("injected body failure") } + +func TestAgentTelemetryCompleteBodyLimits(t *testing.T) { + sp := &store.Speaker{ID: "legacy", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})} + for _, tc := range []struct { + name, prefix string + limit int + call func(nodedispatch.Options) string + }{ + {"health", `{"ok":true}`, 64 << 10, func(opts nodedispatch.Options) string { + return nodedispatch.FetchAgentHealth(context.Background(), sp, opts).Error + }}, + {"protocols", `{"sessions":[]}`, 8 << 20, func(opts nodedispatch.Options) string { + return nodedispatch.FetchBirdProtocols(context.Background(), sp, opts).Error + }}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, known := range []bool{true, false} { + for _, extra := range []int{0, 1} { + data := tc.prefix + strings.Repeat(" ", tc.limit-len(tc.prefix)+extra) + body := &telemetryTestBody{Reader: strings.NewReader(data)} + client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) { + length := int64(-1) + if known { + length = int64(len(data)) + } + return &http.Response{StatusCode: 200, ContentLength: length, Body: body, Header: make(http.Header)}, nil + })} + err := tc.call(nodedispatch.Options{HTTPClient: client}) + if (extra == 0 && err != "") || (extra == 1 && err == "") || body.read > tc.limit+1 || !body.closed { + t.Fatalf("known=%v extra=%d read=%d closed=%v error=%s", known, extra, body.read, body.closed, err) + } + } + } + for _, status := range []int{200, 400} { + reader := io.MultiReader(strings.NewReader(tc.prefix), telemetryErrorReader{}) + if status == 400 { + reader = io.MultiReader(strings.NewReader(strings.Repeat("x", 16000))) + } + body := &telemetryTestBody{Reader: reader} + client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: status, ContentLength: -1, Body: body, Header: make(http.Header)}, nil + })} + err := tc.call(nodedispatch.Options{HTTPClient: client}) + if err == "" || !body.closed || (status == 400 && body.read > 4<<10) { + t.Fatalf("status=%d read=%d closed=%v error=%s", status, body.read, body.closed, err) + } + } + }) + } +} diff --git a/internal/observability/metrics.go b/internal/observability/metrics.go index b8995e2..1aa7d6d 100644 --- a/internal/observability/metrics.go +++ b/internal/observability/metrics.go @@ -20,10 +20,12 @@ import ( const namespace = "evobgp" var ( - metricsStore atomic.Value // store.Backend + metricsStore atomic.Value // storeBackendHolder registerCollectorOnce sync.Once ) +type storeBackendHolder struct{ backend store.Backend } + var ( httpRequests = promauto.NewCounterVec( prometheus.CounterOpts{ @@ -117,7 +119,7 @@ var ( upstreamBreakerOpen = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Name: "upstream_breaker_open", - Help: "1 if per-host HTTP circuit breaker is open.", + Help: "Last request-time observation of the per-host circuit breaker (1=open, 0=closed); idle host series expire with the bounded cache.", }, []string{"host"}) ) @@ -173,6 +175,9 @@ func RecordJobTerminal(kind, status string) { // RecordJobAuditDepth updates durable job_audit depth gauges. func RecordJobAuditDepth(counts map[string]int64) { + for _, status := range []string{"queued", "running", "succeeded", "failed", "cancelled"} { + jobAuditDepth.WithLabelValues(status).Set(float64(counts[status])) + } for status, n := range counts { if status == "" { status = "unknown" @@ -193,6 +198,15 @@ func SetUpstreamBreakerOpen(host string, open bool) { upstreamBreakerOpen.WithLabelValues(host).Set(v) } +// RemoveUpstreamBreaker removes an expired host series together with its cache entry. +// Callers must retain active entries until their final metric update completes. +func RemoveUpstreamBreaker(host string) { + if host == "" { + host = "_" + } + upstreamBreakerOpen.DeleteLabelValues(host) +} + // SetBuildInfo sets evobgp_build_info gauge (idempotent labels). func SetBuildInfo(version, gitSHA string) { if version == "" { @@ -205,34 +219,49 @@ func SetBuildInfo(version, gitSHA string) { } type memoryStoreCollector struct { - prefixMaxDesc *prometheus.Desc - prefixSumDesc *prometheus.Desc - peersDesc *prometheus.Desc - peerStateDesc *prometheus.Desc + prefixMaxDesc *prometheus.Desc + prefixSumDesc *prometheus.Desc + peersDesc *prometheus.Desc + peerStateDesc *prometheus.Desc + storedMaxDesc *prometheus.Desc + storedSumDesc *prometheus.Desc + inventoryDesc *prometheus.Desc + storedStateDesc *prometheus.Desc + collectionSuccessDesc *prometheus.Desc } func newMemoryStoreCollector() *memoryStoreCollector { return &memoryStoreCollector{ prefixMaxDesc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "materialized_prefixes_max"), - "Maximum materialized_prefix_count among all revisions in the store.", + "Deprecated: maximum materialized_prefix_count among all stored revisions; use stored_revision_materialized_prefixes_max.", nil, nil, ), prefixSumDesc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "materialized_prefixes_sum"), - "Sum of materialized_prefix_count over revisions (development aggregate).", + "Deprecated: sum of materialized_prefix_count over all stored revisions; use stored_revision_materialized_prefixes_sum.", nil, nil, ), peersDesc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "bgp_peers_configured_total"), - "BGP peers configured in the control-plane store.", + "Deprecated: BGP peers configured in the control-plane store; use inventory_bgp_peers.", nil, nil, ), peerStateDesc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "bgp_peer_sessions"), - "Configured BGP peers in the store by session_state (intent / last known, not live BIRD).", + "Deprecated: configured BGP peers by persisted session_state, not live BIRD; use stored_bgp_peer_session_state.", []string{"state"}, nil, ), + storedMaxDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_max"), + "Maximum materialized prefix count across all stored revisions; historical, not current routes.", nil, nil), + storedSumDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_sum"), + "Sum of materialized prefix counts across all stored revisions; historical, not current routes.", nil, nil), + inventoryDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "inventory_bgp_peers"), + "Configured BGP peer inventory in the control-plane store, regardless of live state.", nil, nil), + storedStateDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_bgp_peer_session_state"), + "BGP peer inventory grouped by persisted session_state; not a live observation.", []string{"state"}, nil), + collectionSuccessDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "store_metrics_collection_success"), + "1 if the latest store metrics collection succeeded, 0 on database aggregation error; other store metrics are omitted on error.", nil, nil), } } @@ -241,6 +270,11 @@ func (c *memoryStoreCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.prefixSumDesc ch <- c.peersDesc ch <- c.peerStateDesc + ch <- c.storedMaxDesc + ch <- c.storedSumDesc + ch <- c.inventoryDesc + ch <- c.storedStateDesc + ch <- c.collectionSuccessDesc } func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) { @@ -248,19 +282,44 @@ func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) { if v == nil { return } - b, ok := v.(store.Backend) - if !ok || b == nil { + holder, ok := v.(storeBackendHolder) + if !ok || holder.backend == nil { return } - maxN, sumN := b.MaterializedPrefixStats() + b := holder.backend + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + maxN, sumN, peers, states, err := b.ObserveStoreMetrics(ctx) + cancel() + if err != nil { + ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 0) + return + } + ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 1) ch <- prometheus.MustNewConstMetric(c.prefixMaxDesc, prometheus.GaugeValue, float64(maxN)) ch <- prometheus.MustNewConstMetric(c.prefixSumDesc, prometheus.GaugeValue, float64(sumN)) - ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(b.PeerCount())) - for state, n := range b.PeerSessionCountsByState() { + ch <- prometheus.MustNewConstMetric(c.storedMaxDesc, prometheus.GaugeValue, float64(maxN)) + ch <- prometheus.MustNewConstMetric(c.storedSumDesc, prometheus.GaugeValue, float64(sumN)) + ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(peers)) + ch <- prometheus.MustNewConstMetric(c.inventoryDesc, prometheus.GaugeValue, float64(peers)) + canonicalStates := make(map[string]int) + for state, n := range states { if state == "" { state = "unknown" } ch <- prometheus.MustNewConstMetric(c.peerStateDesc, prometheus.GaugeValue, float64(n), state) + canonicalStates[canonicalPeerState(state)] += n + } + for state, n := range canonicalStates { + ch <- prometheus.MustNewConstMetric(c.storedStateDesc, prometheus.GaugeValue, float64(n), state) + } +} + +func canonicalPeerState(state string) string { + switch strings.ToLower(strings.TrimSpace(state)) { + case "idle", "connect", "active", "opensent", "openconfirm", "established", "unknown": + return strings.ToLower(strings.TrimSpace(state)) + default: + return "other" } } @@ -269,7 +328,7 @@ func RegisterStoreBackend(b store.Backend) { if b == nil { return } - metricsStore.Store(b) + metricsStore.Store(storeBackendHolder{backend: b}) registerCollectorOnce.Do(func() { prometheus.DefaultRegisterer.MustRegister(newMemoryStoreCollector()) }) diff --git a/internal/observability/metrics_store_test.go b/internal/observability/metrics_store_test.go new file mode 100644 index 0000000..5bb1e14 --- /dev/null +++ b/internal/observability/metrics_store_test.go @@ -0,0 +1,116 @@ +package observability + +import ( + "context" + "errors" + "testing" + + "github.com/prometheus/client_golang/prometheus" + + "evobgp/internal/store" +) + +type failingMetricsBackend struct{ store.Backend } + +func (failingMetricsBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) { + return 0, 0, 0, nil, errors.New("database unavailable") +} + +type storedMetricsBackend struct{ store.Backend } + +func (storedMetricsBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) { + return 12, 24, 1, map[string]int{"established": 1}, nil +} + +type legacyStateBackend struct{ store.Backend } + +func (legacyStateBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) { + return 0, 0, 2, map[string]int{" Established ": 1, "established": 1}, nil +} + +func TestStoreMetricsCollectionErrorOmitsAggregates(t *testing.T) { + metricsStore.Store(storeBackendHolder{backend: failingMetricsBackend{}}) + registry := prometheus.NewRegistry() + if err := registry.Register(newMemoryStoreCollector()); err != nil { + t.Fatal(err) + } + families, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + if len(families) != 1 || families[0].GetName() != "evobgp_store_metrics_collection_success" { + t.Fatalf("expected only collection status on error, got %v", families) + } + if got := families[0].Metric[0].Gauge.GetValue(); got != 0 { + t.Fatalf("collection status = %v, want 0", got) + } +} + +func TestStoredRevisionMetricsDoNotClaimCurrentRoutes(t *testing.T) { + metricsStore.Store(storeBackendHolder{backend: storedMetricsBackend{}}) + registry := prometheus.NewRegistry() + if err := registry.Register(newMemoryStoreCollector()); err != nil { + t.Fatal(err) + } + families, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + values := make(map[string]float64) + for _, family := range families { + if len(family.Metric) > 0 && family.Metric[0].Gauge != nil { + values[family.GetName()] = family.Metric[0].Gauge.GetValue() + } + } + if values["evobgp_stored_revision_materialized_prefixes_sum"] != 24 || values["evobgp_materialized_prefixes_sum"] != 24 { + t.Fatalf("stored revision and legacy sums must match: %v", values) + } + if values["evobgp_store_metrics_collection_success"] != 1 { + t.Fatalf("successful collection not reported: %v", values) + } + if _, ok := values["evobgp_current_routes"]; ok { + t.Fatal("historical revision sum must not be published as current routes") + } +} + +func TestCanonicalPeerStateBoundsNewLabelValues(t *testing.T) { + if got := canonicalPeerState("Established"); got != "established" { + t.Fatalf("state = %q", got) + } + if got := canonicalPeerState("unexpected peer text"); got != "other" { + t.Fatalf("state = %q", got) + } +} + +func TestLegacyPeerStateLabelPreservedWhileNewLabelIsCanonical(t *testing.T) { + metricsStore.Store(storeBackendHolder{backend: legacyStateBackend{}}) + registry := prometheus.NewRegistry() + if err := registry.Register(newMemoryStoreCollector()); err != nil { + t.Fatal(err) + } + families, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + legacyRaw, newCanonical := false, false + for _, family := range families { + for _, metric := range family.Metric { + if len(metric.Label) != 1 { + continue + } + switch family.GetName() { + case "evobgp_bgp_peer_sessions": + if metric.Label[0].GetValue() == " Established " && metric.Gauge.GetValue() == 1 { + legacyRaw = true + } + case "evobgp_stored_bgp_peer_session_state": + if metric.Label[0].GetValue() == "established" && metric.Gauge.GetValue() == 2 { + newCanonical = true + } + } + } + } + if !legacyRaw || !newCanonical { + t.Fatalf("legacy raw state preserved=%v, new canonical state aggregated=%v", legacyRaw, newCanonical) + } +} diff --git a/internal/pipeline/aggregate.go b/internal/pipeline/aggregate.go index 60d85fd..e93d8fa 100644 --- a/internal/pipeline/aggregate.go +++ b/internal/pipeline/aggregate.go @@ -86,20 +86,26 @@ func aggregateTenantPrefixRowsAll(ctx context.Context, st store.Backend, hc *htt } func rowsForModule(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module) ([]store.PrefixRow, error) { - if rows, ok, err := moduleRowsFromSnapshot(st, tenantID, mod); err != nil { - return nil, err - } else if ok { - return rows, nil - } - var prior []store.PrefixRow - if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil { - prior = snap.Prefixes - } - rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, prior) + snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID) if err != nil { return nil, err } - if err := persistModuleSnapshot(st, tenantID, mod, rows); err != nil { + if ok && snap != nil { + if rows, matched, err := snapshotRowsIfCurrent(ctx, st, tenantID, mod, snap); err != nil { + return nil, err + } else if matched { + return rows, nil + } + } + var priorRows []store.PrefixRow + if ok && snap != nil { + priorRows = snap.Prefixes + } + rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, modulePrior{provided: true, rows: priorRows}) + if err != nil { + return nil, err + } + if err := persistModuleSnapshot(ctx, st, tenantID, mod, rows); err != nil { return nil, err } return rows, nil diff --git a/internal/pipeline/cdn_body.go b/internal/pipeline/cdn_body.go new file mode 100644 index 0000000..623fec1 --- /dev/null +++ b/internal/pipeline/cdn_body.go @@ -0,0 +1,49 @@ +package pipeline + +import ( + "errors" + "io" +) + +const cdnBodyLimit = 8 << 20 + +// ErrCDNBodyTooLarge reports a CDN response larger than the 8 MiB decoded-body limit. +var ErrCDNBodyTooLarge = errors.New("pipeline: cdn response body exceeds 8 MiB") + +type limitedCDNBody struct { + io.ReadCloser + remaining int64 +} + +func (b *limitedCDNBody) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if b.remaining == 0 { + var one [1]byte + n, err := b.ReadCloser.Read(one[:]) + if n > 0 { + return 0, ErrCDNBodyTooLarge + } + return 0, err + } + if int64(len(p)) > b.remaining { + p = p[:int(b.remaining)] + } + n, err := b.ReadCloser.Read(p) + b.remaining -= int64(n) + return n, err +} + +// ReadCDNBody reads a CDN response and rejects decoded bodies above 8 MiB. +// The extra byte distinguishes an exact-limit response from a truncated one. +func ReadCDNBody(body io.Reader) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(body, cdnBodyLimit+1)) + if err != nil { + return nil, err + } + if len(data) > cdnBodyLimit { + return nil, ErrCDNBodyTooLarge + } + return data, nil +} diff --git a/internal/pipeline/cdn_body_test.go b/internal/pipeline/cdn_body_test.go new file mode 100644 index 0000000..a6321c1 --- /dev/null +++ b/internal/pipeline/cdn_body_test.go @@ -0,0 +1,79 @@ +package pipeline + +import ( + "compress/gzip" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "evobgp/internal/store" +) + +func TestReadCDNBodyLimit(t *testing.T) { + tests := []struct { + name string + extra int64 + large bool + }{ + {"exact limit", 0, false}, + {"one byte too large", 1, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + reader := io.MultiReader(io.LimitReader(strings.NewReader(strings.Repeat("x", cdnBodyLimit)), cdnBodyLimit), strings.NewReader(strings.Repeat("y", int(tc.extra)))) + body, err := ReadCDNBody(reader) + if tc.large { + if !errors.Is(err, ErrCDNBodyTooLarge) || body != nil { + t.Fatalf("body=%d err=%v", len(body), err) + } + } else if err != nil || len(body) != cdnBodyLimit { + t.Fatalf("body=%d err=%v", len(body), err) + } + }) + } +} + +func TestFetchCDNSourceRowsRejectsOversizedBody(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1") + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.Copy(w, strings.NewReader(strings.Repeat("x", cdnBodyLimit+1))) + })) + defer srv.Close() + _, err := fetchCDNSourceRows(context.Background(), nil, srv.Client(), "", "", nil, + &store.CDNSource{ID: "source", URL: srv.URL, SourceKind: "plain"}, modulePrior{}, time.Now()) + if !errors.Is(err, ErrCDNBodyTooLarge) { + t.Fatalf("oversized CDN source must fail: %v", err) + } +} + +func TestCDNTransportBoundsDecodedGzipBody(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1") + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Encoding", "gzip") + zw := gzip.NewWriter(w) + _, _ = io.Copy(zw, io.LimitReader(strings.NewReader(strings.Repeat("x", cdnBodyLimit+1)), cdnBodyLimit+1)) + _ = zw.Close() + })) + defer srv.Close() + + client, err := secureCDNClient(srv.Client()) + if err != nil { + t.Fatal(err) + } + resp, err := client.Get(srv.URL) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + if !resp.Uncompressed { + t.Fatal("expected net/http gzip decompression") + } + if _, err := ReadCDNBody(resp.Body); !errors.Is(err, ErrCDNBodyTooLarge) { + t.Fatalf("oversized decoded body: %v", err) + } +} diff --git a/internal/pipeline/cdn_prefetch_test.go b/internal/pipeline/cdn_prefetch_test.go index 575cf91..aa53d6f 100644 --- a/internal/pipeline/cdn_prefetch_test.go +++ b/internal/pipeline/cdn_prefetch_test.go @@ -44,7 +44,7 @@ func TestCollectModulePrefixRows_CDNSendsIfNoneMatch(t *testing.T) { t.Fatal(err) } - collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, nil) + collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, modulePrior{}) if err != nil { t.Fatal(err) } @@ -89,11 +89,11 @@ func TestCollectModulePrefixRows_CDN304UsesSnapshot(t *testing.T) { Prefix: "203.0.113.0/24", Source: cdnSourceKey(src.ID), }} - if err := mergeCDNSourceIntoModuleSnapshot(m, tenant, mod, src.ID, prior); err != nil { + if err := mergeCDNSourceIntoModuleSnapshot(context.Background(), m, tenant, mod, src.ID, prior); err != nil { t.Fatal(err) } - collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, prior) + collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, modulePrior{provided: true, rows: prior}) if err != nil { t.Fatal(err) } @@ -135,7 +135,7 @@ func TestRefreshModuleIngest_CDN304UsesStoredSnapshot(t *testing.T) { Prefix: "203.0.113.0/24", Source: cdnSourceKey(src.ID), }} - if err := mergeCDNSourceIntoModuleSnapshot(m, tenant, mod, src.ID, prior); err != nil { + if err := mergeCDNSourceIntoModuleSnapshot(context.Background(), m, tenant, mod, src.ID, prior); err != nil { t.Fatal(err) } @@ -185,7 +185,7 @@ func TestCollectModulePrefixRows_CDN304RetriesWithoutETag(t *testing.T) { t.Fatal(err) } - collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, nil) + collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, modulePrior{}) if err != nil { t.Fatal(err) } diff --git a/internal/pipeline/cdn_snapshot.go b/internal/pipeline/cdn_snapshot.go index 65bbb45..477cf01 100644 --- a/internal/pipeline/cdn_snapshot.go +++ b/internal/pipeline/cdn_snapshot.go @@ -2,6 +2,7 @@ package pipeline import ( "context" + "errors" "fmt" "io" "net/http" @@ -11,22 +12,86 @@ import ( "evobgp/internal/store" ) +var errCDNSnapshotRead = errors.New("pipeline: CDN snapshot read failed") + +// modulePrior is the module snapshot the caller already loaded. +// provided distinguishes that read from a caller that did not load one. +type modulePrior struct { + provided bool + rows []store.PrefixRow +} + func cdnSourceKey(sourceID string) string { return "cdn:" + strings.TrimSpace(sourceID) } -func cachedCDNPrefixRows(st store.Backend, tenantID, moduleID string, priorSnapshot []store.PrefixRow, sourceKey string) []store.PrefixRow { - if cached := prefixRowsForSource(priorSnapshot, sourceKey); len(cached) > 0 { - return cached +// narrowSnapshotReadCheaper reports whether per-source reads move less data than +// one full module snapshot. One skipped source is filtered in SQL. Several skipped +// sources share one snapshot read once they are a large share of the module. +func narrowSnapshotReadCheaper(skipped, total int) bool { + if skipped <= 0 || total <= 0 { + return false } - if st != nil { - if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil { - if cached := prefixRowsForSource(snap.Prefixes, sourceKey); len(cached) > 0 { - return cached - } + if skipped == 1 { + return true + } + return skipped*2 < total +} + +func listModuleSourceRows(ctx context.Context, st store.Backend, tenantID, moduleID, source string) ([]store.PrefixRow, error) { + if st == nil { + return nil, nil + } + rows, err := st.ListModuleSnapshotSourceRows(ctx, tenantID, moduleID, source) + if err != nil { + return nil, fmt.Errorf("%w: %w", errCDNSnapshotRead, err) + } + return rows, nil +} + +func cachedCDNPrefixRows(ctx context.Context, st store.Backend, tenantID, moduleID string, prior modulePrior, sourceKey string) ([]store.PrefixRow, error) { + if prior.provided { + return prefixRowsForSource(prior.rows, sourceKey), nil + } + return listModuleSourceRows(ctx, st, tenantID, moduleID, sourceKey) +} + +// loadSkippedCDNRows resolves interval-skipped sources without a second full snapshot +// read when the caller already passed prior rows. +func loadSkippedCDNRows(ctx context.Context, st store.Backend, tenantID, moduleID string, sources []*store.CDNSource, prior modulePrior, now time.Time) (modulePrior, map[string][]store.PrefixRow, error) { + if prior.provided || st == nil { + return prior, nil, nil + } + var skipped []*store.CDNSource + for _, src := range sources { + if src != nil && shouldSkipCDNSourceFetch(src, now) { + skipped = append(skipped, src) } } - return nil + if len(skipped) == 0 { + return prior, nil, nil + } + if narrowSnapshotReadCheaper(len(skipped), len(sources)) { + out := make(map[string][]store.PrefixRow, len(skipped)) + for _, src := range skipped { + key := cdnSourceKey(src.ID) + rows, err := listModuleSourceRows(ctx, st, tenantID, moduleID, key) + if err != nil { + return prior, nil, err + } + out[key] = rows + } + return prior, out, nil + } + snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, moduleID) + if err != nil { + return prior, nil, fmt.Errorf("%w: %w", errCDNSnapshotRead, err) + } + rows := []store.PrefixRow(nil) + if ok && snap != nil { + rows = snap.Prefixes + } + return modulePrior{provided: true, rows: rows}, nil, nil } func mergeSnapshotDropSource(rows []store.PrefixRow, sourceKey string) []store.PrefixRow { @@ -69,23 +134,16 @@ func mergeSnapshotKeepSkippedCDN(rows []store.PrefixRow, fetchedSourceIDs, allSo return out } -// mergeAllCDNSourcesIntoModuleSnapshot replaces fetched CDN source rows in one write. -// skipped sources (errors with EVOBGP_CDN_PARTIAL_OK) keep their prior rows. -func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) error { - if st == nil || mod == nil { - return nil +// mergeCDNSnapshotRows builds the full module snapshot after a CDN collect. +// Fetched sources replace their rows. Skipped sources (partial OK) keep prior rows. +// Sources absent from allSourceIDs are dropped. Non-CDN rows in prior are kept. +// The caller persists the result once; this function does not write. +func mergeCDNSnapshotRows(prior, fetched []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) []store.PrefixRow { + kept := mergeSnapshotKeepSkippedCDN(prior, fetchedSourceIDs, allSourceIDs) + if len(fetched) == 0 { + return kept } - unlock := st.LockModuleSnapshot(tenantID, mod.ID) - defer unlock() - var base []store.PrefixRow - if len(priorSnapshot) > 0 { - base = priorSnapshot - } else if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil { - base = snap.Prefixes - } - kept := mergeSnapshotKeepSkippedCDN(base, fetchedSourceIDs, allSourceIDs) - merged := append(kept, cdnRows...) - return persistModuleSnapshot(st, tenantID, mod, merged) + return append(kept, fetched...) } func cdnRowsFromParsed(mod *store.Module, src *store.CDNSource, pfxStrings []string) []store.PrefixRow { @@ -106,19 +164,23 @@ func cdnRowsFromParsed(mod *store.Module, src *store.CDNSource, pfxStrings []str } // mergeCDNSourceIntoModuleSnapshot replaces rows for one CDN source in the module snapshot. -func mergeCDNSourceIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, sourceID string, newRows []store.PrefixRow) error { +func mergeCDNSourceIntoModuleSnapshot(ctx context.Context, st store.Backend, tenantID string, mod *store.Module, sourceID string, newRows []store.PrefixRow) error { if st == nil || mod == nil { return nil } - unlock := st.LockModuleSnapshot(tenantID, mod.ID) - defer unlock() - sourceKey := cdnSourceKey(sourceID) - var base []store.PrefixRow - if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil { - base = mergeSnapshotDropSource(snap.Prefixes, sourceKey) + hash, err := moduleIngestInputHash(st, tenantID, mod) + if err != nil { + return err } - merged := append(base, newRows...) - return persistModuleSnapshot(st, tenantID, mod, merged) + sourceKey := cdnSourceKey(sourceID) + return st.UpdateModulePrefixSnapshot(ctx, tenantID, mod.ID, func(current *store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) { + var base []store.PrefixRow + if current != nil { + base = mergeSnapshotDropSource(current.Prefixes, sourceKey) + } + merged := append(base, newRows...) + return &store.ModulePrefixSnapshot{InputHash: hash, Prefixes: merged}, nil + }) } func parseCDNBody(body string, src *store.CDNSource) ([]string, error) { @@ -134,7 +196,7 @@ func parseCDNBody(body string, src *store.CDNSource) ([]string, error) { } // fetchCDNSourceRows loads CDN prefixes without persisting the module snapshot (caller merges once). -func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) { +func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, prior modulePrior, now time.Time) ([]store.PrefixRow, error) { u := strings.TrimSpace(src.URL) if u == "" { return nil, nil @@ -159,7 +221,12 @@ func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, } if resp.StatusCode == http.StatusNotModified { - if cached := cachedCDNPrefixRows(st, tenantID, moduleID, priorSnapshot, sourceKey); len(cached) > 0 { + cached, cacheErr := cachedCDNPrefixRows(ctx, st, tenantID, moduleID, prior, sourceKey) + if cacheErr != nil { + _ = resp.Body.Close() + return nil, cacheErr + } + if len(cached) > 0 { _ = resp.Body.Close() return cached, nil } @@ -183,7 +250,7 @@ func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, _, _ = io.Copy(io.Discard, resp.Body) return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status) } - body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + body, err := ReadCDNBody(resp.Body) if err != nil { return nil, err } diff --git a/internal/pipeline/cdn_snapshot_lock_test.go b/internal/pipeline/cdn_snapshot_lock_test.go new file mode 100644 index 0000000..c7c3373 --- /dev/null +++ b/internal/pipeline/cdn_snapshot_lock_test.go @@ -0,0 +1,32 @@ +package pipeline + +import ( + "context" + "errors" + "testing" + + "evobgp/internal/store" +) + +type failingSnapshotUpdate struct{ store.Backend } + +func (f failingSnapshotUpdate) UpdateModulePrefixSnapshot(context.Context, string, string, store.ModuleSnapshotUpdate) error { + return context.DeadlineExceeded +} + +func TestMergeCDNSourceRefusesWriteWhenAtomicUpdateFails(t *testing.T) { + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-lock-failure", Enabled: true}) + if err != nil { + t.Fatal(err) + } + err = mergeCDNSourceIntoModuleSnapshot(context.Background(), failingSnapshotUpdate{m}, tenant, mod, "source-a", []store.PrefixRow{{Prefix: "192.0.2.0/24", Source: "cdn:source-a"}}) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected update failure, got %v", err) + } + if _, ok, err := m.GetModulePrefixSnapshot(context.Background(), tenant, mod.ID); err != nil || ok { + t.Fatalf("snapshot written without lock: ok=%v err=%v", ok, err) + } +} diff --git a/internal/pipeline/cdn_snapshot_read_test.go b/internal/pipeline/cdn_snapshot_read_test.go new file mode 100644 index 0000000..36ebdb0 --- /dev/null +++ b/internal/pipeline/cdn_snapshot_read_test.go @@ -0,0 +1,141 @@ +package pipeline + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "evobgp/internal/store" +) + +type snapCalls struct { + store.Backend + full atomic.Int32 + narrow atomic.Int32 +} + +func (s *snapCalls) GetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) { + s.full.Add(1) + return s.Backend.GetModulePrefixSnapshot(ctx, tenantID, moduleID) +} + +func (s *snapCalls) ListModuleSnapshotSourceRows(ctx context.Context, tenantID, moduleID, source string) ([]store.PrefixRow, error) { + s.narrow.Add(1) + return s.Backend.ListModuleSnapshotSourceRows(ctx, tenantID, moduleID, source) +} + +func TestNarrowSnapshotReadCheaper(t *testing.T) { + if !narrowSnapshotReadCheaper(1, 8) { + t.Fatal("one skipped source should use a narrow read") + } + if !narrowSnapshotReadCheaper(3, 8) { + t.Fatal("a minority of skipped sources should use narrow reads") + } + if narrowSnapshotReadCheaper(6, 8) { + t.Fatal("a majority of skipped sources should share one snapshot read") + } + if narrowSnapshotReadCheaper(0, 4) { + t.Fatal("nothing to read") + } +} + +func TestRefreshModuleIngestDoesNotRereadSnapshot(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1") + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-once", Enabled: true}) + if err != nil { + t.Fatal(err) + } + interval := 3600 + fresh := time.Now().UTC().Add(-30 * time.Second) + src, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{ + SourceKind: "txt", + URL: "https://cdn.example/prefixes.txt", + RefreshIntervalSec: &interval, + LastRefreshedAt: &fresh, + }) + if err != nil { + t.Fatal(err) + } + rows := []store.PrefixRow{{Prefix: "203.0.113.0/24", Source: cdnSourceKey(src.ID)}} + if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash", rows); err != nil { + t.Fatal(err) + } + counted := &snapCalls{Backend: m} + if err := RefreshModuleIngest(context.Background(), counted, nil, tenant, mod.ID); err != nil { + t.Fatal(err) + } + if counted.full.Load() != 1 || counted.narrow.Load() != 0 { + t.Fatalf("snapshot reads full=%d narrow=%d", counted.full.Load(), counted.narrow.Load()) + } +} + +func TestSkippedCDNSourceUsesNarrowReadWithoutPrior(t *testing.T) { + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-narrow", Enabled: true}) + if err != nil { + t.Fatal(err) + } + interval := 3600 + fresh := time.Now().UTC().Add(-30 * time.Second) + src := &store.CDNSource{ID: "only", SourceKind: "txt", URL: "https://cdn.example/one", RefreshIntervalSec: &interval, LastRefreshedAt: &fresh} + want := []store.PrefixRow{{Prefix: "203.0.113.8/32", Source: "cdn:only"}} + if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash", append([]store.PrefixRow{{Prefix: "198.51.100.0/24", Source: "cdn:other"}}, want...)); err != nil { + t.Fatal(err) + } + counted := &snapCalls{Backend: m} + got, err := collectCDNPrefixRows(context.Background(), counted, nil, tenant, mod, []*store.CDNSource{src}, modulePrior{}) + if err != nil { + t.Fatal(err) + } + if counted.full.Load() != 0 || counted.narrow.Load() != 1 { + t.Fatalf("snapshot reads full=%d narrow=%d", counted.full.Load(), counted.narrow.Load()) + } + if len(got) != 1 || got[0].Prefix != "203.0.113.8/32" || got[0].Source != "cdn:only" { + t.Fatalf("rows=%+v", got) + } +} + +func TestManySkippedCDNSourcesShareOneSnapshotRead(t *testing.T) { + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-shared", Enabled: true}) + if err != nil { + t.Fatal(err) + } + interval := 3600 + fresh := time.Now().UTC().Add(-30 * time.Second) + sources := []*store.CDNSource{ + {ID: "a", URL: "https://cdn.example/a", RefreshIntervalSec: &interval, LastRefreshedAt: &fresh}, + {ID: "b", URL: "https://cdn.example/b", RefreshIntervalSec: &interval, LastRefreshedAt: &fresh}, + } + stored := []store.PrefixRow{ + {Prefix: "203.0.113.1/32", Source: "cdn:a"}, + {Prefix: "203.0.113.2/32", Source: "cdn:b"}, + {Prefix: "198.51.100.0/24", Source: "cdn:gone"}, + } + if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash", stored); err != nil { + t.Fatal(err) + } + counted := &snapCalls{Backend: m} + got, err := collectCDNPrefixRows(context.Background(), counted, nil, tenant, mod, sources, modulePrior{}) + if err != nil { + t.Fatal(err) + } + if counted.full.Load() != 1 || counted.narrow.Load() != 0 { + t.Fatalf("snapshot reads full=%d narrow=%d", counted.full.Load(), counted.narrow.Load()) + } + seen := map[string]bool{} + for _, row := range got { + seen[row.Source] = true + } + if !seen["cdn:a"] || !seen["cdn:b"] || seen["cdn:gone"] { + t.Fatalf("rows=%+v", got) + } +} diff --git a/internal/pipeline/cdn_transport.go b/internal/pipeline/cdn_transport.go new file mode 100644 index 0000000..4cb63e0 --- /dev/null +++ b/internal/pipeline/cdn_transport.go @@ -0,0 +1,137 @@ +package pipeline + +import ( + "context" + "fmt" + "net" + "net/http" + "net/netip" + "time" + + "evobgp/internal/httpclient" +) + +type cdnRoundTripper struct { + transport *http.Transport +} + +func (t *cdnRoundTripper) CloseIdleConnections() { t.transport.CloseIdleConnections() } + +func (t *cdnRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if _, err := ValidateCDNURL(req.URL.String()); err != nil { + return nil, err + } + resp, err := t.transport.RoundTrip(req) + if err != nil { + return nil, err + } + // Transport has already decoded gzip here when it was negotiated by net/http. + resp.Body = &limitedCDNBody{ReadCloser: resp.Body, remaining: cdnBodyLimit} + return resp, nil +} + +type cdnLookupIP func(context.Context, string) ([]net.IP, error) +type cdnDialContext func(context.Context, string, string) (net.Conn, error) + +// cdnDial resolves every new connection and dials only an address from that checked answer. +// A mixed public/private DNS answer is rejected in full, regardless of address ordering. +func cdnDial(ctx context.Context, network, address string, lookup cdnLookupIP, dial cdnDialContext) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, fmt.Errorf("pipeline: cdn dial address: %w", err) + } + if isBlockedCDNHostname(host) { + return nil, fmt.Errorf("pipeline: cdn url blocked host") + } + var ips []net.IP + if addr, err := netip.ParseAddr(host); err == nil { + ips = []net.IP{net.IP(addr.AsSlice())} + } else { + resolveCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + ips, err = lookup(resolveCtx, host) + if err != nil { + return nil, fmt.Errorf("pipeline: cdn url dns lookup: %w", err) + } + } + if len(ips) == 0 { + return nil, fmt.Errorf("pipeline: cdn url dns lookup: no addresses") + } + checked := make([]netip.Addr, 0, len(ips)) + for _, ip := range ips { + addr, ok := netip.AddrFromSlice(ip) + if !ok || isBlockedCDNIP(addr) { + return nil, fmt.Errorf("pipeline: cdn url resolves to blocked address") + } + checked = append(checked, addr.Unmap()) + } + var dialErr error + for _, addr := range checked { + if err := ctx.Err(); err != nil { + return nil, err + } + conn, err := dial(ctx, network, net.JoinHostPort(addr.String(), port)) + if err == nil { + return conn, nil + } + dialErr = err + } + return nil, fmt.Errorf("pipeline: cdn dial: %w", dialErr) +} + +func secureCDNClient(hc *http.Client) (*http.Client, error) { + if hc == nil { + hc = httpclient.New(httpclient.DefaultTimeout) + } + client := *hc + if pools, ok := client.Transport.(*pipelineTransport); ok { + client.Transport = pools.cdn + } + if client.Timeout <= 0 { + client.Timeout = httpclient.DefaultTimeout + } + if _, ok := client.Transport.(*cdnRoundTripper); !ok { + base := http.DefaultTransport.(*http.Transport).Clone() + switch tr := client.Transport.(type) { + case nil: + case *http.Transport: + // Keep the caller's trust roots for TLS test servers. Other transport + // hooks (including custom protocols and dialers) are not inherited. + if tr.TLSClientConfig != nil { + base.TLSClientConfig = tr.TLSClientConfig.Clone() + } + default: + return nil, fmt.Errorf("pipeline: cdn client requires *http.Transport") + } + // A proxy connects to the destination outside this process, where its DNS answer + // cannot be checked. CDN requests therefore always connect directly. + base.Proxy = nil + base.DialTLSContext = nil + base.DialTLS = nil + base.TLSNextProto = nil + dialer := &net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second} + base.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + return cdnDial(ctx, network, address, func(ctx context.Context, host string) ([]net.IP, error) { + return net.DefaultResolver.LookupIP(ctx, "ip", host) + }, dialer.DialContext) + } + client.Transport = &cdnRoundTripper{transport: base} + } + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if _, err := ValidateCDNURL(req.URL.String()); err != nil { + return err + } + if len(via) >= 10 { + return fmt.Errorf("pipeline: too many cdn redirects") + } + return nil + } + return &client, nil +} + +// NewCDNHTTPClient returns a reusable HTTPS client that pins each CDN connection +// to a checked DNS answer. Environment HTTP proxies are intentionally ignored. +func NewCDNHTTPClient() *http.Client { + client, _ := secureCDNClient(httpclient.New(httpclient.DefaultTimeout)) + return client +} diff --git a/internal/pipeline/cdn_transport_test.go b/internal/pipeline/cdn_transport_test.go new file mode 100644 index 0000000..f7ead68 --- /dev/null +++ b/internal/pipeline/cdn_transport_test.go @@ -0,0 +1,197 @@ +package pipeline + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" +) + +func TestCDNDialPinsAndChecksEveryAnswer(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0") + answers := [][]net.IP{ + {net.ParseIP("203.0.113.10")}, + {net.ParseIP("127.0.0.1")}, + } + lookups := 0 + var dialed []string + lookup := func(context.Context, string) ([]net.IP, error) { + answer := answers[lookups] + lookups++ + return answer, nil + } + dial := func(_ context.Context, _, address string) (net.Conn, error) { + dialed = append(dialed, address) + return nil, nil + } + if _, err := cdnDial(context.Background(), "tcp", "cdn.example:443", lookup, dial); err != nil { + t.Fatal(err) + } + if _, err := cdnDial(context.Background(), "tcp", "cdn.example:443", lookup, dial); err == nil { + t.Fatal("changed DNS answer must be rejected before dial") + } + if lookups != 2 || len(dialed) != 1 || dialed[0] != "203.0.113.10:443" { + t.Fatalf("lookups=%d dialed=%v", lookups, dialed) + } +} + +func TestCDNDialRejectsMixedAndMappedPrivateAnswers(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0") + tests := []struct { + name string + ips []net.IP + }{ + {"mixed", []net.IP{net.ParseIP("203.0.113.10"), net.ParseIP("10.0.0.5")}}, + {"mapped loopback", []net.IP{net.ParseIP("::ffff:127.0.0.1")}}, + {"mapped private", []net.IP{net.ParseIP("::ffff:10.0.0.1")}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + called := false + _, err := cdnDial(context.Background(), "tcp", "cdn.example:443", func(context.Context, string) ([]net.IP, error) { + return tc.ips, nil + }, func(context.Context, string, string) (net.Conn, error) { + called = true + return nil, nil + }) + if err == nil || called { + t.Fatalf("want rejection before dial, got err=%v called=%v", err, called) + } + }) + } + if !isBlockedCDNIP(netip.MustParseAddr("::ffff:169.254.169.254")) { + t.Fatal("mapped metadata address must be blocked") + } +} + +func TestCDNDialFallsBackOnlyToCheckedIP(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0") + var dialed []string + _, err := cdnDial(context.Background(), "tcp", "cdn.example:443", func(context.Context, string) ([]net.IP, error) { + return []net.IP{net.ParseIP("2001:db8::1"), net.ParseIP("203.0.113.10")}, nil + }, func(_ context.Context, _, address string) (net.Conn, error) { + dialed = append(dialed, address) + if len(dialed) == 1 { + return nil, errors.New("unreachable") + } + return nil, nil + }) + if err != nil || len(dialed) != 2 || dialed[0] != "[2001:db8::1]:443" || dialed[1] != "203.0.113.10:443" { + t.Fatalf("err=%v dialed=%v", err, dialed) + } +} + +func TestCDNDialStopsOnCancellation(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + called := 0 + _, err := cdnDial(ctx, "tcp", "cdn.example:443", func(context.Context, string) ([]net.IP, error) { + return []net.IP{net.ParseIP("203.0.113.10"), net.ParseIP("203.0.113.11")}, nil + }, func(context.Context, string, string) (net.Conn, error) { + called++ + cancel() + return nil, errors.New("unreachable") + }) + if !errors.Is(err, context.Canceled) || called != 1 { + t.Fatalf("err=%v dial attempts=%d", err, called) + } +} + +func TestCDNClientRejectsUnsafeRedirectsAndProxy(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0") + t.Setenv("HTTPS_PROXY", "http://127.0.0.1:8080") + client := NewCDNHTTPClient() + tr, ok := client.Transport.(*cdnRoundTripper) + if !ok || tr.transport.Proxy != nil { + t.Fatal("CDN transport must dial directly") + } + for _, raw := range []string{ + "https://127.0.0.1/secret", + "https://[::ffff:127.0.0.1]/secret", + "http://cdn.example/unsafe", + "https://user:password@cdn.example/secret", + } { + t.Run(raw, func(t *testing.T) { + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + if err := client.CheckRedirect(&http.Request{URL: u}, nil); err == nil { + t.Fatal("unsafe redirect accepted") + } + }) + } +} + +type cdnTestRoundTripper func(*http.Request) (*http.Response, error) + +func (f cdnTestRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + +func TestCDNClientRejectsOpaqueTransport(t *testing.T) { + _, err := secureCDNClient(&http.Client{Transport: cdnTestRoundTripper(func(*http.Request) (*http.Response, error) { + return nil, errors.New("must not be called") + })}) + if err == nil { + t.Fatal("opaque transport could bypass checked dial") + } +} + +func TestCDNClientRejectsPrivateTLSRedirectBeforeDial(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0") + var hits atomic.Int32 + var secretHits atomic.Int32 + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + if r.URL.Path == "/secret" { + secretHits.Add(1) + } + http.Redirect(w, r, "https://private.example.com/secret", http.StatusFound) + })) + defer srv.Close() + + client := NewCDNHTTPClient() + tr := client.Transport.(*cdnRoundTripper).transport + tr.TLSClientConfig = srv.Client().Transport.(*http.Transport).TLSClientConfig.Clone() + actual := srv.Listener.Addr().String() + var dialed []string + var dialedMu sync.Mutex + dialer := &net.Dialer{} + tr.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + return cdnDial(ctx, network, address, func(_ context.Context, host string) ([]net.IP, error) { + if host == "private.example.com" { + return []net.IP{net.ParseIP("127.0.0.1")}, nil + } + return []net.IP{net.ParseIP("203.0.113.10")}, nil + }, func(ctx context.Context, network, checkedAddress string) (net.Conn, error) { + dialedMu.Lock() + dialed = append(dialed, checkedAddress) + dialedMu.Unlock() + return dialer.DialContext(ctx, network, actual) + }) + } + _, port, err := net.SplitHostPort(actual) + if err != nil { + t.Fatal(err) + } + resp, err := client.Get("https://cdn.example.com:" + port + "/redirect") + if resp != nil { + _ = resp.Body.Close() + } + if err == nil || !strings.Contains(err.Error(), "blocked address") { + t.Fatalf("private TLS redirect was not rejected: %v", err) + } + dialedMu.Lock() + dialedSnapshot := append([]string(nil), dialed...) + dialedMu.Unlock() + if hits.Load() != 1 || secretHits.Load() != 0 || len(dialedSnapshot) != 1 || dialedSnapshot[0] != "203.0.113.10:"+port { + t.Fatalf("hits=%d secretHits=%d dialed=%v", hits.Load(), secretHits.Load(), dialedSnapshot) + } +} diff --git a/internal/pipeline/cdn_url.go b/internal/pipeline/cdn_url.go index ae0cb39..4b6c099 100644 --- a/internal/pipeline/cdn_url.go +++ b/internal/pipeline/cdn_url.go @@ -20,6 +20,7 @@ func isBlockedCDNIP(ip netip.Addr) bool { if !ip.IsValid() { return true } + ip = ip.Unmap() return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsMulticast() || ip.IsUnspecified() || ip == netip.MustParseAddr("169.254.169.254") } diff --git a/internal/pipeline/collect_parallel.go b/internal/pipeline/collect_parallel.go index bc532ae..40807b5 100644 --- a/internal/pipeline/collect_parallel.go +++ b/internal/pipeline/collect_parallel.go @@ -2,6 +2,7 @@ package pipeline import ( "context" + "errors" "evobgp/internal/logging" "fmt" "net/http" @@ -171,7 +172,7 @@ func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, return out, nil } -func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, sources []*store.CDNSource, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) { +func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, sources []*store.CDNSource, prior modulePrior) ([]store.PrefixRow, error) { moduleID := mod.ID now := time.Now().UTC() @@ -181,6 +182,10 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client valid = append(valid, s) } } + prior, skipRows, err := loadSkippedCDNRows(ctx, st, tenantID, moduleID, valid, prior, now) + if err != nil { + return nil, err + } type srcResult struct { rows []store.PrefixRow err error @@ -196,23 +201,26 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client sem <- struct{}{} defer func() { <-sem }() - sourceKey := "cdn:" + src.ID + sourceKey := cdnSourceKey(src.ID) if shouldSkipCDNSourceFetch(src, now) { - if cached := prefixRowsForSource(priorSnapshot, sourceKey); len(cached) > 0 { + cached := prefixRowsForSource(prior.rows, sourceKey) + if !prior.provided && skipRows != nil { + cached = skipRows[sourceKey] + } + if len(cached) > 0 { results[idx] = srcResult{rows: cached} return } - if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil { - if cached := prefixRowsForSource(snap.Prefixes, sourceKey); len(cached) > 0 { - results[idx] = srcResult{rows: cached} - return - } - } } - rows, err := fetchCDNSourceRows(ctx, st, hc, tenantID, moduleID, mod, src, priorSnapshot, now) + rows, err := fetchCDNSourceRows(ctx, st, hc, tenantID, moduleID, mod, src, prior, now) if err != nil { if staleOnUpstreamError() { - if cached, ok := staleCDNPrefixes(st, tenantID, moduleID, priorSnapshot, src.ID); ok { + cached, ok, cacheErr := staleCDNPrefixes(ctx, st, tenantID, moduleID, prior, src.ID) + if cacheErr != nil { + results[idx] = srcResult{err: cacheErr} + return + } + if ok { logStaleUpstream("cdn", fmt.Sprintf("source %s: %v", src.ID, err)) results[idx] = srcResult{rows: cached} return @@ -235,6 +243,9 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client } for i, r := range results { if r.err != nil { + if errors.Is(r.err, errCDNSnapshotRead) { + return nil, r.err + } if cdnPartialOK() { logging.Default().Info(fmt.Sprintf("pipeline: CDN partial skip source error: %v", r.err)) skipped++ @@ -248,26 +259,15 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client if skipped > 0 && len(fetchedRows) == 0 && len(valid) > 0 { return nil, fmt.Errorf("cdn: all %d source(s) failed (partial ok)", len(valid)) } - out := fetchedRows - if skipped > 0 { - base := priorSnapshot - if len(base) == 0 { - if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil { - base = snap.Prefixes - } - } - for _, row := range mergeSnapshotKeepSkippedCDN(base, fetchedIDs, allIDs) { - if strings.HasPrefix(strings.TrimSpace(row.Source), "cdn:") { - out = append(out, row) - } - } - } - if len(valid) > 0 { - if err := mergeAllCDNSourcesIntoModuleSnapshot(st, tenantID, mod, priorSnapshot, fetchedRows, fetchedIDs, allIDs); err != nil { + base := prior.rows + if skipped > 0 && len(base) == 0 && !prior.provided && st != nil { + if snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, moduleID); err != nil { return nil, err + } else if ok && snap != nil { + base = snap.Prefixes } } - return out, nil + return mergeCDNSnapshotRows(base, fetchedRows, fetchedIDs, allIDs), nil } func collectDomainPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) { diff --git a/internal/pipeline/collect_stale.go b/internal/pipeline/collect_stale.go index e2ead58..de54441 100644 --- a/internal/pipeline/collect_stale.go +++ b/internal/pipeline/collect_stale.go @@ -1,6 +1,7 @@ package pipeline import ( + "context" "evobgp/internal/logging" "fmt" "net/netip" @@ -63,10 +64,10 @@ func staleDomainPrefixes(priorSnapshot []store.PrefixRow, fqdn string) ([]store. return cached, len(cached) > 0 } -func staleCDNPrefixes(st store.Backend, tenantID, moduleID string, priorSnapshot []store.PrefixRow, sourceID string) ([]store.PrefixRow, bool) { +func staleCDNPrefixes(ctx context.Context, st store.Backend, tenantID, moduleID string, prior modulePrior, sourceID string) ([]store.PrefixRow, bool, error) { sourceKey := cdnSourceKey(sourceID) - cached := cachedCDNPrefixRows(st, tenantID, moduleID, priorSnapshot, sourceKey) - return cached, len(cached) > 0 + cached, err := cachedCDNPrefixRows(ctx, st, tenantID, moduleID, prior, sourceKey) + return cached, len(cached) > 0, err } // asnCacheExpired returns cached ASN prefixes even past TTL (for stale fallback only). diff --git a/internal/pipeline/collect_stale_test.go b/internal/pipeline/collect_stale_test.go index 7299e7d..1f7cd35 100644 --- a/internal/pipeline/collect_stale_test.go +++ b/internal/pipeline/collect_stale_test.go @@ -34,7 +34,7 @@ func TestCollectCDNPrefixRows_StaleOnFetchError(t *testing.T) { prior := []store.PrefixRow{ {Prefix: "203.0.113.0/24", Source: "cdn:s1"}, } - rows, err := collectCDNPrefixRows(context.Background(), st, srv.Client(), tenant, mod, []*store.CDNSource{{ID: "s1", URL: srv.URL, SourceKind: "plain"}}, prior) + rows, err := collectCDNPrefixRows(context.Background(), st, srv.Client(), tenant, mod, []*store.CDNSource{{ID: "s1", URL: srv.URL, SourceKind: "plain"}}, modulePrior{provided: true, rows: prior}) if err != nil { t.Fatalf("expected stale fallback, got err: %v", err) } @@ -65,7 +65,7 @@ func TestCollectCDNPrefixRows_FailFastWhenNoStale(t *testing.T) { t.Fatal(err) } - _, err = collectCDNPrefixRows(context.Background(), st, srv.Client(), tenant, mod, []*store.CDNSource{{ID: "s1", URL: srv.URL, SourceKind: "plain"}}, nil) + _, err = collectCDNPrefixRows(context.Background(), st, srv.Client(), tenant, mod, []*store.CDNSource{{ID: "s1", URL: srv.URL, SourceKind: "plain"}}, modulePrior{}) if err == nil { t.Fatal("expected error when stale disabled and no cache") } diff --git a/internal/pipeline/doh_limits_test.go b/internal/pipeline/doh_limits_test.go new file mode 100644 index 0000000..9d78d71 --- /dev/null +++ b/internal/pipeline/doh_limits_test.go @@ -0,0 +1,147 @@ +package pipeline + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "testing" + "time" + + "evobgp/internal/httpclient" + "github.com/miekg/dns" +) + +type dohLimitRoundTrip func(*http.Request) (*http.Response, error) + +func (f dohLimitRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +type dohLimitBody struct { + io.Reader + read int + closed bool +} + +func (b *dohLimitBody) Read(p []byte) (int, error) { + n, e := b.Reader.Read(p) + b.read += n + return n, e +} +func (b *dohLimitBody) Close() error { b.closed = true; return nil } + +func dohLimitCases(t *testing.T) []struct { + name string + limit int + prefix []byte + call func(context.Context, *http.Client, string) ([]netip.Addr, error) + pad byte +} { + t.Helper() + msg := new(dns.Msg) + msg.Response = true + msg.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: "example.test.", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.IPv4(192, 0, 2, 1)}} + wire, err := msg.Pack() + if err != nil { + t.Fatal(err) + } + return []struct { + name string + limit int + prefix []byte + call func(context.Context, *http.Client, string) ([]netip.Addr, error) + pad byte + }{ + {"wire", 2 << 20, wire, func(ctx context.Context, hc *http.Client, url string) ([]netip.Addr, error) { + return resolveDomainWithDOHMessage(ctx, hc, url, "example.test", dns.TypeA) + }, 0}, + {"json", 1 << 20, []byte(`{"Answer":[{"type":1,"data":"192.0.2.1"}]}`), func(ctx context.Context, hc *http.Client, url string) ([]netip.Addr, error) { + return resolveDomainWithDOHJSON(ctx, hc, url, "example.test", "A") + }, ' '}, + } +} + +func TestDOHCompleteResponseLimits(t *testing.T) { + for _, tc := range dohLimitCases(t) { + t.Run(tc.name, func(t *testing.T) { + for _, known := range []bool{true, false} { + for _, extra := range []int{0, 1} { + data := append(append([]byte{}, tc.prefix...), bytes.Repeat([]byte{tc.pad}, tc.limit-len(tc.prefix)+extra)...) + body := &dohLimitBody{Reader: bytes.NewReader(data)} + hc := &http.Client{Transport: dohLimitRoundTrip(func(r *http.Request) (*http.Response, error) { + if _, ok := r.Context().Deadline(); !ok { + t.Error("missing deadline") + } + length := int64(-1) + if known { + length = int64(len(data)) + } + return &http.Response{StatusCode: 200, ContentLength: length, Body: body, Header: make(http.Header)}, nil + })} + ips, err := tc.call(context.Background(), hc, "https://resolver.example/dns-query") + if extra == 0 && (err != nil || len(ips) != 1) { + t.Fatalf("exact limit rejected known=%v ips=%v: %v", known, ips, err) + } + if extra == 1 && !errors.Is(err, httpclient.ErrResponseTooLarge) { + t.Fatalf("valid prefix bypassed size cap known=%v: %v", known, err) + } + if body.read > tc.limit+1 || !body.closed { + t.Fatalf("read=%d closed=%v", body.read, body.closed) + } + } + } + }) + } +} + +func TestDOHGzipDecodedLimit(t *testing.T) { + for _, tc := range dohLimitCases(t) { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Encoding", "gzip") + gz := gzip.NewWriter(w) + _, _ = gz.Write(tc.prefix) + _, _ = gz.Write(bytes.Repeat([]byte{tc.pad}, tc.limit)) + _ = gz.Close() + })) + defer server.Close() + if _, err := tc.call(context.Background(), server.Client(), server.URL); !errors.Is(err, httpclient.ErrResponseTooLarge) { + t.Fatalf("gzip bypass: %v", err) + } + }) + } +} + +func TestDOHNon200AndCancellationBounds(t *testing.T) { + for _, tc := range dohLimitCases(t) { + t.Run(tc.name, func(t *testing.T) { + body := &dohLimitBody{Reader: strings.NewReader(strings.Repeat("x", 1<<20))} + hc := &http.Client{Transport: dohLimitRoundTrip(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 400, Status: "400 Bad Request", Body: body, Header: make(http.Header)}, nil + })} + if _, err := tc.call(context.Background(), hc, "https://resolver.example/dns-query"); err == nil || body.read > 1024 || !body.closed { + t.Fatalf("non200 read=%d closed=%v error=%v", body.read, body.closed, err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + hc.Transport = dohLimitRoundTrip(func(r *http.Request) (*http.Response, error) { <-r.Context().Done(); return nil, r.Context().Err() }) + if _, err := tc.call(ctx, hc, "https://resolver.example/dns-query"); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("caller cancellation lost: %v", err) + } + }) + } +} + +func TestDOHJSONRejectsTrailingObject(t *testing.T) { + hc := &http.Client{Transport: dohLimitRoundTrip(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200, ContentLength: -1, Body: io.NopCloser(strings.NewReader(`{"Answer":[]} {"ignored":true}`)), Header: make(http.Header)}, nil + })} + if _, err := resolveDomainWithDOHJSON(context.Background(), hc, "https://resolver.example/dns-query", "example.test", "A"); err == nil { + t.Fatal("trailing JSON was ignored") + } +} diff --git a/internal/pipeline/doh_resolve.go b/internal/pipeline/doh_resolve.go index e07d550..f1411f4 100644 --- a/internal/pipeline/doh_resolve.go +++ b/internal/pipeline/doh_resolve.go @@ -146,3 +146,10 @@ func dohProfileTimeout(profile *store.DohProfile) time.Duration { } return timeout } + +func dohRequestContext(ctx context.Context) (context.Context, context.CancelFunc) { + if _, ok := ctx.Deadline(); ok { + return context.WithCancel(ctx) + } + return context.WithTimeout(ctx, 10*time.Second) +} diff --git a/internal/pipeline/http_clients.go b/internal/pipeline/http_clients.go new file mode 100644 index 0000000..bbfbc0a --- /dev/null +++ b/internal/pipeline/http_clients.go @@ -0,0 +1,35 @@ +package pipeline + +import ( + "net/http" + + "evobgp/internal/httpclient" +) + +// pipelineTransport owns two reusable pools. Ordinary requests retain the +// explicitly configured DoH/RIPE policy; CDN entry points select the guarded pool. +type pipelineTransport struct { + general http.RoundTripper + cdn http.RoundTripper +} + +func (t *pipelineTransport) RoundTrip(r *http.Request) (*http.Response, error) { + return t.general.RoundTrip(r) +} + +func (t *pipelineTransport) CloseIdleConnections() { + for _, transport := range []http.RoundTripper{t.general, t.cdn} { + if closer, ok := transport.(interface{ CloseIdleConnections() }); ok { + closer.CloseIdleConnections() + } + } +} + +// NewPipelineHTTPClient preserves configured DoH/RIPE access while giving CDN +// fetches a separate reusable guarded transport without an unbounded client cache. +func NewPipelineHTTPClient() *http.Client { + client := httpclient.New(httpclient.DefaultTimeout) + cdn := NewCDNHTTPClient() + client.Transport = &pipelineTransport{general: client.Transport, cdn: cdn.Transport} + return client +} diff --git a/internal/pipeline/http_clients_test.go b/internal/pipeline/http_clients_test.go new file mode 100644 index 0000000..1240e2c --- /dev/null +++ b/internal/pipeline/http_clients_test.go @@ -0,0 +1,66 @@ +package pipeline + +import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +func TestPipelineClientReusesSeparateCDNPool(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1") + var connections atomic.Int32 + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "192.0.2.0/24\n") })) + server.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateNew { + connections.Add(1) + } + } + server.StartTLS() + defer server.Close() + client := NewPipelineHTTPClient() + defer client.CloseIdleConnections() + // Configure only test trust roots; the normal dial/redirect guards remain. + pools := client.Transport.(*pipelineTransport) + pools.cdn.(*cdnRoundTripper).transport.TLSClientConfig = server.Client().Transport.(*http.Transport).TLSClientConfig.Clone() + for range 3 { + req, _ := http.NewRequest(http.MethodGet, server.URL, nil) + response, err := UpstreamHTTPDo(context.Background(), client, req) + if err != nil { + t.Fatal(err) + } + _, err = ReadCDNBody(response.Body) + _ = response.Body.Close() + if err != nil { + t.Fatal(err) + } + } + if connections.Load() != 1 { + t.Fatalf("CDN requests created %d connections", connections.Load()) + } +} + +func TestPipelineClientSeparatesPrivateProfileAndCDN(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0") + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { calls.Add(1); _, _ = io.WriteString(w, "ok") })) + defer server.Close() + client := NewPipelineHTTPClient() + defer client.CloseIdleConnections() + response, err := client.Get(server.URL) + if err != nil { + t.Fatal(err) + } + _, _ = io.Copy(io.Discard, response.Body) + _ = response.Body.Close() + req, _ := http.NewRequest(http.MethodGet, server.URL, nil) + if _, err = UpstreamHTTPDo(context.Background(), client, req); err == nil { + t.Fatal("CDN used trusted profile pool") + } + if calls.Load() != 1 { + t.Fatalf("CDN reached private profile server: %d", calls.Load()) + } +} diff --git a/internal/pipeline/module_hash.go b/internal/pipeline/module_hash.go index 2bce9cb..37a5e28 100644 --- a/internal/pipeline/module_hash.go +++ b/internal/pipeline/module_hash.go @@ -1,6 +1,7 @@ package pipeline import ( + "context" "strings" "evobgp/internal/store" @@ -11,7 +12,7 @@ func moduleIngestInputHash(st store.Backend, tenantID string, mod *store.Module) return store.ComputeModuleInputHash(st, tenantID, mod) } -func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, rows []store.PrefixRow) error { +func persistModuleSnapshot(ctx context.Context, st store.Backend, tenantID string, mod *store.Module, rows []store.PrefixRow) error { if st == nil || mod == nil { return nil } @@ -19,17 +20,14 @@ func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, if err != nil { return err } - if err := st.SetModulePrefixSnapshot(tenantID, mod.ID, hash, rows); err != nil { - return err - } - _ = st.SetModuleInputHash(tenantID, mod.ID, hash) - return nil + return st.SetModulePrefixSnapshot(ctx, tenantID, mod.ID, hash, rows) } -func moduleRowsFromSnapshot(st store.Backend, tenantID string, mod *store.Module) ([]store.PrefixRow, bool, error) { - snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID) - if err != nil || !ok || snap == nil { - return nil, false, err +// snapshotRowsIfCurrent returns rows when snap still matches the module input hash. +// The caller already loaded snap, so this does not read the snapshot again. +func snapshotRowsIfCurrent(ctx context.Context, st store.Backend, tenantID string, mod *store.Module, snap *store.ModulePrefixSnapshot) ([]store.PrefixRow, bool, error) { + if snap == nil { + return nil, false, nil } if h := strings.TrimSpace(mod.InputHash); h != "" { if snap.InputHash != h { @@ -45,6 +43,8 @@ func moduleRowsFromSnapshot(st store.Backend, tenantID string, mod *store.Module if snap.InputHash != hash { return nil, false, nil } - _ = st.SetModuleInputHash(tenantID, mod.ID, hash) + if err := st.SetModuleInputHash(ctx, tenantID, mod.ID, hash); err != nil { + return nil, false, err + } return append([]store.PrefixRow(nil), snap.Prefixes...), true, nil } diff --git a/internal/pipeline/opt_collect_test.go b/internal/pipeline/opt_collect_test.go index f79f7ff..13b9a31 100644 --- a/internal/pipeline/opt_collect_test.go +++ b/internal/pipeline/opt_collect_test.go @@ -98,7 +98,7 @@ func TestCollectCDNPrefixRows_PartialSkipKeepsPrior(t *testing.T) { {ID: "good", URL: good.URL, SourceKind: "plain"}, {ID: "bad", URL: bad.URL, SourceKind: "plain"}, } - rows, err := collectCDNPrefixRows(context.Background(), st, good.Client(), tenant, mod, sources, prior) + rows, err := collectCDNPrefixRows(context.Background(), st, good.Client(), tenant, mod, sources, modulePrior{provided: true, rows: prior}) if err != nil { t.Fatalf("partial skip should succeed: %v", err) } @@ -126,18 +126,18 @@ func TestMergeCDNSource_ParallelNoLostUpdate(t *testing.T) { wg.Add(2) go func() { defer wg.Done() - _ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s1", []store.PrefixRow{ + _ = mergeCDNSourceIntoModuleSnapshot(context.Background(), st, tenant, mod, "s1", []store.PrefixRow{ {Prefix: "1.0.0.0/24", Source: "cdn:s1"}, }) }() go func() { defer wg.Done() - _ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s2", []store.PrefixRow{ + _ = mergeCDNSourceIntoModuleSnapshot(context.Background(), st, tenant, mod, "s2", []store.PrefixRow{ {Prefix: "2.0.0.0/24", Source: "cdn:s2"}, }) }() wg.Wait() - snap, ok, err := st.GetModulePrefixSnapshot(tenant, mod.ID) + snap, ok, err := st.GetModulePrefixSnapshot(context.Background(), tenant, mod.ID) if err != nil || !ok || snap == nil { t.Fatalf("snapshot missing: ok=%v err=%v", ok, err) } @@ -173,7 +173,7 @@ func TestInputHashInvalidatedOnEntryCRUD(t *testing.T) { if err != nil { t.Fatal(err) } - if err := st.SetModuleInputHash(tenant, mod.ID, "pre"); err != nil { + if err := st.SetModuleInputHash(context.Background(), tenant, mod.ID, "pre"); err != nil { t.Fatal(err) } if _, err := st.CreateIPRangeEntry(tenant, mod.ID, &store.IPRangeEntry{Prefix: "10.0.0.0/8"}); err != nil { diff --git a/internal/pipeline/prefetch.go b/internal/pipeline/prefetch.go index afe60d3..ca5fd36 100644 --- a/internal/pipeline/prefetch.go +++ b/internal/pipeline/prefetch.go @@ -2,6 +2,7 @@ package pipeline import ( "context" + "errors" "net/http" "strings" "sync" @@ -57,6 +58,7 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie return nil } sem := make(chan struct{}, collectConcurrency()) + errs := make(chan error, len(tasks)) var wg sync.WaitGroup for _, task := range tasks { wg.Add(1) @@ -64,23 +66,32 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie defer wg.Done() sem <- struct{}{} defer func() { <-sem }() - prefetchOneCDNSource(ctx, st, hc, t) + if err := prefetchOneCDNSource(ctx, st, hc, t); err != nil { + errs <- err + } }(task) } wg.Wait() + close(errs) + for err := range errs { + return err + } return nil } -func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) { +func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) error { now := time.Now().UTC() tid, mod, src := t.tenantID, t.mod, t.src omod, err := st.GetModule(tid, mod.ID) if err != nil || omod == nil { - return + return nil } - rows, err := fetchCDNSourceRows(ctx, st, hc, tid, mod.ID, omod, src, nil, now) + rows, err := fetchCDNSourceRows(ctx, st, hc, tid, mod.ID, omod, src, modulePrior{}, now) if err != nil { - return + if errors.Is(err, errCDNSnapshotRead) { + return err + } + return nil } - _ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows) + return mergeCDNSourceIntoModuleSnapshot(ctx, st, tid, omod, src.ID, rows) } diff --git a/internal/pipeline/refresh.go b/internal/pipeline/refresh.go index 805724a..925d526 100644 --- a/internal/pipeline/refresh.go +++ b/internal/pipeline/refresh.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/json" + "errors" "fmt" "io" "net" @@ -61,15 +62,19 @@ func RefreshModuleIngest(ctx context.Context, st store.Backend, hc *http.Client, return fmt.Errorf("pipeline: module disabled") } - var prior []store.PrefixRow - if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil { - prior = snap.Prefixes - } - rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, prior) + snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, moduleID) if err != nil { return err } - if err := persistModuleSnapshot(st, tenantID, mod, rows); err != nil { + var priorRows []store.PrefixRow + if ok && snap != nil { + priorRows = snap.Prefixes + } + rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, modulePrior{provided: true, rows: priorRows}) + if err != nil { + return err + } + if err := persistModuleSnapshot(ctx, st, tenantID, mod, rows); err != nil { return err } refreshedAt := time.Now().UTC() @@ -83,6 +88,11 @@ func RenderTenantRevision(ctx context.Context, st store.Backend, hc *http.Client if hc == nil { hc = httpclient.New(httpclient.DefaultTimeout) } + if rev, ok, err := unchangedRenderRevision(ctx, st, tenantID); err != nil { + return "", err + } else if ok { + return rev, nil + } agg, err := aggregateTenantPrefixRowsAll(ctx, st, hc, tenantID) if err != nil { return "", err @@ -102,13 +112,71 @@ func RenderTenantRevision(ctx context.Context, st store.Backend, hc *http.Client if err != nil { return "", err } - if err := st.CreateRenderRevision(revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil { + if err := st.CreateRenderRevision(ctx, revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil { return "", err } applyRevisionRetention(st, tenantID) return revisionID, nil } +// unchangedRenderRevision returns the current revision when generation and snapshot +// versions still match the latest manifest. A generation change during the read +// refuses the skip so the caller rebuilds from the snapshots already written. +func unchangedRenderRevision(ctx context.Context, st store.Backend, tenantID string) (string, bool, error) { + if st == nil || strings.TrimSpace(tenantID) == "" { + return "", false, nil + } + gen1, err := st.GetRenderGeneration(ctx, tenantID) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return "", false, nil + } + return "", false, err + } + headers, err := st.ListModuleSnapshotHeaders(ctx, tenantID) + if err != nil { + return "", false, err + } + gen2, err := st.GetRenderGeneration(ctx, tenantID) + if err != nil { + return "", false, err + } + if gen1 != gen2 { + return "", false, nil + } + man, ok, err := st.LatestRenderManifest(ctx, tenantID) + if err != nil || !ok || man == nil { + return "", false, err + } + if man.Generation != gen1 || !sameSnapshotUses(man.Snapshots, headers) { + return "", false, nil + } + gen3, err := st.GetRenderGeneration(ctx, tenantID) + if err != nil { + return "", false, err + } + if gen3 != gen1 || strings.TrimSpace(man.RevisionID) == "" { + return "", false, nil + } + return man.RevisionID, true, nil +} + +func sameSnapshotUses(a, b []store.ModuleSnapshotUse) bool { + if len(a) != len(b) { + return false + } + aa := append([]store.ModuleSnapshotUse(nil), a...) + bb := append([]store.ModuleSnapshotUse(nil), b...) + sort.Slice(aa, func(i, j int) bool { return aa[i].ModuleID < aa[j].ModuleID }) + sort.Slice(bb, func(i, j int) bool { return bb[i].ModuleID < bb[j].ModuleID }) + for i := range aa { + if aa[i].ModuleID != bb[i].ModuleID || aa[i].SnapshotVersion != bb[i].SnapshotVersion { + return false + } + } + return true +} + // RenderTenantRevisionFromPrefixes renders one tenant-wide revision from already materialized prefixes. // This is used for fast paths (e.g. peer-only changes) to avoid ingest/external fetches. func RenderTenantRevisionFromPrefixes(ctx context.Context, st store.Backend, hc *http.Client, tenantID, triggerModuleID string, rows []store.PrefixRow) (revisionID string, err error) { @@ -132,7 +200,7 @@ func RenderTenantRevisionFromPrefixes(ctx context.Context, st store.Backend, hc if err != nil { return "", err } - if err := st.CreateRenderRevision(revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil { + if err := st.CreateRenderRevision(ctx, revisionID, tenantID, triggerModuleID, parent, hash, preview, agg); err != nil { return "", err } applyRevisionRetention(st, tenantID) @@ -148,8 +216,9 @@ func RefreshModule(ctx context.Context, st store.Backend, hc *http.Client, tenan } // collectModulePrefixRows returns materialized prefix rows for a single module (source of truth from store / ASN resolve / CDN fetch). -// priorSnapshot is the last stored module snapshot (used to skip CDN fetches when refresh interval has not elapsed). -func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) { +// prior is the last stored module snapshot when the caller already loaded it. +// CDN collect must not read that snapshot again. +func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, prior modulePrior) ([]store.PrefixRow, error) { moduleID := mod.ID switch mod.Type { case "IP_RANGES": @@ -173,13 +242,13 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli return nil, err } sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN }) - return collectASPrefixRows(ctx, st, hc, tenantID, mod, list, priorSnapshot) + return collectASPrefixRows(ctx, st, hc, tenantID, mod, list, prior.rows) case "CDN_CIDRS": sources, err := st.ListCDNSources(tenantID, moduleID) if err != nil { return nil, err } - return collectCDNPrefixRows(ctx, st, hc, tenantID, mod, sources, priorSnapshot) + return collectCDNPrefixRows(ctx, st, hc, tenantID, mod, sources, prior) case "DOMAINS": entries, err := st.ListDomainEntries(tenantID, moduleID) if err != nil { @@ -189,7 +258,7 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli if err != nil { return nil, err } - return collectDomainPrefixRows(ctx, st, hc, mod, profiles, policy, entries, priorSnapshot) + return collectDomainPrefixRows(ctx, st, hc, mod, profiles, policy, entries, prior.rows) default: return nil, fmt.Errorf("pipeline: unknown module type %q", mod.Type) } @@ -260,6 +329,8 @@ func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohPr } func resolveDomainWithDOHMessage(ctx context.Context, hc *http.Client, baseURL, host string, qtype uint16) ([]netip.Addr, error) { + ctx, cancel := dohRequestContext(ctx) + defer cancel() msg := new(dns.Msg) msg.SetQuestion(dns.Fqdn(host), qtype) wire, err := msg.Pack() @@ -289,7 +360,7 @@ func resolveDomainWithDOHMessage(ctx context.Context, hc *http.Client, baseURL, body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) return nil, fmt.Errorf("doh dns-message status %s: %s", resp.Status, strings.TrimSpace(string(body))) } - raw, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + raw, err := httpclient.ReadResponseBody(resp, 2<<20) if err != nil { return nil, err } @@ -337,6 +408,8 @@ func resolveDomainWithSystemDNS(ctx context.Context, host string) ([]netip.Addr, } func resolveDomainWithDOHJSON(ctx context.Context, hc *http.Client, baseURL, host, qtype string) ([]netip.Addr, error) { + ctx, cancel := dohRequestContext(ctx) + defer cancel() u, err := url.Parse(baseURL) if err != nil { return nil, err @@ -362,7 +435,11 @@ func resolveDomainWithDOHJSON(ctx context.Context, hc *http.Client, baseURL, hos return nil, fmt.Errorf("doh status %s: %s", resp.Status, strings.TrimSpace(string(body))) } var payload dohJSONResponse - if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&payload); err != nil { + body, err := httpclient.ReadResponseBody(resp, 1<<20) + if err != nil { + return nil, err + } + if err := json.Unmarshal(body, &payload); err != nil { return nil, err } var out []netip.Addr @@ -846,15 +923,6 @@ func intFromSettingsMap(m map[string]any, key string) int { return 0 } -func applyRevisionRetention(st store.Backend, tenantID string) { - settings, err := st.ListGlobalSettings(tenantID) - if err != nil { - return - } - cutoff := RevisionRetentionCutoff(settings) - _, _ = st.PruneRevisionsBefore(tenantID, cutoff) -} - type peerPolicyJSON struct { LocalIPv4 string `json:"local_ipv4"` LocalIPv6 string `json:"local_ipv6"` diff --git a/internal/pipeline/refresh_snapshot_order_test.go b/internal/pipeline/refresh_snapshot_order_test.go new file mode 100644 index 0000000..a6382d2 --- /dev/null +++ b/internal/pipeline/refresh_snapshot_order_test.go @@ -0,0 +1,121 @@ +package pipeline + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "evobgp/internal/store" +) + +func TestCDNRefreshSameSetDoesNotRewriteOrRevise(t *testing.T) { + t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1") + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + disabled := false + for _, mod := range m.ListModules(tenant) { + if mod == nil { + continue + } + if _, err := m.UpdateModule(tenant, mod.ID, &store.ModulePatch{Enabled: &disabled}); err != nil { + t.Fatal(err) + } + } + + var body atomic.Value + body.Store("203.0.113.0/24\n198.51.100.0/24\n") + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(body.Load().(string))) + })) + defer srv.Close() + + mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-order", Enabled: true}) + if err != nil { + t.Fatal(err) + } + if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{SourceKind: "plain", URL: srv.URL}); err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err := RefreshModuleIngest(ctx, m, srv.Client(), tenant, mod.ID); err != nil { + t.Fatal(err) + } + snap1, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, mod.ID) + if err != nil || !ok || snap1 == nil { + t.Fatalf("snapshot: ok=%v err=%v", ok, err) + } + gen1, err := m.GetRenderGeneration(ctx, tenant) + if err != nil { + t.Fatal(err) + } + + if err := RefreshModuleIngest(ctx, m, srv.Client(), tenant, mod.ID); err != nil { + t.Fatal(err) + } + snapSame, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, mod.ID) + if err != nil || !ok || snapSame == nil { + t.Fatal(err) + } + genSame, err := m.GetRenderGeneration(ctx, tenant) + if err != nil { + t.Fatal(err) + } + if snapSame.Version != snap1.Version || !snapSame.CollectedAt.Equal(snap1.CollectedAt) || genSame != gen1 { + t.Fatalf("unchanged CDN rewritten: version %d->%d gen %d->%d", snap1.Version, snapSame.Version, gen1, genSame) + } + + body.Store("198.51.100.0/24\n203.0.113.0/24\n") + if err := RefreshModuleIngest(ctx, m, srv.Client(), tenant, mod.ID); err != nil { + t.Fatal(err) + } + snap2, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, mod.ID) + if err != nil || !ok || snap2 == nil { + t.Fatal(err) + } + gen2, err := m.GetRenderGeneration(ctx, tenant) + if err != nil { + t.Fatal(err) + } + if snap2.Version != snap1.Version || !snap2.CollectedAt.Equal(snap1.CollectedAt) || gen2 != gen1 { + t.Fatalf("reordered CDN rewritten: version %d->%d gen %d->%d", snap1.Version, snap2.Version, gen1, gen2) + } + if len(snap2.Prefixes) != len(snap1.Prefixes) || snap2.Prefixes[0].Prefix != snap1.Prefixes[0].Prefix { + t.Fatalf("row order changed without a content change: %+v", snap2.Prefixes) + } + + before, _, _ := m.ListRevisions(tenant, "", "", 50) + rev1, err := RenderTenantRevision(ctx, m, srv.Client(), tenant, mod.ID) + if err != nil { + t.Fatal(err) + } + rev2, err := RenderTenantRevision(ctx, m, srv.Client(), tenant, mod.ID) + if err != nil { + t.Fatal(err) + } + if rev2 != rev1 { + t.Fatalf("order-stable inputs created another revision: %s vs %s", rev1, rev2) + } + mid, _, _ := m.ListRevisions(tenant, "", "", 50) + if len(mid) != len(before)+1 { + t.Fatalf("revisions after unchanged renders: before=%d after=%d", len(before), len(mid)) + } + + body.Store("203.0.113.0/24\n192.0.2.0/24\n") + if err := RefreshModuleIngest(ctx, m, srv.Client(), tenant, mod.ID); err != nil { + t.Fatal(err) + } + rev3, err := RenderTenantRevision(ctx, m, srv.Client(), tenant, mod.ID) + if err != nil { + t.Fatal(err) + } + if rev3 == rev1 { + t.Fatal("prefix change did not create a revision") + } + after, _, _ := m.ListRevisions(tenant, "", "", 50) + if len(after) != len(mid)+1 { + t.Fatalf("prefix change revisions: before=%d after=%d", len(mid), len(after)) + } +} diff --git a/internal/pipeline/revision_prune.go b/internal/pipeline/revision_prune.go new file mode 100644 index 0000000..ec25aaa --- /dev/null +++ b/internal/pipeline/revision_prune.go @@ -0,0 +1,108 @@ +package pipeline + +import ( + "context" + "strings" + "sync" + "time" + + "evobgp/internal/store" +) + +const ( + revisionPruneMinInterval = 5 * time.Minute + revisionPruneTimeout = 2 * time.Minute +) + +// revisionPruneGate runs at most one prune per tenant per interval and drops +// overlapping attempts so renders do not stack prune transactions. +type revisionPruneGate struct { + mu sync.Mutex + last map[string]time.Time + inflight map[string]chan struct{} + now func() time.Time +} + +var revisionPrunes revisionPruneGate + +func (g *revisionPruneGate) start(tenantID string) bool { + g.mu.Lock() + defer g.mu.Unlock() + now := time.Now() + if g.now != nil { + now = g.now() + } + if g.last == nil { + g.last = make(map[string]time.Time) + g.inflight = make(map[string]chan struct{}) + } + if _, running := g.inflight[tenantID]; running { + return false + } + if last, ok := g.last[tenantID]; ok && now.Sub(last) < revisionPruneMinInterval { + return false + } + g.last[tenantID] = now + g.inflight[tenantID] = make(chan struct{}) + return true +} + +func (g *revisionPruneGate) done(tenantID string) { + g.mu.Lock() + defer g.mu.Unlock() + ch, ok := g.inflight[tenantID] + if !ok { + return + } + close(ch) + delete(g.inflight, tenantID) +} + +// applyRevisionRetention schedules retention for tenantID and returns without +// waiting. Prune uses its own short context, at most once per tenant per five +// minutes, and overlapping renders do not start another transaction. +func applyRevisionRetention(st store.Backend, tenantID string) { + tenantID = strings.TrimSpace(tenantID) + if st == nil || tenantID == "" { + return + } + if !revisionPrunes.start(tenantID) { + return + } + go func() { + defer revisionPrunes.done(tenantID) + ctx, cancel := context.WithTimeout(context.Background(), revisionPruneTimeout) + defer cancel() + pruneTenantRevisions(ctx, st, tenantID) + }() +} + +// pruneTenantRevisions applies the tenant retention cutoff immediately. +func pruneTenantRevisions(ctx context.Context, st store.Backend, tenantID string) { + if st == nil || ctx == nil { + return + } + if err := ctx.Err(); err != nil { + return + } + settings, err := st.ListGlobalSettings(tenantID) + if err != nil { + return + } + _, _ = st.PruneRevisionsBefore(ctx, tenantID, RevisionRetentionCutoff(settings)) +} + +func waitRevisionPrune(ctx context.Context, tenantID string) error { + revisionPrunes.mu.Lock() + ch := revisionPrunes.inflight[strings.TrimSpace(tenantID)] + revisionPrunes.mu.Unlock() + if ch == nil { + return nil + } + select { + case <-ch: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/internal/pipeline/revision_prune_async_test.go b/internal/pipeline/revision_prune_async_test.go new file mode 100644 index 0000000..74e8d50 --- /dev/null +++ b/internal/pipeline/revision_prune_async_test.go @@ -0,0 +1,120 @@ +package pipeline + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "evobgp/internal/store" +) + +type blockingPrune struct { + store.Backend + calls atomic.Int32 + started chan struct{} + release chan struct{} + once sync.Once +} + +func (b *blockingPrune) PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (int, error) { + n := b.calls.Add(1) + if n == 1 { + b.once.Do(func() { close(b.started) }) + select { + case <-b.release: + case <-ctx.Done(): + return 0, ctx.Err() + } + } + return b.Backend.PruneRevisionsBefore(ctx, tenantID, cutoff) +} + +func TestRenderDoesNotWaitForRevisionPrune(t *testing.T) { + m := store.NewMemory() + m.SeedDemo() + tenant, _, modIP, _, _ := m.DemoIDs() + release := make(chan struct{}) + var releaseOnce sync.Once + closeRelease := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(closeRelease) + st := &blockingPrune{Backend: m, started: make(chan struct{}), release: release} + var shift atomic.Int64 + revisionPrunes.mu.Lock() + prevNow := revisionPrunes.now + revisionPrunes.now = func() time.Time { + return time.Now().Add(time.Duration(shift.Load()) * time.Minute) + } + revisionPrunes.mu.Unlock() + t.Cleanup(func() { + revisionPrunes.mu.Lock() + revisionPrunes.now = prevNow + revisionPrunes.mu.Unlock() + }) + + rows := []store.PrefixRow{{Prefix: "203.0.113.55/32", Source: "ip_range"}} + renderDone := make(chan error, 1) + go func() { + _, err := RenderTenantRevisionFromPrefixes(context.Background(), st, nil, tenant, modIP, rows) + renderDone <- err + }() + select { + case err := <-renderDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("render waited for revision prune") + } + select { + case <-st.started: + case <-time.After(5 * time.Second): + t.Fatal("prune did not start") + } + if st.calls.Load() != 1 { + t.Fatalf("prune calls=%d", st.calls.Load()) + } + applyRevisionRetention(st, tenant) + if st.calls.Load() != 1 { + t.Fatalf("overlapping render stacked prune calls=%d", st.calls.Load()) + } + closeRelease() + waitCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := waitRevisionPrune(waitCtx, tenant); err != nil { + t.Fatal(err) + } + shift.Store(10) + applyRevisionRetention(st, tenant) + waitCtx2, cancel2 := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel2() + if err := waitRevisionPrune(waitCtx2, tenant); err != nil { + t.Fatal(err) + } + if st.calls.Load() != 2 { + t.Fatalf("prune after interval calls=%d", st.calls.Load()) + } +} + +func TestPruneTenantRevisionsDirect(t *testing.T) { + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + var calls atomic.Int32 + st := pruneCountBackend{Backend: m, calls: &calls} + pruneTenantRevisions(context.Background(), st, tenant) + if calls.Load() != 1 { + t.Fatalf("calls=%d", calls.Load()) + } +} + +type pruneCountBackend struct { + store.Backend + calls *atomic.Int32 +} + +func (p pruneCountBackend) PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (int, error) { + p.calls.Add(1) + return p.Backend.PruneRevisionsBefore(ctx, tenantID, cutoff) +} diff --git a/internal/pipeline/scheduler_due.go b/internal/pipeline/scheduler_due.go index 8f50020..3b0ff55 100644 --- a/internal/pipeline/scheduler_due.go +++ b/internal/pipeline/scheduler_due.go @@ -8,7 +8,8 @@ import ( ) // SchedulerTickSec matches the scheduler ticker interval (internal/scheduler). -const SchedulerTickSec = 30 +// The module refresh window stays at least 60s; a shorter tick only notices the boundary sooner. +const SchedulerTickSec = 10 // ModuleDueForScheduler reports whether a module's refresh interval bucket rolled since the last scheduler tick. // A stable per-module offset (fnv32 of ID) spreads bucket boundaries so modules with the same interval diff --git a/internal/pipeline/upstream_http.go b/internal/pipeline/upstream_http.go index 994d3f1..861e4b1 100644 --- a/internal/pipeline/upstream_http.go +++ b/internal/pipeline/upstream_http.go @@ -14,8 +14,9 @@ func UpstreamHTTPDo(ctx context.Context, hc *http.Client, req *http.Request) (*h } func upstreamHTTPDo(ctx context.Context, hc *http.Client, req *http.Request) (*http.Response, error) { - if hc == nil { - hc = httpclient.New(httpclient.DefaultTimeout) + hc, err := secureCDNClient(hc) + if err != nil { + return nil, err } resp, err := httpclient.DoWithBreaker(ctx, hc, req, 3) if err != nil { diff --git a/internal/repository/asn_cache.go b/internal/repository/asn_cache.go index 58e4b5f..11c944f 100644 --- a/internal/repository/asn_cache.go +++ b/internal/repository/asn_cache.go @@ -4,6 +4,10 @@ import ( "context" "encoding/json" "errors" + "fmt" + "net/netip" + "sort" + "strings" "sync" "time" @@ -78,39 +82,110 @@ func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, boo func (p *Postgres) SetASNPrefixCache(asn int64, holder string, prefixes []string) error { ctx := context.Background() + normalized, err := canonicalASNPrefixes(prefixes) + if err != nil { + return err + } tx, err := p.pool.Begin(ctx) if err != nil { return err } defer func() { _ = tx.Rollback(ctx) }() _, err = tx.Exec(ctx, ` - INSERT INTO asn_prefix_cache (asn, holder, fetched_at) - VALUES ($1, $2, now()) - ON CONFLICT (asn) DO UPDATE SET - holder = EXCLUDED.holder, - fetched_at = EXCLUDED.fetched_at`, asn, holder) + INSERT INTO asn_prefix_cache (asn, holder, fetched_at) + VALUES ($1, $2, now()) + ON CONFLICT (asn) DO UPDATE SET + holder = EXCLUDED.holder, + fetched_at = EXCLUDED.fetched_at`, asn, holder) if err != nil { return err } if asnCacheRowTableExists(ctx, tx) { + existing, err := listASNCachePrefixes(ctx, tx, asn) + if err != nil { + return err + } + if prefixSetEqual(existing, normalized) { + return tx.Commit(ctx) + } if _, err := tx.Exec(ctx, `DELETE FROM asn_prefix_cache_row WHERE asn = $1`, asn); err != nil { return err } - for _, pfx := range prefixes { - if _, err := tx.Exec(ctx, ` - INSERT INTO asn_prefix_cache_row (asn, prefix) VALUES ($1, $2::cidr)`, asn, pfx); err != nil { - return err + if len(normalized) > 0 { + if _, err := tx.CopyFrom(ctx, pgx.Identifier{"asn_prefix_cache_row"}, []string{"asn", "prefix"}, pgx.CopyFromSlice(len(normalized), func(i int) ([]any, error) { + pfx, err := netip.ParsePrefix(normalized[i]) + if err != nil { + return nil, fmt.Errorf("repository: asn cache prefix %q: %w", normalized[i], err) + } + return []any{asn, pfx.Masked()}, nil + })); err != nil { + return fmt.Errorf("repository: copy asn cache rows: %w", err) } } } else { - raw, err := json.Marshal(prefixes) + raw, err := json.Marshal(normalized) if err != nil { return err } if _, err := tx.Exec(ctx, ` - UPDATE asn_prefix_cache SET prefixes_json = $2::jsonb WHERE asn = $1`, asn, string(raw)); err != nil { + UPDATE asn_prefix_cache SET prefixes_json = $2::jsonb WHERE asn = $1`, asn, string(raw)); err != nil { return err } } return tx.Commit(ctx) } + +func listASNCachePrefixes(ctx context.Context, tx pgx.Tx, asn int64) ([]string, error) { + rows, err := tx.Query(ctx, `SELECT prefix::text FROM asn_prefix_cache_row WHERE asn = $1`, asn) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var s string + if err := rows.Scan(&s); err != nil { + return nil, err + } + out = append(out, s) + } + if err := rows.Err(); err != nil { + return nil, err + } + return canonicalASNPrefixes(out) +} + +func canonicalASNPrefixes(in []string) ([]string, error) { + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, raw := range in { + raw = strings.TrimSpace(raw) + if raw == "" { + continue + } + pfx, err := netip.ParsePrefix(raw) + if err != nil { + return nil, fmt.Errorf("repository: asn cache prefix %q: %w", raw, err) + } + s := pfx.Masked().String() + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + out = append(out, s) + } + sort.Strings(out) + return out, nil +} + +func prefixSetEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/repository/backend_test.go b/internal/repository/backend_test.go index 939bac5..cad0d42 100644 --- a/internal/repository/backend_test.go +++ b/internal/repository/backend_test.go @@ -7,6 +7,8 @@ import ( "evobgp/internal/db" "evobgp/internal/store" + + "github.com/google/uuid" ) func TestPostgresImplementsBackend(t *testing.T) { @@ -48,7 +50,11 @@ func TestPostgresModuleCRUDIntegration(t *testing.T) { if err != nil { t.Fatal(err) } - tenant := "01TESTTENANT00000000000001" + tenant := uuid.NewString() + if _, err := pool.Exec(ctx, `INSERT INTO tenant (id, name, slug) VALUES ($1, $2, $3)`, tenant, "integration", "integration-"+tenant); err != nil { + t.Fatal(err) + } + defer func() { _, _ = pool.Exec(ctx, `DELETE FROM tenant WHERE id = $1`, tenant) }() mod, err := pg.CreateModule(tenant, &store.Module{ Name: "audit-test", Type: "IP_RANGES", diff --git a/internal/repository/deployment.go b/internal/repository/deployment.go new file mode 100644 index 0000000..20fdc9f --- /dev/null +++ b/internal/repository/deployment.go @@ -0,0 +1,238 @@ +package repository + +import ( + "context" + "errors" + + "evobgp/internal/store" + "github.com/jackc/pgx/v5" +) + +const deploymentColumns = `job_id::text,tenant_id::text,speaker_id::text,revision_id::text,generation,request_sequence,state,confirmation,error_message,dispatch_attempts,retry_round,health_received_at,health_revision_id,health_source_at,health_error,health_observation_token` + +func scanDeployment(row pgx.Row) (store.DeploymentSpeaker, error) { + var d store.DeploymentSpeaker + err := row.Scan(&d.JobID, &d.TenantID, &d.SpeakerID, &d.RevisionID, &d.Generation, &d.RequestSequence, &d.State, &d.Confirmation, &d.Error, &d.DispatchAttempts, &d.RetryRound, &d.HealthReceivedAt, &d.HealthRevisionID, &d.HealthSourceAt, &d.HealthError, &d.ObservationToken) + return d, err +} + +func deploymentFence(ctx context.Context, tx pgx.Tx, f store.JobFence) (int64, error) { + var seq int64 + err := tx.QueryRow(ctx, `SELECT request_sequence FROM job_audit WHERE id=$1::uuid AND tenant_id=$2::uuid AND owner_id=$3 AND fencing_token=$4 AND status='running' AND NOT cancel_requested AND lease_until>clock_timestamp() FOR UPDATE`, f.JobID, f.TenantID, f.OwnerID, f.Token).Scan(&seq) + if errors.Is(err, pgx.ErrNoRows) { + return 0, store.ErrJobLeaseLost + } + return seq, err +} + +func (p *Postgres) BeginDeploymentSpeaker(ctx context.Context, f store.JobFence, speaker, revision string) (store.DeploymentSpeaker, error) { + tx, err := p.pool.Begin(ctx) + if err != nil { + return store.DeploymentSpeaker{}, err + } + defer rollbackWithTimeout(tx) + seq, err := deploymentFence(ctx, tx, f) + if err != nil { + return store.DeploymentSpeaker{}, err + } + var generation, currentSeq int64 + var currentJob, currentRevision string + err = tx.QueryRow(ctx, `SELECT deployment_generation,deployment_request_sequence,COALESCE(deployment_job_id::text,''),COALESCE(published_revision_id::text,'') FROM bgp_speaker WHERE id=$1::uuid AND tenant_id=$2::uuid FOR UPDATE`, speaker, f.TenantID).Scan(&generation, ¤tSeq, ¤tJob, ¤tRevision) + if err != nil { + return store.DeploymentSpeaker{}, err + } + if f.ExpectedGeneration != 0 && currentJob != f.JobID && (generation != f.ExpectedGeneration || currentRevision != revision) { + return store.DeploymentSpeaker{}, store.ErrJobLeaseLost + } + reserve := func() error { + tag, e := tx.Exec(ctx, `UPDATE bgp_speaker SET delivery_owner=$2,delivery_token=$3,delivery_until=clock_timestamp()+($4::bigint*interval '1 millisecond') WHERE id=$1::uuid AND (delivery_until IS NULL OR delivery_until<=clock_timestamp())`, speaker, f.OwnerID, f.Token, store.DeploymentDeliveryWindow.Milliseconds()) + if e != nil { + return e + } + if tag.RowsAffected() == 0 { + return store.ErrDeploymentBusy + } + return nil + } + existing, err := scanDeployment(tx.QueryRow(ctx, `SELECT `+deploymentColumns+` FROM deployment_speaker WHERE job_id=$1::uuid AND speaker_id=$2::uuid`, f.JobID, speaker)) + if err == nil { + if currentJob != f.JobID || generation != existing.Generation || existing.RevisionID != revision { + return store.DeploymentSpeaker{}, store.ErrJobLeaseLost + } + if existing.State != "applied" { + if err = reserve(); err != nil { + return store.DeploymentSpeaker{}, err + } + } + if _, err = deploymentFence(ctx, tx, f); err != nil { + return store.DeploymentSpeaker{}, err + } + return existing, tx.Commit(ctx) + } + if !errors.Is(err, pgx.ErrNoRows) { + return store.DeploymentSpeaker{}, err + } + if f.ExpectedGeneration != 0 { + seq = currentSeq + } + if currentSeq >= seq && f.ExpectedGeneration == 0 { + return store.DeploymentSpeaker{}, store.ErrJobLeaseLost + } + var valid bool + err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM config_revision WHERE id=$1::uuid AND tenant_id=$2::uuid)`, revision, f.TenantID).Scan(&valid) + if err != nil { + return store.DeploymentSpeaker{}, err + } + if !valid { + return store.DeploymentSpeaker{}, store.ErrNotFound + } + if err = reserve(); err != nil { + return store.DeploymentSpeaker{}, err + } + d, err := scanDeployment(tx.QueryRow(ctx, `INSERT INTO deployment_speaker(job_id,tenant_id,speaker_id,revision_id,generation,request_sequence,state,retry_round) VALUES($1::uuid,$2::uuid,$3::uuid,$4::uuid,$5,$6,'published',$7) RETURNING `+deploymentColumns, f.JobID, f.TenantID, speaker, revision, generation+1, seq, f.RetryRound)) + if err != nil { + return d, err + } + _, err = tx.Exec(ctx, `UPDATE bgp_speaker SET deployment_generation=$2,deployment_request_sequence=$3,deployment_job_id=$4::uuid,published_revision_id=$5::uuid,published_at=now(),updated_at=now() WHERE id=$1::uuid`, speaker, d.Generation, seq, f.JobID, revision) + if err != nil { + return d, err + } + if _, err = deploymentFence(ctx, tx, f); err != nil { + return store.DeploymentSpeaker{}, err + } + return d, tx.Commit(ctx) +} + +func (p *Postgres) RecordDeploymentSpeaker(ctx context.Context, f store.JobFence, d store.DeploymentSpeaker, state, confirmation, message string) error { + if d.JobID != f.JobID || d.TenantID != f.TenantID { + return store.ErrJobLeaseLost + } + if state == "applied" && confirmation != "agent_reported" && confirmation != "local_applied" { + return store.ErrInvalidInput + } + tx, err := p.pool.Begin(ctx) + if err != nil { + return err + } + defer rollbackWithTimeout(tx) + if _, err = deploymentFence(ctx, tx, f); err != nil { + return err + } + var speaker string + err = tx.QueryRow(ctx, `SELECT id::text FROM bgp_speaker WHERE id=$1::uuid AND tenant_id=$2::uuid AND deployment_job_id=$3::uuid AND deployment_generation=$4 AND published_revision_id=$5::uuid AND delivery_owner=$6 AND delivery_token=$7 AND delivery_until>clock_timestamp() FOR UPDATE`, d.SpeakerID, f.TenantID, f.JobID, d.Generation, d.RevisionID, f.OwnerID, f.Token).Scan(&speaker) + if errors.Is(err, pgx.ErrNoRows) { + return store.ErrJobLeaseLost + } + if err != nil { + return err + } + _, err = tx.Exec(ctx, `UPDATE deployment_speaker SET state=$3,confirmation=$4,error_message=$5,dispatch_attempts=dispatch_attempts+CASE WHEN $3='dispatched' THEN 1 ELSE 0 END,updated_at=now() WHERE job_id=$1::uuid AND speaker_id=$2::uuid AND generation=$6`, f.JobID, speaker, state, confirmation, message, d.Generation) + if err != nil { + return err + } + if state != "dispatched" && state != "unknown" { + if _, err = tx.Exec(ctx, `UPDATE bgp_speaker SET delivery_until=NULL,delivery_owner=NULL WHERE id=$1::uuid`, speaker); err != nil { + return err + } + } + if state == "applied" { + _, err = tx.Exec(ctx, `UPDATE bgp_speaker SET last_applied_revision_id=$2::uuid,updated_at=now() WHERE id=$1::uuid`, speaker, d.RevisionID) + if err != nil { + return err + } + } + if _, err = deploymentFence(ctx, tx, f); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (p *Postgres) ListDeploymentSpeakers(ctx context.Context, tenant, job string) ([]store.DeploymentSpeaker, error) { + rows, err := p.pool.Query(ctx, `SELECT `+deploymentColumns+` FROM deployment_speaker WHERE tenant_id=$1::uuid AND job_id=$2::uuid ORDER BY speaker_id`, tenant, job) + if err != nil { + return nil, err + } + defer rows.Close() + var out []store.DeploymentSpeaker + for rows.Next() { + d, e := scanDeployment(rows) + if e != nil { + return nil, e + } + out = append(out, d) + } + return out, rows.Err() +} + +func (p *Postgres) ListPendingDeployments(ctx context.Context, limit int) ([]store.DeploymentSpeaker, error) { + if limit <= 0 || limit > 100 { + limit = 100 + } + rows, err := p.pool.Query(ctx, `SELECT `+deploymentColumns+` FROM deployment_speaker WHERE (job_id,speaker_id) IN (SELECT d.job_id,d.speaker_id FROM deployment_speaker d JOIN bgp_speaker s ON s.id=d.speaker_id AND s.deployment_job_id=d.job_id AND s.deployment_generation=d.generation JOIN job_audit j ON j.id=d.job_id WHERE d.state<>'applied' AND d.retry_round<3 AND j.status IN ('succeeded','failed') AND NOT j.cancel_requested AND (s.delivery_until IS NULL OR s.delivery_until<=clock_timestamp()) AND d.updated_at < clock_timestamp()-((10*(d.retry_round+1))::bigint*interval '1 second')) ORDER BY updated_at LIMIT $1`, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []store.DeploymentSpeaker + for rows.Next() { + d, e := scanDeployment(rows) + if e != nil { + return nil, e + } + out = append(out, d) + } + return out, rows.Err() +} + +// ListDeploymentObservations rotates through current generations by oldest CP +// receipt time; the bounded batch prevents a large fleet monopolizing a worker. +func (p *Postgres) ListDeploymentObservations(ctx context.Context, limit int) ([]store.DeploymentSpeaker, error) { + if limit <= 0 || limit > 100 { + limit = 10 + } + rows, err := p.pool.Query(ctx, `WITH picked AS (SELECT d.job_id AS picked_job,d.speaker_id AS picked_speaker FROM deployment_speaker d JOIN bgp_speaker s ON s.id=d.speaker_id AND s.deployment_job_id=d.job_id AND s.deployment_generation=d.generation JOIN job_audit j ON j.id=d.job_id WHERE j.status IN ('succeeded','failed') AND NOT j.cancel_requested AND (s.delivery_until IS NULL OR s.delivery_until<=clock_timestamp()) ORDER BY d.health_received_at NULLS FIRST,d.speaker_id FOR UPDATE OF d SKIP LOCKED LIMIT $1) UPDATE deployment_speaker d SET health_observation_token=d.health_observation_token+1 FROM picked WHERE d.job_id=picked.picked_job AND d.speaker_id=picked.picked_speaker RETURNING `+deploymentColumns, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []store.DeploymentSpeaker + for rows.Next() { + d, e := scanDeployment(rows) + if e != nil { + return nil, e + } + out = append(out, d) + } + return out, rows.Err() +} + +// ObserveDeployment accepts only a health report for the still-current generation +// outside any delivery reservation. It never treats a legacy report as verified. +func (p *Postgres) ObserveDeployment(ctx context.Context, d store.DeploymentSpeaker, o store.DeploymentObservation) error { + tx, err := p.pool.Begin(ctx) + if err != nil { + return err + } + defer rollbackWithTimeout(tx) + var speaker string + err = tx.QueryRow(ctx, `SELECT id::text FROM bgp_speaker WHERE id=$1::uuid AND tenant_id=$2::uuid AND deployment_job_id=$3::uuid AND deployment_generation=$4 AND published_revision_id=$5::uuid AND (delivery_until IS NULL OR delivery_until<=clock_timestamp()) FOR UPDATE`, d.SpeakerID, d.TenantID, d.JobID, d.Generation, d.RevisionID).Scan(&speaker) + if errors.Is(err, pgx.ErrNoRows) { + return store.ErrJobLeaseLost + } + if err != nil { + return err + } + tag, err := tx.Exec(ctx, `UPDATE deployment_speaker SET health_observation_token=health_observation_token+1,health_received_at=clock_timestamp(),health_revision_id=$3,health_source_at=$4,health_error=$5,state=CASE WHEN $5='' AND $3<>'' THEN CASE WHEN $3=revision_id::text THEN 'applied' ELSE 'unknown' END ELSE state END,confirmation=CASE WHEN $5='' AND $3<>'' THEN CASE WHEN $3=revision_id::text THEN 'agent_reported' ELSE 'unconfirmed' END ELSE confirmation END WHERE job_id=$1::uuid AND speaker_id=$2::uuid AND health_observation_token=$6`, d.JobID, speaker, o.RevisionID, o.SourceAt, o.Error, d.ObservationToken) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return store.ErrJobLeaseLost + } + if o.Error == "" && o.RevisionID == d.RevisionID { + if _, err = tx.Exec(ctx, `UPDATE bgp_speaker SET last_applied_revision_id=$2::uuid,updated_at=now() WHERE id=$1::uuid`, speaker, d.RevisionID); err != nil { + return err + } + } + return tx.Commit(ctx) +} diff --git a/internal/repository/deployment_integration_test.go b/internal/repository/deployment_integration_test.go new file mode 100644 index 0000000..cee9224 --- /dev/null +++ b/internal/repository/deployment_integration_test.go @@ -0,0 +1,135 @@ +package repository + +import ( + "context" + "errors" + "testing" + "time" + + "evobgp/internal/store" + "github.com/google/uuid" +) + +func TestDeploymentLeaseGenerationAndRestartIntegration(t *testing.T) { + p, tenant := queueIntegration(t) + ctx := context.Background() + speaker, revision, newer := uuid.NewString(), uuid.NewString(), uuid.NewString() + for _, id := range []string{revision, newer} { + if _, err := p.pool.Exec(ctx, `INSERT INTO config_revision(id,tenant_id,content_hash) VALUES($1,$2,'deployment-test')`, id, tenant); err != nil { + t.Fatal(err) + } + } + if _, err := p.pool.Exec(ctx, `INSERT INTO bgp_speaker(id,tenant_id,role) VALUES($1,$2,'replica')`, speaker, tenant); err != nil { + t.Fatal(err) + } + enqueue := func() store.JobRecord { + t.Helper() + j, _, err := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "deploy_apply"}) + if err != nil { + t.Fatal(err) + } + return j + } + claim := func(owner string) store.JobRecord { + t.Helper() + rows, err := p.ClaimJobs(ctx, owner, 1, time.Minute) + if err != nil || len(rows) != 1 { + t.Fatalf("claim: %+v %v", rows, err) + } + return rows[0] + } + fence := func(j store.JobRecord) store.JobFence { + return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken} + } + first := enqueue() + first = claim("deployment-one") + d, err := p.BeginDeploymentSpeaker(ctx, fence(first), speaker, revision) + if err != nil { + t.Fatal(err) + } + second := enqueue() + second = claim("deployment-two") + if _, err = p.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer); !errors.Is(err, store.ErrDeploymentBusy) { + t.Fatalf("overlapping network ownership: %v", err) + } + if err = p.RecordDeploymentSpeaker(ctx, fence(first), d, "unknown", "unconfirmed", "response lost"); err != nil { + t.Fatal(err) + } + if _, err = p.pool.Exec(ctx, `UPDATE job_audit SET lease_until=clock_timestamp()-interval '1 second' WHERE id=$1`, first.ID); err != nil { + t.Fatal(err) + } + if err = p.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "agent_reported", ""); !errors.Is(err, store.ErrJobLeaseLost) { + t.Fatalf("expired ACK accepted: %v", err) + } + reclaimed := claim("deployment-reclaimed") + if reclaimed.ID != first.ID || reclaimed.FencingToken <= first.FencingToken { + t.Fatal("job not reclaimed with fresh token") + } + if _, err = p.BeginDeploymentSpeaker(ctx, fence(reclaimed), speaker, revision); !errors.Is(err, store.ErrDeploymentBusy) { + t.Fatalf("reclaim overlaps uncertain request: %v", err) + } + if _, err = p.pool.Exec(ctx, `UPDATE bgp_speaker SET delivery_until=clock_timestamp()-interval '1 second' WHERE id=$1`, speaker); err != nil { + t.Fatal(err) + } + // A fresh repository object sees the committed partial progress after restart. + restarted := &Postgres{pool: p.pool} + rows, err := restarted.ListDeploymentSpeakers(ctx, tenant, first.ID) + if err != nil || len(rows) != 1 || rows[0].State != "unknown" { + t.Fatalf("restart progress: %+v %v", rows, err) + } + d2, err := restarted.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer) + if err != nil { + t.Fatal(err) + } + if d2.Generation <= d.Generation { + t.Fatal("speaker generation did not advance") + } + if err = p.RecordDeploymentSpeaker(ctx, fence(reclaimed), d, "applied", "agent_reported", ""); !errors.Is(err, store.ErrJobLeaseLost) { + t.Fatalf("late old-generation ACK accepted: %v", err) + } + if err = p.RecordDeploymentSpeaker(ctx, fence(second), d2, "applied", "agent_reported", ""); err != nil { + t.Fatal(err) + } + var published, applied string + if err = p.pool.QueryRow(ctx, `SELECT published_revision_id::text,last_applied_revision_id::text FROM bgp_speaker WHERE id=$1`, speaker).Scan(&published, &applied); err != nil || published != newer || applied != newer { + t.Fatalf("fenced pointers: %s %s %v", published, applied, err) + } + d2Replay, err := p.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer) + if err != nil || d2Replay.State != "applied" || d2Replay.Generation != d2.Generation { + t.Fatalf("applied replay: %+v %v", d2Replay, err) + } + second.Status = "succeeded" + if err = p.FinishJob(ctx, second); err != nil { + t.Fatal(err) + } + oldPoll, err := p.ListDeploymentObservations(ctx, 10) + if err != nil || len(oldPoll) != 1 { + t.Fatalf("old poll %+v %v", oldPoll, err) + } + newPoll, err := p.ListDeploymentObservations(ctx, 10) + if err != nil || len(newPoll) != 1 { + t.Fatalf("new poll %+v %v", newPoll, err) + } + // A stale snapshot must not downgrade the current acknowledged state when + // the latest poll times out or the old agent omits revision fields. + newPoll[0].State = "unknown" + newPoll[0].Confirmation = "unconfirmed" + if err = p.ObserveDeployment(ctx, newPoll[0], store.DeploymentObservation{Error: "timeout"}); err != nil { + t.Fatal(err) + } + if err = p.ObserveDeployment(ctx, oldPoll[0], store.DeploymentObservation{RevisionID: revision}); !errors.Is(err, store.ErrJobLeaseLost) { + t.Fatalf("late health accepted: %v", err) + } + progress, err := p.ListDeploymentSpeakers(ctx, tenant, second.ID) + if err != nil || len(progress) != 1 || progress[0].State != "applied" || progress[0].Confirmation != "agent_reported" || progress[0].HealthError != "timeout" { + t.Fatalf("health lost ACK %+v %v", progress, err) + } + // Historical deployment records must survive ordinary revision retention. + if _, err = p.pool.Exec(ctx, `DELETE FROM config_revision WHERE id=$1::uuid`, revision); err != nil { + t.Fatalf("history blocks revision pruning: %v", err) + } + history, err := p.ListDeploymentSpeakers(ctx, tenant, first.ID) + if err != nil || len(history) != 1 || history[0].RevisionID != revision { + t.Fatalf("pruning removed history %+v %v", history, err) + } +} diff --git a/internal/repository/job_audit_integration_test.go b/internal/repository/job_audit_integration_test.go new file mode 100644 index 0000000..775eae4 --- /dev/null +++ b/internal/repository/job_audit_integration_test.go @@ -0,0 +1,260 @@ +package repository + +import ( + "context" + "errors" + "evobgp/internal/db" + "evobgp/internal/store" + "evobgp/migrations" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "os" + "sync" + "testing" + "time" +) + +func TestJobQueueFreshSeedIntegration(t *testing.T) { + dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("EVOBGP_TEST_DATABASE_URL not set") + } + ctx := context.Background() + admin, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer admin.Close() + schema := "queue_seed_" + uuid.NewString() + quoted := `"` + schema + `"` + if _, err = admin.Exec(ctx, `CREATE SCHEMA `+quoted); err != nil { + t.Fatal(err) + } + defer func() { + if _, e := admin.Exec(ctx, `DROP SCHEMA `+quoted+` CASCADE`); e != nil { + t.Error(e) + } + }() + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err = db.ApplyPostgresMigrations(ctx, pool, migrations.Postgres, "postgres"); err != nil { + t.Fatal(err) + } + p, err := NewPostgres(ctx, pool, true) + if err != nil { + t.Fatal(err) + } + tenant, cdn, ip, revision, speaker := p.DemoIDs() + if tenant == "" || tenant == store.SystemJobTenantID || cdn == "" || ip == "" || revision == "" || speaker == "" { + t.Fatalf("system tenant blocked demo seed: %q %q %q %q %q", tenant, cdn, ip, revision, speaker) + } +} + +func queueIntegration(t *testing.T) (*Postgres, string) { + t.Helper() + dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("EVOBGP_TEST_DATABASE_URL not set") + } + pool, e := db.OpenPostgresPool(context.Background(), dsn) + if e != nil { + t.Fatal(e) + } + t.Cleanup(pool.Close) + tenant := uuid.NewString() + if _, e = pool.Exec(context.Background(), `INSERT INTO tenant(id,name,slug) VALUES($1,'queue integration',$2)`, tenant, "queue-"+tenant); e != nil { + t.Fatal(e) + } + t.Cleanup(func() { _, _ = pool.Exec(context.Background(), `DELETE FROM tenant WHERE id=$1`, tenant) }) + return &Postgres{pool: pool}, tenant +} + +func TestJobAuditClaimAndRestartIntegration(t *testing.T) { + p, tenant := queueIntegration(t) + ctx := context.Background() + j, _, e := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"}) + if e != nil { + t.Fatal(e) + } + var wg sync.WaitGroup + results := make(chan []store.JobRecord, 2) + errs := make(chan error, 2) + for _, owner := range []string{"replica-a", "replica-b"} { + wg.Add(1) + go func(owner string) { + defer wg.Done() + items, e := p.ClaimJobs(ctx, owner, 1, time.Minute) + results <- items + errs <- e + }(owner) + } + wg.Wait() + close(results) + close(errs) + for e := range errs { + if e != nil { + t.Fatal(e) + } + } + var first store.JobRecord + count := 0 + for items := range results { + for _, item := range items { + if item.ID != j.ID { + t.Fatalf("unexpected job %+v", item) + } + first = item + count++ + } + } + if count != 1 { + t.Fatalf("claims=%d", count) + } + if _, e = p.HeartbeatJob(ctx, j.ID, first.OwnerID, first.FencingToken, time.Hour); e != nil { + t.Fatal(e) + } + if items, e := p.ClaimJobs(ctx, "other", 1, time.Minute); e != nil || len(items) != 0 { + t.Fatalf("stole live lease: %+v %v", items, e) + } + if _, e = p.pool.Exec(ctx, `UPDATE job_audit SET lease_until=now()-interval '1 second' WHERE id=$1`, j.ID); e != nil { + t.Fatal(e) + } + pool, e := db.OpenPostgresPool(ctx, os.Getenv("EVOBGP_TEST_DATABASE_URL")) + if e != nil { + t.Fatal(e) + } + defer pool.Close() + restarted := &Postgres{pool: pool} + items, e := restarted.ClaimJobs(ctx, "restarted", 1, time.Minute) + if e != nil || len(items) != 1 { + t.Fatalf("restart claim: %+v %v", items, e) + } + if items[0].FencingToken <= first.FencingToken { + t.Fatal("token did not advance") + } + first.Status = "succeeded" + if e = p.FinishJob(ctx, first); !errors.Is(e, store.ErrJobLeaseLost) { + t.Fatalf("stale finish: %v", e) + } + if _, e = p.CancelJob(ctx, tenant, j.ID); e != nil { + t.Fatal(e) + } + items[0].Status = "succeeded" + if e = restarted.FinishJob(ctx, items[0]); e != nil { + t.Fatal(e) + } + got, e := p.GetJob(ctx, tenant, j.ID) + if e != nil || got.Status != "cancelled" { + t.Fatalf("cancel/finish race: %+v %v", got, e) + } + listed, _, _, e := restarted.ListJobs(ctx, tenant, "cancelled", "", "", 10) + if e != nil || len(listed) != 1 { + t.Fatalf("shared terminal list %+v %v", listed, e) + } +} + +func TestJobAuditIdempotencyRowIntegration(t *testing.T) { + p, tenant := queueIntegration(t) + ctx := context.Background() + key := uuid.NewString() + in := store.JobEnqueue{TenantID: tenant, Kind: "test", IdempotencyKey: &key, Meta: map[string]any{"input": "same"}} + var wg sync.WaitGroup + ids := make(chan string, 8) + errs := make(chan error, 8) + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { defer wg.Done(); j, _, e := p.EnqueueJob(ctx, in); ids <- j.ID; errs <- e }() + } + wg.Wait() + close(ids) + close(errs) + for e := range errs { + if e != nil { + t.Fatal(e) + } + } + id := "" + for got := range ids { + if id == "" { + id = got + } + if got != id { + t.Fatalf("different IDs %s / %s", id, got) + } + } + items, e := p.ClaimJobs(ctx, "replica", 1, time.Minute) + if e != nil || len(items) != 1 { + t.Fatalf("claim %+v %v", items, e) + } + items[0].Status = "succeeded" + items[0].Meta = map[string]any{"result": "done"} + if e = p.FinishJob(ctx, items[0]); e != nil { + t.Fatal(e) + } + got, created, e := p.EnqueueJob(ctx, in) + if e != nil || created || got.ID != id || got.Status != "succeeded" { + t.Fatalf("terminal replay %+v %v %v", got, created, e) + } + in.Meta = map[string]any{"input": "changed"} + if _, _, e = p.EnqueueJob(ctx, in); !errors.Is(e, store.ErrJobConflict) { + t.Fatalf("changed request accepted: %v", e) + } +} + +func TestJobQueueRetryBoundsAndFailureIntegration(t *testing.T) { + p, tenant := queueIntegration(t) + ctx := context.Background() + t.Setenv("EVOBGP_JOB_MAX_TENANT_PENDING", "1") + j, _, e := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"}) + if e != nil { + t.Fatal(e) + } + if _, _, e = p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"}); !errors.Is(e, store.ErrJobQueueFull) { + t.Fatalf("unbounded queue: %v", e) + } + if _, e = p.pool.Exec(ctx, `UPDATE job_audit SET attempt=3 WHERE id=$1`, j.ID); e != nil { + t.Fatal(e) + } + if e = p.ReapExpiredJobs(ctx); e != nil { + t.Fatal(e) + } + items, e := p.ClaimJobs(ctx, "worker", 1, time.Minute) + if e != nil || len(items) != 0 { + t.Fatalf("exhausted claim %+v %v", items, e) + } + got, e := p.GetJob(ctx, tenant, j.ID) + if e != nil || got.Status != "failed" { + t.Fatalf("exhausted status %+v %v", got, e) + } + if _, e = p.RetryJob(ctx, tenant, j.ID); e != nil { + t.Fatal(e) + } + items, e = p.ClaimJobs(ctx, "worker", 1, time.Minute) + if e != nil || len(items) != 1 || items[0].Attempt != 1 { + t.Fatalf("operator retry %+v %v", items, e) + } + if e = p.ReleaseJob(ctx, j.ID, items[0].OwnerID, items[0].FencingToken); e != nil { + t.Fatal(e) + } + items, e = p.ClaimJobs(ctx, "worker-2", 1, time.Minute) + if e != nil || len(items) != 1 || items[0].Attempt != 1 { + t.Fatalf("shutdown consumed retry %+v %v", items, e) + } + failedPool, e := db.OpenPostgresPool(ctx, os.Getenv("EVOBGP_TEST_DATABASE_URL")) + if e != nil { + t.Fatal(e) + } + failedPool.Close() + broken := &Postgres{pool: failedPool} + if accepted, created, e := broken.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"}); e == nil || created || accepted.ID != "" { + t.Fatalf("accepted on failed DB: %+v %v %v", accepted, created, e) + } +} diff --git a/internal/repository/job_enqueue_lock_test.go b/internal/repository/job_enqueue_lock_test.go new file mode 100644 index 0000000..90fe59b --- /dev/null +++ b/internal/repository/job_enqueue_lock_test.go @@ -0,0 +1,82 @@ +package repository + +import ( + "context" + "errors" + "testing" + "time" + + "evobgp/internal/store" + + "github.com/google/uuid" +) + +func TestJobEnqueueTenantLocksDiffer(t *testing.T) { + const tenantA = "11111111-1111-1111-1111-111111111111" + const tenantB = "22222222-2222-2222-2222-222222222222" + a := jobEnqueueTenantLock(tenantA) + b := jobEnqueueTenantLock(tenantB) + if a == b || a == jobEnqueueGlobalLock || b == jobEnqueueGlobalLock { + t.Fatalf("locks a=%d b=%d global=%d", a, b, jobEnqueueGlobalLock) + } + if jobEnqueueTenantLock(tenantA) != a { + t.Fatal("tenant lock is not stable") + } +} + +func TestEnqueueJobTenantLocksDoNotOverlapIntegration(t *testing.T) { + p, tenantA := queueIntegration(t) + ctx := context.Background() + tenantB := uuid.NewString() + if _, err := p.pool.Exec(ctx, `INSERT INTO tenant(id,name,slug) VALUES($1,'queue b',$2)`, tenantB, "queue-b-"+tenantB); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = p.pool.Exec(context.Background(), `DELETE FROM tenant WHERE id=$1`, tenantB) + }) + + conn, err := p.pool.Acquire(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Release() + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback(context.Background()) }() + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1::bigint)`, jobEnqueueTenantLock(tenantA)); err != nil { + t.Fatal(err) + } + + otherDone := make(chan error, 1) + go func() { + _, _, err := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenantB, Kind: "test"}) + otherDone <- err + }() + select { + case err := <-otherDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("tenant B enqueue shared tenant A lock") + } + + short, cancel := context.WithTimeout(ctx, 400*time.Millisecond) + defer cancel() + if _, _, err = p.EnqueueJob(short, store.JobEnqueue{TenantID: tenantA, Kind: "test"}); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("same tenant did not wait: %v", err) + } + + key := "idem-key" + in := store.JobEnqueue{TenantID: tenantB, Kind: "test", IdempotencyKey: &key, Meta: map[string]any{"n": 1}} + first, created, err := p.EnqueueJob(ctx, in) + if err != nil || !created || first.ID == "" { + t.Fatalf("first=%+v created=%v err=%v", first, created, err) + } + in.Meta = map[string]any{"n": 2} + if _, _, err = p.EnqueueJob(ctx, in); !errors.Is(err, store.ErrJobConflict) { + t.Fatalf("idempotency conflict: %v", err) + } +} diff --git a/internal/repository/job_queue.go b/internal/repository/job_queue.go new file mode 100644 index 0000000..5c09576 --- /dev/null +++ b/internal/repository/job_queue.go @@ -0,0 +1,371 @@ +package repository + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "hash/fnv" + "os" + "strconv" + "strings" + "time" + + "evobgp/internal/store" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// jobEnqueueGlobalLock serializes the global pending-job count. It is not held +// while idempotency is resolved; that uses jobEnqueueTenantLock. +const jobEnqueueGlobalLock int64 = 38042341 + +const jobColumns = `id::text, tenant_id::text, kind, status, idempotency_key, module_id::text, meta_json, created_at, started_at, finished_at, error_message, COALESCE(owner_id,''), fencing_token, attempt, cancel_requested, lease_until, request_sequence, COALESCE(request_hash,'')` + +func scanJob(row pgx.Row) (store.JobRecord, error) { + var j store.JobRecord + var meta []byte + err := row.Scan(&j.ID, &j.TenantID, &j.Kind, &j.Status, &j.IdempotencyKey, &j.ModuleID, &meta, &j.CreatedAt, &j.StartedAt, &j.FinishedAt, &j.Error, &j.OwnerID, &j.FencingToken, &j.Attempt, &j.CancelRequested, &j.LeaseUntil, &j.RequestSequence, &j.RequestHash) + if err != nil { + return j, err + } + if err = json.Unmarshal(meta, &j.Meta); err != nil { + return j, fmt.Errorf("repository: decode job metadata: %w", err) + } + return j, nil +} + +func jobLimit(env string, fallback int) int { + n, e := strconv.Atoi(os.Getenv(env)) + if e == nil && n > 0 { + return n + } + return fallback +} + +func rollbackWithTimeout(tx pgx.Tx) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = tx.Rollback(ctx) +} + +func (p *Postgres) CountJobsByStatus(ctx context.Context) (map[string]int64, error) { + rows, err := p.pool.Query(ctx, `SELECT status,count(*) FROM job_audit GROUP BY status`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]int64{} + for rows.Next() { + var status string + var n int64 + if err := rows.Scan(&status, &n); err != nil { + return nil, err + } + out[status] = n + } + return out, rows.Err() +} + +func (p *Postgres) FindJobByIdempotencyKey(ctx context.Context, tenant, key string) (store.JobRecord, error) { + j, err := scanJob(p.pool.QueryRow(ctx, `SELECT `+jobColumns+` FROM job_audit WHERE tenant_id=$1::uuid AND idempotency_key=$2`, tenant, key)) + if errors.Is(err, pgx.ErrNoRows) { + return j, store.ErrJobNotFound + } + return j, err +} + +// EnqueueJob inserts exactly one immutable request and returns the committed row. +func (p *Postgres) EnqueueJob(ctx context.Context, in store.JobEnqueue) (store.JobRecord, bool, error) { + h, err := store.JobRequestHash(in) + if err != nil { + return store.JobRecord{}, false, fmt.Errorf("repository: hash job: %w", err) + } + meta, err := json.Marshal(in.Meta) + if err != nil { + return store.JobRecord{}, false, fmt.Errorf("repository: encode job: %w", err) + } + tx, err := p.pool.Begin(ctx) + if err != nil { + return store.JobRecord{}, false, err + } + defer rollbackWithTimeout(tx) + if err = lockJobEnqueue(ctx, tx, jobEnqueueTenantLock(in.TenantID)); err != nil { + return store.JobRecord{}, false, err + } + if in.IdempotencyKey != nil && *in.IdempotencyKey != "" { + var existingHash string + j, e := scanJob(tx.QueryRow(ctx, `SELECT `+jobColumns+` FROM job_audit WHERE tenant_id=$1::uuid AND idempotency_key=$2 FOR UPDATE`, in.TenantID, *in.IdempotencyKey)) + if e == nil { + if err = tx.QueryRow(ctx, `SELECT COALESCE(request_hash,'') FROM job_audit WHERE id=$1::uuid`, j.ID).Scan(&existingHash); err != nil { + return store.JobRecord{}, false, err + } + if existingHash != h { + return store.JobRecord{}, false, store.ErrJobConflict + } + if err = tx.Commit(ctx); err != nil { + return store.JobRecord{}, false, err + } + return j, false, nil + } + if !errors.Is(e, pgx.ErrNoRows) { + return store.JobRecord{}, false, e + } + } + var tenantN int + if err = tx.QueryRow(ctx, `SELECT count(*)::int FROM job_audit WHERE status IN ('queued','running') AND tenant_id=$1::uuid`, in.TenantID).Scan(&tenantN); err != nil { + return store.JobRecord{}, false, err + } + if tenantN >= jobLimit("EVOBGP_JOB_MAX_TENANT_PENDING", 1000) { + return store.JobRecord{}, false, store.ErrJobQueueFull + } + if err = lockJobEnqueue(ctx, tx, jobEnqueueGlobalLock); err != nil { + return store.JobRecord{}, false, err + } + var global int + if err = tx.QueryRow(ctx, `SELECT count(*)::int FROM job_audit WHERE status IN ('queued','running')`).Scan(&global); err != nil { + return store.JobRecord{}, false, err + } + if global >= jobLimit("EVOBGP_JOB_MAX_PENDING", 10000) { + return store.JobRecord{}, false, store.ErrJobQueueFull + } + id := uuid.NewString() + var idem, mod any + if in.IdempotencyKey != nil && *in.IdempotencyKey != "" { + idem = *in.IdempotencyKey + } + if in.ModuleID != nil && *in.ModuleID != "" { + mod = *in.ModuleID + } + j, err := scanJob(tx.QueryRow(ctx, `INSERT INTO job_audit(id,tenant_id,kind,status,idempotency_key,module_id,meta_json,request_hash,request_json) VALUES($1::uuid,$2::uuid,$3,'queued',$4,$5::uuid,$6::jsonb,$7,$6::jsonb) RETURNING `+jobColumns, id, in.TenantID, in.Kind, idem, mod, meta, h)) + if err != nil { + return store.JobRecord{}, false, err + } + if err = tx.Commit(ctx); err != nil { + return store.JobRecord{}, false, err + } + return j, true, nil +} + +// ClaimJobs atomically acquires available rows with a fresh fencing token. +func (p *Postgres) ClaimJobs(ctx context.Context, owner string, limit int, lease time.Duration) ([]store.JobRecord, error) { + if limit <= 0 { + return nil, nil + } + if limit > 64 { + limit = 64 + } + if lease <= 0 { + lease = time.Minute + } + rows, err := p.pool.Query(ctx, `WITH picked AS ( + SELECT id AS picked_id FROM job_audit WHERE ((status='queued' AND available_at<=now()) OR (status='running' AND lease_until=3`); err != nil { + return fmt.Errorf("repository: reap exhausted jobs: %w", err) + } + return nil +} + +// HeartbeatJob extends a live lease and reports a cross-replica cancel request. +func (p *Postgres) HeartbeatJob(ctx context.Context, id, owner string, token int64, lease time.Duration) (bool, error) { + if lease <= 0 { + lease = time.Minute + } + var cancelled bool + err := p.pool.QueryRow(ctx, `UPDATE job_audit SET lease_until=clock_timestamp()+$4::bigint * interval '1 millisecond' WHERE id=$1::uuid AND owner_id=$2 AND fencing_token=$3 AND status='running' AND lease_until>clock_timestamp() RETURNING cancel_requested`, id, owner, token, lease.Milliseconds()).Scan(&cancelled) + if errors.Is(err, pgx.ErrNoRows) { + return false, store.ErrJobLeaseLost + } + return cancelled, err +} + +// CheckpointJob persists mutable progress without altering the immutable request. +func (p *Postgres) CheckpointJob(ctx context.Context, j store.JobRecord) error { + meta, err := json.Marshal(j.Meta) + if err != nil { + return err + } + tag, err := p.pool.Exec(ctx, `UPDATE job_audit SET meta_json=$4::jsonb WHERE id=$1::uuid AND owner_id=$2 AND fencing_token=$3 AND status='running' AND lease_until>clock_timestamp()`, j.ID, j.OwnerID, j.FencingToken, meta) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return store.ErrJobLeaseLost + } + return nil +} + +// FinishJob commits terminal status only for the current live lease owner. +func (p *Postgres) FinishJob(ctx context.Context, j store.JobRecord) error { + meta, err := json.Marshal(j.Meta) + if err != nil { + return err + } + tag, err := p.pool.Exec(ctx, `UPDATE job_audit SET status=CASE WHEN cancel_requested THEN 'cancelled' ELSE $4 END,error_message=CASE WHEN cancel_requested THEN NULL ELSE $5 END,finished_at=now(),meta_json=$6::jsonb,owner_id=NULL,lease_until=NULL WHERE id=$1::uuid AND owner_id=$2 AND fencing_token=$3 AND status='running' AND lease_until>clock_timestamp()`, j.ID, j.OwnerID, j.FencingToken, j.Status, j.Error, meta) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return store.ErrJobLeaseLost + } + return nil +} + +// ReleaseJob returns an owned running job to the queue during graceful shutdown. +func (p *Postgres) ReleaseJob(ctx context.Context, id, owner string, token int64) error { + tag, err := p.pool.Exec(ctx, `UPDATE job_audit SET status=CASE WHEN cancel_requested THEN 'cancelled' ELSE 'queued' END,finished_at=CASE WHEN cancel_requested THEN now() ELSE NULL END,owner_id=NULL,lease_until=NULL,available_at=now(),attempt=GREATEST(attempt-1,0) WHERE id=$1::uuid AND owner_id=$2 AND fencing_token=$3 AND status='running'`, id, owner, token) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return store.ErrJobLeaseLost + } + return nil +} + +// GetJob returns a tenant-scoped job from shared storage. +func (p *Postgres) GetJob(ctx context.Context, tenant, id string) (store.JobRecord, error) { + if _, err := uuid.Parse(id); err != nil { + return store.JobRecord{}, store.ErrJobNotFound + } + j, err := scanJob(p.pool.QueryRow(ctx, `SELECT `+jobColumns+` FROM job_audit WHERE id=$1::uuid AND tenant_id=$2::uuid`, id, tenant)) + if errors.Is(err, pgx.ErrNoRows) { + return j, store.ErrJobNotFound + } + return j, err +} + +// ListJobs returns a bounded shared page in creation order. +func (p *Postgres) ListJobs(ctx context.Context, tenant, status, kind, cursor string, limit int) ([]store.JobRecord, string, bool, error) { + if limit <= 0 || limit > 100 { + limit = 50 + } + off, _ := strconv.Atoi(cursor) + if off < 0 { + off = 0 + } + rows, err := p.pool.Query(ctx, `SELECT `+jobColumns+` FROM job_audit WHERE tenant_id=$1::uuid AND ($2='' OR status=$2) AND ($3='' OR kind=$3) ORDER BY created_at DESC,request_sequence DESC,id DESC OFFSET $4 LIMIT $5`, tenant, status, kind, off, limit+1) + if err != nil { + return nil, "", false, err + } + defer rows.Close() + out := make([]store.JobRecord, 0, limit+1) + for rows.Next() { + j, e := scanJob(rows) + if e != nil { + return nil, "", false, e + } + out = append(out, j) + } + if err = rows.Err(); err != nil { + return nil, "", false, err + } + more := len(out) > limit + if more { + out = out[:limit] + } + next := "" + if more { + next = strconv.Itoa(off + limit) + } + return out, next, more, nil +} + +// CancelJob atomically cancels queued work or signals its active owner. +func (p *Postgres) CancelJob(ctx context.Context, tenant, id string) (store.JobRecord, error) { + j, err := scanJob(p.pool.QueryRow(ctx, `UPDATE job_audit SET status=CASE WHEN status='queued' THEN 'cancelled' ELSE status END,finished_at=CASE WHEN status='queued' THEN now() ELSE finished_at END,cancel_requested=CASE WHEN status='running' THEN true ELSE cancel_requested END WHERE id=$1::uuid AND tenant_id=$2::uuid RETURNING `+jobColumns, id, tenant)) + if errors.Is(err, pgx.ErrNoRows) { + return j, store.ErrJobNotFound + } + return j, err +} + +// RetryJob explicitly schedules a failed operation under its original ID. +func (p *Postgres) RetryJob(ctx context.Context, tenant, id string) (store.JobRecord, error) { + tx, err := p.pool.Begin(ctx) + if err != nil { + return store.JobRecord{}, err + } + defer rollbackWithTimeout(tx) + if err = lockJobEnqueue(ctx, tx, jobEnqueueTenantLock(tenant)); err != nil { + return store.JobRecord{}, err + } + old, err := scanJob(tx.QueryRow(ctx, `SELECT `+jobColumns+` FROM job_audit WHERE id=$1::uuid AND tenant_id=$2::uuid FOR UPDATE`, id, tenant)) + if errors.Is(err, pgx.ErrNoRows) { + return store.JobRecord{}, store.ErrJobNotFound + } + if err != nil { + return store.JobRecord{}, err + } + if old.Status != "failed" { + return store.JobRecord{}, store.ErrJobConflict + } + var pending int + if err = tx.QueryRow(ctx, `SELECT count(*)::int FROM job_audit WHERE status IN ('queued','running') AND tenant_id=$1::uuid`, tenant).Scan(&pending); err != nil { + return store.JobRecord{}, err + } + if pending >= jobLimit("EVOBGP_JOB_MAX_TENANT_PENDING", 1000) { + return store.JobRecord{}, store.ErrJobQueueFull + } + if err = lockJobEnqueue(ctx, tx, jobEnqueueGlobalLock); err != nil { + return store.JobRecord{}, err + } + var global int + if err = tx.QueryRow(ctx, `SELECT count(*)::int FROM job_audit WHERE status IN ('queued','running')`).Scan(&global); err != nil { + return store.JobRecord{}, err + } + if global >= jobLimit("EVOBGP_JOB_MAX_PENDING", 10000) { + return store.JobRecord{}, store.ErrJobQueueFull + } + j, err := scanJob(tx.QueryRow(ctx, `UPDATE job_audit SET status='queued',available_at=now(),started_at=NULL,finished_at=NULL,error_message=NULL,cancel_requested=false,attempt=0,owner_id=NULL,lease_until=NULL,meta_json=COALESCE(request_json,meta_json) WHERE id=$1::uuid AND tenant_id=$2::uuid AND status='failed' RETURNING `+jobColumns, id, tenant)) + if errors.Is(err, pgx.ErrNoRows) { + return j, store.ErrJobConflict + } + if err != nil { + return j, err + } + return j, tx.Commit(ctx) +} + +func lockJobEnqueue(ctx context.Context, tx pgx.Tx, key int64) error { + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1::bigint)`, key); err != nil { + return fmt.Errorf("repository: lock job enqueue: %w", err) + } + return nil +} + +// jobEnqueueTenantLock is the per-tenant advisory key for idempotency and the tenant pending limit. +func jobEnqueueTenantLock(tenantID string) int64 { + h := fnv.New64a() + _, _ = h.Write([]byte("evobgp-job-enqueue-v1\x00")) + _, _ = h.Write([]byte(strings.TrimSpace(tenantID))) + key := int64(h.Sum64()) + if key == jobEnqueueGlobalLock { + key ^= 0x5a17 + } + return key +} diff --git a/internal/repository/module_snapshot.go b/internal/repository/module_snapshot.go index 3670615..17ce4db 100644 --- a/internal/repository/module_snapshot.go +++ b/internal/repository/module_snapshot.go @@ -2,10 +2,11 @@ package repository import ( "context" - "encoding/json" "errors" + "fmt" + "hash/fnv" + "strconv" "strings" - "sync" "time" "evobgp/internal/store" @@ -13,155 +14,240 @@ import ( "github.com/jackc/pgx/v5" ) -// moduleSnapshotRowExistsOnce caches the schema check for the process lifetime. -// The table is created by migrations and never disappears at runtime; DB errors are -// not cached so a transient outage falls back to the JSON path only once. -var moduleSnapshotRowExistsOnce sync.Once - -var moduleSnapshotRowExistsCached bool - -func moduleSnapshotRowTableExists(ctx context.Context, q queryRower) bool { - moduleSnapshotRowExistsOnce.Do(func() { - var n int - err := q.QueryRow(ctx, ` - SELECT 1 FROM information_schema.tables - WHERE table_schema = 'public' AND table_name = 'module_prefix_snapshot_row' - LIMIT 1`).Scan(&n) - // A query error means the backend is unreachable or information_schema is hidden; - // treat as "absent" so callers fall back to prefixes_json, but retry next call. - if err != nil { - moduleSnapshotRowExistsOnce = sync.Once{} - return - } - moduleSnapshotRowExistsCached = true - }) - return moduleSnapshotRowExistsCached +// ListModuleSnapshotSourceRows returns persisted rows for one source key. +func (p *Postgres) ListModuleSnapshotSourceRows(ctx context.Context, tenantID, moduleID, source string) ([]store.PrefixRow, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + rows, err := p.pool.Query(ctx, ` + SELECT prefix, community_id::text, source + FROM module_prefix_snapshot_row + WHERE tenant_id = $1::uuid AND module_id = $2::uuid AND source = $3 + ORDER BY ord`, tenantID, moduleID, source) + if err != nil { + return nil, fmt.Errorf("repository: read snapshot source rows: %w", err) + } + defer rows.Close() + out, err := readModuleSnapshotRows(rows) + if err != nil { + return nil, err + } + return out, nil } -func (p *Postgres) GetModulePrefixSnapshot(tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) { - ctx := context.Background() +func (p *Postgres) GetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) { + tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return nil, false, fmt.Errorf("repository: begin snapshot read: %w", err) + } + defer rollbackSnapshotTx(tx) + snap, ok, err := readModuleSnapshotTx(ctx, tx, tenantID, moduleID) + if err != nil { + return nil, false, err + } + if err := tx.Commit(ctx); err != nil { + return nil, false, fmt.Errorf("repository: commit snapshot read: %w", err) + } + return snap, ok, nil +} + +func readModuleSnapshotTx(ctx context.Context, tx pgx.Tx, tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) { var inputHash string var collectedAt time.Time - err := p.pool.QueryRow(ctx, ` - SELECT input_hash, collected_at + var version int64 + err := tx.QueryRow(ctx, ` + SELECT input_hash, collected_at, snapshot_version FROM module_prefix_snapshot WHERE tenant_id = $1 AND module_id = $2`, - tenantID, moduleID).Scan(&inputHash, &collectedAt) + tenantID, moduleID).Scan(&inputHash, &collectedAt, &version) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil, false, nil } return nil, false, err } - var prefixes []store.PrefixRow - if moduleSnapshotRowTableExists(ctx, p.pool) { - rows, qerr := p.pool.Query(ctx, ` + rows, qerr := tx.Query(ctx, ` SELECT prefix, community_id::text, source FROM module_prefix_snapshot_row WHERE tenant_id = $1::uuid AND module_id = $2::uuid ORDER BY ord`, tenantID, moduleID) - if qerr != nil { - return nil, false, qerr - } - defer rows.Close() - for rows.Next() { - var pr store.PrefixRow - var comm *string - if err := rows.Scan(&pr.Prefix, &comm, &pr.Source); err != nil { - continue - } - pr.CommunityID = comm - prefixes = append(prefixes, pr) - } - } else { - var raw []byte - if err := p.pool.QueryRow(ctx, ` - SELECT prefixes_json FROM module_prefix_snapshot - WHERE tenant_id = $1 AND module_id = $2`, tenantID, moduleID).Scan(&raw); err == nil && len(raw) > 0 { - _ = json.Unmarshal(raw, &prefixes) - } + if qerr != nil { + return nil, false, fmt.Errorf("repository: read snapshot rows: %w", qerr) + } + defer rows.Close() + prefixes, err := readModuleSnapshotRows(rows) + if err != nil { + return nil, false, err } return &store.ModulePrefixSnapshot{ InputHash: inputHash, CollectedAt: collectedAt.UTC(), + Version: version, Prefixes: prefixes, }, true, nil } -func (p *Postgres) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []store.PrefixRow) error { +type moduleSnapshotRows interface { + Next() bool + Scan(...any) error + Err() error +} + +func readModuleSnapshotRows(rows moduleSnapshotRows) ([]store.PrefixRow, error) { + var prefixes []store.PrefixRow + for rows.Next() { + var pr store.PrefixRow + var comm *string + if err := rows.Scan(&pr.Prefix, &comm, &pr.Source); err != nil { + return nil, fmt.Errorf("repository: scan snapshot row: %w", err) + } + pr.CommunityID = comm + prefixes = append(prefixes, pr) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("repository: iterate snapshot rows: %w", err) + } + return prefixes, nil +} + +func (p *Postgres) SetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID, inputHash string, prefixes []store.PrefixRow) error { if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" || strings.TrimSpace(inputHash) == "" { return store.ErrInvalidInput } - ctx := context.Background() + return p.UpdateModulePrefixSnapshot(ctx, tenantID, moduleID, func(*store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) { + return &store.ModulePrefixSnapshot{InputHash: inputHash, Prefixes: prefixes}, nil + }) +} + +func rollbackSnapshotTx(tx pgx.Tx) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = tx.Rollback(ctx) +} + +// UpdateModulePrefixSnapshot serializes read, mutation and write on the same transaction connection. +func (p *Postgres) UpdateModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string, update store.ModuleSnapshotUpdate) error { + if update == nil || strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" { + return store.ErrInvalidInput + } + ctx, cancel, timeoutInstalled := store.SnapshotWriteBudget(ctx) + defer cancel() tx, err := p.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("repository: begin snapshot update: %w", err) + } + defer rollbackSnapshotTx(tx) + if err := setSnapshotStatementTimeout(ctx, tx, timeoutInstalled); err != nil { + return err + } + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1::bigint)`, moduleSnapshotAdvisoryKey(tenantID, moduleID)); err != nil { + return fmt.Errorf("repository: lock snapshot transaction: %w", err) + } + current, _, err := readModuleSnapshotTx(ctx, tx, tenantID, moduleID) if err != nil { return err } - defer func() { _ = tx.Rollback(ctx) }() - _, err = tx.Exec(ctx, ` + next, err := update(current) + if err != nil { + return err + } + if next != nil && strings.TrimSpace(next.InputHash) == "" { + return store.ErrInvalidInput + } + if err := ctx.Err(); err != nil { + return err + } + if next == nil { + if _, err := tx.Exec(ctx, `DELETE FROM module_prefix_snapshot WHERE tenant_id=$1::uuid AND module_id=$2::uuid`, tenantID, moduleID); err != nil { + return fmt.Errorf("repository: delete module snapshot: %w", err) + } + } else if !store.SameSnapshotContent(current, next) { + if err := writeModuleSnapshotTx(ctx, tx, tenantID, moduleID, next); err != nil { + return err + } + } + hash := "" + if next != nil { + hash = next.InputHash + } + tag, err := tx.Exec(ctx, `UPDATE module SET input_hash=$3 WHERE tenant_id=$1::uuid AND id=$2::uuid AND deleted_at IS NULL`, tenantID, moduleID, hash) + if err != nil { + return fmt.Errorf("repository: update module input hash: %w", err) + } + if tag.RowsAffected() == 0 { + return store.ErrNotFound + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("repository: commit module snapshot: %w", err) + } + return nil +} + +func writeModuleSnapshotTx(ctx context.Context, tx pgx.Tx, tenantID, moduleID string, next *store.ModulePrefixSnapshot) error { + _, err := tx.Exec(ctx, ` INSERT INTO module_prefix_snapshot (tenant_id, module_id, input_hash, collected_at) VALUES ($1::uuid, $2::uuid, $3, now()) ON CONFLICT (tenant_id, module_id) DO UPDATE SET input_hash = EXCLUDED.input_hash, collected_at = EXCLUDED.collected_at`, - tenantID, moduleID, inputHash) + tenantID, moduleID, next.InputHash) if err != nil { return err } - if moduleSnapshotRowTableExists(ctx, tx) { - if _, err := tx.Exec(ctx, ` + if _, err := tx.Exec(ctx, ` DELETE FROM module_prefix_snapshot_row WHERE tenant_id = $1::uuid AND module_id = $2::uuid`, tenantID, moduleID); err != nil { - return err - } - if len(prefixes) > 0 { - _, err = tx.CopyFrom( - ctx, - pgx.Identifier{"module_prefix_snapshot_row"}, - []string{"tenant_id", "module_id", "ord", "prefix", "community_id", "source"}, - pgx.CopyFromSlice(len(prefixes), func(i int) ([]any, error) { - pr := prefixes[i] - var comm any - if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" { - comm = strings.TrimSpace(*pr.CommunityID) - } - src := pr.Source - if strings.TrimSpace(src) == "" { - src = "render" - } - return []any{tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src}, nil - }), - ) - if err != nil { - return err - } - } - } else { - raw, err := json.Marshal(prefixes) + return err + } + if len(next.Prefixes) > 0 { + _, err = tx.CopyFrom( + ctx, + pgx.Identifier{"module_prefix_snapshot_row"}, + []string{"tenant_id", "module_id", "ord", "prefix", "community_id", "source"}, + pgx.CopyFromSlice(len(next.Prefixes), func(i int) ([]any, error) { + pr := next.Prefixes[i] + var comm any + if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" { + comm = strings.TrimSpace(*pr.CommunityID) + } + src := pr.Source + if strings.TrimSpace(src) == "" { + src = "render" + } + return []any{tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src}, nil + }), + ) if err != nil { return err } - if _, err := tx.Exec(ctx, ` - UPDATE module_prefix_snapshot SET prefixes_json = $3::jsonb - WHERE tenant_id = $1::uuid AND module_id = $2::uuid`, - tenantID, moduleID, string(raw)); err != nil { - return err - } } - return tx.Commit(ctx) + return nil } -func (p *Postgres) DeleteModulePrefixSnapshot(tenantID, moduleID string) error { - ctx := context.Background() - _, err := p.pool.Exec(ctx, ` - DELETE FROM module_prefix_snapshot WHERE tenant_id = $1 AND module_id = $2`, - tenantID, moduleID) - return err +func (p *Postgres) DeleteModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) error { + return p.UpdateModulePrefixSnapshot(ctx, tenantID, moduleID, func(*store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) { + return nil, nil + }) } -func (p *Postgres) SetModuleInputHash(tenantID, moduleID, hash string) error { - ctx := context.Background() - tag, err := p.pool.Exec(ctx, ` +func (p *Postgres) SetModuleInputHash(ctx context.Context, tenantID, moduleID, hash string) error { + ctx, cancel, timeoutInstalled := store.SnapshotWriteBudget(ctx) + defer cancel() + tx, err := p.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("repository: begin module hash update: %w", err) + } + defer rollbackSnapshotTx(tx) + if err := setSnapshotStatementTimeout(ctx, tx, timeoutInstalled); err != nil { + return err + } + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1::bigint)`, moduleSnapshotAdvisoryKey(tenantID, moduleID)); err != nil { + return fmt.Errorf("repository: lock module hash update: %w", err) + } + tag, err := tx.Exec(ctx, ` UPDATE module SET input_hash = $3 WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`, moduleID, tenantID, hash) @@ -171,22 +257,29 @@ func (p *Postgres) SetModuleInputHash(tenantID, moduleID, hash string) error { if tag.RowsAffected() == 0 { return store.ErrNotFound } + return tx.Commit(ctx) +} + +// setSnapshotStatementTimeout bounds a lock when the caller did not set a deadline. +func setSnapshotStatementTimeout(ctx context.Context, tx pgx.Tx, installed bool) error { + if !installed { + return nil + } + ms := store.SnapshotStatementTimeout(ctx).Milliseconds() + if ms < 1 { + ms = 1 + } + if _, err := tx.Exec(ctx, `SELECT set_config('statement_timeout', $1, true)`, strconv.FormatInt(ms, 10)); err != nil { + return fmt.Errorf("repository: set snapshot statement timeout: %w", err) + } return nil } -func (p *Postgres) LockModuleSnapshot(tenantID, moduleID string) func() { - ctx := context.Background() - conn, err := p.pool.Acquire(ctx) - if err != nil { - return func() {} - } - key := strings.TrimSpace(tenantID) + "\x00" + strings.TrimSpace(moduleID) - if _, err := conn.Exec(ctx, `SELECT pg_advisory_lock(hashtext($1))`, key); err != nil { - conn.Release() - return func() {} - } - return func() { - _, _ = conn.Exec(ctx, `SELECT pg_advisory_unlock(hashtext($1))`, key) - conn.Release() - } +func moduleSnapshotAdvisoryKey(tenantID, moduleID string) int64 { + h := fnv.New64a() + _, _ = h.Write([]byte("evobgp-module-snapshot-v1\x00")) + _, _ = h.Write([]byte(strings.TrimSpace(tenantID))) + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(strings.TrimSpace(moduleID))) + return int64(h.Sum64()) } diff --git a/internal/repository/module_snapshot_atomic_integration_test.go b/internal/repository/module_snapshot_atomic_integration_test.go new file mode 100644 index 0000000..6a0ad86 --- /dev/null +++ b/internal/repository/module_snapshot_atomic_integration_test.go @@ -0,0 +1,191 @@ +package repository + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "evobgp/internal/store" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +func snapshotIntegrationModule(t *testing.T, p *Postgres, tenant string) string { + t.Helper() + id := uuid.NewString() + if _, err := p.pool.Exec(context.Background(), `INSERT INTO module(id,tenant_id,type,name) VALUES($1,$2,'CDN_CIDRS',$3)`, id, tenant, id); err != nil { + t.Fatal(err) + } + return id +} + +func TestPostgresModuleSnapshotAtomicRMWMaxPoolOneIntegration(t *testing.T) { + p, tenant := queueIntegration(t) + module := snapshotIntegrationModule(t, p, tenant) + cfg := p.pool.Config() + cfg.MaxConns = 1 + smallPool, err := pgxpool.NewWithConfig(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + defer smallPool.Close() + small := &Postgres{pool: smallPool} + var wg sync.WaitGroup + errs := make(chan error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + errs <- small.UpdateModulePrefixSnapshot(context.Background(), tenant, module, func(cur *store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) { + var rows []store.PrefixRow + if cur != nil { + rows = append(rows, cur.Prefixes...) + } + rows = append(rows, store.PrefixRow{Prefix: fmt.Sprintf("192.0.%d.0/24", i), Source: fmt.Sprintf("cdn:%d", i)}) + return &store.ModulePrefixSnapshot{InputHash: "atomic", Prefixes: rows}, nil + }) + }(i) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + snap, ok, err := small.GetModulePrefixSnapshot(context.Background(), tenant, module) + if err != nil || !ok || len(snap.Prefixes) != 2 { + t.Fatalf("lost concurrent update: %+v %v %v", snap, ok, err) + } +} + +func TestPostgresModuleSnapshotAdvisoryWaitCancellationIntegration(t *testing.T) { + p, tenant := queueIntegration(t) + module := snapshotIntegrationModule(t, p, tenant) + other := snapshotIntegrationModule(t, p, tenant) + cfg := p.pool.Config() + cfg.MaxConns = 3 + testPool, err := pgxpool.NewWithConfig(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + defer testPool.Close() + locked := &Postgres{pool: testPool} + started := make(chan struct{}) + release := make(chan struct{}) + firstDone := make(chan error, 1) + go func() { + firstDone <- locked.UpdateModulePrefixSnapshot(context.Background(), tenant, module, func(*store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) { + close(started) + <-release + return &store.ModulePrefixSnapshot{InputHash: "first"}, nil + }) + }() + select { + case <-started: + case <-time.After(5 * time.Second): + close(release) + t.Fatal("first transaction did not acquire lock") + } + defer func() { + select { + case <-release: + default: + close(release) + } + }() + waitCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + err = locked.UpdateModulePrefixSnapshot(waitCtx, tenant, module, func(*store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) { + return &store.ModulePrefixSnapshot{InputHash: "second"}, nil + }) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("same-key wait did not cancel: %v", err) + } + otherCtx, otherCancel := context.WithTimeout(context.Background(), 3*time.Second) + defer otherCancel() + if err := locked.SetModulePrefixSnapshot(otherCtx, tenant, other, "other", nil); err != nil { + t.Fatalf("unrelated module blocked: %v", err) + } + close(release) + if err := <-firstDone; err != nil { + t.Fatal(err) + } +} + +func TestPostgresModuleSnapshotPartialWriteRollbackIntegration(t *testing.T) { + p, tenant := queueIntegration(t) + module := snapshotIntegrationModule(t, p, tenant) + ctx := context.Background() + if err := p.SetModulePrefixSnapshot(ctx, tenant, module, "before", []store.PrefixRow{{Prefix: "192.0.2.0/24", Source: "cdn:before"}}); err != nil { + t.Fatal(err) + } + err := p.UpdateModulePrefixSnapshot(ctx, tenant, module, func(cur *store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) { + if cur == nil || len(cur.Prefixes) != 1 { + return nil, errors.New("missing prior snapshot") + } + invalidCommunity := "not-a-uuid" + return &store.ModulePrefixSnapshot{InputHash: "after", Prefixes: []store.PrefixRow{{Prefix: "198.51.100.0/24", CommunityID: &invalidCommunity, Source: "cdn:after"}}}, nil + }) + if err == nil { + t.Fatal("invalid copy unexpectedly committed") + } + snap, ok, err := p.GetModulePrefixSnapshot(ctx, tenant, module) + if err != nil || !ok || snap.InputHash != "before" || len(snap.Prefixes) != 1 || snap.Prefixes[0].Prefix != "192.0.2.0/24" { + t.Fatalf("partial write visible: %+v %v %v", snap, ok, err) + } + var hash string + if err := p.pool.QueryRow(ctx, `SELECT input_hash FROM module WHERE id=$1`, module).Scan(&hash); err != nil || hash != "before" { + t.Fatalf("module input hash changed after rollback: %q %v", hash, err) + } +} + +func TestPostgresModuleSnapshotKilledBackendRollsBackIntegration(t *testing.T) { + p, tenant := queueIntegration(t) + module := snapshotIntegrationModule(t, p, tenant) + ctx := context.Background() + if err := p.SetModulePrefixSnapshot(ctx, tenant, module, "before", []store.PrefixRow{{Prefix: "192.0.2.0/24"}}); err != nil { + t.Fatal(err) + } + cfg := p.pool.Config() + cfg.MaxConns = 1 + cfg.ConnConfig.RuntimeParams["application_name"] = "snapshot-kill-" + uuid.NewString() + smallPool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer smallPool.Close() + small := &Postgres{pool: smallPool} + killedBackend := false + err = small.UpdateModulePrefixSnapshot(ctx, tenant, module, func(*store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) { + var pid int + if err := p.pool.QueryRow(ctx, `SELECT pid FROM pg_stat_activity WHERE application_name=$1 AND state='idle in transaction' ORDER BY pid LIMIT 1`, cfg.ConnConfig.RuntimeParams["application_name"]).Scan(&pid); err != nil { + return nil, err + } + var killed bool + if err := p.pool.QueryRow(ctx, `SELECT pg_terminate_backend($1)`, pid).Scan(&killed); err != nil || !killed { + return nil, fmt.Errorf("terminate snapshot backend: killed=%v err=%w", killed, err) + } + killedBackend = true + return &store.ModulePrefixSnapshot{InputHash: "after", Prefixes: []store.PrefixRow{{Prefix: "198.51.100.0/24"}}}, nil + }) + if !killedBackend { + t.Fatalf("backend kill was not exercised: %v", err) + } + if err == nil { + t.Fatal("killed backend committed snapshot") + } + verifyCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + snap, ok, err := p.GetModulePrefixSnapshot(verifyCtx, tenant, module) + if err != nil || !ok || snap.InputHash != "before" || snap.Prefixes[0].Prefix != "192.0.2.0/24" { + t.Fatalf("snapshot changed after backend loss: %+v %v %v", snap, ok, err) + } + var hash string + if err := p.pool.QueryRow(ctx, `SELECT input_hash FROM module WHERE id=$1`, module).Scan(&hash); err != nil || hash != "before" { + t.Fatalf("module input hash changed after backend loss: %q %v", hash, err) + } +} diff --git a/internal/repository/module_snapshot_context_integration_test.go b/internal/repository/module_snapshot_context_integration_test.go new file mode 100644 index 0000000..0672d04 --- /dev/null +++ b/internal/repository/module_snapshot_context_integration_test.go @@ -0,0 +1,44 @@ +package repository + +import ( + "context" + "errors" + "testing" + + "evobgp/internal/store" + "github.com/google/uuid" +) + +func TestPostgresModuleSnapshotMethodsRespectCancellationIntegration(t *testing.T) { + p, tenant := queueIntegration(t) + module := uuid.NewString() + if _, err := p.pool.Exec(context.Background(), `INSERT INTO module(id,tenant_id,type,name) VALUES($1,$2,'CDN_CIDRS','cancel-snapshot')`, module, tenant); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, _, err := p.GetModulePrefixSnapshot(ctx, tenant, module); !errors.Is(err, context.Canceled) { + t.Fatalf("get: %v", err) + } + if err := p.SetModulePrefixSnapshot(ctx, tenant, module, "hash", []store.PrefixRow{{Prefix: "192.0.2.0/24"}}); !errors.Is(err, context.Canceled) { + t.Fatalf("set: %v", err) + } + if err := p.DeleteModulePrefixSnapshot(ctx, tenant, module); !errors.Is(err, context.Canceled) { + t.Fatalf("delete: %v", err) + } + if err := p.SetModuleInputHash(ctx, tenant, module, "hash"); !errors.Is(err, context.Canceled) { + t.Fatalf("input hash: %v", err) + } + rev := uuid.NewString() + err := p.CreateRenderRevision(ctx, rev, tenant, module, nil, "cancel-hash", map[string]string{"bird.conf": "router id 192.0.2.1;\n"}, []store.PrefixRow{{Prefix: "203.0.113.0/24", Source: "test"}}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("revision: %v", err) + } + var n int + if err := p.pool.QueryRow(context.Background(), `SELECT count(*)::int FROM config_revision WHERE id=$1::uuid`, rev).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatal("cancelled revision write committed") + } +} diff --git a/internal/repository/module_snapshot_rows_test.go b/internal/repository/module_snapshot_rows_test.go new file mode 100644 index 0000000..2a936af --- /dev/null +++ b/internal/repository/module_snapshot_rows_test.go @@ -0,0 +1,50 @@ +package repository + +import ( + "errors" + "testing" +) + +type partialSnapshotRows struct { + returned bool + scanErr error + rowsErr error +} + +func (r *partialSnapshotRows) Next() bool { + if r.returned { + return false + } + r.returned = true + return true +} + +func (r *partialSnapshotRows) Scan(dest ...any) error { + if r.scanErr != nil { + return r.scanErr + } + *dest[0].(*string) = "192.0.2.0/24" + *dest[1].(**string) = nil + *dest[2].(*string) = "cdn:test" + return nil +} + +func (r *partialSnapshotRows) Err() error { return r.rowsErr } + +func TestReadModuleSnapshotRowsRejectsPartialResults(t *testing.T) { + sentinel := errors.New("simulated read failure") + for _, tc := range []struct { + name string + rows *partialSnapshotRows + }{ + {"scan", &partialSnapshotRows{scanErr: sentinel}}, + {"iteration", &partialSnapshotRows{rowsErr: sentinel}}, + } { + t.Run(tc.name, func(t *testing.T) { + prefixes, err := readModuleSnapshotRows(tc.rows) + if prefixes != nil || !errors.Is(err, sentinel) { + t.Fatalf("partial snapshot returned: %+v %v", prefixes, err) + } + }) + } +} diff --git a/internal/repository/postgres.go b/internal/repository/postgres.go index f015b0d..06d6b8a 100644 --- a/internal/repository/postgres.go +++ b/internal/repository/postgres.go @@ -115,6 +115,41 @@ func (p *Postgres) PeerSessionCountsByState() map[string]int { return out } +// ObserveStoreMetrics reads persisted revision and peer inventory aggregates for a Prometheus scrape. +// An error invalidates the entire sample so a database outage cannot appear as an empty store. +func (p *Postgres) ObserveStoreMetrics(ctx context.Context) (max, sum, peers int, states map[string]int, err error) { + err = p.pool.QueryRow(ctx, ` + SELECT + COALESCE(MAX((meta_json->>'materialized_prefix_count')::int), 0), + COALESCE(SUM((meta_json->>'materialized_prefix_count')::int), 0) + FROM config_revision`).Scan(&max, &sum) + if err != nil { + return 0, 0, 0, nil, fmt.Errorf("repository: collect stored revision metrics: %w", err) + } + err = p.pool.QueryRow(ctx, `SELECT COUNT(*) FROM bgp_peer`).Scan(&peers) + if err != nil { + return 0, 0, 0, nil, fmt.Errorf("repository: collect peer inventory metrics: %w", err) + } + rows, err := p.pool.Query(ctx, `SELECT COALESCE(meta_json->>'session_state','unknown'), COUNT(*) FROM bgp_peer GROUP BY 1`) + if err != nil { + return 0, 0, 0, nil, fmt.Errorf("repository: collect stored peer states: %w", err) + } + defer rows.Close() + states = make(map[string]int) + for rows.Next() { + var state string + var count int + if err = rows.Scan(&state, &count); err != nil { + return 0, 0, 0, nil, fmt.Errorf("repository: scan stored peer states: %w", err) + } + states[state] = count + } + if err = rows.Err(); err != nil { + return 0, 0, 0, nil, fmt.Errorf("repository: read stored peer states: %w", err) + } + return max, sum, peers, states, nil +} + func (p *Postgres) ListModules(tenantID string) []*store.Module { ctx := context.Background() rows, err := p.pool.Query(ctx, ` @@ -333,7 +368,9 @@ func (p *Postgres) CreateModule(tenantID string, in *store.Module) (*store.Modul } func (p *Postgres) invalidateModuleInputHash(tenantID, moduleID string) { - _ = p.SetModuleInputHash(tenantID, moduleID, "") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = p.SetModuleInputHash(ctx, tenantID, moduleID, "") } func (p *Postgres) invalidateTenantModuleHashes(tenantID string) { @@ -1059,7 +1096,7 @@ func (p *Postgres) LatestPublishedRevision(speakerID string) (string, time.Time, func (p *Postgres) ListTenantIDs() ([]string, error) { ctx := context.Background() - rows, err := p.pool.Query(ctx, `SELECT id::text FROM tenant ORDER BY id`) + rows, err := p.pool.Query(ctx, `SELECT id::text FROM tenant WHERE id <> $1::uuid ORDER BY id`, store.SystemJobTenantID) if err != nil { return nil, err } @@ -1075,14 +1112,24 @@ func (p *Postgres) ListTenantIDs() ([]string, error) { return out, nil } -func (p *Postgres) CreateRenderRevision(revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []store.PrefixRow) error { +func (p *Postgres) CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []store.PrefixRow) error { + if ctx == nil { + ctx = context.Background() + } + ctx, cancel, _ := store.SnapshotWriteBudget(ctx) + defer cancel() + if err := ctx.Err(); err != nil { + return err + } if strings.TrimSpace(revisionID) == "" { return store.ErrInvalidInput } if _, err := p.GetModule(tenantID, moduleID); err != nil { return err } - ctx := context.Background() + if err := ctx.Err(); err != nil { + return err + } if previewFragments == nil { previewFragments = map[string]string{} } @@ -1092,9 +1139,12 @@ func (p *Postgres) CreateRenderRevision(revisionID, tenantID, moduleID string, p } tx, err := p.pool.Begin(ctx) if err != nil { + return fmt.Errorf("repository: begin render revision: %w", err) + } + defer rollbackWithTimeout(tx) + if err := setRevisionStatementTimeout(ctx, tx); err != nil { return err } - defer func() { _ = tx.Rollback(ctx) }() var parent any if parentRevisionID != nil && strings.TrimSpace(*parentRevisionID) != "" { @@ -1102,8 +1152,8 @@ func (p *Postgres) CreateRenderRevision(revisionID, tenantID, moduleID string, p } revID := strings.TrimSpace(revisionID) _, err = tx.Exec(ctx, ` - INSERT INTO config_revision (id, tenant_id, module_id, content_hash, parent_revision_id, meta_json) - VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5::uuid, $6::jsonb)`, + INSERT INTO config_revision (id, tenant_id, module_id, content_hash, parent_revision_id, meta_json) + VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5::uuid, $6::jsonb)`, revID, tenantID, moduleID, strings.TrimSpace(contentHash), parent, meta) if err != nil { return err @@ -1111,16 +1161,32 @@ func (p *Postgres) CreateRenderRevision(revisionID, tenantID, moduleID string, p if err := insertRevisionPreview(ctx, tx, revID, previewFragments); err != nil { return err } - snapID, err := p.ensurePrefixSnapshot(ctx, tx, contentHash, prefixes) + snapID, err := p.ensurePrefixSnapshot(ctx, tx, prefixes) if err != nil { return err } if err := p.linkRevisionPrefixSnapshot(ctx, tx, revID, snapID); err != nil { return err } - if err := tx.Commit(ctx); err != nil { + if err := insertRenderManifestTx(ctx, tx, revID, tenantID); err != nil { return err } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("repository: commit render revision: %w", err) + } + return nil +} + +// setRevisionStatementTimeout bounds the revision transaction the same way snapshot writes do: +// the caller deadline wins, otherwise EVOBGP_SNAPSHOT_TIMEOUT (default 2 minutes). +func setRevisionStatementTimeout(ctx context.Context, tx pgx.Tx) error { + ms := store.SnapshotStatementTimeout(ctx).Milliseconds() + if ms < 1 { + ms = 1 + } + if _, err := tx.Exec(ctx, `SELECT set_config('statement_timeout', $1, true)`, strconv.FormatInt(ms, 10)); err != nil { + return fmt.Errorf("repository: set revision statement timeout: %w", err) + } return nil } diff --git a/internal/repository/postgres_metrics_test.go b/internal/repository/postgres_metrics_test.go new file mode 100644 index 0000000..11bbb37 --- /dev/null +++ b/internal/repository/postgres_metrics_test.go @@ -0,0 +1,21 @@ +package repository + +import ( + "context" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestObserveStoreMetricsReturnsDatabaseError(t *testing.T) { + pool, err := pgxpool.New(context.Background(), "postgres://localhost/evobgp") + if err != nil { + t.Fatal(err) + } + pool.Close() + pg := &Postgres{pool: pool} + _, _, _, _, err = pg.ObserveStoreMetrics(context.Background()) + if err == nil { + t.Fatal("expected closed pool error") + } +} diff --git a/internal/repository/postgres_prefix_snapshot.go b/internal/repository/postgres_prefix_snapshot.go index f9a6ac8..9d511fc 100644 --- a/internal/repository/postgres_prefix_snapshot.go +++ b/internal/repository/postgres_prefix_snapshot.go @@ -3,6 +3,7 @@ package repository import ( "context" "errors" + "fmt" "strings" "evobgp/internal/store" @@ -48,35 +49,39 @@ func (p *Postgres) revisionPrefixSnapshotID(ctx context.Context, revisionID stri return *snap, true } -func (p *Postgres) ensurePrefixSnapshot(ctx context.Context, db execQuerier, contentHash string, prefixes []store.PrefixRow) (string, error) { +func (p *Postgres) ensurePrefixSnapshot(ctx context.Context, db execQuerier, prefixes []store.PrefixRow) (string, error) { if !prefixSnapshotTableExists(ctx, db) { return "", nil } - hash := normalizeSnapshotHash(contentHash) + hash := store.PrefixSnapshotHash(prefixes) snapID, err := p.resolvePrefixSnapshotID(ctx, db, hash) if err != nil { return "", err } var rowCount int _ = db.QueryRow(ctx, `SELECT COUNT(*)::int FROM prefix_snapshot_row WHERE snapshot_id = $1::uuid`, snapID).Scan(&rowCount) - if rowCount > 0 { + if rowCount > 0 || len(prefixes) == 0 { return snapID, nil } - for i, pr := range prefixes { - var comm any - if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" { - comm = strings.TrimSpace(*pr.CommunityID) - } - src := pr.Source - if strings.TrimSpace(src) == "" { - src = "render" - } - if _, err := db.Exec(ctx, ` - INSERT INTO prefix_snapshot_row (snapshot_id, ord, prefix, community_id, source) - VALUES ($1::uuid, $2, $3, $4::uuid, $5)`, - snapID, i, strings.TrimSpace(pr.Prefix), comm, src); err != nil { - return "", err - } + _, err = db.CopyFrom( + ctx, + pgx.Identifier{"prefix_snapshot_row"}, + []string{"snapshot_id", "ord", "prefix", "community_id", "source"}, + pgx.CopyFromSlice(len(prefixes), func(i int) ([]any, error) { + pr := prefixes[i] + var comm any + if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" { + comm = strings.TrimSpace(*pr.CommunityID) + } + src := pr.Source + if strings.TrimSpace(src) == "" { + src = "render" + } + return []any{snapID, i, strings.TrimSpace(pr.Prefix), comm, src}, nil + }), + ) + if err != nil { + return "", fmt.Errorf("repository: copy prefix snapshot rows: %w", err) } return snapID, nil } diff --git a/internal/repository/postgres_prefix_snapshot_test.go b/internal/repository/postgres_prefix_snapshot_test.go index 503016a..27737e9 100644 --- a/internal/repository/postgres_prefix_snapshot_test.go +++ b/internal/repository/postgres_prefix_snapshot_test.go @@ -27,7 +27,8 @@ func TestEnsurePrefixSnapshotFillsEmptyExistingIntegration(t *testing.T) { t.Fatal(err) } - hash := normalizeSnapshotHash("sha256:empty-fill-" + uuid.NewString()) + prefixes := []store.PrefixRow{{Prefix: "203.0.113.1/32", Source: "test"}} + hash := store.PrefixSnapshotHash(prefixes) snapID := uuid.NewString() if _, err := pool.Exec(ctx, `INSERT INTO prefix_snapshot (id, content_hash) VALUES ($1::uuid, $2)`, snapID, hash); err != nil { t.Fatal(err) @@ -42,9 +43,7 @@ func TestEnsurePrefixSnapshotFillsEmptyExistingIntegration(t *testing.T) { } defer func() { _ = tx.Rollback(ctx) }() - got, err := pg.ensurePrefixSnapshot(ctx, tx, "sha256:"+hash, []store.PrefixRow{ - {Prefix: "203.0.113.1/32", Source: "test"}, - }) + got, err := pg.ensurePrefixSnapshot(ctx, tx, prefixes) if err != nil { t.Fatal(err) } @@ -79,14 +78,13 @@ func TestEnsurePrefixSnapshotIdempotentIntegration(t *testing.T) { t.Fatal(err) } - contentHash := "sha256:idempotent-" + uuid.NewString() prefixes := []store.PrefixRow{{Prefix: "198.51.100.0/24", Source: "test"}} tx1, err := pool.Begin(ctx) if err != nil { t.Fatal(err) } - id1, err := pg.ensurePrefixSnapshot(ctx, tx1, contentHash, prefixes) + id1, err := pg.ensurePrefixSnapshot(ctx, tx1, prefixes) if err != nil { t.Fatal(err) } @@ -102,7 +100,7 @@ func TestEnsurePrefixSnapshotIdempotentIntegration(t *testing.T) { t.Fatal(err) } defer func() { _ = tx2.Rollback(ctx) }() - id2, err := pg.ensurePrefixSnapshot(ctx, tx2, contentHash, prefixes) + id2, err := pg.ensurePrefixSnapshot(ctx, tx2, prefixes) if err != nil { t.Fatal(err) } diff --git a/internal/repository/postgres_revision_preview.go b/internal/repository/postgres_revision_preview.go index 5b15de6..edde1c8 100644 --- a/internal/repository/postgres_revision_preview.go +++ b/internal/repository/postgres_revision_preview.go @@ -49,6 +49,7 @@ func loadRevisionPreview(ctx context.Context, q queryRower, revisionID string, m type execQuerier interface { queryRower Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) + CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error) } func insertRevisionPreview(ctx context.Context, db execQuerier, revisionID string, preview map[string]string) error { diff --git a/internal/repository/postgres_revision_prune.go b/internal/repository/postgres_revision_prune.go index 2b7aeaf..b4da3dc 100644 --- a/internal/repository/postgres_revision_prune.go +++ b/internal/repository/postgres_revision_prune.go @@ -27,7 +27,13 @@ func sqlPrunableRevisionsWhere(tenantParam, cutoffParam string) string { } func (p *Postgres) EstimateRevisionPrune(tenantID string, cutoff time.Time, retentionMinutes int) (store.RevisionPruneEstimate, error) { - ctx := context.Background() + return p.estimateRevisionPrune(context.Background(), tenantID, cutoff, retentionMinutes) +} + +func (p *Postgres) estimateRevisionPrune(ctx context.Context, tenantID string, cutoff time.Time, retentionMinutes int) (store.RevisionPruneEstimate, error) { + if ctx == nil { + ctx = context.Background() + } out := store.RevisionPruneEstimate{ RetentionMinutes: retentionMinutes, CutoffAt: cutoff.UTC(), @@ -88,8 +94,14 @@ func (p *Postgres) EstimateRevisionPrune(tenantID string, cutoff time.Time, rete } func (p *Postgres) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (store.RevisionPruneResult, error) { - ctx := context.Background() - est, err := p.EstimateRevisionPrune(tenantID, cutoff, 0) + return p.pruneRevisionsWithStats(context.Background(), tenantID, cutoff) +} + +func (p *Postgres) pruneRevisionsWithStats(ctx context.Context, tenantID string, cutoff time.Time) (store.RevisionPruneResult, error) { + if ctx == nil { + ctx = context.Background() + } + est, err := p.estimateRevisionPrune(ctx, tenantID, cutoff, 0) if err != nil { return store.RevisionPruneResult{}, err } @@ -97,6 +109,9 @@ func (p *Postgres) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (s where := sqlPrunableRevisionsWhere("$1", "$2") for { + if err := ctx.Err(); err != nil { + return out, err + } cmd, err := p.pool.Exec(ctx, ` DELETE FROM config_revision AS cr WHERE cr.id IN ( @@ -116,6 +131,9 @@ func (p *Postgres) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (s } for { + if err := ctx.Err(); err != nil { + return out, err + } snaps, rows, err := p.pruneUnreferencedPrefixSnapshots(ctx, revisionPruneBatchSize) if err != nil { return out, err @@ -129,8 +147,8 @@ func (p *Postgres) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (s return out, nil } -func (p *Postgres) PruneRevisionsBefore(tenantID string, cutoff time.Time) (int, error) { - res, err := p.PruneRevisionsWithStats(tenantID, cutoff) +func (p *Postgres) PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (int, error) { + res, err := p.pruneRevisionsWithStats(ctx, tenantID, cutoff) if err != nil { return 0, err } diff --git a/internal/repository/postgres_seed.go b/internal/repository/postgres_seed.go index 125c9de..1995b82 100644 --- a/internal/repository/postgres_seed.go +++ b/internal/repository/postgres_seed.go @@ -3,13 +3,14 @@ package repository import ( "context" "encoding/json" + "evobgp/internal/store" "github.com/google/uuid" ) func (p *Postgres) seedDemo(ctx context.Context) error { var n int - if err := p.pool.QueryRow(ctx, `SELECT COUNT(*)::int FROM tenant`).Scan(&n); err != nil { + if err := p.pool.QueryRow(ctx, `SELECT COUNT(*)::int FROM tenant WHERE id<>$1::uuid`, store.SystemJobTenantID).Scan(&n); err != nil { return err } if n > 0 { diff --git a/internal/repository/render_manifest.go b/internal/repository/render_manifest.go new file mode 100644 index 0000000..50338e3 --- /dev/null +++ b/internal/repository/render_manifest.go @@ -0,0 +1,127 @@ +package repository + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "evobgp/internal/store" + + "github.com/jackc/pgx/v5" +) + +func (p *Postgres) GetRenderGeneration(ctx context.Context, tenantID string) (int64, error) { + var generation int64 + err := p.pool.QueryRow(ctx, `SELECT generation FROM tenant_render_generation WHERE tenant_id=$1::uuid`, tenantID).Scan(&generation) + if errors.Is(err, pgx.ErrNoRows) { + return 0, store.ErrNotFound + } + if err != nil { + return 0, fmt.Errorf("repository: render generation: %w", err) + } + return generation, nil +} + +func (p *Postgres) ListModuleSnapshotHeaders(ctx context.Context, tenantID string) ([]store.ModuleSnapshotUse, error) { + rows, err := p.pool.Query(ctx, ` + SELECT module_id::text, snapshot_version, COALESCE(input_hash, '') + FROM module_prefix_snapshot + WHERE tenant_id = $1::uuid + ORDER BY module_id`, tenantID) + if err != nil { + return nil, fmt.Errorf("repository: list snapshot headers: %w", err) + } + defer rows.Close() + var out []store.ModuleSnapshotUse + for rows.Next() { + var item store.ModuleSnapshotUse + if err := rows.Scan(&item.ModuleID, &item.SnapshotVersion, &item.InputHash); err != nil { + return nil, fmt.Errorf("repository: scan snapshot header: %w", err) + } + out = append(out, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("repository: iterate snapshot headers: %w", err) + } + return out, nil +} + +func (p *Postgres) LatestRenderManifest(ctx context.Context, tenantID string) (*store.RenderManifestRecord, bool, error) { + var revID string + var gen *int64 + var raw []byte + err := p.pool.QueryRow(ctx, ` + SELECT cr.id::text, m.config_generation, m.manifest_json + FROM config_revision cr + LEFT JOIN render_revision_manifest m ON m.revision_id = cr.id + WHERE cr.tenant_id = $1::uuid + ORDER BY cr.created_at DESC, cr.id DESC + LIMIT 1`, tenantID).Scan(&revID, &gen, &raw) + if errors.Is(err, pgx.ErrNoRows) { + return nil, false, nil + } + if err != nil { + return nil, false, fmt.Errorf("repository: latest render manifest: %w", err) + } + if gen == nil || len(raw) == 0 { + return nil, false, nil + } + var doc struct { + Snapshots []store.ModuleSnapshotUse `json:"snapshots"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, false, fmt.Errorf("repository: decode render manifest: %w", err) + } + return &store.RenderManifestRecord{ + RevisionID: revID, + Generation: *gen, + Snapshots: doc.Snapshots, + }, true, nil +} + +func insertRenderManifestTx(ctx context.Context, tx pgx.Tx, revisionID, tenantID string) error { + var gen int64 + err := tx.QueryRow(ctx, ` + SELECT generation FROM tenant_render_generation + WHERE tenant_id = $1::uuid + FOR UPDATE`, tenantID).Scan(&gen) + if errors.Is(err, pgx.ErrNoRows) { + gen = 1 + } else if err != nil { + return fmt.Errorf("repository: lock render generation: %w", err) + } + rows, err := tx.Query(ctx, ` + SELECT module_id::text, snapshot_version, COALESCE(input_hash, '') + FROM module_prefix_snapshot + WHERE tenant_id = $1::uuid + ORDER BY module_id`, tenantID) + if err != nil { + return fmt.Errorf("repository: read snapshot versions: %w", err) + } + defer rows.Close() + var snaps []store.ModuleSnapshotUse + for rows.Next() { + var item store.ModuleSnapshotUse + if err := rows.Scan(&item.ModuleID, &item.SnapshotVersion, &item.InputHash); err != nil { + return fmt.Errorf("repository: scan snapshot version: %w", err) + } + snaps = append(snaps, item) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("repository: iterate snapshot versions: %w", err) + } + body, err := json.Marshal(struct { + Snapshots []store.ModuleSnapshotUse `json:"snapshots"` + }{Snapshots: snaps}) + if err != nil { + return fmt.Errorf("repository: encode render manifest: %w", err) + } + if _, err := tx.Exec(ctx, ` + INSERT INTO render_revision_manifest (revision_id, tenant_id, config_generation, manifest_json) + VALUES ($1::uuid, $2::uuid, $3, $4::jsonb)`, + revisionID, tenantID, gen, body); err != nil { + return fmt.Errorf("repository: insert render manifest: %w", err) + } + return nil +} diff --git a/internal/repository/telemetry.go b/internal/repository/telemetry.go new file mode 100644 index 0000000..eaa7eea --- /dev/null +++ b/internal/repository/telemetry.go @@ -0,0 +1,324 @@ +package repository + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "evobgp/internal/store" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +const telemetryColumns = `tenant_id::text,target_id,COALESCE(speaker_id::text,''),kind,local_authority,config_fingerprint,bgp_json,health_json,next_poll_at,failure_count,discovery_cursor` +const telemetryClaimColumns = `t.tenant_id::text,t.target_id,COALESCE(t.speaker_id::text,''),t.kind,t.local_authority,t.config_fingerprint,t.bgp_json,t.health_json,t.next_poll_at,t.failure_count,t.discovery_cursor,t.fencing_token` + +func scanTelemetry(row pgx.Row) (store.TelemetrySnapshot, error) { + var s store.TelemetrySnapshot + var bgp, health []byte + err := row.Scan(&s.TenantID, &s.TargetID, &s.SpeakerID, &s.Kind, &s.LocalAuthority, &s.ConfigFingerprint, &bgp, &health, &s.NextPollAt, &s.FailureCount, &s.DiscoveryCursor) + if err != nil { + return s, err + } + if err := json.Unmarshal(bgp, &s.BGP); err != nil { + return s, fmt.Errorf("repository: decode telemetry BGP: %w", err) + } + if err := json.Unmarshal(health, &s.Health); err != nil { + return s, fmt.Errorf("repository: decode telemetry health: %w", err) + } + return s, nil +} + +// ClaimTelemetry inserts at most one bounded batch of new targets and leases due rows. +func (p *Postgres) ClaimTelemetry(ctx context.Context, owner string, limit int, lease time.Duration, localAuthority string) ([]store.TelemetryClaim, error) { + if owner == "" || limit <= 0 { + return nil, store.ErrInvalidInput + } + if limit > 64 { + limit = 64 + } + if lease <= 0 { + lease = 15 * time.Second + } + _, err := p.pool.Exec(ctx, `INSERT INTO telemetry_snapshot(tenant_id,target_id,speaker_id,kind) + SELECT s.tenant_id,s.id::text,s.id,'remote_agent' FROM bgp_speaker s + WHERE s.role<>'master' AND COALESCE(s.meta_json->>'agent_domain','')<>'' AND COALESCE(s.meta_json->>'agent_secret','')<>'' + AND NOT EXISTS (SELECT 1 FROM telemetry_snapshot t WHERE t.tenant_id=s.tenant_id AND t.target_id=s.id::text) + ORDER BY s.id LIMIT $1 ON CONFLICT DO NOTHING`, limit) + if err != nil { + return nil, fmt.Errorf("repository: materialize telemetry targets: %w", err) + } + if localAuthority != "" { + _, err = p.pool.Exec(ctx, `INSERT INTO telemetry_snapshot(tenant_id,target_id,kind,local_authority) + SELECT t.id,'cp-local/'||t.id::text||'/'||$3,'local_bird',$3 FROM tenant t + WHERE t.id<>$2::uuid AND NOT EXISTS (SELECT 1 FROM telemetry_snapshot x WHERE x.tenant_id=t.id AND x.target_id='cp-local/'||t.id::text||'/'||$3) + ORDER BY t.id LIMIT $1 ON CONFLICT DO NOTHING`, limit, store.SystemJobTenantID, localAuthority) + if err != nil { + return nil, fmt.Errorf("repository: materialize local telemetry targets: %w", err) + } + } + rows, err := p.pool.Query(ctx, `WITH picked AS ( + SELECT tenant_id,target_id FROM telemetry_snapshot WHERE next_poll_at<=clock_timestamp() + AND (lease_until IS NULL OR lease_until<=clock_timestamp()) AND (kind<>'local_bird' OR local_authority=$4) + AND (kind='local_bird' OR EXISTS (SELECT 1 FROM bgp_speaker s WHERE s.id=telemetry_snapshot.speaker_id AND s.role<>'master' + AND COALESCE(s.meta_json->>'agent_domain','')<>'' AND COALESCE(s.meta_json->>'agent_secret','')<>'')) + ORDER BY next_poll_at,tenant_id,target_id FOR UPDATE SKIP LOCKED LIMIT $1 + ) UPDATE telemetry_snapshot t SET owner_id=$2,lease_until=clock_timestamp()+$3::bigint*interval '1 millisecond',fencing_token=t.fencing_token+1 + FROM picked WHERE t.tenant_id=picked.tenant_id AND t.target_id=picked.target_id + RETURNING `+telemetryClaimColumns, limit, owner, lease.Milliseconds(), localAuthority) + if err != nil { + return nil, fmt.Errorf("repository: claim telemetry: %w", err) + } + defer rows.Close() + claims := make([]store.TelemetryClaim, 0, limit) + for rows.Next() { + var c store.TelemetryClaim + var bgp, health []byte + if err = rows.Scan(&c.TenantID, &c.TargetID, &c.SpeakerID, &c.Kind, &c.LocalAuthority, &c.ConfigFingerprint, &bgp, &health, &c.NextPollAt, &c.FailureCount, &c.DiscoveryCursor, &c.FencingToken); err != nil { + return nil, fmt.Errorf("repository: scan telemetry claim: %w", err) + } + if err = json.Unmarshal(bgp, &c.BGP); err != nil { + return nil, fmt.Errorf("repository: decode telemetry BGP: %w", err) + } + if err = json.Unmarshal(health, &c.Health); err != nil { + return nil, fmt.Errorf("repository: decode telemetry health: %w", err) + } + c.OwnerID = owner + claims = append(claims, c) + } + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("repository: iterate telemetry claims: %w", err) + } + rows.Close() + for i := range claims { + c := &claims[i] + if c.Kind == "remote_agent" { + c.Speaker, err = p.getTelemetrySpeaker(ctx, c.TenantID, c.SpeakerID) + } else { + var spID string + err = p.pool.QueryRow(ctx, `SELECT id::text FROM bgp_speaker WHERE tenant_id=$1::uuid AND role='master' ORDER BY id LIMIT 1`, c.TenantID).Scan(&spID) + if err == pgx.ErrNoRows { + err = nil + } else if err == nil { + c.SpeakerID = spID + c.Speaker, err = p.getTelemetrySpeaker(ctx, c.TenantID, spID) + } + } + if err != nil { + return nil, fmt.Errorf("repository: resolve telemetry speaker: %w", err) + } + c.ConfigFingerprint = store.TelemetrySpeakerFingerprint(c.Speaker) + } + return claims, nil +} + +func (p *Postgres) getTelemetrySpeaker(ctx context.Context, tenantID, speakerID string) (*store.Speaker, error) { + var sp store.Speaker + err := p.pool.QueryRow(ctx, `SELECT id::text,tenant_id::text,role,COALESCE(endpoint,''),COALESCE(meta_json::text,'{}') FROM bgp_speaker WHERE tenant_id=$1::uuid AND id=$2::uuid`, tenantID, speakerID).Scan(&sp.ID, &sp.TenantID, &sp.Role, &sp.Endpoint, &sp.MetaJSON) + if err != nil { + return nil, err + } + return &sp, nil +} + +// CompleteTelemetry records an attempt only while the claim still owns its lease. +func (p *Postgres) CompleteTelemetry(ctx context.Context, claim store.TelemetryClaim, result store.TelemetryResult, nextPoll time.Time) error { + tx, err := p.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("repository: begin telemetry completion: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + var oldBGP, oldHealth []byte + var token int64 + var owner string + var leaseUntil time.Time + var pending bool + var failures int + var oldFingerprint string + var discoveryCursor int + err = tx.QueryRow(ctx, `SELECT bgp_json,health_json,fencing_token,COALESCE(owner_id,''),COALESCE(lease_until,'epoch'::timestamptz),refresh_pending,failure_count,config_fingerprint,discovery_cursor + FROM telemetry_snapshot WHERE tenant_id=$1::uuid AND target_id=$2 FOR UPDATE`, claim.TenantID, claim.TargetID).Scan(&oldBGP, &oldHealth, &token, &owner, &leaseUntil, &pending, &failures, &oldFingerprint, &discoveryCursor) + if err == pgx.ErrNoRows || err == nil && (token != claim.FencingToken || owner != claim.OwnerID) { + return store.ErrTelemetryLeaseLost + } + if err != nil { + return fmt.Errorf("repository: read telemetry lease: %w", err) + } + var dbNow time.Time + if err = tx.QueryRow(ctx, `SELECT clock_timestamp()`).Scan(&dbNow); err != nil { + return fmt.Errorf("repository: telemetry clock: %w", err) + } + if !leaseUntil.After(dbNow) { + return store.ErrTelemetryLeaseLost + } + if claim.Speaker != nil { + var current store.Speaker + err = tx.QueryRow(ctx, `SELECT id::text,role,COALESCE(endpoint,''),COALESCE(meta_json::text,'{}') FROM bgp_speaker WHERE id=$1::uuid AND tenant_id=$2::uuid FOR SHARE`, claim.Speaker.ID, claim.TenantID).Scan(¤t.ID, ¤t.Role, ¤t.Endpoint, ¤t.MetaJSON) + if err == pgx.ErrNoRows || err == nil && store.TelemetrySpeakerFingerprint(¤t) != claim.ConfigFingerprint { + return store.ErrTelemetryLeaseLost + } + if err != nil { + return fmt.Errorf("repository: verify telemetry config: %w", err) + } + } + if claim.Kind == "local_bird" { + var masterID string + err = tx.QueryRow(ctx, `SELECT id::text FROM bgp_speaker WHERE tenant_id=$1::uuid AND role='master' ORDER BY id LIMIT 1`, claim.TenantID).Scan(&masterID) + if err == pgx.ErrNoRows { + masterID = "" + err = nil + } + if err != nil { + return fmt.Errorf("repository: verify local telemetry master: %w", err) + } + if masterID != claim.SpeakerID { + return store.ErrTelemetryLeaseLost + } + } + var bgp store.TelemetryBGP + var health store.TelemetryHealth + if err = json.Unmarshal(oldBGP, &bgp); err != nil { + return fmt.Errorf("repository: decode previous telemetry BGP: %w", err) + } + if err = json.Unmarshal(oldHealth, &health); err != nil { + return fmt.Errorf("repository: decode previous telemetry health: %w", err) + } + if oldFingerprint != claim.ConfigFingerprint { + bgp = store.TelemetryBGP{} + health = store.TelemetryHealth{} + discoveryCursor = 0 + } + if result.BGP.Source != "" { + if result.BGP.Error != "" { + result.BGP.Sessions = nil + result.BGP.LastSuccessAt = bgp.LastSuccessAt + } else { + result.BGP.LastSuccessAt = result.BGP.ReceivedAt + } + bgp = result.BGP + } + if result.Health.Source != "" { + if result.Health.OK { + result.Health.LastSuccessAt = result.Health.ReceivedAt + } else { + result.Health.LastSuccessAt = health.LastSuccessAt + } + health = result.Health + } + if bgp.Error != "" || health.Source != "" && !health.OK { + failures++ + } else { + failures = 0 + } + if pending { + nextPoll = dbNow + } + bgpJSON, err := json.Marshal(bgp) + if err != nil { + return fmt.Errorf("repository: encode telemetry BGP: %w", err) + } + healthJSON, err := json.Marshal(health) + if err != nil { + return fmt.Errorf("repository: encode telemetry health: %w", err) + } + _, err = tx.Exec(ctx, `UPDATE telemetry_snapshot SET bgp_json=$5::jsonb,health_json=$6::jsonb,next_poll_at=$7,failure_count=$8, + config_fingerprint=$9,speaker_id=$10::uuid + WHERE tenant_id=$1::uuid AND target_id=$2 AND owner_id=$3 AND fencing_token=$4`, claim.TenantID, claim.TargetID, claim.OwnerID, claim.FencingToken, bgpJSON, healthJSON, nextPoll, failures, claim.ConfigFingerprint, nullTelemetryUUID(claim.SpeakerID)) + if err != nil { + return fmt.Errorf("repository: write telemetry: %w", err) + } + if result.BGP.Source != "" && result.BGP.Error == "" { + if discoveryCursor, err = writeTelemetryDiscoveries(ctx, tx, claim, result.BGP, discoveryCursor); err != nil { + return err + } + } + tag, err := tx.Exec(ctx, `UPDATE telemetry_snapshot SET owner_id=NULL,lease_until=NULL,refresh_pending=false,discovery_cursor=$5 WHERE tenant_id=$1::uuid AND target_id=$2 AND owner_id=$3 AND fencing_token=$4 AND lease_until>clock_timestamp()`, claim.TenantID, claim.TargetID, claim.OwnerID, claim.FencingToken, discoveryCursor) + if err != nil { + return fmt.Errorf("repository: release telemetry lease: %w", err) + } + if tag.RowsAffected() != 1 { + return store.ErrTelemetryLeaseLost + } + if err = tx.Commit(ctx); err != nil { + return fmt.Errorf("repository: commit telemetry: %w", err) + } + return nil +} + +func writeTelemetryDiscoveries(ctx context.Context, tx pgx.Tx, claim store.TelemetryClaim, bgp store.TelemetryBGP, cursor int) (int, error) { + sessions, next := store.TelemetryDiscoveryBatch(bgp.Sessions, cursor, 256) + for _, sess := range sessions { + neighbor, ok := store.NormalizePeerNeighborString(sess.Neighbor) + if !ok { + continue + } + var known bool + if err := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM bgp_peer WHERE tenant_id=$1::uuid AND neighbor=$2::inet)`, claim.TenantID, neighbor).Scan(&known); err != nil { + return 0, fmt.Errorf("repository: check discovered neighbor: %w", err) + } + if known { + continue + } + neighborID := strings.TrimSpace(sess.NeighborID) + spID := nullTelemetryUUID(claim.SpeakerID) + seenAt := bgp.ReceivedAt + _, err := tx.Exec(ctx, `INSERT INTO bgp_peer_discovery(id,tenant_id,speaker_id,neighbor_id,neighbor,remote_asn,protocol_name,session_state,first_seen_at,last_seen_at) + VALUES($1,$2::uuid,$3::uuid,$4,$5::inet,$6,$7,$8,$9,$9) ON CONFLICT DO NOTHING`, uuid.NewString(), claim.TenantID, spID, neighborID, neighbor, sess.NeighborAS, sess.Name, sess.State, seenAt) + if err != nil { + return 0, fmt.Errorf("repository: insert discovered neighbor: %w", err) + } + _, err = tx.Exec(ctx, `UPDATE bgp_peer_discovery SET protocol_name=$5,session_state=$6,last_seen_at=$7, + speaker_id=COALESCE($8::uuid,speaker_id),updated_at=clock_timestamp() + WHERE tenant_id=$1::uuid AND status<>'rejected' AND last_seen_at<=$7 + AND ((neighbor_id<>'' AND neighbor_id=$2) OR (neighbor_id='' AND neighbor=$3::inet AND remote_asn=$4))`, claim.TenantID, neighborID, neighbor, sess.NeighborAS, sess.Name, sess.State, seenAt, spID) + if err != nil { + return 0, fmt.Errorf("repository: update discovered neighbor: %w", err) + } + } + return next, nil +} + +func nullTelemetryUUID(id string) any { + if strings.TrimSpace(id) == "" { + return nil + } + return id +} + +// ListTelemetry returns committed snapshots for one tenant without touching the agents. +func (p *Postgres) ListTelemetry(ctx context.Context, tenantID string) ([]store.TelemetrySnapshot, error) { + rows, err := p.pool.Query(ctx, `SELECT `+telemetryColumns+` FROM telemetry_snapshot t WHERE tenant_id=$1::uuid AND (kind='local_bird' OR EXISTS + (SELECT 1 FROM bgp_speaker s WHERE s.id=t.speaker_id AND s.role<>'master' AND COALESCE(s.meta_json->>'agent_domain','')<>'' AND COALESCE(s.meta_json->>'agent_secret','')<>'')) ORDER BY target_id`, tenantID) + if err != nil { + return nil, fmt.Errorf("repository: list telemetry: %w", err) + } + defer rows.Close() + out := make([]store.TelemetrySnapshot, 0) + for rows.Next() { + s, err := scanTelemetry(rows) + if err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + +// RequestTelemetryRefresh advances due times once per cooldown window per target. +func (p *Postgres) RequestTelemetryRefresh(ctx context.Context, tenantID string, cooldown time.Duration) (bool, error) { + if cooldown <= 0 { + cooldown = 10 * time.Second + } + tag, err := p.pool.Exec(ctx, `UPDATE telemetry_snapshot SET last_refresh_requested_at=clock_timestamp(), + refresh_pending=CASE WHEN lease_until>clock_timestamp() THEN true ELSE refresh_pending END, + next_poll_at=CASE WHEN lease_until>clock_timestamp() THEN next_poll_at ELSE clock_timestamp() END + WHERE tenant_id=$1::uuid AND (kind='local_bird' OR speaker_id IS NOT NULL) + AND (last_refresh_requested_at IS NULL OR last_refresh_requested_at<=clock_timestamp()-$2::bigint*interval '1 millisecond')`, tenantID, cooldown.Milliseconds()) + if err != nil { + return false, fmt.Errorf("repository: request telemetry refresh: %w", err) + } + return tag.RowsAffected() > 0, nil +} diff --git a/internal/repository/telemetry_integration_test.go b/internal/repository/telemetry_integration_test.go new file mode 100644 index 0000000..4271169 --- /dev/null +++ b/internal/repository/telemetry_integration_test.go @@ -0,0 +1,96 @@ +package repository + +import ( + "context" + "errors" + "testing" + "time" + + "evobgp/internal/birdfmt" + "evobgp/internal/store" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func TestTelemetryLeaseSnapshotAndDiscoveryIntegration(t *testing.T) { + p, tenant := queueIntegration(t) + ctx := context.Background() + speaker := uuid.NewString() + _, err := p.pool.Exec(ctx, `INSERT INTO bgp_speaker(id,tenant_id,role,meta_json) VALUES($1,$2,'replica',$3::jsonb)`, speaker, tenant, `{"agent_domain":"node.example","agent_secret":"secret"}`) + if err != nil { + t.Fatal(err) + } + first, err := p.ClaimTelemetry(ctx, "owner-a", 1, time.Minute, "") + if err != nil || len(first) != 1 || first[0].SpeakerID != speaker { + t.Fatalf("first claim=%+v %v", first, err) + } + if second, err := p.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, ""); err != nil || len(second) != 0 { + t.Fatalf("overlapping claim=%+v %v", second, err) + } + at := time.Now().UTC() + result := store.TelemetryResult{BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{Source: "agent_bird_protocols", ObservedAt: at, ReceivedAt: at}, Sessions: []birdfmt.BGPSession{{Name: birdfmt.DynamicPeerProtocolPrefix + "test", Neighbor: "192.0.2.8", NeighborAS: 64512, State: "Established"}}}, Health: store.TelemetryHealth{TelemetryObservation: store.TelemetryObservation{Source: "agent_health", ObservedAt: at, ReceivedAt: at}, OK: true}} + if err := p.CompleteTelemetry(ctx, first[0], result, at.Add(time.Hour)); err != nil { + t.Fatal(err) + } + other := &Postgres{pool: p.pool} + snaps, err := other.ListTelemetry(ctx, tenant) + if err != nil || len(snaps) != 1 || len(snaps[0].BGP.Sessions) != 1 || snaps[0].ConfigFingerprint == "" { + t.Fatalf("restarted repository snapshot=%+v %v", snaps, err) + } + discoveries, err := other.ListPeerDiscoveries(tenant, store.PeerDiscoveryPending) + if err != nil || len(discoveries) != 1 { + t.Fatalf("fenced discovery=%+v %v", discoveries, err) + } + if requested, err := other.RequestTelemetryRefresh(ctx, tenant, time.Minute); err != nil || !requested { + t.Fatalf("refresh=%v %v", requested, err) + } + if requested, err := other.RequestTelemetryRefresh(ctx, tenant, time.Minute); err != nil || requested { + t.Fatalf("refresh not coalesced=%v %v", requested, err) + } + newClaim, err := other.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, "") + if err != nil || len(newClaim) != 1 || newClaim[0].FencingToken <= first[0].FencingToken { + t.Fatalf("new lease=%+v %v", newClaim, err) + } + if err := p.CompleteTelemetry(ctx, first[0], result, at.Add(time.Hour)); !errors.Is(err, store.ErrTelemetryLeaseLost) { + t.Fatalf("late result accepted: %v", err) + } + + // Expire the lease inside the discovery write, after the initial fence check. + // The final fence must roll back BOTH snapshot and discovery atomically. + function := pgx.Identifier{"test_telemetry_expiry_" + uuid.NewString()}.Sanitize() + trigger := pgx.Identifier{"test_telemetry_expiry_" + uuid.NewString()}.Sanitize() + _, err = p.pool.Exec(ctx, `CREATE FUNCTION `+function+`() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.tenant_id::text = TG_ARGV[0] THEN + UPDATE telemetry_snapshot SET lease_until='epoch'::timestamptz + WHERE tenant_id=NEW.tenant_id AND target_id=TG_ARGV[1]; + END IF; + RETURN NEW; + END $$`) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _, _ = p.pool.Exec(context.Background(), `DROP FUNCTION IF EXISTS `+function+`() CASCADE`) }) + // Both arguments are generated UUIDs, not untrusted SQL input. + _, err = p.pool.Exec(ctx, `CREATE TRIGGER `+trigger+` BEFORE INSERT ON bgp_peer_discovery FOR EACH ROW EXECUTE FUNCTION `+function+`('`+tenant+`','`+speaker+`')`) + if err != nil { + t.Fatal(err) + } + result.BGP.Sessions = []birdfmt.BGPSession{{Name: birdfmt.DynamicPeerProtocolPrefix + "rollback", Neighbor: "192.0.2.9", NeighborAS: 64512, State: "Established"}} + if err = other.CompleteTelemetry(ctx, newClaim[0], result, at.Add(time.Hour)); !errors.Is(err, store.ErrTelemetryLeaseLost) { + t.Fatalf("completion after lease expired during discovery: %v", err) + } + snaps, err = p.ListTelemetry(ctx, tenant) + if err != nil || len(snaps) != 1 || len(snaps[0].BGP.Sessions) != 1 || snaps[0].BGP.Sessions[0].Neighbor != "192.0.2.8" { + t.Fatalf("expired completion changed snapshot: %+v %v", snaps, err) + } + discoveries, err = p.ListPeerDiscoveries(tenant, store.PeerDiscoveryPending) + if err != nil || len(discoveries) != 1 { + t.Fatalf("expired completion changed discoveries: %+v %v", discoveries, err) + } + neighbor, ok := store.NormalizePeerNeighborString(discoveries[0].Neighbor) + if !ok || neighbor != "192.0.2.8" { + t.Fatalf("expired completion inserted discovery: %+v", discoveries) + } +} diff --git a/internal/scheduler/run.go b/internal/scheduler/run.go index 81e1cb6..c24630f 100644 --- a/internal/scheduler/run.go +++ b/internal/scheduler/run.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "encoding/json" - "evobgp/internal/logging" "fmt" "io" "log" @@ -17,18 +16,20 @@ import ( "evobgp/internal/config" "evobgp/internal/httpclient" "evobgp/internal/jobs" + "evobgp/internal/logging" "evobgp/internal/pipeline" "evobgp/internal/store" ) -// Deps wires the scheduler to the store. Use either Jobs (same process as API, e.g. evobgp-all) or -// APIBase+APIToken to call the control plane over HTTP (separate containers in reference Compose). +// Deps wires the scheduler to a trusted durable Registry (or local evobgp-all), +// or to a single tenant over HTTP using APIBase, APIToken and APITenantID. type Deps struct { - Store store.Backend - Jobs *jobs.Registry - APIBase string // e.g. http://evobgp-api:8080 - APIToken string // Bearer token (operator/editor) - HTTP *http.Client + Store store.Backend + Jobs *jobs.Registry + APIBase string // e.g. http://evobgp-api:8080 + APIToken string // Tenant-scoped bearer token (operator/editor) + APITenantID string // Explicit single tenant for HTTP mode; never enumerate other tenants + HTTP *http.Client } // Run blocks until ctx is cancelled. Misconfigured deps terminate the process (no idle fallback). @@ -38,13 +39,13 @@ func Run(ctx context.Context, deps *Deps) { if deps == nil || deps.Store == nil { log.Fatalf("evobgp-scheduler: missing store (pass scheduler.Deps with Store from BootstrapWorkers or evobgp-all)") } - if deps.Jobs == nil && (strings.TrimSpace(deps.APIBase) == "" || strings.TrimSpace(deps.APIToken) == "") { - log.Fatalf("evobgp-scheduler: need either in-process Jobs (evobgp-all) or both EVOBGP_CONTROL_PLANE_URL and EVOBGP_SCHEDULER_BEARER") + if deps.Jobs == nil && (strings.TrimSpace(deps.APIBase) == "" || strings.TrimSpace(deps.APIToken) == "" || strings.TrimSpace(deps.APITenantID) == "") { + log.Fatalf("evobgp-scheduler: need either in-process Jobs (evobgp-all) or EVOBGP_CONTROL_PLANE_URL, EVOBGP_SCHEDULER_BEARER and EVOBGP_SCHEDULER_TENANT_ID") } - t := time.NewTicker(30 * time.Second) + t := time.NewTicker(time.Duration(pipeline.SchedulerTickSec) * time.Second) defer t.Stop() if deps.Jobs != nil { - logging.Default().Info(fmt.Sprintf("evobgp-scheduler: active (in-process enqueue tenant_refresh)")) + logging.Default().Info(fmt.Sprintf("evobgp-scheduler: active (registry enqueue tenant_refresh)")) } else { logging.Default().Info(fmt.Sprintf("evobgp-scheduler: active (HTTP POST .../tenant/refresh → %s)", strings.TrimSpace(deps.APIBase))) } @@ -54,20 +55,44 @@ func Run(ctx context.Context, deps *Deps) { logging.Default().Info(fmt.Sprintf("evobgp-scheduler: stopped")) return case <-t.C: - tick(context.Background(), deps) + tick(ctx, deps) } } } func tick(ctx context.Context, deps *Deps) { - tenants, err := deps.Store.ListTenantIDs() - if err != nil { - logging.Default().Info(fmt.Sprintf("evobgp-scheduler: list tenants: %v", err)) + tickAt(ctx, deps, time.Now().UTC()) +} + +func tickAt(ctx context.Context, deps *Deps, now time.Time) { + ctx, cancel := context.WithTimeout(ctx, 45*time.Second) + defer cancel() + if ctx.Err() != nil { return } - now := time.Now().UTC() + var tenants []string + if deps.Jobs == nil { + tenant := strings.TrimSpace(deps.APITenantID) + if tenant == "" { + return + } + tenants = []string{tenant} + } else { + var err error + tenants, err = deps.Store.ListTenantIDs() + if err != nil { + logging.Default().Error("scheduler list tenants", "err", err) + return + } + } tickBucket := now.Unix() / pipeline.SchedulerTickSec for _, tid := range tenants { + if ctx.Err() != nil { + return + } + if tid == store.SystemJobTenantID { + continue + } var due []string for _, mod := range deps.Store.ListModules(tid) { if pipeline.ModuleDueForScheduler(mod, now) { @@ -80,7 +105,7 @@ func tick(ctx context.Context, deps *Deps) { key := fmt.Sprintf("sched-tenant-%s-%d", tid, tickBucket) if deps.Jobs != nil { ids := append([]string(nil), due...) - _, created, err := deps.Jobs.Enqueue(tid, jobs.KindTenantRefresh, &key, nil, map[string]any{ + _, created, err := deps.Jobs.EnqueueContext(ctx, tid, jobs.KindTenantRefresh, &key, nil, map[string]any{ "module_ids": ids, "trigger": "scheduler", }) diff --git a/internal/scheduler/run_test.go b/internal/scheduler/run_test.go new file mode 100644 index 0000000..758a855 --- /dev/null +++ b/internal/scheduler/run_test.go @@ -0,0 +1,114 @@ +package scheduler + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "evobgp/internal/jobs" + "evobgp/internal/pipeline" + "evobgp/internal/store" +) + +type schedulerStore struct { + store.Backend + tenants []string + modules map[string][]*store.Module + enumerations int + reads []string +} + +func (s *schedulerStore) ListTenantIDs() ([]string, error) { s.enumerations++; return s.tenants, nil } +func (s *schedulerStore) ListModules(tenant string) []*store.Module { + s.reads = append(s.reads, tenant) + return s.modules[tenant] +} + +type schedulerRoundTrip func(*http.Request) (*http.Response, error) + +func (f schedulerRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } +func schedulerFixture(t *testing.T) (*schedulerStore, time.Time) { + t.Helper() + now := time.Date(2026, 10, 26, 12, 0, 0, 0, time.UTC) + st := &schedulerStore{Backend: store.NewMemory(), tenants: []string{"a", "b", store.SystemJobTenantID}, modules: map[string][]*store.Module{}} + for _, tenant := range st.tenants { + for i := 0; i < 1000; i++ { + mod := &store.Module{ID: fmt.Sprintf("%s-%d", tenant, i), TenantID: tenant, Type: "DOMAINS", Enabled: true, RefreshIntervalSec: 60} + if pipeline.ModuleDueForScheduler(mod, now) { + st.modules[tenant] = []*store.Module{mod} + break + } + } + if len(st.modules[tenant]) == 0 { + t.Fatal("fixture has no due module") + } + } + return st, now +} +func TestSchedulerHTTPExplicitTenantScope(t *testing.T) { + st, now := schedulerFixture(t) + requests := 0 + client := &http.Client{Transport: schedulerRoundTrip(func(r *http.Request) (*http.Response, error) { + requests++ + var body struct { + ModuleIDs []string `json:"module_ids"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + if len(body.ModuleIDs) != 1 || body.ModuleIDs[0] != st.modules["a"][0].ID { + t.Fatalf("cross-tenant payload: %+v", body) + } + if r.Header.Get("Authorization") != "Bearer scoped" { + t.Fatal("missing scoped bearer") + } + if _, ok := r.Context().Deadline(); !ok { + t.Fatal("request has no deadline") + } + return &http.Response{StatusCode: http.StatusAccepted, Body: io.NopCloser(strings.NewReader("{}")), Header: make(http.Header)}, nil + })} + deps := &Deps{Store: st, APIBase: "https://cp.test", APIToken: "scoped", APITenantID: "a", HTTP: client} + tickAt(context.Background(), deps, now) + if requests != 1 || st.enumerations != 0 || len(st.reads) != 1 || st.reads[0] != "a" { + t.Fatalf("scope requests=%d enumerations=%d reads=%v", requests, st.enumerations, st.reads) + } + deps.APITenantID = "" + tickAt(context.Background(), deps, now) + if requests != 1 || len(st.reads) != 1 { + t.Fatal("unscoped HTTP scheduler performed work") + } +} +func TestSchedulerTrustedQueueSeparatesTenantsAndHonorsCancellation(t *testing.T) { + st, now := schedulerFixture(t) + reg := jobs.NewRegistry(nil) + reg.UseDurableQueue(st) + deps := &Deps{Store: st, Jobs: reg} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + tickAt(ctx, deps, now) + if st.enumerations != 0 { + t.Fatal("cancelled scheduler accessed store") + } + tickAt(context.Background(), deps, now) + tickAt(context.Background(), deps, now) + for _, tenant := range []string{"a", "b"} { + rows, _, _, err := st.ListJobs(context.Background(), tenant, "", "", "", 10) + if err != nil || len(rows) != 1 || rows[0].TenantID != tenant { + t.Fatalf("queue isolation/idempotency: %s %+v %v", tenant, rows, err) + } + raw, _ := json.Marshal(rows[0].Meta) + if !strings.Contains(string(raw), st.modules[tenant][0].ID) { + t.Fatalf("wrong tenant module: %s", raw) + } + } + for _, read := range st.reads { + if read == store.SystemJobTenantID { + t.Fatal("scheduler read system tenant") + } + } +} diff --git a/internal/store/backend.go b/internal/store/backend.go index d86ede5..4f68a4f 100644 --- a/internal/store/backend.go +++ b/internal/store/backend.go @@ -8,6 +8,11 @@ import ( // Backend is the persistence abstraction for the control plane (memory, PostgreSQL, SQLite). type Backend interface { + JobQueue + Deployments + TelemetryStore + // ObserveStoreMetrics returns historical revision and peer inventory aggregates or an error. + ObserveStoreMetrics(ctx context.Context) (max, sum, peers int, states map[string]int, err error) MaterializedPrefixStats() (max int, sum int) PeerCount() int PeerSessionCountsByState() map[string]int @@ -16,6 +21,14 @@ type Backend interface { // ListTenantIDs returns distinct tenant identifiers (for background workers). ListTenantIDs() ([]string, error) + // GetRenderGeneration returns the tenant's monotonic render-input barrier. + GetRenderGeneration(ctx context.Context, tenantID string) (int64, error) + // ListModuleSnapshotHeaders returns snapshot versions for the tenant, ordered by module id. + // It does not load prefix rows. + ListModuleSnapshotHeaders(ctx context.Context, tenantID string) ([]ModuleSnapshotUse, error) + // LatestRenderManifest returns the manifest stored with the newest config revision. + // ok is false when that revision has no manifest yet. + LatestRenderManifest(ctx context.Context, tenantID string) (*RenderManifestRecord, bool, error) // ListModules returns all modules for a tenant (control plane may paginate in httpapi). ListModules(tenantID string) []*Module @@ -91,11 +104,15 @@ type Backend interface { ListRevisionPrefixes(tenantID, revisionID string, cursor string, limit int) (prefixes []PrefixRow, next string, more bool) CreateRollbackRevision(tenantID, sourceRevisionID string) (newID string, err error) // CreateRenderRevision inserts a new config_revision (revID must be unique) with materialized prefixes and preview fragments. - CreateRenderRevision(revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error + // The same transaction records config_generation and snapshot versions in render_revision_manifest. + // ctx cancellation aborts the write. prefix_snapshot is keyed by a hash of prefixes only. + CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error RevisionDiff(tenantID, aID, bID string) (map[string]any, error) EstimateRevisionPrune(tenantID string, cutoff time.Time, retentionMinutes int) (RevisionPruneEstimate, error) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (RevisionPruneResult, error) - PruneRevisionsBefore(tenantID string, cutoff time.Time) (deleted int, err error) + // PruneRevisionsBefore deletes revisions older than cutoff, keeping the newest + // revision and any revision a speaker still references. ctx bounds the delete. + PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (deleted int, err error) SetLastAppliedRevision(tenantID, speakerID, revisionID string) error PublishRevisionForSpeaker(speakerID, revisionID string) error LatestPublishedRevision(speakerID string) (revisionID string, publishedAt time.Time, err error) @@ -113,13 +130,16 @@ type Backend interface { TouchAPIKeyLastUsed(id string) error // Module prefix snapshots cache last successful collect per module (pipeline ingest/render). - GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error) - SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error - DeleteModulePrefixSnapshot(tenantID, moduleID string) error + GetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error) + // ListModuleSnapshotSourceRows returns persisted rows for one source key. + // It does not load the rest of the module snapshot. + ListModuleSnapshotSourceRows(ctx context.Context, tenantID, moduleID, source string) ([]PrefixRow, error) + SetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID, inputHash string, prefixes []PrefixRow) error + DeleteModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) error + // UpdateModulePrefixSnapshot runs a pure mutation against the latest snapshot atomically. + UpdateModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string, update ModuleSnapshotUpdate) error // SetModuleInputHash stores the ingest fingerprint on the module row (O(1) snapshot check). - SetModuleInputHash(tenantID, moduleID, hash string) error - // LockModuleSnapshot serializes read-modify-write of one module snapshot; unlock must be called. - LockModuleSnapshot(tenantID, moduleID string) (unlock func()) + SetModuleInputHash(ctx context.Context, tenantID, moduleID, hash string) error // ASNPrefixCache stores RIPEstat announced-prefixes per ASN (global TTL cache). GetASNPrefixCache(asn int64) (*ASNPrefixCacheEntry, bool, error) @@ -197,6 +217,7 @@ type DomainResolveCacheEntry struct { type ModulePrefixSnapshot struct { InputHash string CollectedAt time.Time + Version int64 Prefixes []PrefixRow } diff --git a/internal/store/deployment.go b/internal/store/deployment.go new file mode 100644 index 0000000..338cf73 --- /dev/null +++ b/internal/store/deployment.go @@ -0,0 +1,50 @@ +package store + +import ( + "context" + "errors" + "time" +) + +// ErrDeploymentBusy means a previous legacy request may still be executing. +var ErrDeploymentBusy = errors.New("store: speaker delivery reservation is active") + +// DeploymentDeliveryWindow retains delivery ownership after an ambiguous response. +const DeploymentDeliveryWindow = 2 * time.Minute + +// JobFence identifies the currently leased deployment attempt. +type JobFence struct { + JobID, TenantID, OwnerID string + Token int64 + ExpectedGeneration int64 + RetryRound int +} + +// DeploymentSpeaker preserves delivery progress independently from job result metadata. +// Generation is monotonic per speaker, including across different jobs. +type DeploymentSpeaker struct { + ObservationToken int64 + JobID, TenantID, SpeakerID, RevisionID string + Generation int64 + RequestSequence int64 + State, Confirmation, Error string + DispatchAttempts int + RetryRound int + HealthReceivedAt *time.Time + HealthRevisionID, HealthSourceAt, HealthError string +} + +// DeploymentObservation is a compatible authenticated health report. SourceAt may +// be absent on legacy agents; ReceivedAt is generated by the control-plane store. +type DeploymentObservation struct{ RevisionID, SourceAt, Error string } + +// Deployments atomically fences publication and acknowledgement against the job +// lease and the speaker's current deployment generation. +type Deployments interface { + BeginDeploymentSpeaker(context.Context, JobFence, string, string) (DeploymentSpeaker, error) + RecordDeploymentSpeaker(context.Context, JobFence, DeploymentSpeaker, string, string, string) error + ListDeploymentSpeakers(context.Context, string, string) ([]DeploymentSpeaker, error) + ListPendingDeployments(context.Context, int) ([]DeploymentSpeaker, error) + ListDeploymentObservations(context.Context, int) ([]DeploymentSpeaker, error) + ObserveDeployment(context.Context, DeploymentSpeaker, DeploymentObservation) error +} diff --git a/internal/store/deployment_test.go b/internal/store/deployment_test.go new file mode 100644 index 0000000..9eac516 --- /dev/null +++ b/internal/store/deployment_test.go @@ -0,0 +1,170 @@ +package store + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestDeploymentLeaseGenerationAndReplay(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, _, _, revision, speaker := m.DemoIDs() + ctx := context.Background() + enqueue := func() JobRecord { + t.Helper() + j, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: tenant, Kind: "deploy_apply"}) + if err != nil { + t.Fatal(err) + } + return j + } + claim := func(owner string) JobRecord { + t.Helper() + rows, err := m.ClaimJobs(ctx, owner, 1, time.Minute) + if err != nil || len(rows) != 1 { + t.Fatalf("claim: %v %v", rows, err) + } + return rows[0] + } + fence := func(j JobRecord) JobFence { + return JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken} + } + first := enqueue() + first = claim("one") + d, err := m.BeginDeploymentSpeaker(ctx, fence(first), speaker, revision) + if err != nil { + t.Fatal(err) + } + if err = m.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "verified", ""); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("unproved verified: %v", err) + } + if err = m.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "agent_reported", ""); err != nil { + t.Fatal(err) + } + replayed, err := m.BeginDeploymentSpeaker(ctx, fence(first), speaker, revision) + if err != nil || replayed.Generation != d.Generation || replayed.State != "applied" { + t.Fatalf("replay: %+v %v", replayed, err) + } + if err = m.ReleaseJob(ctx, first.ID, first.OwnerID, first.FencingToken); err != nil { + t.Fatal(err) + } + reclaimed := claim("two") + if err = m.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "agent_reported", ""); !errors.Is(err, ErrJobLeaseLost) { + t.Fatalf("old owner ACK: %v", err) + } + if _, err = m.BeginDeploymentSpeaker(ctx, fence(reclaimed), speaker, revision); err != nil { + t.Fatal(err) + } + second := enqueue() + second = claim("three") + d2, err := m.BeginDeploymentSpeaker(ctx, fence(second), speaker, revision) + if err != nil { + t.Fatal(err) + } + if d2.Generation <= d.Generation { + t.Fatal("generation failed to advance across independent job fencing tokens") + } + if err = m.RecordDeploymentSpeaker(ctx, fence(reclaimed), d, "applied", "agent_reported", ""); !errors.Is(err, ErrJobLeaseLost) { + t.Fatalf("old generation ACK: %v", err) + } + if _, err = m.BeginDeploymentSpeaker(ctx, fence(reclaimed), speaker, revision); !errors.Is(err, ErrJobLeaseLost) { + t.Fatalf("old generation republish: %v", err) + } + rows, err := m.ListDeploymentSpeakers(ctx, tenant, first.ID) + if err != nil || len(rows) != 1 { + t.Fatalf("durable progress: %v %v", rows, err) + } + if err = m.RecordDeploymentSpeaker(ctx, fence(second), d2, "unknown", "unconfirmed", "response lost"); err != nil { + t.Fatal(err) + } +} + +func TestDeploymentObservationTokenRejectsLateHealth(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, _, _, revision, speaker := m.DemoIDs() + ctx := context.Background() + _, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: tenant, Kind: "deploy_apply"}) + if err != nil { + t.Fatal(err) + } + jobs, err := m.ClaimJobs(ctx, "owner", 1, time.Minute) + if err != nil || len(jobs) != 1 { + t.Fatal(err) + } + j := jobs[0] + f := JobFence{JobID: j.ID, TenantID: tenant, OwnerID: j.OwnerID, Token: j.FencingToken} + d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision) + if err != nil { + t.Fatal(err) + } + if err = m.RecordDeploymentSpeaker(ctx, f, d, "applied", "agent_reported", ""); err != nil { + t.Fatal(err) + } + j.Status = "succeeded" + if err = m.FinishJob(ctx, j); err != nil { + t.Fatal(err) + } + old, err := m.ListDeploymentObservations(ctx, 1) + if err != nil || len(old) != 1 { + t.Fatal(err) + } + latest, err := m.ListDeploymentObservations(ctx, 1) + if err != nil || len(latest) != 1 { + t.Fatal(err) + } + if err = m.ObserveDeployment(ctx, latest[0], DeploymentObservation{Error: "timeout"}); err != nil { + t.Fatal(err) + } + if err = m.ObserveDeployment(ctx, old[0], DeploymentObservation{RevisionID: "older"}); !errors.Is(err, ErrJobLeaseLost) { + t.Fatalf("late report accepted: %v", err) + } + rows, err := m.ListDeploymentSpeakers(ctx, tenant, j.ID) + if err != nil || rows[0].State != "applied" || rows[0].Confirmation != "agent_reported" || rows[0].HealthError != "timeout" { + t.Fatalf("error downgraded confirmation: %+v %v", rows, err) + } +} + +func TestDeploymentReconciliationRetryBudget(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, _, _, revision, speaker := m.DemoIDs() + ctx := context.Background() + var previous DeploymentSpeaker + for round := 0; round <= 3; round++ { + _, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: tenant, Kind: "deploy_apply"}) + if err != nil { + t.Fatal(err) + } + jobs, err := m.ClaimJobs(ctx, "worker", 1, time.Minute) + if err != nil || len(jobs) != 1 { + t.Fatal(err) + } + j := jobs[0] + f := JobFence{JobID: j.ID, TenantID: tenant, OwnerID: j.OwnerID, Token: j.FencingToken, RetryRound: round} + if round > 0 { + f.ExpectedGeneration = previous.Generation + } + d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision) + if err != nil { + t.Fatal(err) + } + if err = m.RecordDeploymentSpeaker(ctx, f, d, "failed", "unconfirmed", "unreachable"); err != nil { + t.Fatal(err) + } + j.Status = "failed" + if err = m.FinishJob(ctx, j); err != nil { + t.Fatal(err) + } + pending, err := m.ListPendingDeployments(ctx, 100) + if err != nil { + t.Fatal(err) + } + if (round < 3 && len(pending) != 1) || (round == 3 && len(pending) != 0) { + t.Fatalf("round %d pending=%d", round, len(pending)) + } + previous = d + } +} diff --git a/internal/store/job_queue.go b/internal/store/job_queue.go new file mode 100644 index 0000000..c7fe3b5 --- /dev/null +++ b/internal/store/job_queue.go @@ -0,0 +1,83 @@ +package store + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "time" +) + +// SystemJobTenantID owns global maintenance jobs outside user tenant scope. +const SystemJobTenantID = "00000000-0000-0000-0000-000000000000" + +// JobRequestHash identifies the immutable requested operation, excluding result metadata. +func JobRequestHash(in JobEnqueue) (string, error) { + meta := in.Meta + if in.Request != nil { + meta = in.Request + } + b, err := json.Marshal(struct { + Kind string `json:"kind"` + ModuleID *string `json:"module_id"` + Meta map[string]any `json:"meta"` + }{in.Kind, in.ModuleID, meta}) + if err != nil { + return "", err + } + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]), nil +} + +var ( + ErrJobNotFound = errors.New("store: job not found") + ErrJobConflict = errors.New("store: idempotency key conflicts with another request") + ErrJobQueueFull = errors.New("store: job queue is full") + ErrJobLeaseLost = errors.New("store: job lease lost") +) + +// JobRecord is the shared, tenant-scoped job lifecycle record. +type JobRecord struct { + RequestHash string + ID, TenantID, Kind, Status string + IdempotencyKey, ModuleID *string + Meta map[string]any + CreatedAt time.Time + StartedAt, FinishedAt *time.Time + LeaseUntil *time.Time + Error *string + OwnerID string + FencingToken int64 + RequestSequence int64 + Attempt int + CancelRequested bool +} + +// JobEnqueue is the immutable request used for idempotent insertion. +type JobEnqueue struct { + TenantID, Kind string + IdempotencyKey, ModuleID *string + Meta map[string]any + // Request optionally supplies the original selector before dynamic resolution. + Request map[string]any +} + +// JobQueue provides durable operations; PostgreSQL is authoritative in production. +type JobQueue interface { + CountJobsByStatus(context.Context) (map[string]int64, error) + FindJobByIdempotencyKey(context.Context, string, string) (JobRecord, error) + EnqueueJob(context.Context, JobEnqueue) (JobRecord, bool, error) + ClaimJobs(context.Context, string, int, time.Duration) ([]JobRecord, error) + // ReapExpiredJobs marks orphaned cancellations and exhausted retries as terminal. + // Production callers run it about every 15 seconds, not on every claim. + ReapExpiredJobs(context.Context) error + HeartbeatJob(context.Context, string, string, int64, time.Duration) (bool, error) + CheckpointJob(context.Context, JobRecord) error + FinishJob(context.Context, JobRecord) error + ReleaseJob(context.Context, string, string, int64) error + GetJob(context.Context, string, string) (JobRecord, error) + ListJobs(context.Context, string, string, string, string, int) ([]JobRecord, string, bool, error) + CancelJob(context.Context, string, string) (JobRecord, error) + RetryJob(context.Context, string, string) (JobRecord, error) +} diff --git a/internal/store/memory.go b/internal/store/memory.go index 6f3c07e..5df4c16 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -21,7 +21,18 @@ var ( // Memory is a development-oriented in-memory backend for control-plane API handlers. type Memory struct { - mu sync.RWMutex + mu sync.RWMutex + jobMu sync.Mutex + jobTenantMu sync.Mutex + jobTenantLocks map[string]*sync.Mutex + jobSequence int64 + deploymentReservations map[string]deploymentReservation + deployments map[string]DeploymentSpeaker + deploymentCurrent map[string]DeploymentSpeaker + jobs map[string]JobRecord + jobHashes map[string]string + jobRequests map[string]map[string]any + telemetry map[string]telemetryMemoryRow tenants map[string]*Tenant modules map[string]*Module @@ -35,26 +46,33 @@ type Memory struct { peerDiscoveries map[string]*BGPPeerDiscovery - dohProfiles map[string]*DohProfile - communities map[string]*Community - cdnSources map[string]*CDNSource - asEntries map[string]*ASEntry - domainEnt map[string]*DomainEntry - ipRanges map[string]*IPRangeEntry - settings map[string]map[string]any // tenantID -> key -> JSON-compatible value - revPrefixes map[string][]PrefixRow - moduleSnapshots map[string]*moduleSnapshotRec - snapshotLocks sync.Map // key -> *sync.Mutex (per-module snapshot RMW) - asnPrefixCache map[int64]*ASNPrefixCacheEntry - domainResolveCache map[string]*DomainResolveCacheEntry - apiKeys map[string]*apiKeyRec - firewallClients map[string]*firewallClientRec - firewallRules map[string]*FirewallRule - firewallHashIndex map[string]string // hex hash -> client id - maintenancePolicies map[string]*MaintenancePolicy - maintConfigAudit []*MaintenancePolicyConfigAudit - runtimeLogCleanupAudit []*RuntimeLogCleanupAudit - auditLog []*AuditEntry + dohProfiles map[string]*DohProfile + communities map[string]*Community + cdnSources map[string]*CDNSource + asEntries map[string]*ASEntry + domainEnt map[string]*DomainEntry + ipRanges map[string]*IPRangeEntry + settings map[string]map[string]any // tenantID -> key -> JSON-compatible value + revPrefixes map[string][]PrefixRow + prefixSnapByHash map[string]string + prefixSnapRows map[string][]PrefixRow + revPrefixSnap map[string]string + moduleSnapshots map[string]*moduleSnapshotRec + nextModuleSnapshotVersion int64 + renderGeneration map[string]int64 + renderManifests map[string]*RenderManifestRecord + snapshotLocksMu sync.Mutex + snapshotLocks map[string]*snapshotLockEntry + asnPrefixCache map[int64]*ASNPrefixCacheEntry + domainResolveCache map[string]*DomainResolveCacheEntry + apiKeys map[string]*apiKeyRec + firewallClients map[string]*firewallClientRec + firewallRules map[string]*FirewallRule + firewallHashIndex map[string]string // hex hash -> client id + maintenancePolicies map[string]*MaintenancePolicy + maintConfigAudit []*MaintenancePolicyConfigAudit + runtimeLogCleanupAudit []*RuntimeLogCleanupAudit + auditLog []*AuditEntry // DemoIDs valid after SeedDemo() demoTenantID string @@ -157,7 +175,12 @@ func NewMemory() *Memory { ipRanges: make(map[string]*IPRangeEntry), settings: make(map[string]map[string]any), revPrefixes: make(map[string][]PrefixRow), + prefixSnapByHash: make(map[string]string), + prefixSnapRows: make(map[string][]PrefixRow), + revPrefixSnap: make(map[string]string), moduleSnapshots: make(map[string]*moduleSnapshotRec), + renderGeneration: make(map[string]int64), + renderManifests: make(map[string]*RenderManifestRecord), asnPrefixCache: make(map[int64]*ASNPrefixCacheEntry), domainResolveCache: make(map[string]*DomainResolveCacheEntry), apiKeys: make(map[string]*apiKeyRec), @@ -319,6 +342,31 @@ func (m *Memory) PeerSessionCountsByState() map[string]int { return out } +// ObserveStoreMetrics returns an error-aware snapshot of stored revision and peer inventory aggregates. +func (m *Memory) ObserveStoreMetrics(ctx context.Context) (max, sum, peers int, states map[string]int, err error) { + if err := ctx.Err(); err != nil { + return 0, 0, 0, nil, err + } + m.mu.RLock() + defer m.mu.RUnlock() + for _, revision := range m.revisions { + sum += revision.MaterializedPrefixCount + if revision.MaterializedPrefixCount > max { + max = revision.MaterializedPrefixCount + } + } + states = make(map[string]int) + for _, peer := range m.peers { + peers++ + state := strings.TrimSpace(peer.SessionState) + if state == "" { + state = "unknown" + } + states[state]++ + } + return max, sum, peers, states, nil +} + // DemoIDs returns IDs from SeedDemo; empty strings if SeedDemo was not called. func (m *Memory) DemoIDs() (tenant, moduleCDN, moduleIP, revision, speaker string) { m.mu.RLock() @@ -349,13 +397,22 @@ func (m *Memory) ListTenantIDs() ([]string, error) { return out, nil } -// CreateRenderRevision stores a new revision and its materialized prefixes. -func (m *Memory) CreateRenderRevision(revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error { +// CreateRenderRevision stores a new revision, its prefix snapshot, and the render manifest. +func (m *Memory) CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } if strings.TrimSpace(revisionID) == "" || strings.TrimSpace(moduleID) == "" || strings.TrimSpace(contentHash) == "" { return ErrInvalidInput } m.mu.Lock() defer m.mu.Unlock() + if err := ctx.Err(); err != nil { + return err + } if _, exists := m.revisions[revisionID]; exists { return ErrInvalidInput } @@ -392,16 +449,30 @@ func (m *Memory) CreateRenderRevision(revisionID, tenantID, moduleID string, par MaterializedPrefixCount: len(prefixes), PreviewFragments: frag, } - if len(prefixes) > 0 { - cp := make([]PrefixRow, len(prefixes)) - copy(cp, prefixes) - m.revPrefixes[revisionID] = cp - } else { - m.revPrefixes[revisionID] = nil - } + snapID := m.bindPrefixSnapshotLocked(prefixes) + m.revPrefixSnap[revisionID] = snapID + m.revPrefixes[revisionID] = m.prefixSnapRows[snapID] + m.recordRenderManifestLocked(revisionID, tenantID) return nil } +// bindPrefixSnapshotLocked reuses an existing prefix snapshot when the prefix hash matches. +func (m *Memory) bindPrefixSnapshotLocked(prefixes []PrefixRow) string { + if m.prefixSnapByHash == nil { + m.prefixSnapByHash = make(map[string]string) + m.prefixSnapRows = make(map[string][]PrefixRow) + m.revPrefixSnap = make(map[string]string) + } + h := PrefixSnapshotHash(prefixes) + if id, ok := m.prefixSnapByHash[h]; ok { + return id + } + id := uuid.NewString() + m.prefixSnapByHash[h] = id + m.prefixSnapRows[id] = append([]PrefixRow(nil), prefixes...) + return id +} + // ListModules returns modules for a tenant (sorted by priority, then name). func (m *Memory) ListModules(tenantID string) []*Module { m.mu.RLock() diff --git a/internal/store/memory_asn_cache.go b/internal/store/memory_asn_cache.go index f77a67a..f77dd1d 100644 --- a/internal/store/memory_asn_cache.go +++ b/internal/store/memory_asn_cache.go @@ -1,6 +1,7 @@ package store import ( + "sort" "time" ) @@ -29,6 +30,11 @@ func (m *Memory) SetASNPrefixCache(asn int64, holder string, prefixes []string) if m.asnPrefixCache == nil { m.asnPrefixCache = make(map[int64]*ASNPrefixCacheEntry) } + if cur := m.asnPrefixCache[asn]; cur != nil && sameStringSet(cur.Prefixes, prefixes) { + cur.Holder = holder + cur.FetchedAt = time.Now().UTC() + return nil + } cp := append([]string(nil), prefixes...) m.asnPrefixCache[asn] = &ASNPrefixCacheEntry{ ASN: asn, @@ -38,3 +44,19 @@ func (m *Memory) SetASNPrefixCache(asn int64, holder string, prefixes []string) } return nil } + +func sameStringSet(a, b []string) bool { + if len(a) != len(b) { + return false + } + aa := append([]string(nil), a...) + bb := append([]string(nil), b...) + sort.Strings(aa) + sort.Strings(bb) + for i := range aa { + if aa[i] != bb[i] { + return false + } + } + return true +} diff --git a/internal/store/memory_crud.go b/internal/store/memory_crud.go index b8e3762..1d81d00 100644 --- a/internal/store/memory_crud.go +++ b/internal/store/memory_crud.go @@ -38,6 +38,7 @@ func (m *Memory) CreateModule(tenantID string, in *Module) (*Module, error) { } NormalizeModuleDoh(mod) m.modules[id] = mod + m.bumpRenderGenerationLocked(tenantID) return cloneModule(mod), nil } @@ -81,7 +82,10 @@ func (m *Memory) UpdateModule(tenantID, moduleID string, patch *ModulePatch) (*M t := patch.LastRefreshedAt.UTC() mod.LastRefreshedAt = &t } - mod.InputHash = "" + if patch.Name != nil || patch.Enabled != nil || patch.Priority != nil || patch.RefreshIntervalSec != nil || patch.CronExpr != nil || patch.DefaultCommunityID != nil || patch.DohProfileIDs != nil || patch.DohProfileID != nil || patch.DohResolverPolicy != nil { + mod.InputHash = "" + m.bumpRenderGenerationLocked(tenantID) + } return cloneModule(mod), nil } @@ -97,6 +101,7 @@ func (m *Memory) SoftDeleteModule(tenantID, moduleID string) error { } now := time.Now().UTC() mod.DeletedAt = &now + m.bumpRenderGenerationLocked(tenantID) return nil } @@ -154,6 +159,7 @@ func (m *Memory) CreateCDNSource(tenantID, moduleID string, in *CDNSource) (*CDN } m.cdnSources[id] = s mod.InputHash = "" + m.bumpRenderGenerationLocked(tenantID) return s, nil } @@ -200,6 +206,7 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN } if CDNPatchAffectsInputHash(patch) { mod.InputHash = "" + m.bumpRenderGenerationLocked(tenantID) } return s, nil } @@ -216,6 +223,7 @@ func (m *Memory) DeleteCDNSource(tenantID, moduleID, sourceID string) error { } delete(m.cdnSources, sourceID) m.clearModuleInputHashLocked(moduleID) + m.bumpRenderGenerationLocked(tenantID) return nil } @@ -255,6 +263,7 @@ func (m *Memory) CreateASEntry(tenantID, moduleID string, in *ASEntry) (*ASEntry e := &ASEntry{ID: id, ModuleID: moduleID, ASN: in.ASN, CommunityID: in.CommunityID} m.asEntries[id] = e mod.InputHash = "" + m.bumpRenderGenerationLocked(tenantID) return e, nil } @@ -293,6 +302,7 @@ func (m *Memory) UpdateASEntry(tenantID, moduleID, entryID string, patch *ASEntr e.ASNResolvedAt = nil } mod.InputHash = "" + m.bumpRenderGenerationLocked(tenantID) return e, nil } @@ -335,6 +345,7 @@ func (m *Memory) DeleteASEntry(tenantID, moduleID, entryID string) error { } delete(m.asEntries, entryID) m.clearModuleInputHashLocked(moduleID) + m.bumpRenderGenerationLocked(tenantID) return nil } @@ -791,6 +802,7 @@ func (m *Memory) CreatePeer(tenantID string, in *BGPPeer) (*BGPPeer, error) { CreatedByUserID: strings.TrimSpace(in.CreatedByUserID), } m.peers[id] = p + m.bumpRenderGenerationLocked(tenantID) return p, nil } @@ -804,6 +816,11 @@ func (m *Memory) UpdatePeer(tenantID, id string, patch *PeerPatch) (*BGPPeer, er if !ok || p.TenantID != tenantID { return nil, ErrNotFound } + prevNeighbor, prevASN, prevEnabled, prevPolicies := p.Neighbor, p.RemoteASN, p.Enabled, p.PoliciesJSON + prevSpeaker := "" + if p.SpeakerID != nil { + prevSpeaker = *p.SpeakerID + } if patch.Neighbor != nil { n, ok := NormalizePeerNeighborString(*patch.Neighbor) if !ok { @@ -834,6 +851,13 @@ func (m *Memory) UpdatePeer(tenantID, id string, patch *PeerPatch) (*BGPPeer, er if patch.PoliciesJSON != nil { p.PoliciesJSON = *patch.PoliciesJSON } + speaker := "" + if p.SpeakerID != nil { + speaker = *p.SpeakerID + } + if p.Neighbor != prevNeighbor || p.RemoteASN != prevASN || p.Enabled != prevEnabled || p.PoliciesJSON != prevPolicies || speaker != prevSpeaker { + m.bumpRenderGenerationLocked(tenantID) + } return p, nil } @@ -846,6 +870,7 @@ func (m *Memory) DeletePeer(tenantID, id string) error { } delete(m.peers, id) _ = p + m.bumpRenderGenerationLocked(tenantID) return nil } diff --git a/internal/store/memory_deployment.go b/internal/store/memory_deployment.go new file mode 100644 index 0000000..29fa0e4 --- /dev/null +++ b/internal/store/memory_deployment.go @@ -0,0 +1,235 @@ +package store + +import ( + "context" + "sort" + "time" +) + +type deploymentReservation struct { + Owner string + Token int64 + Until time.Time +} + +func (m *Memory) deploymentFence(f JobFence) (JobRecord, error) { + j, ok := m.jobs[f.JobID] + if !ok || j.TenantID != f.TenantID || j.Status != "running" || j.OwnerID != f.OwnerID || j.FencingToken != f.Token || j.CancelRequested || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) { + return JobRecord{}, ErrJobLeaseLost + } + return j, nil +} + +func (m *Memory) BeginDeploymentSpeaker(ctx context.Context, f JobFence, speaker, revision string) (DeploymentSpeaker, error) { + if err := ctx.Err(); err != nil { + return DeploymentSpeaker{}, err + } + m.jobMu.Lock() + defer m.jobMu.Unlock() + j, err := m.deploymentFence(f) + if err != nil { + return DeploymentSpeaker{}, err + } + m.mu.Lock() + defer m.mu.Unlock() + sp, ok := m.speakers[speaker] + if !ok || sp.TenantID != f.TenantID { + return DeploymentSpeaker{}, ErrNotFound + } + r, ok := m.revisions[revision] + if !ok || r.TenantID != f.TenantID { + return DeploymentSpeaker{}, ErrNotFound + } + if m.deployments == nil { + m.deployments = make(map[string]DeploymentSpeaker) + m.deploymentCurrent = make(map[string]DeploymentSpeaker) + } + key := f.JobID + ":" + speaker + cur := m.deploymentCurrent[speaker] + if f.ExpectedGeneration != 0 && cur.JobID != f.JobID && (cur.Generation != f.ExpectedGeneration || cur.RevisionID != revision) { + return DeploymentSpeaker{}, ErrJobLeaseLost + } + reserve := func() error { + if m.deploymentReservations == nil { + m.deploymentReservations = make(map[string]deploymentReservation) + } + if r := m.deploymentReservations[speaker]; r.Until.After(time.Now()) { + return ErrDeploymentBusy + } + m.deploymentReservations[speaker] = deploymentReservation{Owner: f.OwnerID, Token: f.Token, Until: time.Now().Add(DeploymentDeliveryWindow)} + return nil + } + if existing, ok := m.deployments[key]; ok { + if cur.JobID != f.JobID || cur.Generation != existing.Generation || cur.RevisionID != revision { + return DeploymentSpeaker{}, ErrJobLeaseLost + } + if existing.State != "applied" { + if err := reserve(); err != nil { + return DeploymentSpeaker{}, err + } + } + return existing, nil + } + if f.ExpectedGeneration != 0 { + j.RequestSequence = cur.RequestSequence + } + if cur.RequestSequence >= j.RequestSequence && cur.JobID != "" && f.ExpectedGeneration == 0 { + return DeploymentSpeaker{}, ErrJobLeaseLost + } + if err := reserve(); err != nil { + return DeploymentSpeaker{}, err + } + d := DeploymentSpeaker{JobID: f.JobID, TenantID: f.TenantID, SpeakerID: speaker, RevisionID: revision, Generation: cur.Generation + 1, RequestSequence: j.RequestSequence, State: "published", RetryRound: f.RetryRound} + m.deployments[key] = d + m.deploymentCurrent[speaker] = d + m.publishedRevision[speaker] = publishedInfo{RevisionID: revision, PublishedAt: time.Now().UTC()} + return d, nil +} + +func (m *Memory) RecordDeploymentSpeaker(ctx context.Context, f JobFence, d DeploymentSpeaker, state, confirmation, message string) error { + if err := ctx.Err(); err != nil { + return err + } + m.jobMu.Lock() + defer m.jobMu.Unlock() + if _, err := m.deploymentFence(f); err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + cur := m.deploymentCurrent[d.SpeakerID] + if d.JobID != f.JobID || d.TenantID != f.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID { + return ErrJobLeaseLost + } + r := m.deploymentReservations[d.SpeakerID] + if r.Owner != f.OwnerID || r.Token != f.Token || !r.Until.After(time.Now()) { + return ErrJobLeaseLost + } + if state == "applied" && confirmation != "agent_reported" && confirmation != "local_applied" { + return ErrInvalidInput + } + if state != "dispatched" && state != "unknown" { + delete(m.deploymentReservations, d.SpeakerID) + } + cur.State, cur.Confirmation, cur.Error = state, confirmation, message + if state == "dispatched" { + cur.DispatchAttempts++ + } + if state == "applied" { + revision := d.RevisionID + m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision + } + m.deploymentCurrent[d.SpeakerID] = cur + m.deployments[d.JobID+":"+d.SpeakerID] = cur + return nil +} + +func (m *Memory) ListDeploymentSpeakers(ctx context.Context, tenant, job string) ([]DeploymentSpeaker, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + m.mu.RLock() + defer m.mu.RUnlock() + var out []DeploymentSpeaker + for _, d := range m.deployments { + if d.TenantID == tenant && d.JobID == job { + out = append(out, d) + } + } + return out, nil +} + +func (m *Memory) ListPendingDeployments(ctx context.Context, limit int) ([]DeploymentSpeaker, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + m.jobMu.Lock() + defer m.jobMu.Unlock() + m.mu.RLock() + defer m.mu.RUnlock() + var out []DeploymentSpeaker + for speaker, d := range m.deploymentCurrent { + j := m.jobs[d.JobID] + if d.State != "applied" && d.RetryRound < 3 && !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) { + out = append(out, d) + if len(out) >= limit { + break + } + } + } + return out, nil +} + +func (m *Memory) ListDeploymentObservations(ctx context.Context, limit int) ([]DeploymentSpeaker, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if limit <= 0 || limit > 100 { + limit = 10 + } + m.jobMu.Lock() + defer m.jobMu.Unlock() + m.mu.Lock() + defer m.mu.Unlock() + var out []DeploymentSpeaker + for speaker, d := range m.deploymentCurrent { + j := m.jobs[d.JobID] + if !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) { + out = append(out, d) + } + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i].HealthReceivedAt, out[j].HealthReceivedAt + if a == nil && b == nil { + return out[i].SpeakerID < out[j].SpeakerID + } + if a == nil { + return true + } + if b == nil { + return false + } + return a.Before(*b) + }) + if len(out) > limit { + out = out[:limit] + } + for i := range out { + out[i].ObservationToken++ + m.deploymentCurrent[out[i].SpeakerID] = out[i] + m.deployments[out[i].JobID+":"+out[i].SpeakerID] = out[i] + } + return out, nil +} + +func (m *Memory) ObserveDeployment(ctx context.Context, d DeploymentSpeaker, o DeploymentObservation) error { + if err := ctx.Err(); err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + cur := m.deploymentCurrent[d.SpeakerID] + if cur.TenantID != d.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || cur.ObservationToken != d.ObservationToken || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID || m.deploymentReservations[d.SpeakerID].Until.After(time.Now()) { + return ErrJobLeaseLost + } + now := time.Now().UTC() + cur.ObservationToken++ + cur.HealthReceivedAt = &now + cur.HealthRevisionID = o.RevisionID + cur.HealthSourceAt = o.SourceAt + cur.HealthError = o.Error + if o.Error == "" && o.RevisionID != "" { + if o.RevisionID == d.RevisionID { + cur.State = "applied" + cur.Confirmation = "agent_reported" + revision := d.RevisionID + m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision + } else { + cur.State = "unknown" + cur.Confirmation = "unconfirmed" + } + } + m.deploymentCurrent[d.SpeakerID] = cur + m.deployments[d.JobID+":"+d.SpeakerID] = cur + return nil +} diff --git a/internal/store/memory_job_queue.go b/internal/store/memory_job_queue.go new file mode 100644 index 0000000..f57b160 --- /dev/null +++ b/internal/store/memory_job_queue.go @@ -0,0 +1,405 @@ +package store + +import ( + "context" + "encoding/json" + "os" + "sort" + "strconv" + "sync" + "time" + + "github.com/google/uuid" +) + +func copyJob(j JobRecord) JobRecord { + if j.IdempotencyKey != nil { + v := *j.IdempotencyKey + j.IdempotencyKey = &v + } + if j.ModuleID != nil { + v := *j.ModuleID + j.ModuleID = &v + } + if j.Error != nil { + v := *j.Error + j.Error = &v + } + if j.StartedAt != nil { + v := *j.StartedAt + j.StartedAt = &v + } + if j.FinishedAt != nil { + v := *j.FinishedAt + j.FinishedAt = &v + } + if j.LeaseUntil != nil { + v := *j.LeaseUntil + j.LeaseUntil = &v + } + if j.Meta != nil { + raw, _ := json.Marshal(j.Meta) + var meta map[string]any + _ = json.Unmarshal(raw, &meta) + j.Meta = meta + } + return j +} + +func (m *Memory) EnqueueJob(_ context.Context, in JobEnqueue) (JobRecord, bool, error) { + h, err := JobRequestHash(in) + if err != nil { + return JobRecord{}, false, err + } + unlockTenant := m.lockJobEnqueueTenant(in.TenantID) + defer unlockTenant() + m.jobMu.Lock() + defer m.jobMu.Unlock() + if m.jobs == nil { + m.jobs = make(map[string]JobRecord) + m.jobHashes = make(map[string]string) + m.jobRequests = make(map[string]map[string]any) + } + if in.IdempotencyKey != nil && *in.IdempotencyKey != "" { + for _, j := range m.jobs { + if j.TenantID == in.TenantID && j.IdempotencyKey != nil && *j.IdempotencyKey == *in.IdempotencyKey { + if m.jobHashes[j.ID] != h { + return JobRecord{}, false, ErrJobConflict + } + return copyJob(j), false, nil + } + } + } + global, tenant := 0, 0 + for _, j := range m.jobs { + if j.Status == "queued" || j.Status == "running" { + global++ + if j.TenantID == in.TenantID { + tenant++ + } + } + } + if global >= jobLimitMemory("EVOBGP_JOB_MAX_PENDING", 10000) || tenant >= jobLimitMemory("EVOBGP_JOB_MAX_TENANT_PENDING", 1000) { + return JobRecord{}, false, ErrJobQueueFull + } + j := JobRecord{ID: uuid.NewString(), TenantID: in.TenantID, Kind: in.Kind, Status: "queued", IdempotencyKey: in.IdempotencyKey, ModuleID: in.ModuleID, Meta: in.Meta, CreatedAt: time.Now().UTC()} + j = copyJob(j) + m.jobSequence++ + j.RequestSequence = m.jobSequence + j.RequestHash = h + m.jobs[j.ID] = j + m.jobHashes[j.ID] = h + m.jobRequests[j.ID] = copyJob(j).Meta + return copyJob(j), true, nil +} + +func (m *Memory) ClaimJobs(_ context.Context, owner string, limit int, lease time.Duration) ([]JobRecord, error) { + m.jobMu.Lock() + defer m.jobMu.Unlock() + if limit <= 0 { + return nil, nil + } + if limit > 64 { + limit = 64 + } + if lease <= 0 { + lease = time.Minute + } + now := time.Now().UTC() + m.reapExpiredJobsLocked(now) + var all []JobRecord + for _, j := range m.jobs { + if j.Attempt < 3 && (j.Status == "queued" || (j.Status == "running" && j.LeaseUntil != nil && now.After(*j.LeaseUntil))) { + all = append(all, j) + } + } + sort.Slice(all, func(i, j int) bool { + if !all[i].CreatedAt.Equal(all[j].CreatedAt) { + return all[i].CreatedAt.Before(all[j].CreatedAt) + } + if all[i].RequestSequence != all[j].RequestSequence { + return all[i].RequestSequence < all[j].RequestSequence + } + return all[i].ID < all[j].ID + }) + if len(all) > limit { + all = all[:limit] + } + for i := range all { + j := all[i] + j.Status = "running" + j.OwnerID = owner + j.FencingToken++ + j.Attempt++ + if j.StartedAt == nil { + j.StartedAt = &now + } + until := now.Add(lease) + j.LeaseUntil = &until + m.jobs[j.ID] = j + all[i] = copyJob(j) + } + return all, nil +} + +// ReapExpiredJobs marks orphaned cancellations and exhausted retries terminal. +func (m *Memory) ReapExpiredJobs(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + m.jobMu.Lock() + defer m.jobMu.Unlock() + m.reapExpiredJobsLocked(time.Now().UTC()) + return nil +} + +func (m *Memory) reapExpiredJobsLocked(now time.Time) { + for id, j := range m.jobs { + if j.Status == "running" && j.LeaseUntil != nil && now.After(*j.LeaseUntil) && j.CancelRequested { + j.Status = "cancelled" + j.FinishedAt = &now + j.OwnerID = "" + j.LeaseUntil = nil + m.jobs[id] = j + continue + } + if (j.Status == "queued" || (j.Status == "running" && j.LeaseUntil != nil && now.After(*j.LeaseUntil))) && j.Attempt >= 3 { + j.Status = "failed" + msg := "retry limit reached" + j.Error = &msg + j.FinishedAt = &now + j.OwnerID = "" + j.LeaseUntil = nil + m.jobs[id] = j + } + } +} + +func (m *Memory) HeartbeatJob(_ context.Context, id, owner string, token int64, lease time.Duration) (bool, error) { + if lease <= 0 { + lease = time.Minute + } + m.jobMu.Lock() + defer m.jobMu.Unlock() + j, ok := m.jobs[id] + if !ok || j.OwnerID != owner || j.FencingToken != token || j.Status != "running" || j.LeaseUntil == nil || !time.Now().Before(*j.LeaseUntil) { + return false, ErrJobLeaseLost + } + until := time.Now().Add(lease) + j.LeaseUntil = &until + m.jobs[id] = j + return j.CancelRequested, nil +} + +func (m *Memory) FinishJob(_ context.Context, j JobRecord) error { + m.jobMu.Lock() + defer m.jobMu.Unlock() + old, ok := m.jobs[j.ID] + if !ok || old.OwnerID != j.OwnerID || old.FencingToken != j.FencingToken || old.Status != "running" || old.LeaseUntil == nil || !time.Now().Before(*old.LeaseUntil) { + return ErrJobLeaseLost + } + j.CreatedAt = old.CreatedAt + j.RequestSequence = old.RequestSequence + j.RequestHash = old.RequestHash + j.IdempotencyKey = old.IdempotencyKey + j.ModuleID = old.ModuleID + j.TenantID = old.TenantID + j.Kind = old.Kind + j.StartedAt = old.StartedAt + j.Attempt = old.Attempt + j.CancelRequested = old.CancelRequested + j.OwnerID = "" + j.LeaseUntil = nil + now := time.Now().UTC() + j.FinishedAt = &now + if old.CancelRequested { + j.Status = "cancelled" + j.Error = nil + } + m.jobs[j.ID] = copyJob(j) + return nil +} + +func (m *Memory) CheckpointJob(_ context.Context, in JobRecord) error { + m.jobMu.Lock() + defer m.jobMu.Unlock() + j, ok := m.jobs[in.ID] + if !ok || j.OwnerID != in.OwnerID || j.FencingToken != in.FencingToken || j.Status != "running" || j.LeaseUntil == nil || !time.Now().Before(*j.LeaseUntil) { + return ErrJobLeaseLost + } + j.Meta = copyJob(in).Meta + m.jobs[j.ID] = j + return nil +} + +func (m *Memory) ReleaseJob(_ context.Context, id, owner string, token int64) error { + m.jobMu.Lock() + defer m.jobMu.Unlock() + j, ok := m.jobs[id] + if !ok || j.OwnerID != owner || j.FencingToken != token || j.Status != "running" { + return ErrJobLeaseLost + } + j.Status = "queued" + if j.CancelRequested { + j.Status = "cancelled" + now := time.Now().UTC() + j.FinishedAt = &now + } + j.OwnerID = "" + j.LeaseUntil = nil + if j.Attempt > 0 { + j.Attempt-- + } + m.jobs[id] = j + return nil +} + +func (m *Memory) GetJob(_ context.Context, tenant, id string) (JobRecord, error) { + m.jobMu.Lock() + defer m.jobMu.Unlock() + j, ok := m.jobs[id] + if !ok || j.TenantID != tenant { + return JobRecord{}, ErrJobNotFound + } + return copyJob(j), nil +} + +func (m *Memory) ListJobs(_ context.Context, tenant, status, kind, cursor string, limit int) ([]JobRecord, string, bool, error) { + m.jobMu.Lock() + defer m.jobMu.Unlock() + var all []JobRecord + for _, j := range m.jobs { + if j.TenantID == tenant && (status == "" || j.Status == status) && (kind == "" || j.Kind == kind) { + all = append(all, copyJob(j)) + } + } + sort.Slice(all, func(i, j int) bool { + if !all[i].CreatedAt.Equal(all[j].CreatedAt) { + return all[i].CreatedAt.After(all[j].CreatedAt) + } + if all[i].RequestSequence != all[j].RequestSequence { + return all[i].RequestSequence > all[j].RequestSequence + } + return all[i].ID > all[j].ID + }) + off, _ := strconv.Atoi(cursor) + if off < 0 { + off = 0 + } + if limit <= 0 || limit > 100 { + limit = 50 + } + if off >= len(all) { + return nil, "", false, nil + } + end := off + limit + more := end < len(all) + if end > len(all) { + end = len(all) + } + next := "" + if more { + next = strconv.Itoa(end) + } + return all[off:end], next, more, nil +} + +func (m *Memory) CancelJob(_ context.Context, tenant, id string) (JobRecord, error) { + m.jobMu.Lock() + defer m.jobMu.Unlock() + j, ok := m.jobs[id] + if !ok || j.TenantID != tenant { + return JobRecord{}, ErrJobNotFound + } + if j.Status == "queued" { + now := time.Now().UTC() + j.Status = "cancelled" + j.FinishedAt = &now + } + if j.Status == "running" { + j.CancelRequested = true + } + m.jobs[id] = j + return copyJob(j), nil +} + +func (m *Memory) RetryJob(_ context.Context, tenant, id string) (JobRecord, error) { + m.jobMu.Lock() + defer m.jobMu.Unlock() + j, ok := m.jobs[id] + if !ok || j.TenantID != tenant { + return JobRecord{}, ErrJobNotFound + } + if j.Status != "failed" { + return JobRecord{}, ErrJobConflict + } + global, pending := 0, 0 + for _, item := range m.jobs { + if item.Status == "queued" || item.Status == "running" { + global++ + if item.TenantID == tenant { + pending++ + } + } + } + if global >= jobLimitMemory("EVOBGP_JOB_MAX_PENDING", 10000) || pending >= jobLimitMemory("EVOBGP_JOB_MAX_TENANT_PENDING", 1000) { + return JobRecord{}, ErrJobQueueFull + } + j.Status = "queued" + j.Error = nil + j.FinishedAt = nil + j.CancelRequested = false + j.Attempt = 0 + j.StartedAt = nil + j.OwnerID = "" + j.LeaseUntil = nil + j.Meta = copyJob(JobRecord{Meta: m.jobRequests[id]}).Meta + m.jobs[id] = j + return copyJob(j), nil +} + +// lockJobEnqueueTenant serializes idempotency and the tenant pending limit. +// The global jobMu is still required to mutate the shared job map. +func (m *Memory) lockJobEnqueueTenant(tenantID string) func() { + m.jobTenantMu.Lock() + if m.jobTenantLocks == nil { + m.jobTenantLocks = make(map[string]*sync.Mutex) + } + mu := m.jobTenantLocks[tenantID] + if mu == nil { + mu = &sync.Mutex{} + m.jobTenantLocks[tenantID] = mu + } + m.jobTenantMu.Unlock() + mu.Lock() + return mu.Unlock +} + +func jobLimitMemory(name string, fallback int) int { + n, e := strconv.Atoi(os.Getenv(name)) + if e == nil && n > 0 { + return n + } + return fallback +} + +func (m *Memory) CountJobsByStatus(_ context.Context) (map[string]int64, error) { + m.jobMu.Lock() + defer m.jobMu.Unlock() + out := map[string]int64{} + for _, j := range m.jobs { + out[j.Status]++ + } + return out, nil +} + +func (m *Memory) FindJobByIdempotencyKey(_ context.Context, tenant, key string) (JobRecord, error) { + m.jobMu.Lock() + defer m.jobMu.Unlock() + for _, j := range m.jobs { + if j.TenantID == tenant && j.IdempotencyKey != nil && *j.IdempotencyKey == key { + return copyJob(j), nil + } + } + return JobRecord{}, ErrJobNotFound +} diff --git a/internal/store/memory_job_queue_test.go b/internal/store/memory_job_queue_test.go new file mode 100644 index 0000000..0c93b0a --- /dev/null +++ b/internal/store/memory_job_queue_test.go @@ -0,0 +1,122 @@ +package store + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +func TestEnqueueJobTenantsDoNotShareLock(t *testing.T) { + m := NewMemory() + ctx := context.Background() + unlock := m.lockJobEnqueueTenant("tenant-a") + var once sync.Once + release := func() { once.Do(unlock) } + t.Cleanup(release) + + otherDone := make(chan error, 1) + go func() { + _, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: "tenant-b", Kind: "test"}) + otherDone <- err + }() + select { + case err := <-otherDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("other tenant waited on the held tenant lock") + } + + sameDone := make(chan struct{}) + go func() { + _, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: "tenant-a", Kind: "test"}) + if err != nil { + t.Error(err) + } + close(sameDone) + }() + select { + case <-sameDone: + t.Fatal("same tenant did not wait on its lock") + case <-time.After(200 * time.Millisecond): + } + release() + select { + case <-sameDone: + case <-time.After(2 * time.Second): + t.Fatal("same tenant did not finish after unlock") + } +} + +func TestEnqueueJobIdempotencyKeyConflicts(t *testing.T) { + m := NewMemory() + ctx := context.Background() + key := "same-key" + in := JobEnqueue{TenantID: "tenant-a", Kind: "refresh", IdempotencyKey: &key, Meta: map[string]any{"module": "a"}} + first, created, err := m.EnqueueJob(ctx, in) + if err != nil || !created || first.ID == "" { + t.Fatalf("first=%+v created=%v err=%v", first, created, err) + } + in.Meta = map[string]any{"module": "b"} + if _, _, err = m.EnqueueJob(ctx, in); !errors.Is(err, ErrJobConflict) { + t.Fatalf("same key accepted: %v", err) + } + other := JobEnqueue{TenantID: "tenant-b", Kind: "refresh", IdempotencyKey: &key, Meta: map[string]any{"module": "b"}} + if _, created, err = m.EnqueueJob(ctx, other); err != nil || !created { + t.Fatalf("other tenant created=%v err=%v", created, err) + } +} + +func TestMemoryJobQueueEqualTimestampOrdering(t *testing.T) { + m := NewMemory() + ctx := context.Background() + stamp := time.Date(2026, 10, 26, 0, 0, 0, 0, time.UTC) + var ids []string + for range 32 { + j, _, err := m.EnqueueJob(ctx, JobEnqueue{TenantID: "tenant", Kind: "test"}) + if err != nil { + t.Fatal(err) + } + ids = append(ids, j.ID) + m.jobMu.Lock() + saved := m.jobs[j.ID] + saved.CreatedAt = stamp + m.jobs[j.ID] = saved + m.jobMu.Unlock() + } + // Repeated offset pages must use the same total order despite map iteration. + for range 5 { + cursor := "" + var listed []string + for { + page, next, more, err := m.ListJobs(ctx, "tenant", "", "", cursor, 3) + if err != nil { + t.Fatal(err) + } + for _, j := range page { + listed = append(listed, j.ID) + } + if !more { + break + } + cursor = next + } + if len(listed) != len(ids) { + t.Fatalf("listed %d jobs", len(listed)) + } + for i, id := range listed { + if id != ids[len(ids)-1-i] { + t.Fatalf("list[%d]=%s expected reverse enqueue sequence %s", i, id, ids[len(ids)-1-i]) + } + } + } + for i, want := range ids { + claimed, err := m.ClaimJobs(ctx, "owner", 1, time.Minute) + if err != nil || len(claimed) != 1 || claimed[0].ID != want { + t.Fatalf("claim[%d]=%+v %v want %s", i, claimed, err, want) + } + } +} diff --git a/internal/store/memory_revision_prune.go b/internal/store/memory_revision_prune.go index dbdac0c..609995f 100644 --- a/internal/store/memory_revision_prune.go +++ b/internal/store/memory_revision_prune.go @@ -1,6 +1,7 @@ package store import ( + "context" "strings" "time" ) @@ -109,7 +110,13 @@ func (m *Memory) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (Rev }, nil } -func (m *Memory) PruneRevisionsBefore(tenantID string, cutoff time.Time) (int, error) { +func (m *Memory) PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (int, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return 0, err + } res, err := m.PruneRevisionsWithStats(tenantID, cutoff) if err != nil { return 0, err diff --git a/internal/store/memory_revision_prune_test.go b/internal/store/memory_revision_prune_test.go new file mode 100644 index 0000000..e8c22bc --- /dev/null +++ b/internal/store/memory_revision_prune_test.go @@ -0,0 +1,22 @@ +package store + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestPruneRevisionsBeforeCancelledContext(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, _, _, rev, _ := m.DemoIDs() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := m.PruneRevisionsBefore(ctx, tenant, time.Now().Add(time.Hour)); !errors.Is(err, context.Canceled) { + t.Fatalf("err=%v", err) + } + if _, err := m.GetRevision(tenant, rev); err != nil { + t.Fatal(err) + } +} diff --git a/internal/store/memory_snapshot.go b/internal/store/memory_snapshot.go index 18898e2..f6a7c06 100644 --- a/internal/store/memory_snapshot.go +++ b/internal/store/memory_snapshot.go @@ -1,12 +1,43 @@ package store import ( + "context" "strings" "sync" "time" ) -func (m *Memory) GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error) { +// ModuleSnapshotUpdate computes a replacement from the current snapshot; nil deletes it. +// The callback must not perform store I/O or mutate its input. +type ModuleSnapshotUpdate func(current *ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) + +// ListModuleSnapshotSourceRows returns persisted rows for one source key. +func (m *Memory) ListModuleSnapshotSourceRows(ctx context.Context, tenantID, moduleID, source string) ([]PrefixRow, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + m.mu.RLock() + defer m.mu.RUnlock() + snap := m.moduleSnapshots[moduleSnapshotKey(tenantID, moduleID)] + if snap == nil { + return nil, nil + } + var out []PrefixRow + for _, row := range snap.Prefixes { + if row.Source == source { + out = append(out, row) + } + } + return out, nil +} + +func (m *Memory) GetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, err + } m.mu.RLock() defer m.mu.RUnlock() key := moduleSnapshotKey(tenantID, moduleID) @@ -19,42 +50,95 @@ func (m *Memory) GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePref return &ModulePrefixSnapshot{ InputHash: snap.InputHash, CollectedAt: snap.CollectedAt, + Version: snap.Version, Prefixes: cp, }, true, nil } -func (m *Memory) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error { +func (m *Memory) SetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID, inputHash string, prefixes []PrefixRow) error { if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" || strings.TrimSpace(inputHash) == "" { return ErrInvalidInput } + return m.UpdateModulePrefixSnapshot(ctx, tenantID, moduleID, func(*ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) { + return &ModulePrefixSnapshot{InputHash: inputHash, Prefixes: prefixes}, nil + }) +} + +func (m *Memory) DeleteModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) error { + return m.UpdateModulePrefixSnapshot(ctx, tenantID, moduleID, func(*ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) { + return nil, nil + }) +} + +// UpdateModulePrefixSnapshot serializes a pure RMW callback and commits snapshot plus input hash together. +func (m *Memory) UpdateModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string, update ModuleSnapshotUpdate) error { + if update == nil || strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" { + return ErrInvalidInput + } + unlock, err := m.lockModuleSnapshot(ctx, tenantID, moduleID) + if err != nil { + return err + } + defer unlock() + current, _, err := m.GetModulePrefixSnapshot(ctx, tenantID, moduleID) + if err != nil { + return err + } + next, err := update(current) + if err != nil { + return err + } + if next != nil && strings.TrimSpace(next.InputHash) == "" { + return ErrInvalidInput + } + if err := ctx.Err(); err != nil { + return err + } m.mu.Lock() defer m.mu.Unlock() - if m.moduleSnapshots == nil { - m.moduleSnapshots = make(map[string]*moduleSnapshotRec) + if err := ctx.Err(); err != nil { + return err + } + mod := m.modules[moduleID] + if mod == nil || mod.DeletedAt != nil { + return ErrNotFound + } + if mod.TenantID != tenantID { + return ErrTenantScope } - cp := make([]PrefixRow, len(prefixes)) - copy(cp, prefixes) key := moduleSnapshotKey(tenantID, moduleID) - m.moduleSnapshots[key] = &moduleSnapshotRec{ - InputHash: inputHash, - CollectedAt: time.Now().UTC(), - Prefixes: cp, + if next == nil { + if m.moduleSnapshots[key] != nil { + m.bumpRenderGenerationLocked(tenantID) + } + delete(m.moduleSnapshots, key) + } else if !SameSnapshotContent(current, next) { + m.bumpRenderGenerationLocked(tenantID) + if m.moduleSnapshots == nil { + m.moduleSnapshots = make(map[string]*moduleSnapshotRec) + } + m.nextModuleSnapshotVersion++ + m.moduleSnapshots[key] = &moduleSnapshotRec{InputHash: next.InputHash, CollectedAt: time.Now().UTC(), Version: m.nextModuleSnapshotVersion, Prefixes: append([]PrefixRow(nil), next.Prefixes...)} + } + if next == nil { + mod.InputHash = "" + } else { + mod.InputHash = next.InputHash } return nil } -func (m *Memory) DeleteModulePrefixSnapshot(tenantID, moduleID string) error { - m.mu.Lock() - defer m.mu.Unlock() - if m.moduleSnapshots != nil { - delete(m.moduleSnapshots, moduleSnapshotKey(tenantID, moduleID)) +func (m *Memory) SetModuleInputHash(ctx context.Context, tenantID, moduleID, hash string) error { + unlock, err := m.lockModuleSnapshot(ctx, tenantID, moduleID) + if err != nil { + return err } - return nil -} - -func (m *Memory) SetModuleInputHash(tenantID, moduleID, hash string) error { + defer unlock() m.mu.Lock() defer m.mu.Unlock() + if err := ctx.Err(); err != nil { + return err + } mod, ok := m.modules[moduleID] if !ok || mod.DeletedAt != nil { return ErrNotFound @@ -66,20 +150,88 @@ func (m *Memory) SetModuleInputHash(tenantID, moduleID, hash string) error { return nil } -func (m *Memory) LockModuleSnapshot(tenantID, moduleID string) func() { +type snapshotLockEntry struct { + gate chan struct{} + refs int +} + +// lockModuleSnapshot waits until the caller deadline, or EVOBGP_SNAPSHOT_TIMEOUT when unset. +func (m *Memory) lockModuleSnapshot(ctx context.Context, tenantID, moduleID string) (func(), error) { + if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" { + return nil, ErrInvalidInput + } + if ctx == nil { + ctx = context.Background() + } + ctx, cancel, _ := SnapshotWriteBudget(ctx) + defer cancel() key := moduleSnapshotKey(tenantID, moduleID) - v, _ := m.snapshotLocks.LoadOrStore(key, &sync.Mutex{}) - mu := v.(*sync.Mutex) - mu.Lock() - return func() { mu.Unlock() } + m.snapshotLocksMu.Lock() + if m.snapshotLocks == nil { + m.snapshotLocks = make(map[string]*snapshotLockEntry) + } + entry := m.snapshotLocks[key] + if entry == nil { + entry = &snapshotLockEntry{gate: make(chan struct{}, 1)} + entry.gate <- struct{}{} + m.snapshotLocks[key] = entry + } + entry.refs++ + m.snapshotLocksMu.Unlock() + select { + case <-ctx.Done(): + m.releaseSnapshotLockRef(key, entry) + return nil, ctx.Err() + case <-entry.gate: + if err := ctx.Err(); err != nil { + entry.gate <- struct{}{} + m.releaseSnapshotLockRef(key, entry) + return nil, err + } + } + var once sync.Once + return func() { + once.Do(func() { + entry.gate <- struct{}{} + m.releaseSnapshotLockRef(key, entry) + }) + }, nil +} + +func (m *Memory) releaseSnapshotLockRef(key string, entry *snapshotLockEntry) { + m.snapshotLocksMu.Lock() + defer m.snapshotLocksMu.Unlock() + entry.refs-- + if entry.refs == 0 { + delete(m.snapshotLocks, key) + } } type moduleSnapshotRec struct { InputHash string CollectedAt time.Time + Version int64 Prefixes []PrefixRow } +// SameSnapshotContent compares persisted rows, ignoring timestamps, versions, and order. +// Rows are matched by (prefix, community, source). +func SameSnapshotContent(a, b *ModulePrefixSnapshot) bool { + if a == nil || b == nil || a.InputHash != b.InputHash || len(a.Prefixes) != len(b.Prefixes) { + return false + } + ap := append([]PrefixRow(nil), a.Prefixes...) + bp := append([]PrefixRow(nil), b.Prefixes...) + sortPrefixRows(ap) + sortPrefixRows(bp) + for i := range ap { + if !samePrefixRow(ap[i], bp[i]) { + return false + } + } + return true +} + func moduleSnapshotKey(tenantID, moduleID string) string { return strings.TrimSpace(tenantID) + "\x00" + strings.TrimSpace(moduleID) } diff --git a/internal/store/memory_snapshot_atomic_test.go b/internal/store/memory_snapshot_atomic_test.go new file mode 100644 index 0000000..2e7c35c --- /dev/null +++ b/internal/store/memory_snapshot_atomic_test.go @@ -0,0 +1,76 @@ +package store + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" +) + +func TestMemoryModuleSnapshotAtomicConcurrentRMW(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + mod, err := m.CreateModule(tenant, &Module{Type: "CDN_CIDRS", Name: "atomic", Enabled: true}) + if err != nil { + t.Fatal(err) + } + const writers = 12 + var wg sync.WaitGroup + errCh := make(chan error, writers) + for i := 0; i < writers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + errCh <- m.UpdateModulePrefixSnapshot(context.Background(), tenant, mod.ID, func(current *ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) { + var rows []PrefixRow + if current != nil { + rows = append(rows, current.Prefixes...) + } + rows = append(rows, PrefixRow{Prefix: fmt.Sprintf("192.0.2.%d/32", i)}) + return &ModulePrefixSnapshot{InputHash: "atomic", Prefixes: rows}, nil + }) + }(i) + } + wg.Wait() + close(errCh) + for err := range errCh { + if err != nil { + t.Fatal(err) + } + } + snap, ok, err := m.GetModulePrefixSnapshot(context.Background(), tenant, mod.ID) + if err != nil || !ok || len(snap.Prefixes) != writers { + t.Fatalf("lost concurrent update: %+v ok=%v err=%v", snap, ok, err) + } + if got, err := m.GetModule(tenant, mod.ID); err != nil || got == nil || got.InputHash != snap.InputHash { + t.Fatalf("module input hash differs from snapshot: %+v %+v", got, snap) + } +} + +func TestMemoryModuleSnapshotAtomicCallbackErrorRollsBack(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + mod, err := m.CreateModule(tenant, &Module{Type: "CDN_CIDRS", Name: "rollback", Enabled: true}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err := m.SetModulePrefixSnapshot(ctx, tenant, mod.ID, "before", []PrefixRow{{Prefix: "192.0.2.0/24"}}); err != nil { + t.Fatal(err) + } + want := errors.New("update failed") + err = m.UpdateModulePrefixSnapshot(ctx, tenant, mod.ID, func(current *ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) { + current.Prefixes[0].Prefix = "198.51.100.0/24" + return nil, want + }) + if !errors.Is(err, want) { + t.Fatalf("callback error: %v", err) + } + snap, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, mod.ID) + if err != nil || !ok || snap.InputHash != "before" || snap.Prefixes[0].Prefix != "192.0.2.0/24" { + t.Fatalf("partial snapshot visible: %+v ok=%v err=%v", snap, ok, err) + } +} diff --git a/internal/store/memory_snapshot_context_test.go b/internal/store/memory_snapshot_context_test.go new file mode 100644 index 0000000..799a3ee --- /dev/null +++ b/internal/store/memory_snapshot_context_test.go @@ -0,0 +1,33 @@ +package store + +import ( + "context" + "errors" + "testing" +) + +func TestMemorySnapshotMethodsRespectCancellation(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, _, module, _, _ := m.DemoIDs() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if snap, ok, err := m.GetModulePrefixSnapshot(ctx, tenant, module); snap != nil || ok || !errors.Is(err, context.Canceled) { + t.Fatalf("get: %+v %v %v", snap, ok, err) + } + if err := m.SetModulePrefixSnapshot(ctx, tenant, module, "hash", []PrefixRow{{Prefix: "192.0.2.0/24"}}); !errors.Is(err, context.Canceled) { + t.Fatalf("set: %v", err) + } + if err := m.DeleteModulePrefixSnapshot(ctx, tenant, module); !errors.Is(err, context.Canceled) { + t.Fatalf("delete: %v", err) + } + if err := m.SetModuleInputHash(ctx, tenant, module, "hash"); !errors.Is(err, context.Canceled) { + t.Fatalf("input hash: %v", err) + } + if rows, err := m.ListModuleSnapshotSourceRows(ctx, tenant, module, "ip_range"); rows != nil || !errors.Is(err, context.Canceled) { + t.Fatalf("source rows: %+v %v", rows, err) + } + if snap, ok, err := m.GetModulePrefixSnapshot(context.Background(), tenant, module); snap != nil || ok || err != nil { + t.Fatalf("cancelled write committed: %+v %v %v", snap, ok, err) + } +} diff --git a/internal/store/memory_snapshot_lock_test.go b/internal/store/memory_snapshot_lock_test.go new file mode 100644 index 0000000..2b5f502 --- /dev/null +++ b/internal/store/memory_snapshot_lock_test.go @@ -0,0 +1,59 @@ +package store + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestMemoryModuleSnapshotLockCancellationAndCleanup(t *testing.T) { + m := NewMemory() + first, err := m.lockModuleSnapshot(context.Background(), "tenant-a", "module-a") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + second, err := m.lockModuleSnapshot(ctx, "tenant-a", "module-a") + if second != nil || !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("waiter not cancelled: unlock=%v err=%v", second != nil, err) + } + if len(m.snapshotLocks) != 1 || m.snapshotLocks[moduleSnapshotKey("tenant-a", "module-a")].refs != 1 { + t.Fatalf("waiter leaked lock ref: %+v", m.snapshotLocks) + } + first() + first() + if len(m.snapshotLocks) != 0 { + t.Fatalf("idle lock leaked: %+v", m.snapshotLocks) + } + third, err := m.lockModuleSnapshot(context.Background(), "tenant-a", "module-a") + if err != nil { + t.Fatal(err) + } + third() +} + +func TestMemoryModuleSnapshotLockScopesTenantAndModule(t *testing.T) { + m := NewMemory() + first, err := m.lockModuleSnapshot(context.Background(), "tenant-a", "module-x") + if err != nil { + t.Fatal(err) + } + defer first() + for _, pair := range [][2]string{{"tenant-b", "module-x"}, {"tenant-a", "module-y"}} { + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + unlock, err := m.lockModuleSnapshot(ctx, pair[0], pair[1]) + cancel() + if err != nil { + t.Fatalf("unrelated key blocked %v: %v", pair, err) + } + unlock() + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + unlock, err := m.lockModuleSnapshot(ctx, "tenant-a", "module-x") + if unlock != nil || !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled context acquired: unlock=%v err=%v", unlock != nil, err) + } +} diff --git a/internal/store/memory_telemetry.go b/internal/store/memory_telemetry.go new file mode 100644 index 0000000..775ba67 --- /dev/null +++ b/internal/store/memory_telemetry.go @@ -0,0 +1,284 @@ +package store + +import ( + "context" + "sort" + "strings" + "time" + + "github.com/google/uuid" +) + +type telemetryMemoryRow struct { + snapshot TelemetrySnapshot + owner string + leaseUntil time.Time + token int64 + lastRequested time.Time + refreshPending bool +} + +func telemetryKey(tenantID, targetID string) string { return tenantID + "/" + targetID } + +func (m *Memory) ensureTelemetryRows(localAuthority string) { + if m.telemetry == nil { + m.telemetry = make(map[string]telemetryMemoryRow) + } + for _, sp := range m.speakers { + if sp == nil || !SpeakerNeedsRemoteDispatch(sp.Role, ParseSpeakerMeta(sp.MetaJSON)) { + continue + } + key := telemetryKey(sp.TenantID, sp.ID) + if _, ok := m.telemetry[key]; !ok { + m.telemetry[key] = telemetryMemoryRow{snapshot: TelemetrySnapshot{TenantID: sp.TenantID, TargetID: sp.ID, SpeakerID: sp.ID, Kind: "remote_agent"}} + } + } + if localAuthority != "" { + for tenantID := range m.tenants { + if tenantID == SystemJobTenantID { + continue + } + targetID := TelemetryLocalTargetID(tenantID, localAuthority) + key := telemetryKey(tenantID, targetID) + if _, ok := m.telemetry[key]; !ok { + m.telemetry[key] = telemetryMemoryRow{snapshot: TelemetrySnapshot{TenantID: tenantID, TargetID: targetID, Kind: "local_bird", LocalAuthority: localAuthority}} + } + } + } +} + +// ClaimTelemetry acquires at most limit due targets with a short fencing lease. +func (m *Memory) ClaimTelemetry(_ context.Context, owner string, limit int, lease time.Duration, localAuthority string) ([]TelemetryClaim, error) { + if strings.TrimSpace(owner) == "" || limit <= 0 { + return nil, ErrInvalidInput + } + if limit > 64 { + limit = 64 + } + if lease <= 0 { + lease = 15 * time.Second + } + m.mu.Lock() + defer m.mu.Unlock() + m.ensureTelemetryRows(localAuthority) + keys := make([]string, 0, len(m.telemetry)) + for key := range m.telemetry { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + a, b := m.telemetry[keys[i]], m.telemetry[keys[j]] + if !a.snapshot.NextPollAt.Equal(b.snapshot.NextPollAt) { + return a.snapshot.NextPollAt.Before(b.snapshot.NextPollAt) + } + return keys[i] < keys[j] + }) + now := time.Now().UTC() + out := make([]TelemetryClaim, 0, limit) + for _, key := range keys { + row := m.telemetry[key] + if row.snapshot.Kind == "local_bird" && row.snapshot.LocalAuthority != localAuthority || row.snapshot.NextPollAt.After(now) || row.leaseUntil.After(now) { + continue + } + var speaker *Speaker + if row.snapshot.Kind == "remote_agent" { + sp := m.speakers[row.snapshot.SpeakerID] + if sp == nil || !SpeakerNeedsRemoteDispatch(sp.Role, ParseSpeakerMeta(sp.MetaJSON)) { + delete(m.telemetry, key) + continue + } + cp := *sp + speaker = &cp + } else { + for _, id := range sortedSpeakerIDs(m.speakers) { + sp := m.speakers[id] + if sp.TenantID == row.snapshot.TenantID && strings.EqualFold(sp.Role, "master") { + cp := *sp + speaker = &cp + break + } + } + } + row.owner = owner + row.token++ + row.leaseUntil = now.Add(lease) + m.telemetry[key] = row + claim := TelemetryClaim{TelemetrySnapshot: row.snapshot, Speaker: speaker, OwnerID: owner, FencingToken: row.token} + if speaker != nil { + claim.SpeakerID = speaker.ID + } + claim.ConfigFingerprint = TelemetrySpeakerFingerprint(speaker) + out = append(out, claim) + if len(out) == limit { + break + } + } + return out, nil +} + +// CompleteTelemetry commits only the current live lease and preserves last success on failure. +func (m *Memory) CompleteTelemetry(_ context.Context, claim TelemetryClaim, result TelemetryResult, nextPoll time.Time) error { + m.mu.Lock() + defer m.mu.Unlock() + key := telemetryKey(claim.TenantID, claim.TargetID) + row, ok := m.telemetry[key] + now := time.Now().UTC() + if !ok || row.owner != claim.OwnerID || row.token != claim.FencingToken || !row.leaseUntil.After(now) { + return ErrTelemetryLeaseLost + } + if claim.Speaker != nil { + current := m.speakers[claim.Speaker.ID] + if TelemetrySpeakerFingerprint(current) != claim.ConfigFingerprint { + return ErrTelemetryLeaseLost + } + } + if claim.Kind == "local_bird" { + masterID := "" + for _, id := range sortedSpeakerIDs(m.speakers) { + sp := m.speakers[id] + if sp.TenantID == claim.TenantID && strings.EqualFold(sp.Role, "master") { + masterID = id + break + } + } + if masterID != claim.SpeakerID { + return ErrTelemetryLeaseLost + } + } + if row.snapshot.ConfigFingerprint != claim.ConfigFingerprint { + row.snapshot.BGP = TelemetryBGP{} + row.snapshot.Health = TelemetryHealth{} + row.snapshot.DiscoveryCursor = 0 + } + row.snapshot.ConfigFingerprint = claim.ConfigFingerprint + row.snapshot.SpeakerID = claim.SpeakerID + if result.BGP.Source != "" { + if result.BGP.Error != "" { + result.BGP.Sessions = nil + result.BGP.LastSuccessAt = row.snapshot.BGP.LastSuccessAt + } else { + result.BGP.LastSuccessAt = result.BGP.ReceivedAt + } + row.snapshot.BGP = result.BGP + } + if result.Health.Source != "" { + if !result.Health.OK { + result.Health.LastSuccessAt = row.snapshot.Health.LastSuccessAt + } else { + result.Health.LastSuccessAt = result.Health.ReceivedAt + } + row.snapshot.Health = result.Health + } + if row.snapshot.BGP.Error != "" || row.snapshot.Health.Source != "" && !row.snapshot.Health.OK { + row.snapshot.FailureCount++ + } else { + row.snapshot.FailureCount = 0 + } + row.snapshot.NextPollAt = nextPoll.UTC() + if row.refreshPending { + row.snapshot.NextPollAt = now + row.refreshPending = false + } + row.owner = "" + row.leaseUntil = time.Time{} + if result.BGP.Source != "" && result.BGP.Error == "" { + row.snapshot.DiscoveryCursor = m.writeTelemetryDiscoveriesLocked(claim, result.BGP, row.snapshot.DiscoveryCursor) + } + m.telemetry[key] = row + return nil +} + +func sortedSpeakerIDs(speakers map[string]*Speaker) []string { + ids := make([]string, 0, len(speakers)) + for id := range speakers { + ids = append(ids, id) + } + sort.Strings(ids) + return ids +} + +func (m *Memory) writeTelemetryDiscoveriesLocked(claim TelemetryClaim, bgp TelemetryBGP, cursor int) int { + sessions, next := TelemetryDiscoveryBatch(bgp.Sessions, cursor, 256) + known := make(map[string]struct{}) + for _, p := range m.peers { + if p == nil || p.TenantID != claim.TenantID { + continue + } + if neighbor, ok := NormalizePeerNeighborString(p.Neighbor); ok { + known[neighbor] = struct{}{} + } + } + for _, sess := range sessions { + neighbor, ok := NormalizePeerNeighborString(sess.Neighbor) + if !ok { + continue + } + if _, ok := known[neighbor]; ok { + continue + } + neighborID := strings.TrimSpace(sess.NeighborID) + d := m.findPeerDiscoveryLocked(claim.TenantID, neighborID, neighbor, sess.NeighborAS) + if d == nil { + id := uuid.NewString() + m.peerDiscoveries[id] = &BGPPeerDiscovery{ID: id, TenantID: claim.TenantID, SpeakerID: claim.SpeakerID, NeighborID: neighborID, Neighbor: neighbor, RemoteASN: sess.NeighborAS, ProtocolName: sess.Name, SessionState: sess.State, Status: PeerDiscoveryPending, FirstSeenAt: bgp.ReceivedAt, LastSeenAt: bgp.ReceivedAt} + continue + } + if d.Status == PeerDiscoveryRejected || d.LastSeenAt.After(bgp.ReceivedAt) { + continue + } + d.ProtocolName = sess.Name + d.SessionState = sess.State + d.LastSeenAt = bgp.ReceivedAt + if claim.SpeakerID != "" { + d.SpeakerID = claim.SpeakerID + } + } + return next +} + +// ListTelemetry reads the last committed snapshots without polling agents. +func (m *Memory) ListTelemetry(_ context.Context, tenantID string) ([]TelemetrySnapshot, error) { + m.mu.RLock() + defer m.mu.RUnlock() + out := make([]TelemetrySnapshot, 0) + for _, row := range m.telemetry { + if row.snapshot.TenantID != tenantID { + continue + } + if row.snapshot.Kind == "remote_agent" { + sp := m.speakers[row.snapshot.SpeakerID] + if sp == nil || !SpeakerNeedsRemoteDispatch(sp.Role, ParseSpeakerMeta(sp.MetaJSON)) { + continue + } + } + snap := row.snapshot + snap.BGP.Sessions = append(snap.BGP.Sessions[:0:0], snap.BGP.Sessions...) + out = append(out, snap) + } + sort.Slice(out, func(i, j int) bool { return out[i].TargetID < out[j].TargetID }) + return out, nil +} + +// RequestTelemetryRefresh coalesces repeated requests per tenant within cooldown. +func (m *Memory) RequestTelemetryRefresh(_ context.Context, tenantID string, cooldown time.Duration) (bool, error) { + if cooldown <= 0 { + cooldown = 10 * time.Second + } + m.mu.Lock() + defer m.mu.Unlock() + now := time.Now().UTC() + changed := false + for key, row := range m.telemetry { + if row.snapshot.TenantID != tenantID || now.Sub(row.lastRequested) < cooldown { + continue + } + row.lastRequested = now + if row.leaseUntil.After(now) { + row.refreshPending = true + } else { + row.snapshot.NextPollAt = now + } + m.telemetry[key] = row + changed = true + } + return changed, nil +} diff --git a/internal/store/memory_telemetry_test.go b/internal/store/memory_telemetry_test.go new file mode 100644 index 0000000..aef6540 --- /dev/null +++ b/internal/store/memory_telemetry_test.go @@ -0,0 +1,172 @@ +package store + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "evobgp/internal/birdfmt" +) + +func TestMemoryTelemetryFencingRefreshAndDiscovery(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + sp, err := m.CreateSpeaker(tenant, &Speaker{Role: "replica", MetaJSON: `{"agent_domain":"node.test","agent_secret":"secret"}`}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + claims, err := m.ClaimTelemetry(ctx, "owner-a", 2, time.Minute, "") + if err != nil || len(claims) != 1 || claims[0].SpeakerID != sp.ID { + t.Fatalf("claims=%+v err=%v", claims, err) + } + first := claims[0] + if snaps, err := m.ListTelemetry(ctx, tenant); err != nil || len(snaps) != 1 || snaps[0].ConfigFingerprint != "" { + t.Fatalf("uncommitted config visible: %+v %v", snaps, err) + } + if other, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, ""); err != nil || len(other) != 0 { + t.Fatalf("stole lease: %+v %v", other, err) + } + if requested, err := m.RequestTelemetryRefresh(ctx, tenant, time.Minute); err != nil || !requested { + t.Fatalf("refresh=%v err=%v", requested, err) + } + if requested, err := m.RequestTelemetryRefresh(ctx, tenant, time.Minute); err != nil || requested { + t.Fatalf("not coalesced: %v %v", requested, err) + } + at := time.Now().UTC() + result := TelemetryResult{BGP: TelemetryBGP{TelemetryObservation: TelemetryObservation{Source: "agent_bird_protocols", ObservedAt: at, ReceivedAt: at}, Sessions: []birdfmt.BGPSession{{Name: birdfmt.DynamicPeerProtocolPrefix + "test", Neighbor: "192.0.2.1", NeighborAS: 64501, State: "Established"}}}, Health: TelemetryHealth{TelemetryObservation: TelemetryObservation{Source: "agent_health", ObservedAt: at, ReceivedAt: at}, OK: true}} + if err := m.CompleteTelemetry(ctx, first, result, at.Add(time.Hour)); err != nil { + t.Fatal(err) + } + if err := m.CompleteTelemetry(ctx, first, result, at.Add(time.Hour)); !errors.Is(err, ErrTelemetryLeaseLost) { + t.Fatalf("reused lease: %v", err) + } + snaps, err := m.ListTelemetry(ctx, tenant) + if err != nil || len(snaps) != 1 || snaps[0].ConfigFingerprint == "" || snaps[0].BGP.LastSuccessAt.IsZero() || snaps[0].NextPollAt.After(time.Now().Add(time.Second)) { + t.Fatalf("snapshot=%+v err=%v", snaps, err) + } + discoveries, err := m.ListPeerDiscoveries(tenant, PeerDiscoveryPending) + if err != nil || len(discoveries) != 1 { + t.Fatalf("discovery=%+v err=%v", discoveries, err) + } + next, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, "") + if err != nil || len(next) != 1 || next[0].FencingToken <= first.FencingToken { + t.Fatalf("new claim=%+v %v", next, err) + } + meta := `{"agent_domain":"new.test","agent_secret":"secret"}` + if _, err := m.UpdateSpeaker(tenant, sp.ID, &SpeakerPatch{MetaJSON: &meta}); err != nil { + t.Fatal(err) + } + if err := m.CompleteTelemetry(ctx, next[0], result, at.Add(time.Hour)); !errors.Is(err, ErrTelemetryLeaseLost) { + t.Fatalf("config changed, late result accepted: %v", err) + } + m.mu.Lock() + row := m.telemetry[telemetryKey(tenant, sp.ID)] + row.leaseUntil = time.Now().Add(-time.Second) + m.telemetry[telemetryKey(tenant, sp.ID)] = row + m.mu.Unlock() + newConfig, err := m.ClaimTelemetry(ctx, "owner-c", 1, time.Minute, "") + if err != nil || len(newConfig) != 1 { + t.Fatalf("new config claim=%+v %v", newConfig, err) + } + failed := TelemetryResult{BGP: TelemetryBGP{TelemetryObservation: TelemetryObservation{Source: "agent_bird_protocols", Error: "timeout", ReceivedAt: time.Now().UTC()}}, Health: TelemetryHealth{TelemetryObservation: TelemetryObservation{Source: "agent_health", Error: "timeout", ReceivedAt: time.Now().UTC()}}} + if err := m.CompleteTelemetry(ctx, newConfig[0], failed, time.Now().Add(time.Hour)); err != nil { + t.Fatal(err) + } + snaps, err = m.ListTelemetry(ctx, tenant) + if err != nil || len(snaps) != 1 || !snaps[0].BGP.LastSuccessAt.IsZero() || !snaps[0].Health.LastSuccessAt.IsZero() { + t.Fatalf("old endpoint success survived config change: %+v %v", snaps, err) + } +} + +func TestMemoryTelemetryLocalAuthority(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + sp, err := m.CreateSpeaker(tenant, &Speaker{Role: "master"}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + first, err := m.ClaimTelemetry(ctx, "owner-a", 1, time.Minute, "site-a") + if err != nil || len(first) != 1 || first[0].TargetID != TelemetryLocalTargetID(tenant, "site-a") || first[0].SpeakerID != sp.ID { + t.Fatalf("local claim=%+v %v", first, err) + } + second, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, "site-b") + if err != nil || len(second) != 1 || second[0].TargetID != TelemetryLocalTargetID(tenant, "site-b") { + t.Fatalf("other authority=%+v %v", second, err) + } +} + +func TestTelemetryDiscoveryBatchRotatesPastStaticPrefix(t *testing.T) { + sessions := make([]birdfmt.BGPSession, 300) + for i := range sessions { + sessions[i] = birdfmt.BGPSession{Name: fmt.Sprintf("static-%d", i)} + } + sessions[299] = birdfmt.BGPSession{Name: birdfmt.DynamicPeerProtocolPrefix + "last", Neighbor: "192.0.2.9"} + batch, cursor := TelemetryDiscoveryBatch(sessions, 0, 256) + if len(batch) != 1 || batch[0].Neighbor != "192.0.2.9" || cursor != 0 { + t.Fatalf("batch=%+v cursor=%d", batch, cursor) + } + for i := range sessions { + sessions[i].Name = birdfmt.DynamicPeerProtocolPrefix + fmt.Sprint(i) + } + first, cursor := TelemetryDiscoveryBatch(sessions, 0, 256) + second, next := TelemetryDiscoveryBatch(sessions, cursor, 256) + if len(first) != 256 || len(second) != 256 || second[0].Name != sessions[256].Name || next != 212 { + t.Fatalf("rotation first=%d second=%d cursor=%d next=%d", len(first), len(second), cursor, next) + } +} + +func TestTelemetryFingerprintIgnoresDispatchStatus(t *testing.T) { + sp := &Speaker{Role: "replica", Endpoint: "https://node.test", MetaJSON: `{"agent_domain":"node.test","agent_secret":"secret"}`} + before := TelemetrySpeakerFingerprint(sp) + sp.MetaJSON = `{"agent_domain":"node.test","agent_secret":"secret","last_dispatch_at":"2026-10-26T10:00:00Z","last_dispatch_status":"ok"}` + if TelemetrySpeakerFingerprint(sp) != before { + t.Fatal("dispatch status invalidated telemetry config") + } + sp.Role = "master" + before = TelemetrySpeakerFingerprint(sp) + sp.MetaJSON = `{"agent_domain":"other.test","agent_secret":"changed"}` + if TelemetrySpeakerFingerprint(sp) != before { + t.Fatal("agent metadata invalidated local BIRD config") + } +} + +func TestMemoryTelemetryExpiredClaimCannotDiscover(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + sp, err := m.CreateSpeaker(tenant, &Speaker{Role: "replica", MetaJSON: `{"agent_domain":"node.test","agent_secret":"secret"}`}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + first, err := m.ClaimTelemetry(ctx, "owner-a", 1, time.Minute, "") + if err != nil || len(first) != 1 || first[0].SpeakerID != sp.ID { + t.Fatalf("first=%+v %v", first, err) + } + m.mu.Lock() + row := m.telemetry[telemetryKey(tenant, sp.ID)] + row.leaseUntil = time.Now().Add(-time.Second) + m.telemetry[telemetryKey(tenant, sp.ID)] = row + m.mu.Unlock() + second, err := m.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, "") + if err != nil || len(second) != 1 { + t.Fatalf("second=%+v %v", second, err) + } + at := time.Now().UTC() + result := TelemetryResult{BGP: TelemetryBGP{TelemetryObservation: TelemetryObservation{Source: "agent_bird_protocols", ReceivedAt: at}, Sessions: []birdfmt.BGPSession{{Name: birdfmt.DynamicPeerProtocolPrefix + "late", Neighbor: "192.0.2.13"}}}} + if err := m.CompleteTelemetry(ctx, first[0], result, at.Add(time.Hour)); !errors.Is(err, ErrTelemetryLeaseLost) { + t.Fatalf("old claim accepted: %v", err) + } + if list, err := m.ListPeerDiscoveries(tenant, ""); err != nil || len(list) != 0 { + t.Fatalf("stale discovery persisted: %+v %v", list, err) + } + if err := m.CompleteTelemetry(ctx, second[0], result, at.Add(time.Hour)); err != nil { + t.Fatal(err) + } +} diff --git a/internal/store/module_input_hash.go b/internal/store/module_input_hash.go index 3e20697..9068bde 100644 --- a/internal/store/module_input_hash.go +++ b/internal/store/module_input_hash.go @@ -1,11 +1,13 @@ package store import ( + "context" "crypto/sha256" "encoding/hex" "fmt" "sort" "strings" + "time" ) // ComputeModuleInputHash fingerprints module config and child entries so snapshots @@ -109,7 +111,9 @@ func TouchModuleInputHash(st Backend, tenantID, moduleID string) { if err != nil { return } - _ = st.SetModuleInputHash(tenantID, moduleID, h) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = st.SetModuleInputHash(ctx, tenantID, moduleID, h) } // CDNPatchAffectsInputHash reports whether a CDN source patch changes ingest fingerprint diff --git a/internal/store/prefix_snapshot_hash.go b/internal/store/prefix_snapshot_hash.go new file mode 100644 index 0000000..a6815d2 --- /dev/null +++ b/internal/store/prefix_snapshot_hash.go @@ -0,0 +1,69 @@ +package store + +import ( + "crypto/sha256" + "encoding/hex" + "sort" +) + +// PrefixSnapshotHash is the content key of prefix_snapshot. +// It fingerprints prefix rows only. Peer configuration stays on +// config_revision.content_hash so a peer-only change can reuse the snapshot. +// The result is 64 lowercase hex characters. Row order does not affect the hash. +func PrefixSnapshotHash(rows []PrefixRow) string { + cp := append([]PrefixRow(nil), rows...) + sortPrefixRows(cp) + h := sha256.New() + for _, row := range cp { + _, _ = h.Write([]byte(row.Prefix)) + _, _ = h.Write([]byte{1}) + if row.CommunityID == nil { + _, _ = h.Write([]byte{0}) + } else { + _, _ = h.Write([]byte{1}) + _, _ = h.Write([]byte(*row.CommunityID)) + } + _, _ = h.Write([]byte{1}) + _, _ = h.Write([]byte(row.Source)) + _, _ = h.Write([]byte{0}) + } + sum := h.Sum(nil) + return hex.EncodeToString(sum) +} + +func sortPrefixRows(rows []PrefixRow) { + sort.Slice(rows, func(i, j int) bool { + if rows[i].Prefix != rows[j].Prefix { + return rows[i].Prefix < rows[j].Prefix + } + ni, vi := communitySortKey(rows[i].CommunityID) + nj, vj := communitySortKey(rows[j].CommunityID) + if ni != nj { + return ni + } + if vi != vj { + return vi < vj + } + return rows[i].Source < rows[j].Source + }) +} + +func communitySortKey(id *string) (isNil bool, val string) { + if id == nil { + return true, "" + } + return false, *id +} + +func samePrefixRow(a, b PrefixRow) bool { + if a.Prefix != b.Prefix || a.Source != b.Source { + return false + } + if (a.CommunityID == nil) != (b.CommunityID == nil) { + return false + } + if a.CommunityID != nil && *a.CommunityID != *b.CommunityID { + return false + } + return true +} diff --git a/internal/store/prefix_snapshot_reuse_test.go b/internal/store/prefix_snapshot_reuse_test.go new file mode 100644 index 0000000..198445b --- /dev/null +++ b/internal/store/prefix_snapshot_reuse_test.go @@ -0,0 +1,153 @@ +package store + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/uuid" +) + +func TestSameSnapshotContentIgnoresOrder(t *testing.T) { + comm := "c1" + a := &ModulePrefixSnapshot{InputHash: "h", Prefixes: []PrefixRow{ + {Prefix: "203.0.113.0/24", CommunityID: &comm, Source: "cdn:a"}, + {Prefix: "198.51.100.0/24", Source: "cdn:b"}, + }} + b := &ModulePrefixSnapshot{InputHash: "h", Prefixes: []PrefixRow{ + {Prefix: "198.51.100.0/24", Source: "cdn:b"}, + {Prefix: "203.0.113.0/24", CommunityID: &comm, Source: "cdn:a"}, + }} + if !SameSnapshotContent(a, b) { + t.Fatal("same set in another order was treated as new") + } + b.Prefixes[0].Prefix = "192.0.2.0/24" + if SameSnapshotContent(a, b) { + t.Fatal("different prefix compared equal") + } +} + +func TestPrefixSnapshotHashIgnoresOrder(t *testing.T) { + rows := []PrefixRow{ + {Prefix: "203.0.113.0/24", Source: "ip_range"}, + {Prefix: "198.51.100.0/24", Source: "cdn:a"}, + } + rev := []PrefixRow{rows[1], rows[0]} + if PrefixSnapshotHash(rows) != PrefixSnapshotHash(rev) { + t.Fatal("prefix hash depends on order") + } + rev = append(rev, PrefixRow{Prefix: "2001:db8::/32", Source: "ip_range"}) + if PrefixSnapshotHash(rows) == PrefixSnapshotHash(rev) { + t.Fatal("prefix change kept the same hash") + } +} + +func TestCreateRenderRevisionReusesPrefixSnapshot(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, _, mod, _, _ := m.DemoIDs() + ctx := context.Background() + rows := []PrefixRow{ + {Prefix: "203.0.113.0/24", Source: "ip_range"}, + {Prefix: "198.51.100.0/24", Source: "cdn:a"}, + } + rev1 := uuid.NewString() + if err := m.CreateRenderRevision(ctx, rev1, tenant, mod, nil, "hash-peer-a", map[string]string{"bird.conf": "a"}, rows); err != nil { + t.Fatal(err) + } + rev2 := uuid.NewString() + reversed := []PrefixRow{rows[1], rows[0]} + if err := m.CreateRenderRevision(ctx, rev2, tenant, mod, nil, "hash-peer-b", map[string]string{"bird.conf": "b"}, reversed); err != nil { + t.Fatal(err) + } + if m.revPrefixSnap[rev1] == "" || m.revPrefixSnap[rev1] != m.revPrefixSnap[rev2] { + t.Fatalf("peer-only change copied prefix rows: %q vs %q", m.revPrefixSnap[rev1], m.revPrefixSnap[rev2]) + } + if len(m.prefixSnapRows) != 1 { + t.Fatalf("stored prefix snapshots: %d", len(m.prefixSnapRows)) + } + page, _, _ := m.ListRevisionPrefixes(tenant, rev2, "", 10) + if len(page) != 2 { + t.Fatalf("reused snapshot not readable: %+v", page) + } + changed := append(append([]PrefixRow(nil), rows...), PrefixRow{Prefix: "2001:db8::/32", Source: "ip_range"}) + rev3 := uuid.NewString() + if err := m.CreateRenderRevision(ctx, rev3, tenant, mod, nil, "hash-prefix", nil, changed); err != nil { + t.Fatal(err) + } + if m.revPrefixSnap[rev3] == m.revPrefixSnap[rev1] || len(m.prefixSnapRows) != 2 { + t.Fatalf("prefix change did not allocate a snapshot: id=%s count=%d", m.revPrefixSnap[rev3], len(m.prefixSnapRows)) + } +} + +func TestUpdateModulePrefixSnapshotCancelledContext(t *testing.T) { + m := NewMemory() + m.SeedDemo() + tenant, mod, _, _, _ := m.DemoIDs() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := m.UpdateModulePrefixSnapshot(ctx, tenant, mod, func(*ModulePrefixSnapshot) (*ModulePrefixSnapshot, error) { + t.Fatal("snapshot update ran after cancel") + return nil, nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err=%v", err) + } +} + +func TestMemorySnapshotLockUsesSnapshotTimeout(t *testing.T) { + t.Setenv("EVOBGP_SNAPSHOT_TIMEOUT", "80ms") + m := NewMemory() + hold, err := m.lockModuleSnapshot(context.Background(), "tenant", "module") + if err != nil { + t.Fatal(err) + } + defer hold() + start := time.Now() + _, err = m.lockModuleSnapshot(context.Background(), "tenant", "module") + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err=%v", err) + } + if time.Since(start) > time.Second { + t.Fatalf("lock wait ignored snapshot timeout: %s", time.Since(start)) + } +} + +func TestSnapshotWriteBudgetKeepsCallerDeadline(t *testing.T) { + parent, cancel := context.WithTimeout(context.Background(), time.Hour) + defer cancel() + got, stop, installed := SnapshotWriteBudget(parent) + defer stop() + if installed { + t.Fatal("caller deadline was replaced") + } + dl, ok := got.Deadline() + if !ok || time.Until(dl) < 30*time.Minute { + t.Fatal("caller deadline was shortened") + } +} + +func TestSetASNPrefixCacheKeepsRowsWhenSetUnchanged(t *testing.T) { + m := NewMemory() + if err := m.SetASNPrefixCache(64500, "holder-a", []string{"203.0.113.0/24", "198.51.100.0/24"}); err != nil { + t.Fatal(err) + } + before := m.asnPrefixCache[64500] + ptr := &before.Prefixes[0] + fetched := before.FetchedAt + time.Sleep(5 * time.Millisecond) + if err := m.SetASNPrefixCache(64500, "holder-b", []string{"198.51.100.0/24", "203.0.113.0/24"}); err != nil { + t.Fatal(err) + } + after := m.asnPrefixCache[64500] + if &after.Prefixes[0] != ptr { + t.Fatal("unchanged ASN prefixes were rewritten") + } + if !after.FetchedAt.After(fetched) { + t.Fatal("fetched_at was not updated") + } + if after.Holder != "holder-b" { + t.Fatalf("holder=%q", after.Holder) + } +} diff --git a/internal/store/render_manifest.go b/internal/store/render_manifest.go new file mode 100644 index 0000000..79b0ae4 --- /dev/null +++ b/internal/store/render_manifest.go @@ -0,0 +1,363 @@ +package store + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "strings" +) + +func (m *Memory) GetRenderGeneration(ctx context.Context, tenantID string) (int64, error) { + if err := ctx.Err(); err != nil { + return 0, err + } + m.mu.RLock() + defer m.mu.RUnlock() + if m.tenants[tenantID] == nil { + return 0, ErrNotFound + } + gen := m.renderGeneration[tenantID] + if gen == 0 { + gen = 1 + } + return gen, nil +} + +func (m *Memory) ListModuleSnapshotHeaders(ctx context.Context, tenantID string) ([]ModuleSnapshotUse, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + m.mu.RLock() + defer m.mu.RUnlock() + return append([]ModuleSnapshotUse(nil), m.snapshotHeadersLocked(tenantID)...), nil +} + +func (m *Memory) snapshotHeadersLocked(tenantID string) []ModuleSnapshotUse { + prefix := moduleSnapshotKey(tenantID, "") + var out []ModuleSnapshotUse + for key, snap := range m.moduleSnapshots { + if snap == nil || !strings.HasPrefix(key, prefix) { + continue + } + out = append(out, ModuleSnapshotUse{ + ModuleID: strings.TrimPrefix(key, prefix), + InputHash: snap.InputHash, + SnapshotVersion: snap.Version, + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].ModuleID < out[j].ModuleID }) + return out +} + +func (m *Memory) LatestRenderManifest(ctx context.Context, tenantID string) (*RenderManifestRecord, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, err + } + m.mu.RLock() + defer m.mu.RUnlock() + var latest *Revision + for _, rev := range m.revisions { + if rev == nil || rev.TenantID != tenantID { + continue + } + if latest == nil || rev.CreatedAt.After(latest.CreatedAt) || (rev.CreatedAt.Equal(latest.CreatedAt) && rev.ID > latest.ID) { + latest = rev + } + } + if latest == nil { + return nil, false, nil + } + rec := m.renderManifests[latest.ID] + if rec == nil { + return nil, false, nil + } + return cloneRenderManifest(rec), true, nil +} + +func (m *Memory) recordRenderManifestLocked(revisionID, tenantID string) { + if m.renderManifests == nil { + m.renderManifests = make(map[string]*RenderManifestRecord) + } + gen := m.renderGeneration[tenantID] + if gen == 0 { + gen = 1 + } + m.renderManifests[revisionID] = &RenderManifestRecord{ + RevisionID: revisionID, + Generation: gen, + Snapshots: m.snapshotHeadersLocked(tenantID), + } +} + +func cloneRenderManifest(rec *RenderManifestRecord) *RenderManifestRecord { + if rec == nil { + return nil + } + out := *rec + out.Snapshots = append([]ModuleSnapshotUse(nil), rec.Snapshots...) + return &out +} + +// bumpRenderGenerationLocked is called only while holding m.mu for a semantic write. +func (m *Memory) bumpRenderGenerationLocked(tenantID string) { + if m.renderGeneration == nil { + m.renderGeneration = make(map[string]int64) + } + if m.renderGeneration[tenantID] == 0 { + m.renderGeneration[tenantID] = 1 + } + m.renderGeneration[tenantID]++ +} + +// RenderInputs is the complete, immutable configuration read by tenant rendering. +// Fetch observations such as CDN ETags, resolve metadata and dispatch status are +// retained in the captured rows where the ingest path needs them, but excluded +// from SemanticHash. Callers must not mutate a captured value. +type RenderInputs struct { + TenantID string `json:"tenant_id"` + Generation int64 `json:"generation"` + Modules []RenderModule `json:"modules"` + Profiles []*DohProfile `json:"profiles"` + Peers []*BGPPeer `json:"peers"` + Communities []*Community `json:"communities"` + Settings map[string]any `json:"settings"` + Speakers []RenderSpeaker `json:"speakers"` +} + +// RenderModule contains all ingestion inputs for one module in one capture. +type RenderModule struct { + Module *Module `json:"module"` + CDNSources []*CDNSource `json:"cdn_sources"` + ASEntries []*ASEntry `json:"as_entries"` + DomainEntries []*DomainEntry `json:"domain_entries"` + IPRanges []*IPRangeEntry `json:"ip_ranges"` +} + +// RenderSpeaker holds only values that affect speaker-specific BIRD fragments. +type RenderSpeaker struct { + ID string `json:"id"` + NodeIPv4 string `json:"node_ipv4"` + BirdBgpSourceIPv4 string `json:"bird_bgp_source_ipv4"` + BirdBgpSourceIPv6 string `json:"bird_bgp_source_ipv6"` +} + +// ModuleSnapshotUse identifies the exact module materialization used by a render. +type ModuleSnapshotUse struct { + ModuleID string `json:"module_id"` + InputHash string `json:"input_hash"` + SnapshotVersion int64 `json:"snapshot_version"` +} + +// RenderManifestRecord is the admission barrier stored with a config revision. +// Generation and snapshot versions let a later render return the revision +// without loading prefix rows when nothing material changed. +type RenderManifestRecord struct { + RevisionID string + Generation int64 + Snapshots []ModuleSnapshotUse +} + +// RenderManifest is persisted with a revision so new publish admission can +// distinguish a current render from a historical rollback or durable replay. +type RenderManifest struct { + Inputs RenderInputs `json:"inputs"` + ConfigHash string `json:"config_hash"` + Snapshots []ModuleSnapshotUse `json:"snapshots"` + OverlayBySpeaker map[string]map[string]string `json:"overlay_by_speaker"` +} + +// SemanticHash fingerprints captured render settings and ingest configuration. +// It deliberately excludes fetch timestamps, ETags, health and dispatch state. +func (in *RenderInputs) SemanticHash() (string, error) { + if in == nil || strings.TrimSpace(in.TenantID) == "" { + return "", ErrInvalidInput + } + semantic := struct { + TenantID string `json:"tenant_id"` + Modules []any `json:"modules"` + Profiles []any `json:"profiles"` + Peers []any `json:"peers"` + Communities []any `json:"communities"` + Settings map[string]any `json:"settings"` + Speakers []RenderSpeaker `json:"speakers"` + }{TenantID: in.TenantID, Settings: in.Settings, Speakers: append([]RenderSpeaker(nil), in.Speakers...)} + modules := append([]RenderModule(nil), in.Modules...) + for _, item := range modules { + if item.Module == nil { + return "", fmt.Errorf("store: render inputs: nil module") + } + } + sort.Slice(modules, func(i, j int) bool { return modules[i].Module.ID < modules[j].Module.ID }) + for _, item := range modules { + m := item.Module + semantic.Modules = append(semantic.Modules, struct { + ID string + Type string + Enabled bool + DefaultCommunityID *string + DohResolverPolicy string + DohProfileIDs []string + CDNSources []any + ASEntries []any + DomainEntries []any + IPRanges []any + }{ID: m.ID, Type: m.Type, Enabled: m.Enabled, DefaultCommunityID: m.DefaultCommunityID, DohResolverPolicy: m.DohResolverPolicy, + DohProfileIDs: append([]string(nil), m.EffectiveDohProfileIDs()...), + CDNSources: semanticCDNSources(item.CDNSources), ASEntries: semanticASEntries(item.ASEntries), DomainEntries: semanticDomainEntries(item.DomainEntries), IPRanges: semanticIPRanges(item.IPRanges)}) + } + profiles := append([]*DohProfile(nil), in.Profiles...) + sort.Slice(profiles, func(i, j int) bool { + if profiles[i] == nil { + return false + } + if profiles[j] == nil { + return true + } + return profiles[i].ID < profiles[j].ID + }) + for _, p := range profiles { + if p != nil { + semantic.Profiles = append(semantic.Profiles, struct { + ID, URL string + TimeoutMs *int + SecretRef *string + }{p.ID, p.URL, p.TimeoutMs, p.SecretRef}) + } + } + peers := append([]*BGPPeer(nil), in.Peers...) + sort.Slice(peers, func(i, j int) bool { + if peers[i] == nil { + return false + } + if peers[j] == nil { + return true + } + return peers[i].ID < peers[j].ID + }) + for _, p := range peers { + if p != nil { + semantic.Peers = append(semantic.Peers, struct { + ID, Neighbor, PoliciesJSON string + RemoteASN int64 + Enabled bool + SpeakerID *string + }{p.ID, p.Neighbor, p.PoliciesJSON, p.RemoteASN, p.Enabled, p.SpeakerID}) + } + } + communities := append([]*Community(nil), in.Communities...) + sort.Slice(communities, func(i, j int) bool { + if communities[i] == nil { + return false + } + if communities[j] == nil { + return true + } + return communities[i].ID < communities[j].ID + }) + for _, c := range communities { + if c != nil { + semantic.Communities = append(semantic.Communities, struct{ ID, Community, ValueJSON string }{c.ID, c.Community, c.ValueJSON}) + } + } + sort.Slice(semantic.Speakers, func(i, j int) bool { return semantic.Speakers[i].ID < semantic.Speakers[j].ID }) + data, err := json.Marshal(semantic) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +func semanticCDNSources(items []*CDNSource) []any { + items = append([]*CDNSource(nil), items...) + sort.Slice(items, func(i, j int) bool { + if items[i] == nil { + return false + } + if items[j] == nil { + return true + } + return items[i].ID < items[j].ID + }) + out := make([]any, 0, len(items)) + for _, s := range items { + if s != nil { + out = append(out, struct { + ID, SourceKind, URL, PrefixPath string + RefreshIntervalSec *int + CommunityID *string + }{s.ID, s.SourceKind, s.URL, s.PrefixPath, s.RefreshIntervalSec, s.CommunityID}) + } + } + return out +} +func semanticASEntries(items []*ASEntry) []any { + items = append([]*ASEntry(nil), items...) + sort.Slice(items, func(i, j int) bool { + if items[i] == nil { + return false + } + if items[j] == nil { + return true + } + return items[i].ID < items[j].ID + }) + out := make([]any, 0, len(items)) + for _, s := range items { + if s != nil { + out = append(out, struct { + ID string + ASN int64 + CommunityID *string + }{s.ID, s.ASN, s.CommunityID}) + } + } + return out +} +func semanticDomainEntries(items []*DomainEntry) []any { + items = append([]*DomainEntry(nil), items...) + sort.Slice(items, func(i, j int) bool { + if items[i] == nil { + return false + } + if items[j] == nil { + return true + } + return items[i].ID < items[j].ID + }) + out := make([]any, 0, len(items)) + for _, s := range items { + if s != nil { + out = append(out, struct { + ID, FQDN string + CommunityID *string + }{s.ID, s.FQDN, s.CommunityID}) + } + } + return out +} +func semanticIPRanges(items []*IPRangeEntry) []any { + items = append([]*IPRangeEntry(nil), items...) + sort.Slice(items, func(i, j int) bool { + if items[i] == nil { + return false + } + if items[j] == nil { + return true + } + return items[i].ID < items[j].ID + }) + out := make([]any, 0, len(items)) + for _, s := range items { + if s != nil { + out = append(out, struct { + ID, Prefix string + CommunityID *string + }{s.ID, s.Prefix, s.CommunityID}) + } + } + return out +} diff --git a/internal/store/render_manifest_test.go b/internal/store/render_manifest_test.go new file mode 100644 index 0000000..2bafe17 --- /dev/null +++ b/internal/store/render_manifest_test.go @@ -0,0 +1,37 @@ +package store + +import ( + "testing" + "time" +) + +func TestRenderInputsSemanticHashCanonicalAndIgnoresFetchState(t *testing.T) { + now := time.Now().UTC() + a := &RenderInputs{TenantID: "tenant", Modules: []RenderModule{ + {Module: &Module{ID: "b", Type: "CDN_CIDRS", Enabled: true}, CDNSources: []*CDNSource{{ID: "s2", URL: "https://b.test", Etag: "old"}, {ID: "s1", URL: "https://a.test", LastRefreshedAt: &now}}}, + {Module: &Module{ID: "a", Type: "IP_RANGES", Enabled: true}, IPRanges: []*IPRangeEntry{{ID: "r2", Prefix: "192.0.2.0/24"}, {ID: "r1", Prefix: "198.51.100.0/24"}}}, + }} + b := &RenderInputs{TenantID: "tenant", Generation: 99, Modules: []RenderModule{ + {Module: &Module{ID: "a", Type: "IP_RANGES", Enabled: true}, IPRanges: []*IPRangeEntry{{ID: "r1", Prefix: "198.51.100.0/24"}, {ID: "r2", Prefix: "192.0.2.0/24"}}}, + {Module: &Module{ID: "b", Type: "CDN_CIDRS", Enabled: true}, CDNSources: []*CDNSource{{ID: "s1", URL: "https://a.test"}, {ID: "s2", URL: "https://b.test", Etag: "new"}}}, + }} + ha, err := a.SemanticHash() + if err != nil { + t.Fatal(err) + } + hb, err := b.SemanticHash() + if err != nil { + t.Fatal(err) + } + if ha != hb { + t.Fatalf("same semantic inputs have different hashes: %s %s", ha, hb) + } + b.Modules[1].CDNSources[0].URL = "https://changed.test" + hb, err = b.SemanticHash() + if err != nil { + t.Fatal(err) + } + if ha == hb { + t.Fatal("source URL change did not change hash") + } +} diff --git a/internal/store/snapshot_budget.go b/internal/store/snapshot_budget.go new file mode 100644 index 0000000..0086640 --- /dev/null +++ b/internal/store/snapshot_budget.go @@ -0,0 +1,51 @@ +package store + +import ( + "context" + "os" + "strings" + "time" +) + +const defaultSnapshotTimeout = 2 * time.Minute + +// SnapshotWriteBudget keeps a caller deadline. When ctx has none, it applies +// EVOBGP_SNAPSHOT_TIMEOUT (default 2 minutes). installed is true when this +// function created the deadline; callers use that to set statement_timeout. +func SnapshotWriteBudget(ctx context.Context) (context.Context, context.CancelFunc, bool) { + if ctx == nil { + ctx = context.Background() + } + if _, ok := ctx.Deadline(); ok { + return ctx, func() {}, false + } + ctx, cancel := context.WithTimeout(ctx, snapshotTimeout()) + return ctx, cancel, true +} + +// SnapshotStatementTimeout is the PostgreSQL statement_timeout that matches +// SnapshotWriteBudget. A caller deadline wins; otherwise the env default applies. +func SnapshotStatementTimeout(ctx context.Context) time.Duration { + if ctx != nil { + if dl, ok := ctx.Deadline(); ok { + d := time.Until(dl) + if d < time.Millisecond { + return time.Millisecond + } + return d + } + } + return snapshotTimeout() +} + +func snapshotTimeout() time.Duration { + s := strings.TrimSpace(os.Getenv("EVOBGP_SNAPSHOT_TIMEOUT")) + if s == "" { + return defaultSnapshotTimeout + } + d, err := time.ParseDuration(s) + if err != nil || d <= 0 { + return defaultSnapshotTimeout + } + return d +} diff --git a/internal/store/telemetry.go b/internal/store/telemetry.go new file mode 100644 index 0000000..7376b9f --- /dev/null +++ b/internal/store/telemetry.go @@ -0,0 +1,113 @@ +package store + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "strconv" + "strings" + "time" + + "evobgp/internal/birdfmt" +) + +// ErrTelemetryLeaseLost means a collector result belongs to an expired or replaced lease. +var ErrTelemetryLeaseLost = errors.New("store: telemetry lease lost") + +// TelemetryLocalTargetID identifies the synthetic local BIRD target of a tenant. +func TelemetryLocalTargetID(tenantID, authority string) string { + return "cp-local/" + tenantID + "/" + authority +} + +// TelemetryObservation is the most recent attempt and last successful reading of one source. +type TelemetryObservation struct { + ObservedAt time.Time `json:"observed_at"` + ReceivedAt time.Time `json:"received_at"` + LastSuccessAt time.Time `json:"last_success_at"` + Source string `json:"source"` + Error string `json:"error,omitempty"` +} + +// TelemetryBGP is one BIRD protocols measurement. +type TelemetryBGP struct { + TelemetryObservation + Sessions []birdfmt.BGPSession `json:"sessions,omitempty"` +} + +// TelemetryHealth is one legacy agent health measurement. +type TelemetryHealth struct { + TelemetryObservation + OK bool `json:"ok"` + LastSyncAt string `json:"last_sync_at,omitempty"` + LastAppliedRevisionID string `json:"last_applied_revision_id,omitempty"` +} + +// TelemetrySnapshot contains independent BGP and agent health readings for one target. +type TelemetrySnapshot struct { + TenantID, TargetID, SpeakerID, Kind string + LocalAuthority string + ConfigFingerprint string + BGP TelemetryBGP + Health TelemetryHealth + NextPollAt time.Time + FailureCount int + DiscoveryCursor int +} + +// TelemetrySpeakerFingerprint binds a snapshot to the configuration it observed. +func TelemetrySpeakerFingerprint(sp *Speaker) string { + if sp == nil { + return "" + } + if strings.EqualFold(strings.TrimSpace(sp.Role), "master") { + sum := sha256.Sum256([]byte("master\x00" + sp.Endpoint)) + return hex.EncodeToString(sum[:]) + } + meta := ParseSpeakerMeta(sp.MetaJSON) + sum := sha256.Sum256([]byte(sp.Role + "\x00" + sp.Endpoint + "\x00" + meta.AgentDomain + "\x00" + meta.AgentSecret + "\x00" + strconv.Itoa(meta.AgentPort))) + return hex.EncodeToString(sum[:]) +} + +// TelemetryClaim fences a bounded collector attempt. Speaker is the current config. +type TelemetryClaim struct { + TelemetrySnapshot + Speaker *Speaker + OwnerID string + FencingToken int64 +} + +// TelemetryResult is the atomic pair of results of one collector attempt. +type TelemetryResult struct { + BGP TelemetryBGP + Health TelemetryHealth +} + +// TelemetryDiscoveryBatch rotates over long protocol lists without starving later entries. +func TelemetryDiscoveryBatch(sessions []birdfmt.BGPSession, cursor, limit int) ([]birdfmt.BGPSession, int) { + if len(sessions) == 0 || limit <= 0 { + return nil, 0 + } + if cursor < 0 || cursor >= len(sessions) { + cursor = 0 + } + out := make([]birdfmt.BGPSession, 0, limit) + for visited := 0; visited < len(sessions); visited++ { + idx := (cursor + visited) % len(sessions) + if birdfmt.IsDynamicDiscoverySession(sessions[idx].Name) { + out = append(out, sessions[idx]) + if len(out) == limit { + return out, (idx + 1) % len(sessions) + } + } + } + return out, 0 +} + +// TelemetryStore coordinates polling and read-only snapshots across API replicas. +type TelemetryStore interface { + ClaimTelemetry(context.Context, string, int, time.Duration, string) ([]TelemetryClaim, error) + CompleteTelemetry(context.Context, TelemetryClaim, TelemetryResult, time.Time) error + ListTelemetry(context.Context, string) ([]TelemetrySnapshot, error) + RequestTelemetryRefresh(context.Context, string, time.Duration) (bool, error) +} diff --git a/internal/telemetry/collector.go b/internal/telemetry/collector.go new file mode 100644 index 0000000..a472756 --- /dev/null +++ b/internal/telemetry/collector.go @@ -0,0 +1,221 @@ +// Package telemetry polls legacy agent and BIRD endpoints into shared snapshots. +package telemetry + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "errors" + "math/rand" + "net/http" + "os" + "strconv" + "strings" + "sync" + "time" + + "evobgp/internal/birdfmt" + "evobgp/internal/logging" + "evobgp/internal/nodedispatch" + "evobgp/internal/store" + + "github.com/google/uuid" +) + +// Config bounds collector concurrency, freshness and external request duration. +type Config struct { + Interval, StaleAfter, Timeout, Lease, BackoffMax time.Duration + Workers int + BirdSocket, BirdBin, LocalAuthority string + HTTPClient *http.Client +} + +// ConfigFromEnv returns safe defaults and derives a stable local BIRD authority. +func ConfigFromEnv() Config { + cfg := Config{Interval: 20 * time.Second, StaleAfter: time.Minute, Timeout: 10 * time.Second, Lease: 30 * time.Second, BackoffMax: 2 * time.Minute, Workers: 4} + if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_TELEMETRY_INTERVAL_SEC"))); err == nil && n >= 15 && n <= 300 { + cfg.Interval = time.Duration(n) * time.Second + } + if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_TELEMETRY_WORKERS"))); err == nil && n >= 1 && n <= 32 { + cfg.Workers = n + } + cfg.StaleAfter = 3 * cfg.Interval + if cfg.BackoffMax < 4*cfg.Interval { + cfg.BackoffMax = 4 * cfg.Interval + } + cfg.BirdSocket = strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")) + cfg.BirdBin = strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")) + if cfg.BirdSocket != "" { + identity := strings.TrimSpace(os.Getenv("EVOBGP_TELEMETRY_LOCAL_ID")) + if identity == "" { + host, err := os.Hostname() + if err == nil && host != "" { + identity = host + "\x00" + cfg.BirdSocket + "\x00" + cfg.BirdBin + } + } + if identity != "" { + sum := sha256.Sum256([]byte(identity)) + cfg.LocalAuthority = hex.EncodeToString(sum[:12]) + } + } + return cfg +} + +// Collector owns a fixed number of lease-claiming workers. +type Collector struct { + cancel context.CancelFunc + done chan struct{} + client *http.Client +} + +// Start launches bounded polling until ctx cancellation or Close. +func Start(ctx context.Context, st store.TelemetryStore, cfg Config) *Collector { + workerCtx, cancel := context.WithCancel(ctx) + c := &Collector{cancel: cancel, done: make(chan struct{})} + if st == nil { + close(c.done) + return c + } + if cfg.Interval <= 0 { + cfg.Interval = 20 * time.Second + } + if cfg.Timeout <= 0 { + cfg.Timeout = 10 * time.Second + } + if cfg.Lease <= cfg.Timeout+5*time.Second { + cfg.Lease = cfg.Timeout + 20*time.Second + } + if cfg.BackoffMax < 4*cfg.Interval { + cfg.BackoffMax = 4 * cfg.Interval + } + if cfg.Workers <= 0 { + cfg.Workers = 4 + } + if cfg.Workers > 32 { + cfg.Workers = 32 + } + if cfg.HTTPClient != nil { + c.client = cfg.HTTPClient + } else { + c.client = newPollHTTPClient(cfg) + } + var wg sync.WaitGroup + for i := 0; i < cfg.Workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + worker(workerCtx, st, cfg, c.client, uuid.NewString()) + }() + } + go func() { wg.Wait(); close(c.done) }() + return c +} + +// Close stops claims and waits for in-flight polls to finish or honor cancellation. +func (c *Collector) Close(ctx context.Context) error { + if c == nil { + return nil + } + c.cancel() + select { + case <-c.done: + if c.client != nil { + c.client.CloseIdleConnections() + } + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func newPollHTTPClient(cfg Config) *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConns = cfg.Workers * 4 + transport.MaxIdleConnsPerHost = cfg.Workers * 2 + if strings.TrimSpace(os.Getenv("EVOBGP_NODE_DISPATCH_INSECURE_TLS")) == "1" { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // lab-only legacy dispatch setting + } + return &http.Client{Transport: transport, Timeout: cfg.Timeout} +} + +func worker(ctx context.Context, st store.TelemetryStore, cfg Config, client *http.Client, owner string) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + claimCtx, claimCancel := context.WithTimeout(ctx, 3*time.Second) + claims, err := st.ClaimTelemetry(claimCtx, owner, 1, cfg.Lease, cfg.LocalAuthority) + claimCancel() + if err != nil { + logging.With("component", "telemetry.collector").Warn("claim failed", "err", err) + continue + } + if len(claims) == 0 { + continue + } + claim := claims[0] + pollCtx, cancel := context.WithTimeout(ctx, cfg.Timeout) + result := poll(pollCtx, claim, cfg, client) + cancel() + failed := result.BGP.Error != "" || result.Health.Source != "" && !result.Health.OK + next := time.Now().Add(nextDelay(cfg, claim.FailureCount, failed)) + persistCtx, persistCancel := context.WithTimeout(ctx, 3*time.Second) + err = st.CompleteTelemetry(persistCtx, claim, result, next) + persistCancel() + if err != nil && !errors.Is(err, store.ErrTelemetryLeaseLost) && ctx.Err() == nil { + logging.With("component", "telemetry.collector").Warn("completion failed", "err", err) + } + } +} + +func nextDelay(cfg Config, priorFailures int, failed bool) time.Duration { + delay := cfg.Interval + if failed { + for i := 0; i <= priorFailures && delay < cfg.BackoffMax; i++ { + delay *= 2 + } + if delay > cfg.BackoffMax { + delay = cfg.BackoffMax + } + } + return delay + time.Duration(rand.Int63n(int64(delay/5+1))) +} + +func poll(ctx context.Context, claim store.TelemetryClaim, cfg Config, client *http.Client) store.TelemetryResult { + if claim.Kind == "local_bird" { + out, err := birdfmt.ShowProtocols(ctx, cfg.BirdSocket, cfg.BirdBin) + received := time.Now().UTC() + bgp := store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ObservedAt: received, ReceivedAt: received, Source: "cp_birdc"}} + if err != nil { + bgp.Error = err.Error() + } else { + bgp.Sessions = birdfmt.ParseBGPSessions(out) + } + return store.TelemetryResult{BGP: bgp} + } + if claim.Speaker == nil { + return store.TelemetryResult{BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ObservedAt: time.Now().UTC(), ReceivedAt: time.Now().UTC(), Source: "agent_bird_protocols", Error: "speaker missing"}}} + } + var result store.TelemetryResult + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + res := nodedispatch.FetchBirdProtocols(ctx, claim.Speaker, nodedispatch.Options{Timeout: cfg.Timeout, HTTPClient: client}) + received := time.Now().UTC() + result.BGP = store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ObservedAt: received, ReceivedAt: received, Source: "agent_bird_protocols", Error: res.Error}, Sessions: res.Sessions} + }() + go func() { + defer wg.Done() + res := nodedispatch.FetchAgentHealth(ctx, claim.Speaker, nodedispatch.Options{Timeout: cfg.Timeout, HTTPClient: client}) + received := time.Now().UTC() + result.Health = store.TelemetryHealth{TelemetryObservation: store.TelemetryObservation{ObservedAt: received, ReceivedAt: received, Source: "agent_health", Error: res.Error}, OK: res.OK, LastSyncAt: res.LastSyncAt, LastAppliedRevisionID: res.LastAppliedRevisionID} + }() + wg.Wait() + return result +} diff --git a/internal/telemetry/collector_test.go b/internal/telemetry/collector_test.go new file mode 100644 index 0000000..6c34ec0 --- /dev/null +++ b/internal/telemetry/collector_test.go @@ -0,0 +1,126 @@ +package telemetry + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "evobgp/internal/store" +) + +func TestNextDelayBounded(t *testing.T) { + cfg := Config{Interval: 20 * time.Second, BackoffMax: 2 * time.Minute} + for _, tc := range []struct { + failures int + failed bool + min, max time.Duration + }{ + {0, false, 20 * time.Second, 24 * time.Second}, + {0, true, 40 * time.Second, 48 * time.Second}, + {8, true, 2 * time.Minute, 144 * time.Second}, + } { + got := nextDelay(cfg, tc.failures, tc.failed) + if got < tc.min || got > tc.max { + t.Fatalf("delay(%d,%v)=%s outside [%s,%s]", tc.failures, tc.failed, got, tc.min, tc.max) + } + } +} + +func TestCollectorNilStoreCloses(t *testing.T) { + c := Start(context.Background(), nil, Config{}) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := c.Close(ctx); err != nil { + t.Fatal(err) + } +} + +type collectorTestStore struct { + mu sync.Mutex + claims []store.TelemetryClaim + claimCall int + finished chan struct{} +} + +func (s *collectorTestStore) ClaimTelemetry(_ context.Context, owner string, limit int, _ time.Duration, _ string) ([]store.TelemetryClaim, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.claimCall++ + if len(s.claims) == 0 { + return nil, nil + } + claim := s.claims[0] + s.claims = s.claims[1:] + claim.OwnerID = owner + return []store.TelemetryClaim{claim}, nil +} +func (s *collectorTestStore) CompleteTelemetry(_ context.Context, _ store.TelemetryClaim, _ store.TelemetryResult, _ time.Time) error { + s.finished <- struct{}{} + return nil +} +func (s *collectorTestStore) ListTelemetry(context.Context, string) ([]store.TelemetrySnapshot, error) { + return nil, nil +} +func (s *collectorTestStore) RequestTelemetryRefresh(context.Context, string, time.Duration) (bool, error) { + return false, nil +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestCollectorBoundsPollsAndStopsClaims(t *testing.T) { + var active, maxActive, requests atomic.Int32 + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + current := active.Add(1) + for { + old := maxActive.Load() + if current <= old || maxActive.CompareAndSwap(old, current) { + break + } + } + defer active.Add(-1) + requests.Add(1) + time.Sleep(100 * time.Millisecond) + body := `{"sessions":[]}` + if strings.HasSuffix(r.URL.Path, "/health") { + body = `{"ok":true}` + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil + })} + host := "node.test" + st := &collectorTestStore{finished: make(chan struct{}, 4)} + for i := 0; i < 4; i++ { + st.claims = append(st.claims, store.TelemetryClaim{TelemetrySnapshot: store.TelemetrySnapshot{TenantID: "tenant", TargetID: fmt.Sprint(i), Kind: "remote_agent"}, Speaker: &store.Speaker{ID: fmt.Sprint(i), Role: "replica", MetaJSON: fmt.Sprintf(`{"agent_domain":%q,"agent_secret":"secret"}`, host)}}) + } + c := Start(context.Background(), st, Config{Workers: 2, Interval: 20 * time.Second, Timeout: 3 * time.Second, Lease: 8 * time.Second, BackoffMax: time.Minute, HTTPClient: client}) + for i := 0; i < 4; i++ { + select { + case <-st.finished: + case <-time.After(6 * time.Second): + t.Fatal("collector did not finish bounded claims") + } + } + if maxActive.Load() > 4 || requests.Load() != 8 { + t.Fatalf("requests=%d maxActive=%d", requests.Load(), maxActive.Load()) + } + if err := c.Close(context.Background()); err != nil { + t.Fatal(err) + } + st.mu.Lock() + before := st.claimCall + st.mu.Unlock() + time.Sleep(1100 * time.Millisecond) + st.mu.Lock() + after := st.claimCall + st.mu.Unlock() + if after != before { + t.Fatalf("claims continued after Close: %d -> %d", before, after) + } +} diff --git a/migrations/postgres/000034_job_queue_lease.down.sql b/migrations/postgres/000034_job_queue_lease.down.sql new file mode 100644 index 0000000..71ed63e --- /dev/null +++ b/migrations/postgres/000034_job_queue_lease.down.sql @@ -0,0 +1,12 @@ +DROP INDEX IF EXISTS idx_job_audit_expired_lease; +DROP INDEX IF EXISTS idx_job_audit_claim; +ALTER TABLE job_audit DROP COLUMN request_sequence; +ALTER TABLE job_audit DROP COLUMN cancel_requested; +ALTER TABLE job_audit DROP COLUMN available_at; +ALTER TABLE job_audit DROP COLUMN fencing_token; +ALTER TABLE job_audit DROP COLUMN attempt; +ALTER TABLE job_audit DROP COLUMN lease_until; +ALTER TABLE job_audit DROP COLUMN owner_id; +ALTER TABLE job_audit DROP COLUMN request_hash; +ALTER TABLE job_audit DROP COLUMN request_json; +DELETE FROM tenant WHERE id = '00000000-0000-0000-0000-000000000000' AND slug = 'evobgp-system-jobs'; diff --git a/migrations/postgres/000034_job_queue_lease.up.sql b/migrations/postgres/000034_job_queue_lease.up.sql new file mode 100644 index 0000000..7597629 --- /dev/null +++ b/migrations/postgres/000034_job_queue_lease.up.sql @@ -0,0 +1,15 @@ +ALTER TABLE job_audit ADD COLUMN request_hash TEXT; +ALTER TABLE job_audit ADD COLUMN request_sequence BIGINT GENERATED ALWAYS AS IDENTITY; +ALTER TABLE job_audit ADD COLUMN request_json JSONB; +ALTER TABLE job_audit ADD COLUMN owner_id TEXT; +ALTER TABLE job_audit ADD COLUMN lease_until TIMESTAMPTZ; +ALTER TABLE job_audit ADD COLUMN attempt INTEGER NOT NULL DEFAULT 0; +ALTER TABLE job_audit ADD COLUMN fencing_token BIGINT NOT NULL DEFAULT 0; +ALTER TABLE job_audit ADD COLUMN available_at TIMESTAMPTZ NOT NULL DEFAULT now(); +ALTER TABLE job_audit ADD COLUMN cancel_requested BOOLEAN NOT NULL DEFAULT false; +INSERT INTO tenant (id, name, slug) VALUES ('00000000-0000-0000-0000-000000000000', 'EvoBGP system jobs', 'evobgp-system-jobs') ON CONFLICT (id) DO NOTHING; +CREATE INDEX idx_job_audit_claim ON job_audit (available_at, created_at) WHERE status = 'queued'; +CREATE INDEX idx_job_audit_expired_lease ON job_audit (lease_until) WHERE status = 'running'; +-- Give pre-upgrade workers their old 15-minute grace; later claims can recover them. +UPDATE job_audit SET owner_id = 'legacy-unfenced', lease_until = now() + interval '15 minutes', attempt = 1 +WHERE status = 'running' AND lease_until IS NULL; diff --git a/migrations/postgres/000035_deployment_progress.down.sql b/migrations/postgres/000035_deployment_progress.down.sql new file mode 100644 index 0000000..4ca2d10 --- /dev/null +++ b/migrations/postgres/000035_deployment_progress.down.sql @@ -0,0 +1,7 @@ +DROP TABLE deployment_speaker; +ALTER TABLE bgp_speaker DROP COLUMN deployment_job_id; +ALTER TABLE bgp_speaker DROP COLUMN deployment_request_sequence; +ALTER TABLE bgp_speaker DROP COLUMN deployment_generation; +ALTER TABLE bgp_speaker DROP COLUMN delivery_owner; +ALTER TABLE bgp_speaker DROP COLUMN delivery_token; +ALTER TABLE bgp_speaker DROP COLUMN delivery_until; diff --git a/migrations/postgres/000035_deployment_progress.up.sql b/migrations/postgres/000035_deployment_progress.up.sql new file mode 100644 index 0000000..f7bffd9 --- /dev/null +++ b/migrations/postgres/000035_deployment_progress.up.sql @@ -0,0 +1,27 @@ +ALTER TABLE bgp_speaker ADD COLUMN deployment_generation BIGINT NOT NULL DEFAULT 0; +ALTER TABLE bgp_speaker ADD COLUMN deployment_request_sequence BIGINT NOT NULL DEFAULT 0; +ALTER TABLE bgp_speaker ADD COLUMN deployment_job_id UUID; +ALTER TABLE bgp_speaker ADD COLUMN delivery_owner TEXT; +ALTER TABLE bgp_speaker ADD COLUMN delivery_token BIGINT NOT NULL DEFAULT 0; +ALTER TABLE bgp_speaker ADD COLUMN delivery_until TIMESTAMPTZ; +CREATE TABLE deployment_speaker ( + job_id UUID NOT NULL REFERENCES job_audit(id) ON DELETE CASCADE, + tenant_id UUID NOT NULL REFERENCES tenant(id) ON DELETE CASCADE, + speaker_id UUID NOT NULL REFERENCES bgp_speaker(id) ON DELETE CASCADE, + revision_id UUID NOT NULL, + generation BIGINT NOT NULL, + request_sequence BIGINT NOT NULL, + state TEXT NOT NULL, + confirmation TEXT NOT NULL DEFAULT '', + error_message TEXT NOT NULL DEFAULT '', + health_received_at TIMESTAMPTZ, + health_observation_token BIGINT NOT NULL DEFAULT 0, + health_revision_id TEXT NOT NULL DEFAULT '', + health_source_at TEXT NOT NULL DEFAULT '', + health_error TEXT NOT NULL DEFAULT '', + retry_round INTEGER NOT NULL DEFAULT 0, + dispatch_attempts INTEGER NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY(job_id,speaker_id) +); +CREATE INDEX deployment_speaker_tenant_job ON deployment_speaker(tenant_id,job_id); diff --git a/migrations/postgres/000036_telemetry_snapshot.down.sql b/migrations/postgres/000036_telemetry_snapshot.down.sql new file mode 100644 index 0000000..2797324 --- /dev/null +++ b/migrations/postgres/000036_telemetry_snapshot.down.sql @@ -0,0 +1 @@ +DROP TABLE telemetry_snapshot; diff --git a/migrations/postgres/000036_telemetry_snapshot.up.sql b/migrations/postgres/000036_telemetry_snapshot.up.sql new file mode 100644 index 0000000..6a6df74 --- /dev/null +++ b/migrations/postgres/000036_telemetry_snapshot.up.sql @@ -0,0 +1,21 @@ +CREATE TABLE telemetry_snapshot ( + tenant_id UUID NOT NULL REFERENCES tenant(id) ON DELETE CASCADE, + target_id TEXT NOT NULL, + speaker_id UUID REFERENCES bgp_speaker(id) ON DELETE SET NULL, + kind TEXT NOT NULL CHECK (kind IN ('remote_agent','local_bird')), + local_authority TEXT NOT NULL DEFAULT '', + config_fingerprint TEXT NOT NULL DEFAULT '', + bgp_json JSONB NOT NULL DEFAULT '{}', + health_json JSONB NOT NULL DEFAULT '{}', + next_poll_at TIMESTAMPTZ NOT NULL DEFAULT now(), + failure_count INTEGER NOT NULL DEFAULT 0, + discovery_cursor INTEGER NOT NULL DEFAULT 0, + last_refresh_requested_at TIMESTAMPTZ, + refresh_pending BOOLEAN NOT NULL DEFAULT false, + owner_id TEXT, + fencing_token BIGINT NOT NULL DEFAULT 0, + lease_until TIMESTAMPTZ, + PRIMARY KEY (tenant_id,target_id) +); +CREATE INDEX telemetry_snapshot_due ON telemetry_snapshot(next_poll_at) WHERE owner_id IS NULL; +CREATE INDEX telemetry_snapshot_speaker ON telemetry_snapshot(speaker_id); diff --git a/migrations/postgres/000037_render_manifest.down.sql b/migrations/postgres/000037_render_manifest.down.sql new file mode 100644 index 0000000..8130822 --- /dev/null +++ b/migrations/postgres/000037_render_manifest.down.sql @@ -0,0 +1,23 @@ +DROP TRIGGER IF EXISTS render_snapshot_changed ON module_prefix_snapshot; +DROP TRIGGER IF EXISTS render_speaker_changed ON bgp_speaker; +DROP TRIGGER IF EXISTS render_settings_changed ON global_settings; +DROP TRIGGER IF EXISTS render_community_changed ON bgp_community; +DROP TRIGGER IF EXISTS render_peer_changed ON bgp_peer; +DROP TRIGGER IF EXISTS render_module_doh_changed ON module_doh_profile; +DROP TRIGGER IF EXISTS render_doh_profile_changed ON doh_profile; +DROP TRIGGER IF EXISTS render_ip_range_entry_changed ON module_ip_range_entry; +DROP TRIGGER IF EXISTS render_domain_entry_changed ON module_domain_entry; +DROP TRIGGER IF EXISTS render_as_entry_changed ON module_as_entry; +DROP TRIGGER IF EXISTS render_cdn_source_changed ON module_cdn_source; +DROP TRIGGER IF EXISTS render_module_changed ON module; +DROP TRIGGER IF EXISTS module_snapshot_version ON module_prefix_snapshot; +DROP TRIGGER IF EXISTS tenant_render_created ON tenant; +DROP FUNCTION IF EXISTS render_snapshot_version(); +DROP FUNCTION IF EXISTS render_semantic_change(); +DROP FUNCTION IF EXISTS render_tenant_created(); +DROP FUNCTION IF EXISTS render_bump_module_generation(UUID); +DROP FUNCTION IF EXISTS render_bump_tenant_generation(UUID); +DROP TABLE IF EXISTS render_revision_manifest; +ALTER TABLE module_prefix_snapshot DROP COLUMN IF EXISTS snapshot_version; +DROP SEQUENCE IF EXISTS module_snapshot_version_seq; +DROP TABLE IF EXISTS tenant_render_generation; diff --git a/migrations/postgres/000037_render_manifest.up.sql b/migrations/postgres/000037_render_manifest.up.sql new file mode 100644 index 0000000..24d2090 --- /dev/null +++ b/migrations/postgres/000037_render_manifest.up.sql @@ -0,0 +1,152 @@ +-- A tenant clock is the short commit barrier for revision admission. Semantic +-- writers update it in their own transaction, after their entity row change. +CREATE TABLE tenant_render_generation ( + tenant_id UUID PRIMARY KEY REFERENCES tenant(id) ON DELETE CASCADE, + generation BIGINT NOT NULL DEFAULT 1 +); +INSERT INTO tenant_render_generation(tenant_id, generation) +SELECT id, 1 FROM tenant; + +CREATE SEQUENCE module_snapshot_version_seq AS BIGINT; +ALTER TABLE module_prefix_snapshot + ADD COLUMN snapshot_version BIGINT NOT NULL DEFAULT nextval('module_snapshot_version_seq'); + +CREATE TABLE render_revision_manifest ( + revision_id UUID PRIMARY KEY REFERENCES config_revision(id) ON DELETE CASCADE, + tenant_id UUID NOT NULL REFERENCES tenant(id) ON DELETE CASCADE, + config_generation BIGINT NOT NULL, + manifest_json JSONB NOT NULL +); +CREATE INDEX render_revision_manifest_tenant ON render_revision_manifest(tenant_id, revision_id); + +CREATE FUNCTION render_bump_tenant_generation(p_tenant UUID) RETURNS void +LANGUAGE plpgsql AS $$ +BEGIN + IF p_tenant IS NULL THEN RETURN; END IF; + INSERT INTO tenant_render_generation(tenant_id, generation) + VALUES (p_tenant, 2) + ON CONFLICT (tenant_id) DO UPDATE + SET generation = tenant_render_generation.generation + 1; +END; +$$; + +CREATE FUNCTION render_bump_module_generation(p_module UUID) RETURNS void +LANGUAGE plpgsql AS $$ +DECLARE v_tenant UUID; +BEGIN + IF p_module IS NULL THEN RETURN; END IF; + SELECT tenant_id INTO v_tenant FROM module WHERE id = p_module; + PERFORM render_bump_tenant_generation(v_tenant); +END; +$$; + +CREATE FUNCTION render_tenant_created() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + INSERT INTO tenant_render_generation(tenant_id, generation) VALUES(NEW.id, 1); + RETURN NEW; +END; +$$; +CREATE TRIGGER tenant_render_created AFTER INSERT ON tenant +FOR EACH ROW EXECUTE FUNCTION render_tenant_created(); + +CREATE FUNCTION render_semantic_change() RETURNS trigger +LANGUAGE plpgsql AS $$ +DECLARE v_old JSONB; v_new JSONB; + v_old_tenant UUID; v_new_tenant UUID; + v_old_module UUID; v_new_module UUID; +BEGIN + IF TG_OP = 'UPDATE' THEN + v_old := to_jsonb(OLD); + v_new := to_jsonb(NEW); + IF TG_TABLE_NAME = 'module' THEN + v_old := v_old - ARRAY['input_hash','last_refreshed_at','updated_at']; + v_new := v_new - ARRAY['input_hash','last_refreshed_at','updated_at']; + ELSIF TG_TABLE_NAME = 'module_cdn_source' THEN + v_old := v_old - ARRAY['etag','last_refreshed_at','updated_at']; + v_new := v_new - ARRAY['etag','last_refreshed_at','updated_at']; + ELSIF TG_TABLE_NAME = 'module_as_entry' THEN + v_old := v_old - ARRAY['asn_name','prefix_count','asn_resolved_at','updated_at']; + v_new := v_new - ARRAY['asn_name','prefix_count','asn_resolved_at','updated_at']; + ELSIF TG_TABLE_NAME = 'module_domain_entry' THEN + v_old := v_old - ARRAY['resolve_meta','updated_at']; + v_new := v_new - ARRAY['resolve_meta','updated_at']; + ELSIF TG_TABLE_NAME = 'bgp_peer' THEN + v_old := v_old - ARRAY['meta_json','updated_at']; + v_new := v_new - ARRAY['meta_json','updated_at']; + ELSIF TG_TABLE_NAME = 'bgp_community' THEN + v_old := v_old - ARRAY['title','updated_at']; + v_new := v_new - ARRAY['title','updated_at']; + ELSIF TG_TABLE_NAME = 'bgp_speaker' THEN + v_old := jsonb_build_object('node_ipv4',OLD.meta_json->>'node_ipv4', + 'bird_bgp_source_ipv4',OLD.meta_json->>'bird_bgp_source_ipv4', + 'bird_bgp_source_ipv6',OLD.meta_json->>'bird_bgp_source_ipv6'); + v_new := jsonb_build_object('node_ipv4',NEW.meta_json->>'node_ipv4', + 'bird_bgp_source_ipv4',NEW.meta_json->>'bird_bgp_source_ipv4', + 'bird_bgp_source_ipv6',NEW.meta_json->>'bird_bgp_source_ipv6'); + ELSE + v_old := v_old - 'updated_at'; + v_new := v_new - 'updated_at'; + END IF; + IF v_old = v_new THEN RETURN NEW; END IF; + END IF; + IF TG_TABLE_NAME IN ('module','doh_profile','bgp_peer','bgp_community','bgp_speaker','global_settings') THEN + IF TG_OP = 'INSERT' THEN v_new_tenant := NEW.tenant_id; + ELSIF TG_OP = 'DELETE' THEN v_old_tenant := OLD.tenant_id; + ELSE v_old_tenant := OLD.tenant_id; v_new_tenant := NEW.tenant_id; END IF; + ELSIF TG_TABLE_NAME IN ('module_cdn_source','module_domain_entry','module_as_entry','module_ip_range_entry','module_doh_profile') THEN + IF TG_OP = 'INSERT' THEN v_new_module := NEW.module_id; + ELSIF TG_OP = 'DELETE' THEN v_old_module := OLD.module_id; + ELSE v_old_module := OLD.module_id; v_new_module := NEW.module_id; END IF; + ELSIF TG_TABLE_NAME = 'module_prefix_snapshot' THEN + IF TG_OP = 'INSERT' THEN v_new_tenant := NEW.tenant_id; + ELSIF TG_OP = 'DELETE' THEN v_old_tenant := OLD.tenant_id; + ELSE v_old_tenant := OLD.tenant_id; v_new_tenant := NEW.tenant_id; END IF; + END IF; + IF v_old_module IS NOT NULL THEN PERFORM render_bump_module_generation(v_old_module); END IF; + IF v_new_module IS NOT NULL AND v_new_module IS DISTINCT FROM v_old_module THEN + PERFORM render_bump_module_generation(v_new_module); + END IF; + IF v_old_tenant IS NOT NULL THEN PERFORM render_bump_tenant_generation(v_old_tenant); END IF; + IF v_new_tenant IS NOT NULL AND v_new_tenant IS DISTINCT FROM v_old_tenant THEN + PERFORM render_bump_tenant_generation(v_new_tenant); + END IF; + IF TG_OP = 'DELETE' THEN RETURN OLD; END IF; + RETURN NEW; +END; +$$; + +CREATE FUNCTION render_snapshot_version() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + NEW.snapshot_version := nextval('module_snapshot_version_seq'); + RETURN NEW; +END; +$$; +CREATE TRIGGER module_snapshot_version BEFORE INSERT OR UPDATE ON module_prefix_snapshot +FOR EACH ROW EXECUTE FUNCTION render_snapshot_version(); + +CREATE TRIGGER render_module_changed AFTER INSERT OR UPDATE OR DELETE ON module +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); +CREATE TRIGGER render_cdn_source_changed AFTER INSERT OR UPDATE OR DELETE ON module_cdn_source +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); +CREATE TRIGGER render_as_entry_changed AFTER INSERT OR UPDATE OR DELETE ON module_as_entry +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); +CREATE TRIGGER render_domain_entry_changed AFTER INSERT OR UPDATE OR DELETE ON module_domain_entry +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); +CREATE TRIGGER render_ip_range_entry_changed AFTER INSERT OR UPDATE OR DELETE ON module_ip_range_entry +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); +CREATE TRIGGER render_doh_profile_changed AFTER INSERT OR UPDATE OR DELETE ON doh_profile +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); +CREATE TRIGGER render_module_doh_changed AFTER INSERT OR UPDATE OR DELETE ON module_doh_profile +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); +CREATE TRIGGER render_peer_changed AFTER INSERT OR UPDATE OR DELETE ON bgp_peer +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); +CREATE TRIGGER render_community_changed AFTER INSERT OR UPDATE OR DELETE ON bgp_community +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); +CREATE TRIGGER render_settings_changed AFTER INSERT OR UPDATE OR DELETE ON global_settings +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); +CREATE TRIGGER render_speaker_changed AFTER INSERT OR UPDATE OR DELETE ON bgp_speaker +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); +CREATE TRIGGER render_snapshot_changed AFTER INSERT OR UPDATE OR DELETE ON module_prefix_snapshot +FOR EACH ROW EXECUTE FUNCTION render_semantic_change(); diff --git a/migrations/sqlite/000034_job_queue_lease.down.sql b/migrations/sqlite/000034_job_queue_lease.down.sql new file mode 100644 index 0000000..71ed63e --- /dev/null +++ b/migrations/sqlite/000034_job_queue_lease.down.sql @@ -0,0 +1,12 @@ +DROP INDEX IF EXISTS idx_job_audit_expired_lease; +DROP INDEX IF EXISTS idx_job_audit_claim; +ALTER TABLE job_audit DROP COLUMN request_sequence; +ALTER TABLE job_audit DROP COLUMN cancel_requested; +ALTER TABLE job_audit DROP COLUMN available_at; +ALTER TABLE job_audit DROP COLUMN fencing_token; +ALTER TABLE job_audit DROP COLUMN attempt; +ALTER TABLE job_audit DROP COLUMN lease_until; +ALTER TABLE job_audit DROP COLUMN owner_id; +ALTER TABLE job_audit DROP COLUMN request_hash; +ALTER TABLE job_audit DROP COLUMN request_json; +DELETE FROM tenant WHERE id = '00000000-0000-0000-0000-000000000000' AND slug = 'evobgp-system-jobs'; diff --git a/migrations/sqlite/000034_job_queue_lease.up.sql b/migrations/sqlite/000034_job_queue_lease.up.sql new file mode 100644 index 0000000..d8a1174 --- /dev/null +++ b/migrations/sqlite/000034_job_queue_lease.up.sql @@ -0,0 +1,14 @@ +ALTER TABLE job_audit ADD COLUMN request_hash TEXT; +ALTER TABLE job_audit ADD COLUMN request_sequence INTEGER NOT NULL DEFAULT 0; +ALTER TABLE job_audit ADD COLUMN request_json TEXT; +ALTER TABLE job_audit ADD COLUMN owner_id TEXT; +ALTER TABLE job_audit ADD COLUMN lease_until TEXT; +ALTER TABLE job_audit ADD COLUMN attempt INTEGER NOT NULL DEFAULT 0; +ALTER TABLE job_audit ADD COLUMN fencing_token INTEGER NOT NULL DEFAULT 0; +ALTER TABLE job_audit ADD COLUMN available_at TEXT NOT NULL DEFAULT '1970-01-01T00:00:00Z'; +ALTER TABLE job_audit ADD COLUMN cancel_requested INTEGER NOT NULL DEFAULT 0; +INSERT OR IGNORE INTO tenant (id, name, slug) VALUES ('00000000-0000-0000-0000-000000000000', 'EvoBGP system jobs', 'evobgp-system-jobs'); +CREATE INDEX idx_job_audit_claim ON job_audit (available_at, created_at) WHERE status = 'queued'; +CREATE INDEX idx_job_audit_expired_lease ON job_audit (lease_until) WHERE status = 'running'; +UPDATE job_audit SET owner_id = 'legacy-unfenced', lease_until = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '+15 minutes'), attempt = 1 +WHERE status = 'running' AND lease_until IS NULL; diff --git a/migrations/sqlite/000035_deployment_progress.down.sql b/migrations/sqlite/000035_deployment_progress.down.sql new file mode 100644 index 0000000..4ca2d10 --- /dev/null +++ b/migrations/sqlite/000035_deployment_progress.down.sql @@ -0,0 +1,7 @@ +DROP TABLE deployment_speaker; +ALTER TABLE bgp_speaker DROP COLUMN deployment_job_id; +ALTER TABLE bgp_speaker DROP COLUMN deployment_request_sequence; +ALTER TABLE bgp_speaker DROP COLUMN deployment_generation; +ALTER TABLE bgp_speaker DROP COLUMN delivery_owner; +ALTER TABLE bgp_speaker DROP COLUMN delivery_token; +ALTER TABLE bgp_speaker DROP COLUMN delivery_until; diff --git a/migrations/sqlite/000035_deployment_progress.up.sql b/migrations/sqlite/000035_deployment_progress.up.sql new file mode 100644 index 0000000..34315ee --- /dev/null +++ b/migrations/sqlite/000035_deployment_progress.up.sql @@ -0,0 +1,27 @@ +ALTER TABLE bgp_speaker ADD COLUMN deployment_generation INTEGER NOT NULL DEFAULT 0; +ALTER TABLE bgp_speaker ADD COLUMN deployment_request_sequence INTEGER NOT NULL DEFAULT 0; +ALTER TABLE bgp_speaker ADD COLUMN deployment_job_id TEXT; +ALTER TABLE bgp_speaker ADD COLUMN delivery_owner TEXT; +ALTER TABLE bgp_speaker ADD COLUMN delivery_token INTEGER NOT NULL DEFAULT 0; +ALTER TABLE bgp_speaker ADD COLUMN delivery_until TEXT; +CREATE TABLE deployment_speaker ( + job_id TEXT NOT NULL REFERENCES job_audit(id) ON DELETE CASCADE, + tenant_id TEXT NOT NULL REFERENCES tenant(id) ON DELETE CASCADE, + speaker_id TEXT NOT NULL REFERENCES bgp_speaker(id) ON DELETE CASCADE, + revision_id TEXT NOT NULL, + generation INTEGER NOT NULL, + request_sequence INTEGER NOT NULL, + state TEXT NOT NULL, + confirmation TEXT NOT NULL DEFAULT '', + error_message TEXT NOT NULL DEFAULT '', + health_received_at TEXT, + health_observation_token INTEGER NOT NULL DEFAULT 0, + health_revision_id TEXT NOT NULL DEFAULT '', + health_source_at TEXT NOT NULL DEFAULT '', + health_error TEXT NOT NULL DEFAULT '', + retry_round INTEGER NOT NULL DEFAULT 0, + dispatch_attempts INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(job_id,speaker_id) +); +CREATE INDEX deployment_speaker_tenant_job ON deployment_speaker(tenant_id,job_id); diff --git a/migrations/sqlite/000036_telemetry_snapshot.down.sql b/migrations/sqlite/000036_telemetry_snapshot.down.sql new file mode 100644 index 0000000..2797324 --- /dev/null +++ b/migrations/sqlite/000036_telemetry_snapshot.down.sql @@ -0,0 +1 @@ +DROP TABLE telemetry_snapshot; diff --git a/migrations/sqlite/000036_telemetry_snapshot.up.sql b/migrations/sqlite/000036_telemetry_snapshot.up.sql new file mode 100644 index 0000000..6efdbc2 --- /dev/null +++ b/migrations/sqlite/000036_telemetry_snapshot.up.sql @@ -0,0 +1,21 @@ +CREATE TABLE telemetry_snapshot ( + tenant_id TEXT NOT NULL REFERENCES tenant(id) ON DELETE CASCADE, + target_id TEXT NOT NULL, + speaker_id TEXT REFERENCES bgp_speaker(id) ON DELETE SET NULL, + kind TEXT NOT NULL CHECK (kind IN ('remote_agent','local_bird')), + local_authority TEXT NOT NULL DEFAULT '', + config_fingerprint TEXT NOT NULL DEFAULT '', + bgp_json TEXT NOT NULL DEFAULT '{}', + health_json TEXT NOT NULL DEFAULT '{}', + next_poll_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + failure_count INTEGER NOT NULL DEFAULT 0, + discovery_cursor INTEGER NOT NULL DEFAULT 0, + last_refresh_requested_at TEXT, + refresh_pending INTEGER NOT NULL DEFAULT 0, + owner_id TEXT, + fencing_token INTEGER NOT NULL DEFAULT 0, + lease_until TEXT, + PRIMARY KEY (tenant_id,target_id) +); +CREATE INDEX telemetry_snapshot_due ON telemetry_snapshot(next_poll_at) WHERE owner_id IS NULL; +CREATE INDEX telemetry_snapshot_speaker ON telemetry_snapshot(speaker_id); diff --git a/migrations/sqlite/000037_render_manifest.down.sql b/migrations/sqlite/000037_render_manifest.down.sql new file mode 100644 index 0000000..b4ca5f5 --- /dev/null +++ b/migrations/sqlite/000037_render_manifest.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS render_revision_manifest; +ALTER TABLE module_prefix_snapshot DROP COLUMN snapshot_version; +DROP TABLE IF EXISTS tenant_render_generation; diff --git a/migrations/sqlite/000037_render_manifest.up.sql b/migrations/sqlite/000037_render_manifest.up.sql new file mode 100644 index 0000000..1b78495 --- /dev/null +++ b/migrations/sqlite/000037_render_manifest.up.sql @@ -0,0 +1,14 @@ +-- SQLite is used for schema parity and local fixtures; the production render +-- admission barrier is implemented by the PostgreSQL repository. +CREATE TABLE tenant_render_generation ( + tenant_id TEXT PRIMARY KEY REFERENCES tenant(id) ON DELETE CASCADE, + generation INTEGER NOT NULL DEFAULT 1 +); +INSERT INTO tenant_render_generation(tenant_id, generation) SELECT id, 1 FROM tenant; +ALTER TABLE module_prefix_snapshot ADD COLUMN snapshot_version INTEGER NOT NULL DEFAULT 0; +CREATE TABLE render_revision_manifest ( + revision_id TEXT PRIMARY KEY REFERENCES config_revision(id) ON DELETE CASCADE, + tenant_id TEXT NOT NULL REFERENCES tenant(id) ON DELETE CASCADE, + config_generation INTEGER NOT NULL, + manifest_json TEXT NOT NULL +); diff --git a/scripts/ci/legacy-node-compatibility.sh b/scripts/ci/legacy-node-compatibility.sh new file mode 100644 index 0000000..18f6ff6 --- /dev/null +++ b/scripts/ci/legacy-node-compatibility.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Build historical source, not today's implementation with a legacy DTO. +# Keep this commit pinned; a moved release tag must not silently change the gate. +case "${EVOBGP_LEGACY_VERSION:-2.56.2}" in + 2.56.2) legacy_ref=924f111e4871e1d6d47c9916d7da1b9051718f1c ;; + 1.0.0) legacy_ref=6ca34b900859665eede22044d734b698059bdf84 ;; + *) echo 'Unsupported legacy test version' >&2; exit 2 ;; +esac +if ! git cat-file -e "$legacy_ref^{commit}" 2>/dev/null; then + git fetch --no-tags --depth=1 origin "$legacy_ref" +fi +legacy_tmp="$(mktemp -d)" +trap 'rm -rf -- "$legacy_tmp"' EXIT +mkdir "$legacy_tmp/source" +archive_paths=(go.mod go.sum cmd/evobgp-node internal migrations) +if [ "${EVOBGP_LEGACY_VERSION:-2.56.2}" = 2.56.2 ]; then + archive_paths+=(cmd/evobgp-agent) +fi +git archive "$legacy_ref" "${archive_paths[@]}" | + tar -x -C "$legacy_tmp/source" +( + cd "$legacy_tmp/source" + go build -mod=readonly -o "$legacy_tmp/evobgp-node-legacy" ./cmd/evobgp-node + if [ "${EVOBGP_LEGACY_VERSION:-2.56.2}" = 2.56.2 ]; then + go build -mod=readonly -o "$legacy_tmp/evobgp-agent-legacy" ./cmd/evobgp-agent + fi +) +export EVOBGP_TEST_LEGACY_NODE_BIN="$legacy_tmp/evobgp-node-legacy" +go test -race -count=1 -json -run '^TestLegacyNodeBinaryCurrentControlPlane$' ./internal/httpapi | + tee "$legacy_tmp/result.jsonl" +assert_test() { +python3 - "$1" "$2" "$3" <<'PY' +import json +import sys + +events = [json.loads(line) for line in open(sys.argv[1], encoding="utf-8")] +package, expected = sys.argv[2:4] +if any(e.get("Package") != package for e in events): + raise SystemExit("unexpected legacy compatibility package") +if any(e.get("Action") in ("skip", "fail") for e in events): + raise SystemExit("legacy compatibility test skipped or failed") +if not any(e.get("Test") == expected and e.get("Action") == "pass" for e in events): + raise SystemExit("required legacy binary compatibility test did not execute") +if not any("Test" not in e and e.get("Action") == "pass" for e in events): + raise SystemExit("legacy compatibility package did not pass") +print(f"Pinned legacy executable passed {expected}") +PY +} +assert_test "$legacy_tmp/result.jsonl" evobgp/internal/httpapi TestLegacyNodeBinaryCurrentControlPlane +if [ "${EVOBGP_LEGACY_VERSION:-2.56.2}" = 2.56.2 ]; then + export EVOBGP_TEST_LEGACY_AGENT_BIN="$legacy_tmp/evobgp-agent-legacy" + go test -race -count=1 -json -run '^TestLegacyAgentBinaryCurrentDispatch$' ./internal/nodedispatch | + tee "$legacy_tmp/agent-result.jsonl" + assert_test "$legacy_tmp/agent-result.jsonl" evobgp/internal/nodedispatch TestLegacyAgentBinaryCurrentDispatch +fi diff --git a/scripts/ci/postgres-integration.sh b/scripts/ci/postgres-integration.sh new file mode 100644 index 0000000..86c8344 --- /dev/null +++ b/scripts/ci/postgres-integration.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${EVOBGP_TEST_DATABASE_URL:?PostgreSQL integration tests require EVOBGP_TEST_DATABASE_URL}" + +assert_passes() { + local result_file="$1" package="$2" label="$3" + shift 3 + python3 - "$result_file" "$package" "$label" "$@" <<'PY' +import json +import sys + +path, package, label = sys.argv[1:4] +required = set(sys.argv[4:]) +passed = set() +skipped = set() +failed = set() +package_passed = False +unexpected_packages = set() +with open(path, encoding="utf-8") as events: + for line in events: + event = json.loads(line) + if event.get("Package") != package: + unexpected_packages.add(event.get("Package", "")) + continue + action = event.get("Action") + test = event.get("Test") + if action == "pass" and test: + passed.add(test) + elif action == "skip" and test: + skipped.add(test) + elif action == "fail": + failed.add(test or "") + elif action == "pass" and not test: + package_passed = True +missing = required - passed +if failed or skipped or missing or unexpected_packages or not package_passed: + raise SystemExit( + f"{label}: required PostgreSQL tests did not pass: " + f"missing={sorted(missing)}, skipped={sorted(skipped)}, " + f"failed={sorted(failed)}, unexpected_packages={sorted(unexpected_packages)}, " + f"package_passed={package_passed}" + ) +print(f"{label}: all {len(required)} required PostgreSQL integration tests passed, 0 skipped") +PY +} + +db_result="$(mktemp)" +repo_result="$(mktemp)" +trap 'rm -f "$db_result" "$repo_result"' EXIT + +# Run packages serially against the ephemeral database. Migration concurrency is +# exercised explicitly inside the isolated-schema DB tests. +go test -race -count=1 -p 1 -json -run '^TestPostgresMigration.*Integration$' ./internal/db | tee "$db_result" +assert_passes "$db_result" evobgp/internal/db 'DB migrations' \ + TestPostgresMigrationsIntegration \ + TestPostgresMigrationConcurrentApplyIntegration \ + TestPostgresMigrationRollbackIntegration \ + TestPostgresMigrationIdempotentIntegration \ + TestPostgresMigrationLockCancellationIntegration +go test -race -count=1 -p 1 -json -run 'Integration$' ./internal/repository | tee "$repo_result" +assert_passes "$repo_result" evobgp/internal/repository 'Repository' \ + TestPostgresPingIntegration \ + TestPostgresModuleCRUDIntegration \ + TestPostgresFirewallClientCreateAndGetIntegration \ + TestEnsurePrefixSnapshotFillsEmptyExistingIntegration \ + TestEnsurePrefixSnapshotIdempotentIntegration \ + TestPostgresPruneUnreferencedPrefixSnapshotsIntegration \ + TestPostgresEstimateRevisionPruneIntegration \ + TestJobAuditClaimAndRestartIntegration \ + TestJobAuditIdempotencyRowIntegration \ + TestJobQueueRetryBoundsAndFailureIntegration \ + TestJobQueueFreshSeedIntegration \ + TestDeploymentLeaseGenerationAndRestartIntegration \ + TestTelemetryLeaseSnapshotAndDiscoveryIntegration \ + TestPostgresModuleSnapshotAtomicRMWMaxPoolOneIntegration \ + TestPostgresModuleSnapshotAdvisoryWaitCancellationIntegration \ + TestPostgresModuleSnapshotPartialWriteRollbackIntegration \ + TestPostgresModuleSnapshotKilledBackendRollsBackIntegration \ + TestPostgresModuleSnapshotMethodsRespectCancellationIntegration diff --git a/scripts/collect-backend-baseline.ps1 b/scripts/collect-backend-baseline.ps1 new file mode 100644 index 0000000..7892864 --- /dev/null +++ b/scripts/collect-backend-baseline.ps1 @@ -0,0 +1,133 @@ +# Read-only Docker snapshot. Never persist raw Compose config, inspect or stderr. +[CmdletBinding()] +param( + [string[]]$ComposeFile = @('deploy/compose/docker-compose.yaml'), + [string]$Profile = 'microvps-full', + [string]$ProjectName, + [string]$OutputPath, + [ValidateRange(1, 120)][int]$Samples = 1, + [ValidateRange(1, 60)][int]$IntervalSeconds = 15 +) + +$ErrorActionPreference = 'Stop' +$repoRoot = Split-Path $PSScriptRoot -Parent +if (-not $OutputPath) { + $OutputPath = Join-Path $repoRoot ('artifacts/backend-baseline/{0}.json' -f (Get-Date -Format 'yyyyMMdd-HHmmss-fff')) +} +$report = [ordered]@{ + schema_version = 1 + collected_at_utc = [DateTime]::UtcNow.ToString('o') + status = 'unavailable' + git_commit = $null + profile = $Profile + requested_samples = $Samples + interval_seconds = $IntervalSeconds + host = $null + configured_services = @() + containers = @() + samples = @() + issues = @() + limitations = @('Read-only point samples, not a workload benchmark or peak-memory measurement.', + 'No raw environment, commands, labels, mounts, logs or Compose configuration is stored.', + 'CPU/RAM/IO measurements require a reachable Docker daemon and running project containers.') +} + +function Invoke-DockerJSON { + param([string[]]$DockerArgs, [switch]$JsonLines) + # Native errors may contain expanded secrets; expose only a fixed category. + $priorPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + $raw = & docker @DockerArgs 2>$null + $code = $LASTEXITCODE + } finally { $ErrorActionPreference = $priorPreference } + if ($code -ne 0) { throw 'docker-command-failed' } + try { + if ($JsonLines) { + foreach ($line in $raw) { if ("$line".Trim()) { $line | ConvertFrom-Json } } + return + } + return (($raw -join "`n") | ConvertFrom-Json) + } + catch { throw 'docker-json-invalid' } +} + +try { + if (Get-Command git -ErrorAction SilentlyContinue) { + $revision = & git -C $repoRoot rev-parse HEAD 2>$null + if ($LASTEXITCODE -eq 0) { $report.git_commit = "$revision".Trim() } + } + if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { throw 'docker-cli-unavailable' } + $composeArgs = @('compose') + foreach ($file in $ComposeFile) { + $absolute = if ([IO.Path]::IsPathRooted($file)) { $file } else { Join-Path $repoRoot $file } + if (-not (Test-Path -LiteralPath $absolute -PathType Leaf)) { throw 'compose-file-unavailable' } + $composeArgs += @('-f', $absolute) + } + if ($Profile) { $composeArgs += @('--profile', $Profile) } + if ($ProjectName) { $composeArgs += @('--project-name', $ProjectName) } + # Config is parsed only in memory. Project metadata/environment are not copied. + $config = Invoke-DockerJSON -DockerArgs ($composeArgs + @('config', '--format', 'json')) + foreach ($service in $config.services.PSObject.Properties) { + $value = $service.Value + $report.configured_services += [ordered]@{ + service = $service.Name + image = $value.image + profiles = @($value.profiles) + memory_limit = $value.deploy.resources.limits.memory + cpu_limit = $value.deploy.resources.limits.cpus + mem_limit = $value.mem_limit + cpus = $value.cpus + } + } + $config = $null + $report.host = Invoke-DockerJSON -DockerArgs @('info', '--format', '{"server_version":{{json .ServerVersion}},"os_type":{{json .OSType}},"architecture":{{json .Architecture}},"cpus":{{json .NCPU}},"memory_bytes":{{json .MemTotal}},"cgroup_version":{{json .CgroupVersion}}}') + $priorPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + $containerIDs = @(& docker @composeArgs ps --all --quiet 2>$null) + $code = $LASTEXITCODE + } finally { $ErrorActionPreference = $priorPreference } + if ($code -ne 0) { throw 'compose-ps-failed' } + $containerIDs = @($containerIDs | Where-Object { $_ -match '^[a-f0-9]{12,64}$' }) + if ($containerIDs.Count -eq 0) { $report.issues += 'no-project-containers' } + $runningIDs = @() + foreach ($id in $containerIDs) { + $container = Invoke-DockerJSON -DockerArgs @('inspect', '--format', '{"id":{{json .Id}},"image_id":{{json .Image}},"status":{{json .State.Status}},"running":{{json .State.Running}},"oom_killed":{{json .State.OOMKilled}},"restart_count":{{json .RestartCount}},"memory_limit_bytes":{{json .HostConfig.Memory}},"nano_cpus":{{json .HostConfig.NanoCpus}},"cpu_quota":{{json .HostConfig.CpuQuota}},"cpu_period":{{json .HostConfig.CpuPeriod}}}', $id) + $report.containers += $container + if ($container.running) { $runningIDs += $id } + } + $allSamplesComplete = $true + for ($i = 0; $i -lt $Samples -and $runningIDs.Count -gt 0; $i++) { + if ($i -gt 0) { Start-Sleep -Seconds $IntervalSeconds } + $stats = @(Invoke-DockerJSON -JsonLines -DockerArgs (@('stats', '--no-stream', '--format', '{"id":{{json .ID}},"cpu_percent":{{json .CPUPerc}},"memory_usage":{{json .MemUsage}},"memory_percent":{{json .MemPerc}},"network_io":{{json .NetIO}},"block_io":{{json .BlockIO}},"pids":{{json .PIDs}}}') + $runningIDs)) + $missingIDs = @() + foreach ($expectedID in $runningIDs) { + $matches = @($stats | Where-Object { + $sampleID = [string]$_.id + $sampleID -match '^[a-f0-9]{12,64}$' -and $expectedID.StartsWith($sampleID) + }) + if ($matches.Count -ne 1) { $missingIDs += $expectedID } + } + $complete = $missingIDs.Count -eq 0 -and $stats.Count -eq $runningIDs.Count + if (-not $complete) { + $allSamplesComplete = $false + $report.issues += 'incomplete-stats-sample' + } + $report.samples += [ordered]@{ observed_at_utc = [DateTime]::UtcNow.ToString('o'); complete = $complete; containers = $stats } + } + $report.status = if ($runningIDs.Count -gt 0 -and $allSamplesComplete) { 'collected' } else { 'partial' } +} catch { + # Do not expose arbitrary exception text, which may include native output. + $known = @('docker-cli-unavailable', 'compose-file-unavailable', 'docker-command-failed', 'docker-json-invalid', 'compose-ps-failed') + $reason = $_.Exception.Message + if ($reason -notin $known) { $reason = 'collection-failed' } + $report.issues += $reason +} finally { + $absoluteOutput = [IO.Path]::GetFullPath($OutputPath) + $parent = Split-Path $absoluteOutput -Parent + if (-not (Test-Path -LiteralPath $parent)) { New-Item -ItemType Directory -Path $parent -Force | Out-Null } + $report | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $absoluteOutput -Encoding UTF8 + Write-Output ('Baseline status: {0}; report: {1}' -f $report.status, $absoluteOutput) +} +if ($report.status -ne 'collected') { exit 2 } diff --git a/scripts/lint-go.ps1 b/scripts/lint-go.ps1 index 79f9b71..af95752 100644 --- a/scripts/lint-go.ps1 +++ b/scripts/lint-go.ps1 @@ -2,11 +2,24 @@ $ErrorActionPreference = 'Stop' Set-Location (Join-Path $PSScriptRoot '..') -$unfmt = gofmt -l . 2>$null +# Enumerate packages in this module instead of recursively formatting unrelated +# checkouts such as .claude/worktrees. Include tagged tests in each package dir. +$packageDirs = @(go list -f '{{.Dir}}' ./...) +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +$unfmt = @( + foreach ($packageDir in ($packageDirs | Sort-Object -Unique)) { + $sourceFiles = @(Get-ChildItem -LiteralPath $packageDir -File -Filter '*.go' | Select-Object -ExpandProperty FullName) + if ($sourceFiles.Count -gt 0) { + gofmt -l @sourceFiles + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + } +) if ($unfmt) { Write-Error "gofmt: unformatted files:`n$unfmt" } go vet ./... +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $golangci = Get-Command golangci-lint -ErrorAction SilentlyContinue if (-not $golangci) { $golangciPath = Join-Path $env:USERPROFILE 'go\bin\golangci-lint.exe' @@ -14,6 +27,7 @@ if (-not $golangci) { } if ($golangci) { & $golangci.Source run + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } } else { Write-Warning 'lint-go: golangci-lint not found, skipping (CI will run it)' } diff --git a/scripts/tests/collect-backend-baseline.test.ps1 b/scripts/tests/collect-backend-baseline.test.ps1 new file mode 100644 index 0000000..c90dd4b --- /dev/null +++ b/scripts/tests/collect-backend-baseline.test.ps1 @@ -0,0 +1,46 @@ +# Offline contract test: no Docker daemon or production access needed. +$ErrorActionPreference = 'Stop' +$repoRoot = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent +$output = Join-Path $repoRoot 'artifacts/backend-baseline/offline-test.json' +$mockState = @{ Calls = [Collections.Generic.List[object]]::new(); EmptyStats = $false } +function docker { + param([Parameter(ValueFromRemainingArguments = $true)][string[]]$DockerArgs) + $mockState.Calls.Add($DockerArgs) + $global:LASTEXITCODE = 0 + switch ($DockerArgs[0]) { + 'compose' { + if ($DockerArgs -contains 'config') { + '{"services":{"api":{"image":"example/api:test","environment":{"PASSWORD":"MUST_NOT_LEAK"},"command":"MUST_NOT_LEAK","deploy":{"resources":{"limits":{"memory":"280M","cpus":"0.85"}}}}}}' + } else { 'aaaaaaaaaaaa'; 'bbbbbbbbbbbb' } + } + 'info' { '{"server_version":"test","os_type":"linux","cpus":2,"memory_bytes":1073741824}' } + 'inspect' { + '{"id":"' + $DockerArgs[-1] + '","image_id":"sha256:test","running":true,"memory_limit_bytes":293601280,"oom_killed":false,"restart_count":0}' + } + 'stats' { + if ($mockState.EmptyStats) { return } + '{"id":"aaaaaaaaaaaa","cpu_percent":"1.0%","memory_usage":"20MiB / 280MiB"}' + '{"id":"bbbbbbbbbbbb","cpu_percent":"2.0%","memory_usage":"30MiB / 280MiB"}' + } + default { throw 'unexpected Docker operation' } + } +} +& (Join-Path $repoRoot 'scripts/collect-backend-baseline.ps1') -OutputPath $output -Samples 1 +$raw = Get-Content -LiteralPath $output -Raw +$result = $raw | ConvertFrom-Json +if ($result.status -ne 'collected') { throw 'collector did not succeed' } +if ($result.containers.Count -ne 2 -or $result.samples[0].containers.Count -ne 2) { throw 'NDJSON multi-container stats were lost' } +if ($raw.Contains('MUST_NOT_LEAK') -or $raw.Contains('PASSWORD')) { throw 'raw Compose secrets leaked' } +if ($result.configured_services[0].memory_limit -ne '280M') { throw 'effective resource limit missing' } +if ($mockState.Calls.Count -eq 0) { throw 'Docker mock was not exercised' } +foreach ($call in $mockState.Calls) { + if ($call[0] -notin @('compose', 'info', 'inspect', 'stats')) { throw 'unexpected operation' } + if ($call[0] -eq 'compose' -and -not ($call -contains 'config' -or $call -contains 'ps')) { throw 'mutating Compose operation' } +} +$mockState.EmptyStats = $true +& (Join-Path $repoRoot 'scripts/collect-backend-baseline.ps1') -OutputPath $output -Samples 1 +if ($LASTEXITCODE -ne 2) { throw 'missing stats must exit 2' } +$partial = Get-Content -LiteralPath $output -Raw | ConvertFrom-Json +if ($partial.status -ne 'partial' -or $partial.samples[0].complete -ne $false) { throw 'empty stats reported as complete' } +Write-Output 'PASS: multi-container snapshot, resource projection, secret exclusion, read-only operations, incomplete stats detection' +exit 0 diff --git a/scripts/tests/mirror-base-images.test.sh b/scripts/tests/mirror-base-images.test.sh new file mode 100644 index 0000000..601befc --- /dev/null +++ b/scripts/tests/mirror-base-images.test.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env sh +# Offline behavioral checks for deploy/docker/mirror-base-images.sh. +set -eu + +repo_root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" +script="$repo_root/deploy/docker/mirror-base-images.sh" +test_dir="$(mktemp -d)" +case_env="$test_dir/mirror.env" +case_log="$test_dir/calls.log" +case_out="$test_dir/output.log" +copy_count_file="$test_dir/copy-count" + +cleanup() { + rm -f "$test_dir/docker" "$test_dir/sleep" "$case_env" "$case_log" "$case_out" "$copy_count_file" + rmdir "$test_dir" +} +trap cleanup EXIT + +cat > "$test_dir/docker" <<'EOF' +#!/usr/bin/env sh +set -eu +case "$1 $2 $3" in + 'buildx imagetools inspect') + if [ "${4:-}" = '--format' ]; then + printf 'digest %s\n' "$6" >> "$CASE_LOG" + [ "$TEST_BAD_DIGEST" = 0 ] || exit 1 + printf 'sha256:%s\n' 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + else + printf 'inspect %s\n' "$4" >> "$CASE_LOG" + [ "$TEST_EXISTING" = 1 ] + fi + ;; + 'buildx imagetools create') + n="$(cat "$COPY_COUNT_FILE")" + n="$((n + 1))" + printf '%s\n' "$n" > "$COPY_COUNT_FILE" + printf 'copy %s\n' "$8" >> "$CASE_LOG" + [ "$TEST_COPY_ALWAYS_FAIL" = 0 ] && [ "$n" -gt "$TEST_FAIL_FIRST" ] + ;; + *) + printf 'unexpected docker call: %s\n' "$*" >&2 + exit 2 + ;; +esac +EOF +cat > "$test_dir/sleep" <<'EOF' +#!/usr/bin/env sh +printf 'sleep %s\n' "$1" >> "$CASE_LOG" +EOF +chmod +x "$test_dir/docker" "$test_dir/sleep" + +fail() { + printf 'FAIL: %s\n' "$1" >&2 + cat "$case_out" >&2 + exit 1 +} + +count_calls() { + awk -v kind="$1" '$1 == kind { count++ } END { print count + 0 }' "$case_log" +} + +run_case() { + label="$1" + TEST_EXISTING="$2" + MIRROR_REFRESH="$3" + TEST_FAIL_FIRST="$4" + TEST_COPY_ALWAYS_FAIL="$5" + TEST_BAD_DIGEST="$6" + expected_status="$7" + export TEST_EXISTING MIRROR_REFRESH TEST_FAIL_FIRST TEST_COPY_ALWAYS_FAIL TEST_BAD_DIGEST + : > "$case_log" + printf '0\n' > "$copy_count_file" + rm -f "$case_env" + if REGISTRY=example.test/owner MIRROR_ENV_FILE="$case_env" MIRROR_RETRIES=2 \ + CASE_LOG="$case_log" COPY_COUNT_FILE="$copy_count_file" PATH="$test_dir:$PATH" \ + sh "$script" > "$case_out" 2>&1; then + status=0 + else + status=$? + fi + if [ "$expected_status" = success ] && [ "$status" -ne 0 ]; then + fail "$label returned $status, expected success" + fi + if [ "$expected_status" = failure ] && [ "$status" -eq 0 ]; then + fail "$label succeeded, expected failure" + fi +} + +assert_calls() { + kind="$1" + expected="$2" + actual="$(count_calls "$kind")" + [ "$actual" -eq "$expected" ] || fail "expected $expected $kind calls, got $actual" +} + +assert_immutable_refs() { + [ -f "$case_env" ] || fail 'mirror env file missing' + lines="$(wc -l < "$case_env" | tr -d ' ')" + [ "$lines" -eq 6 ] || fail "expected 6 mirror refs, got $lines" + if grep -v -E '^BASE_[A-Z_]+=example\.test/owner/evobgp-buildcache@sha256:[a-f0-9]{64}$' "$case_env"; then + fail 'mirror env contains mutable or malformed image ref' + fi +} + +run_case existing 1 0 0 0 0 success +assert_calls copy 0 +assert_calls sleep 0 +assert_immutable_refs + +run_case new 0 0 0 0 0 success +assert_calls copy 6 +assert_calls sleep 0 +assert_immutable_refs + +run_case retry 0 0 1 0 0 success +assert_calls copy 7 +assert_calls sleep 1 +assert_immutable_refs + +run_case refresh 1 1 0 0 0 success +assert_calls copy 6 +assert_calls sleep 0 +assert_immutable_refs + +run_case refresh_copy_failure 1 1 0 1 0 failure +assert_calls copy 2 +assert_calls sleep 1 +[ ! -f "$case_env" ] || fail 'failed refresh wrote mirror refs' + +run_case refresh_digest_failure 1 1 0 0 1 failure +assert_calls copy 1 +assert_calls sleep 0 +[ ! -f "$case_env" ] || fail 'failed digest lookup wrote mirror refs' + +run_case fallback 0 0 0 1 0 success +assert_calls copy 12 +assert_calls sleep 6 +[ ! -s "$case_env" ] || fail 'fallback wrote stale mirror refs' + +run_case digest_fallback 1 0 0 0 1 success +assert_calls copy 0 +assert_calls sleep 0 +[ ! -s "$case_env" ] || fail 'digest fallback wrote mutable mirror refs' + +printf 'mirror-base-images offline tests passed\n'