diff --git a/apps/web/src/components/network/speaker-form-dialog.tsx b/apps/web/src/components/network/speaker-form-dialog.tsx index 4809253..4955354 100644 --- a/apps/web/src/components/network/speaker-form-dialog.tsx +++ b/apps/web/src/components/network/speaker-form-dialog.tsx @@ -284,7 +284,8 @@ function SpeakerInstallStep({ Сохраните сейчас agent_secret и node_token больше не будут показаны. Traefik на ноде выпускает - сертификат через DNS-01 (Cloudflare). + сертификат через DNS-01 (Cloudflare). MikroTik стучится на IP ноды:179; 80/443 — + только агент панели. Логи: docker compose logs -f bird2 evobgp-agent.
diff --git a/deploy/bird/bird.conf b/deploy/bird/bird.conf index 25c4bb2..ecd4a77 100644 --- a/deploy/bird/bird.conf +++ b/deploy/bird/bird.conf @@ -1,5 +1,6 @@ # Default BIRD 2 config for EvoBGP Docker stack (operator extends with include "bird.d/*.conf";). router id 192.0.2.1; +log stderr all; protocol device { } diff --git a/deploy/compose/docker-compose.remote-speaker.yaml b/deploy/compose/docker-compose.remote-speaker.yaml index f150464..f120728 100644 --- a/deploy/compose/docker-compose.remote-speaker.yaml +++ b/deploy/compose/docker-compose.remote-speaker.yaml @@ -7,9 +7,12 @@ # --env-file .env.remote-speaker --env-file .env.remote-speaker-tls up -d # # Profiles: -# production (default) — bird2 host + agent + evobgp-edge +# production (default) — bird2 (speaker-net, 179:179) + agent + evobgp-edge # plain — bird2 + agent без Traefik (lab) # fallback — + sync-bundle polling +# +# BGP TCP/179 as on the control plane. Overlay sets router id / local. +# Logs: docker compose logs -f bird2 evobgp-agent name: evobgp-remote-speaker @@ -24,13 +27,18 @@ services: profiles: ["production", "plain", "fallback"] image: ${EVOBGP_REGISTRY:-git.shx.one/denozord}/evobgp-bird2:${EVOBGP_IMAGE_TAG:-latest} restart: unless-stopped - network_mode: host cap_add: - NET_ADMIN - # sysctls нельзя с network_mode: host — включите ip_forward на VPS (см. docs/remote-speakers.md) + sysctls: + net.ipv4.ip_forward: "1" + net.ipv6.conf.all.forwarding: "1" + ports: + - "179:179/tcp" volumes: - bird_etc:/etc/bird - bird_run:/run/bird + networks: + - speaker-net logging: *default-logging evobgp-agent: diff --git a/docs/remote-speakers.md b/docs/remote-speakers.md index 420238e..c9e5087 100644 --- a/docs/remote-speakers.md +++ b/docs/remote-speakers.md @@ -42,9 +42,11 @@ Runbook для реплик **bird2 + evobgp-agent** на отдельных VPS | Порт | Кто | Зачем | |------|-----|-------| | **443** | IP CP (`PANEL_IP_WHITELIST`) | HTTPS dispatch, health, `GET /v1/agent/bird/protocols` | -| **179** | BGP peers | Data plane | +| **179** | BGP peers | Data plane — Docker `ports: 179:179/tcp`, как на панели | | **80** | любой | редирект HTTP → HTTPS (не HTTP-01 ACME) | +В панели хостера / security group откройте **TCP 179** (скрипт compose это не делает). Overlay (`bird_bgp_source_ipv4` / `node_ipv4`) задаёт `router id`; host-сеть bird2 не используется. + DNS-01 ходит **исходящим** к Cloudflare API и Let's Encrypt; inbound 80 для выпуска сертификата не нужен. Agent слушает `:8443` только во внутренней docker-сети; снаружи — Traefik 443. Токен Cloudflare для панели (`evobgp-edge` на CP) в процесс API **не проброшен** — для реплики его задают в форме создания. @@ -55,15 +57,15 @@ Profile `plain` в файле репозитория — только lab без | Profile | Состав | |---------|--------| -| `production` | bird2 (host) + agent + Traefik LE | -| `plain` | bird2 + agent на хосте без Traefik (только lab) | +| `production` | bird2 (`speaker-net`, `179:179`) + agent + Traefik LE | +| `plain` | bird2 + agent без Traefik (lab; agent на хосте) | | `fallback` | + `sync-bundle` polling (`scripts/sync-bundle.sh`) | Команда из UI — самодостаточный yaml **без profiles** (эквивалент production). ## Подготовка VPS -`bird2` — **`network_mode: host`**. Docker **не может** задать `net.ipv4.ip_forward` в таком контейнере. Команда из UI включает sysctl; для постоянства: +`bird2` в docker-сети с `ports: 179:179/tcp` и `sysctls` ip_forward (как панель). Команда из UI дополнительно включает sysctl на хосте: ```bash sysctl -w net.ipv4.ip_forward=1 @@ -73,6 +75,18 @@ echo 'net.ipv6.conf.all.forwarding=1' >> /etc/sysctl.d/99-evobgp-bird.conf sysctl --system ``` +## Логи на реплике + +BIRD пишет в stderr (`log stderr all`), agent — в stdout. На VPS: + +```bash +cd /opt/evobgp-speaker +docker compose logs -f bird2 +docker compose logs -f evobgp-agent +``` + +До первого apply бандла с `protocol bgp` порт 179 может быть CLOSED (нет listener). После sync в логах agent: `sync start` / `sync ok` / `sync failed`. + ## Безопасность (три участка) 1. **CP → реплика:** HTTPS (LE) + Traefik ipallowlist + `agent_secret`. @@ -105,13 +119,12 @@ Tenant `/v1/settings` (`bird_bgp_source_ipv4`) — fallback для master / ес | Симптом | Проверка | |---------|----------| -| `sysctl net.ipv4.ip_forward not allowed in host network` | Уберите sysctls из compose (уже так в main); включите ip_forward на VPS (см. выше) | | `CHANGE_ME_*` в yaml | В форме не заполнены email LE / CF token / IP панели / домен | | Traefik отдаёт дефолтный сертификат | DNS only; token `Zone:DNS:Edit`; логи `evobgp-edge`; том acme.json | | Offline в UI | `GET https://AGENT_DOMAIN/v1/agent/health` с CP; LE cert; whitelist | | dispatch error | CP logs job meta; firewall 443; `agent_secret` | | verify-bundle fail | pubkey совпадает с CP seed; пересоберите pubkey после смены seed | -| BGP не поднимается | bird2 `network_mode: host`; peers; MD5 BGP отдельно от HTTP sync | +| BGP не поднимается / сканер CLOSED | `179:179` в compose; SG хостера; `docker compose logs bird2`; пир MikroTik на IP ноды; бандл применён (`sync ok`) | ## Ограничения (scale-review) diff --git a/internal/agentserver/server.go b/internal/agentserver/server.go index e8c7e65..f7a39ec 100644 --- a/internal/agentserver/server.go +++ b/internal/agentserver/server.go @@ -7,6 +7,7 @@ import ( "fmt" "log" "net/http" + "net/url" "os" "strings" "time" @@ -117,6 +118,9 @@ func (s *Server) handleSync(w http.ResponseWriter, r *http.Request) { if timeout <= 0 { timeout = 45 * time.Second } + revID := strings.TrimSpace(req.RevisionID) + log.Printf("agentserver: sync start speaker_id=%s revision_id=%q control_plane=%s", + strings.TrimSpace(s.cfg.SpeakerID), revID, controlPlaneHost(s.cfg.ControlPlaneURL)) ctx, cancel := context.WithTimeout(r.Context(), timeout) defer cancel() @@ -124,7 +128,7 @@ func (s *Server) handleSync(w http.ResponseWriter, r *http.Request) { BaseURL: s.cfg.ControlPlaneURL, Token: s.cfg.NodeToken, SpeakerID: s.cfg.SpeakerID, - RevisionID: strings.TrimSpace(req.RevisionID), + RevisionID: revID, PubKeyB64: s.cfg.PubKeyB64, PubKeyHex: s.cfg.PubKeyHex, ExtractDir: s.cfg.ExtractDir, @@ -134,13 +138,16 @@ func (s *Server) handleSync(w http.ResponseWriter, r *http.Request) { Timeout: timeout, }) if err != nil { - log.Printf("agentserver: sync: %v", err) + log.Printf("agentserver: sync failed speaker_id=%s revision_id=%q err=%v", + strings.TrimSpace(s.cfg.SpeakerID), revID, err) writeProblem(w, http.StatusBadGateway, upstreamErrorDetail) return } if s.cfg.OnSyncSuccess != nil { s.cfg.OnSyncSuccess(res.RevisionID) } + log.Printf("agentserver: sync ok speaker_id=%s applied_revision_id=%s", + strings.TrimSpace(s.cfg.SpeakerID), res.RevisionID) writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "applied_revision_id": res.RevisionID, @@ -148,6 +155,18 @@ func (s *Server) handleSync(w http.ResponseWriter, r *http.Request) { }) } +func controlPlaneHost(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + u, err := url.Parse(raw) + if err != nil || strings.TrimSpace(u.Host) == "" { + return raw + } + return u.Host +} + func (s *Server) authorize(r *http.Request) bool { secret := strings.TrimSpace(s.cfg.Secret) if secret == "" { diff --git a/internal/birdfmt/layout.go b/internal/birdfmt/layout.go index 4e6b25d..5325f87 100644 --- a/internal/birdfmt/layout.go +++ b/internal/birdfmt/layout.go @@ -72,7 +72,8 @@ func RenderMainBirdConf(opts MainBirdConfOptions) (string, error) { } b.WriteString("router id ") b.WriteString(strings.TrimSpace(opts.RouterID)) - b.WriteString(";\n\n") + b.WriteString(";\n") + b.WriteString("log stderr all;\n\n") for _, inc := range opts.Includes { inc = strings.TrimSpace(inc) if inc == "" { diff --git a/internal/birdfmt/testdata/golden/main_bird_skeleton.golden b/internal/birdfmt/testdata/golden/main_bird_skeleton.golden index 8b2a658..2ad5c67 100644 --- a/internal/birdfmt/testdata/golden/main_bird_skeleton.golden +++ b/internal/birdfmt/testdata/golden/main_bird_skeleton.golden @@ -1,4 +1,5 @@ router id 192.0.2.1; +log stderr all; include "bird.d/evobgp_filters_v4.conf"; include "bird.d/evobgp_filters_v6.conf"; diff --git a/internal/birdfmt/testdata/scenarios/standard_layout/bird.conf b/internal/birdfmt/testdata/scenarios/standard_layout/bird.conf index 4cd16a3..3ece44b 100644 --- a/internal/birdfmt/testdata/scenarios/standard_layout/bird.conf +++ b/internal/birdfmt/testdata/scenarios/standard_layout/bird.conf @@ -2,6 +2,7 @@ # Standard EvoBGP layout: main skeleton + bird.d fragments (matches StandardIncludeFragments). router id 192.0.2.1; +log stderr all; include "bird.d/evobgp_filters_v4.conf"; include "bird.d/evobgp_filters_v6.conf"; diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 8092882..54fea0a 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -999,9 +999,12 @@ func (s *Server) handleNodeBundle(w http.ResponseWriter, r *http.Request) { return } frags := rev.PreviewFragments - if overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, a.TenantID, sid, rid, frags); err == nil { - frags = overlaid + overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, a.TenantID, sid, rid, frags) + if err != nil { + writeInternalError(w, "bundle overlay", err) + return } + frags = overlaid tgz, err := bundle.BuildGzippedTar(rid, sid, frags, s.bundlePriv) if err != nil { writeInternalError(w, "internal", err) diff --git a/internal/httpapi/speakers_test.go b/internal/httpapi/speakers_test.go index ceb8b29..d9d9164 100644 --- a/internal/httpapi/speakers_test.go +++ b/internal/httpapi/speakers_test.go @@ -149,6 +149,7 @@ func TestPostSpeaker_installCommandsAndMetaObject(t *testing.T) { secret, token, "https://cp.example.com", + `"179:179/tcp"`, } { if !strings.Contains(cmd, want) { t.Errorf("docker_commands missing %q", want) diff --git a/internal/pipeline/speaker_overlay.go b/internal/pipeline/speaker_overlay.go index 477a4e7..173a0a6 100644 --- a/internal/pipeline/speaker_overlay.go +++ b/internal/pipeline/speaker_overlay.go @@ -19,9 +19,13 @@ func BirdLocalsForSpeaker(st store.Backend, tenantID, speakerID string) birdLoca return loc } meta := store.ParseSpeakerMeta(sp.MetaJSON) - if s := strings.TrimSpace(meta.BirdBgpSourceIPv4); s != "" { - loc.routerID = s - loc.localV4 = s + src := strings.TrimSpace(meta.BirdBgpSourceIPv4) + if src == "" { + src = strings.TrimSpace(meta.NodeIPv4) + } + if src != "" { + loc.routerID = src + loc.localV4 = src } if s := strings.TrimSpace(meta.BirdBgpSourceIPv6); s != "" { loc.localV6 = s @@ -32,7 +36,7 @@ func BirdLocalsForSpeaker(st store.Backend, tenantID, speakerID string) birdLoca // OverlayFragmentsForSpeaker re-renders bird.conf and peers fragment with speaker-specific BIRD locals. func OverlayFragmentsForSpeaker(st store.Backend, tenantID, speakerID, revisionID string, frags map[string]string) (map[string]string, error) { if frags == nil { - return nil, fmt.Errorf("pipeline: overlay: nil fragments") + frags = map[string]string{} } locals := BirdLocalsForSpeaker(st, tenantID, speakerID) out := make(map[string]string, len(frags)) diff --git a/internal/pipeline/speaker_overlay_test.go b/internal/pipeline/speaker_overlay_test.go index 26bb9cd..10a0e3a 100644 --- a/internal/pipeline/speaker_overlay_test.go +++ b/internal/pipeline/speaker_overlay_test.go @@ -40,3 +40,48 @@ func TestOverlayFragmentsForSpeaker_differentRouterID(t *testing.T) { t.Fatalf("sp2 router: %s", out2["bird.conf"]) } } + +func TestOverlayFragmentsForSpeaker_nodeIPv4Fallback(t *testing.T) { + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + sp, err := m.CreateSpeaker(tenant, &store.Speaker{ + Role: "replica", + Endpoint: "https://node.example.com", + MetaJSON: `{"node_ipv4":"198.51.100.9"}`, + }) + if err != nil { + t.Fatal(err) + } + out, err := pipeline.OverlayFragmentsForSpeaker(m, tenant, sp.ID, "rev1", nil) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out["bird.conf"], "198.51.100.9") { + t.Fatalf("expected node_ipv4 as router id, got: %s", out["bird.conf"]) + } +} + +func TestOverlayFragmentsForSpeaker_sourceOverridesNodeIPv4(t *testing.T) { + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + sp, err := m.CreateSpeaker(tenant, &store.Speaker{ + Role: "replica", + Endpoint: "https://node.example.com", + MetaJSON: `{"node_ipv4":"198.51.100.9","bird_bgp_source_ipv4":"203.0.113.40"}`, + }) + if err != nil { + t.Fatal(err) + } + out, err := pipeline.OverlayFragmentsForSpeaker(m, tenant, sp.ID, "rev1", map[string]string{}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out["bird.conf"], "203.0.113.40") { + t.Fatalf("source should win: %s", out["bird.conf"]) + } + if strings.Contains(out["bird.conf"], "198.51.100.9") { + t.Fatalf("node_ipv4 should not win over source: %s", out["bird.conf"]) + } +} diff --git a/internal/speakerinstall/install.go b/internal/speakerinstall/install.go index 5dddc48..0c00a17 100644 --- a/internal/speakerinstall/install.go +++ b/internal/speakerinstall/install.go @@ -57,6 +57,7 @@ type renderData struct { const composeTemplate = `# EvoBGP replica: bird2 + evobgp-agent + Traefik (Let's Encrypt DNS-01 / Cloudflare). # Generated by control plane. Do not commit secrets. ACME state: volume evobgp_speaker_traefik_letsencrypt. +# BGP: ports 179:179 like control plane (overlay sets router id). Logs: docker compose logs -f bird2 evobgp-agent name: evobgp-remote-speaker @@ -70,12 +71,18 @@ services: bird2: image: {{.BirdImage}} restart: unless-stopped - network_mode: host cap_add: - NET_ADMIN + sysctls: + net.ipv4.ip_forward: "1" + net.ipv6.conf.all.forwarding: "1" + ports: + - "179:179/tcp" volumes: - bird_etc:/etc/bird - bird_run:/run/bird + networks: + - speaker-net logging: *default-logging evobgp-agent: @@ -200,6 +207,8 @@ func dockerCommands(composeYAML string) string { var b strings.Builder b.WriteString(`# EvoBGP replica: bird2 + agent + Traefik (Let's Encrypt DNS-01 / Cloudflare) # docker login git.shx.one # if images are private +# BGP TCP/179 published like the control plane. Cloud security group must allow 179. +# Logs: cd /opt/evobgp-speaker && docker compose logs -f bird2 evobgp-agent set -euo pipefail sysctl -w net.ipv4.ip_forward=1 sysctl -w net.ipv6.conf.all.forwarding=1 diff --git a/internal/speakerinstall/install_test.go b/internal/speakerinstall/install_test.go index f7d6e4d..ed89401 100644 --- a/internal/speakerinstall/install_test.go +++ b/internal/speakerinstall/install_test.go @@ -34,11 +34,16 @@ func TestBuild_includesTraefikDNS01(t *testing.T) { "docker compose up -d", "sysctl -w net.ipv4.ip_forward=1", "evobgp_speaker_traefik_letsencrypt", + `"179:179/tcp"`, + "net.ipv4.ip_forward: \"1\"", } { if !strings.Contains(cmd, want) { t.Errorf("docker_commands missing %q", want) } } + if strings.Contains(cmd, "network_mode: host") { + t.Error("replica bird2 must not use network_mode: host") + } if strings.Contains(cmd, "?set ") { t.Error("compose must bake values, not ${VAR:?set VAR}") }