feat(firewall): improve blocklist parsing and nft element addition

Enhanced the blocklist parsing function to log when the blocklist file is empty. Introduced new helper functions `nft_join_elements` and `nft_add_v4_chunk` to streamline the addition of elements to the nftables, allowing for batch processing and improved error handling. Adjusted the chunk size for element addition to optimize performance. Updated logging to provide better visibility into the blocklist processing and applied prefixes.

docs/firewall.md

@@ -35,6 +35,16 @@ curl -fsSL https://<api>/v1/firewall/install.sh | \
 
 Файлы: `/etc/evobgp/firewall.conf`, `/usr/local/sbin/evobgp-firewall.sh`, systemd timer `evobgp-firewall.timer`.
 
+После **approve** в UI выполните на сервере (или дождитесь timer):
+
+```bash
+sudo rm -f /var/lib/evobgp-firewall/last_hash
+sudo /usr/local/sbin/evobgp-firewall.sh
+sudo nft list table inet evobgp_blocklist
+```
+
+Для парсинга JSON нужен `jq` или `python3` (install.sh ставит `jq` на Debian/Ubuntu при отсутствии).
+
 ## Failover через speaker
 
 При `EVOBGP_FIREWALL_FAILOVER_ENABLED=1` на speaker-agent CP реплицирует состояние через `POST /v1/agent/firewall-replicate`. Клиенты используют тот же DNS-домен.

internal/firewallscripts/evobgp-firewall.sh

@@ -5,6 +5,7 @@ CONF_FILE=/etc/evobgp/firewall.conf
 LOG_FILE=/var/log/evobgp-firewall.log
 STATE_DIR=/var/lib/evobgp-firewall
 HASH_FILE="${STATE_DIR}/last_hash"
+PREFIX_FILE="${STATE_DIR}/last_prefixes.txt"
 
 log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" | tee -a "$LOG_FILE"; }
 
@@ -17,36 +18,32 @@ source "$CONF_FILE"
 
 : "${EVOBGP_CP_URL:?}"
 : "${CLIENT_TOKEN:?}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\r'/}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\n'/}"
 
 mkdir -p "$STATE_DIR"
 BACKEND="${KERNEL_BACKEND:-auto}"
 
-curl_get_blocklist() {
+curl_get_blocklist_file() {
   local url="$1"
-  local host
-  host=$(echo "$url" | sed -E 's#https?://([^/]+)/?.*#\1#')
-  local tmp
-  tmp=$(mktemp)
+  local dest="$2"
   local code
-  code=$(curl -sS -o "$tmp" -w "%{http_code}" \
+  code=$(curl -sS -o "$dest" -w "%{http_code}" \
     -H "Authorization: Bearer ${CLIENT_TOKEN}" \
     -H "Accept: application/json" \
     "${url}/v1/firewall/blocklist") || return 1
   if [[ "$code" == "403" ]]; then
     log "pending approval"
-    rm -f "$tmp"
-    exit 0
+    return 2
   fi
   if [[ "$code" != "200" ]]; then
     log "blocklist HTTP $code from $url"
-    rm -f "$tmp"
     return 1
   fi
-  cat "$tmp"
-  rm -f "$tmp"
+  return 0
 }
 
-try_urls() {
+try_fetch_blocklist() {
   local urls=()
   if [[ -n "${EVOBGP_FAILOVER_URLS:-}" ]]; then
     IFS=',' read -r -a urls <<<"$EVOBGP_FAILOVER_URLS"
@@ -57,7 +54,12 @@ try_urls() {
   for u in "${urls[@]}"; do
     u="${u// /}"
     u="${u%/}"
-    if OUT=$(curl_get_blocklist "$u"); then
+    local rc=0
+    curl_get_blocklist_file "$u" "$PREFIX_FILE" || rc=$?
+    if [[ "$rc" == 2 ]]; then
+      exit 0
+    fi
+    if [[ "$rc" == 0 ]]; then
       CP_HIT="$u"
       return 0
     fi
@@ -65,22 +67,87 @@ try_urls() {
   return 1
 }
 
-if ! OUT=$(try_urls); then
+parse_blocklist_file() {
+  local f="$1"
+  if [[ ! -s "$f" ]]; then
+    log "blocklist file empty: $f"
+    return 1
+  fi
+  if command -v jq >/dev/null 2>&1; then
+    HASH=$(jq -r '.hash // empty' "$f")
+    TOTAL=$(jq -r '.total // 0' "$f")
+    mapfile -t PREFIXES < <(jq -r '.prefixes[]? // empty' "$f")
+    return 0
+  fi
+  if command -v python3 >/dev/null 2>&1; then
+    local parsed
+    parsed=$(python3 - "$f" <<'PY'
+import json, sys
+with open(sys.argv[1], encoding="utf-8") as fh:
+    data = json.load(fh)
+print(data.get("hash") or "")
+print(data.get("total") or 0)
+for p in data.get("prefixes") or []:
+    if p:
+        print(p)
+PY
+    )
+    HASH=$(echo "$parsed" | sed -n '1p')
+    TOTAL=$(echo "$parsed" | sed -n '2p')
+    mapfile -t PREFIXES < <(echo "$parsed" | sed -n '3,$p')
+    return 0
+  fi
+  HASH=$(grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' "$f" | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
+  TOTAL=$(grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' "$f" | head -1 | grep -o '[0-9]*$' || true)
+  mapfile -t PREFIXES < <(grep -oE '"[0-9]+(\.[0-9]+){3}/[0-9]+"' "$f" | tr -d '"' || true)
+  return 0
+}
+
+nft_join_elements() {
+  local out="" p
+  for p in "$@"; do
+    if [[ -n "$out" ]]; then
+      out+=", "
+    fi
+    out+="$p"
+  done
+  printf '%s' "$out"
+}
+
+nft_add_v4_chunk() {
+  local table=$1 name=$2
+  shift 2
+  local joined
+  joined=$(nft_join_elements "$@")
+  if nft add element "$table" "$name" v4 "{ ${joined} }" 2>>"$LOG_FILE"; then
+    return 0
+  fi
+  log "nft batch add failed (chunk=$#), retrying one-by-one"
+  local p ok=0
+  for p in "$@"; do
+    if nft add element "$table" "$name" v4 "{ $p }" 2>>"$LOG_FILE"; then
+      ok=$((ok + 1))
+    fi
+  done
+  [[ "$ok" -gt 0 ]]
+}
+
+if ! try_fetch_blocklist; then
   log "all endpoints failed"
   exit 1
 fi
 
-if command -v jq >/dev/null 2>&1; then
-  HASH=$(echo "$OUT" | jq -r '.hash // empty')
-  TOTAL=$(echo "$OUT" | jq -r '.total // 0')
-  mapfile -t PREFIXES < <(echo "$OUT" | jq -r '.prefixes[]?')
-else
-  HASH=$(echo "$OUT" | grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
-  TOTAL=$(echo "$OUT" | grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
-  mapfile -t PREFIXES < <(echo "$OUT" | grep -o '"[0-9a-fA-F:.]*/[0-9]*"' | tr -d '"')
+HASH=""
+TOTAL=0
+PREFIXES=()
+parse_blocklist_file "$PREFIX_FILE"
+log "blocklist bytes=$(wc -c <"$PREFIX_FILE" | tr -d ' ') parsed=${#PREFIXES[@]} api_total=${TOTAL:-0}"
+
+if [[ -z "${TOTAL// }" ]]; then
+  TOTAL=${#PREFIXES[@]}
 fi
 
-if [[ -f "$HASH_FILE" && "$(cat "$HASH_FILE")" == "$HASH" ]]; then
+if [[ -f "$HASH_FILE" && "$(tr -d '\r\n' <"$HASH_FILE")" == "$HASH" && -n "$HASH" ]]; then
   log "unchanged hash $HASH — skip kernel apply"
   exit 0
 fi
@@ -88,48 +155,66 @@ fi
 apply_nft() {
   local table=inet
   local name=evobgp_blocklist
+  local v4=()
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    v4+=("$p")
+  done
+
   nft list table "$table" "$name" >/dev/null 2>&1 || nft add table "$table" "$name"
-  nft list set "$table" "$name" v4 >/dev/null 2>&1 || nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
+  nft list set "$table" "$name" v4 >/dev/null 2>&1 || \
+    nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
   nft flush set "$table" "$name" v4
-  if ((${#PREFIXES[@]})); then
-    local v4=()
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      v4+=("$p")
+
+  if ((${#v4[@]})); then
+    local batch=()
+    local chunk=64
+    for p in "${v4[@]}"; do
+      batch+=("$p")
+      if ((${#batch[@]} >= chunk)); then
+        nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft chunk add partial failure"
+        batch=()
+      fi
     done
-    if ((${#v4[@]})); then
-      nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${v4[*]}") }"
+    if ((${#batch[@]})); then
+      nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft tail chunk add partial failure"
     fi
   fi
+
   nft list chain "$table" "$name" input >/dev/null 2>&1 || {
-    nft add chain "$table" "$name" input '{ type filter hook input priority 0; }'
+    nft add chain "$table" "$name" input '{ type filter hook input priority 0; policy accept; }'
     nft add rule "$table" "$name" input ip saddr @v4 drop
   }
+  APPLIED_V4=${#v4[@]}
 }
 
 apply_ipset() {
   local set=evobgp_blocklist_v4
+  local n=0
   ipset list "$set" >/dev/null 2>&1 || ipset create "$set" hash:net family inet hashsize 4096 maxelem 1048576
   ipset flush "$set"
   local p
   for p in "${PREFIXES[@]}"; do
     [[ "$p" == *:* ]] && continue
     ipset add "$set" "$p" -exist
+    n=$((n + 1))
   done
   iptables -C INPUT -m set --match-set "$set" src -j DROP 2>/dev/null || \
     iptables -I INPUT -m set --match-set "$set" src -j DROP
+  APPLIED_V4=$n
 }
 
 apply_iptables_only() {
   iptables -D INPUT -m comment --comment evobgp-block -j DROP 2>/dev/null || true
-  if ((${#PREFIXES[@]})); then
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
-    done
-  fi
+  local n=0
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
+    n=$((n + 1))
+  done
+  APPLIED_V4=$n
 }
 
 clear_block() {
@@ -141,10 +226,13 @@ clear_block() {
       ;;
     iptables) iptables -S INPUT | grep -i evobgp | sed 's/^-A /-D /' | while read -r line; do iptables $line 2>/dev/null || true; done ;;
   esac
+  APPLIED_V4=0
 }
 
-if [[ "$TOTAL" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
+APPLIED_V4=0
+if [[ "${TOTAL:-0}" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
   clear_block
+  log "cleared blocklist (api total=${TOTAL:-0}) backend=$BACKEND"
 else
   case "$BACKEND" in
     nft|auto) if command -v nft >/dev/null 2>&1; then apply_nft; else apply_ipset; fi ;;
@@ -152,12 +240,12 @@ else
     iptables) apply_iptables_only ;;
     *) apply_ipset ;;
   esac
+  log "applied api_total=${TOTAL} ipv4_in_kernel=${APPLIED_V4} from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND hash=${HASH:-empty}"
 fi
 
 echo "$HASH" >"$HASH_FILE"
-log "applied $TOTAL prefixes from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND"
 
-REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":0,"source":"cp"}' "${TOTAL:-0}")
+REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":%s,"source":"cp"}' "${TOTAL:-0}" "${APPLIED_V4:-0}")
 curl -fsS -X POST "${EVOBGP_CP_URL%/}/v1/firewall/apply-report" \
   -H "Authorization: Bearer ${CLIENT_TOKEN}" \
   -H "Content-Type: application/json" \

internal/firewallscripts/install.sh

@@ -10,6 +10,16 @@ for cmd in curl bash; do
   command -v "$cmd" >/dev/null 2>&1 || { echo "missing $cmd" >&2; exit 1; }
 done
 
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  if command -v apt-get >/dev/null 2>&1; then
+    apt-get update -qq && apt-get install -y -qq jq
+  fi
+fi
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  echo "evobgp-firewall install: install jq or python3 for blocklist JSON parsing" >&2
+  exit 1
+fi
+
 : "${EVOBGP_CP_URL:?EVOBGP_CP_URL required}"
 : "${EVOBGP_SEED:?EVOBGP_SEED required}"
 : "${EVOBGP_CLIENT_NAME:?EVOBGP_CLIENT_NAME required}"
@@ -110,6 +120,7 @@ WantedBy=timers.target
 UNIT
   systemctl daemon-reload
   systemctl enable --now evobgp-firewall.timer
+  echo "Tip: after UI approve, run: rm -f /var/lib/evobgp-firewall/last_hash && ${SYNC_SCRIPT}"
 else
   echo "*/5 * * * * root ${SYNC_SCRIPT}" >/etc/cron.d/evobgp-firewall
 fi

internal/repository/postgres_firewall.go

@@ -13,14 +13,17 @@ import (
 	"github.com/jackc/pgx/v5"
 )
 
+const firewallClientSelectCols = `
+	id, name, COALESCE(hostname, ''), token_prefix, status,
+	last_seen_at, COALESCE(last_seen_at_source, ''), COALESCE(last_seen_ip, ''),
+	last_apply_at, COALESCE(last_apply_status, ''), COALESCE(last_apply_error, ''),
+	COALESCE(last_apply_prefix_count, 0), COALESCE(last_apply_ip_count, 0), COALESCE(last_apply_source, ''),
+	COALESCE(client_version, ''), created_at, approved_at, approved_by_api_key_id, revoked_at`
+
 func (p *Postgres) ListFirewallClients(tenantID string) ([]*store.FirewallClient, error) {
 	ctx := context.Background()
 	rows, err := p.pool.Query(ctx, `
-		SELECT id, name, hostname, token_prefix, status,
-			last_seen_at, last_seen_at_source, last_seen_ip,
-			last_apply_at, last_apply_status, last_apply_error,
-			last_apply_prefix_count, last_apply_ip_count, last_apply_source,
-			client_version, created_at, approved_at, approved_by_api_key_id, revoked_at
+		SELECT `+firewallClientSelectCols+`
 		FROM firewall_client WHERE tenant_id=$1 ORDER BY created_at DESC`, tenantID)
 	if err != nil {
 		return nil, err
@@ -40,11 +43,7 @@ func (p *Postgres) ListFirewallClients(tenantID string) ([]*store.FirewallClient
 func (p *Postgres) GetFirewallClient(tenantID, id string) (*store.FirewallClient, error) {
 	ctx := context.Background()
 	row := p.pool.QueryRow(ctx, `
-		SELECT id, name, hostname, token_prefix, status,
-			last_seen_at, last_seen_at_source, last_seen_ip,
-			last_apply_at, last_apply_status, last_apply_error,
-			last_apply_prefix_count, last_apply_ip_count, last_apply_source,
-			client_version, created_at, approved_at, approved_by_api_key_id, revoked_at
+		SELECT `+firewallClientSelectCols+`
 		FROM firewall_client WHERE id=$1 AND tenant_id=$2`, id, tenantID)
 	c, err := scanFirewallClientRow(row.Scan, tenantID)
 	if err != nil {
@@ -147,11 +146,7 @@ func (p *Postgres) LookupFirewallClientByTokenHash(hash []byte) (*store.Firewall
 	}
 	ctx := context.Background()
 	row := p.pool.QueryRow(ctx, `
-		SELECT tenant_id, id, name, hostname, token_prefix, status,
-			last_seen_at, last_seen_at_source, last_seen_ip,
-			last_apply_at, last_apply_status, last_apply_error,
-			last_apply_prefix_count, last_apply_ip_count, last_apply_source,
-			client_version, created_at, approved_at, approved_by_api_key_id, revoked_at
+		SELECT tenant_id, `+firewallClientSelectCols+`
 		FROM firewall_client WHERE token_hash=$1`, hash)
 	c, err := scanFirewallClientLookupRow(row.Scan)
 	if err != nil {

internal/repository/postgres_firewall_test.go

@@ -0,0 +1,52 @@
+package repository
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"evobgp/internal/authkey"
+	"evobgp/internal/db"
+	"evobgp/internal/store"
+)
+
+func TestPostgresFirewallClientCreateAndGetIntegration(t *testing.T) {
+	dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
+	if dsn == "" {
+		t.Skip("EVOBGP_TEST_DATABASE_URL not set")
+	}
+	ctx := context.Background()
+	pool, err := db.OpenPostgresPool(ctx, dsn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pool.Close()
+	pg, err := NewPostgres(ctx, pool, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tenant, _, _, _, _ := pg.DemoIDs()
+	if tenant == "" {
+		t.Fatal("demo tenant required")
+	}
+	tok := "evobgp_fw_pgtest_" + t.Name()
+	hash := authkey.HashToken(tok)
+	client, err := pg.CreateFirewallClient(tenant, &store.FirewallClientCreate{
+		Name:          "pg-firewall-test",
+		Hostname:      "test.local",
+		TokenPrefix:   tok[:12],
+		TokenHash:     hash,
+		ClientVersion: "test/1",
+	})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+	got, err := pg.GetFirewallClient(tenant, client.ID)
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	if got.Name != "pg-firewall-test" || got.Status != "pending" {
+		t.Fatalf("got %+v", got)
+	}
+	_ = pg.DeleteFirewallClient(tenant, client.ID)
+}

scripts/firewall/evobgp-firewall.sh

@@ -5,6 +5,7 @@ CONF_FILE=/etc/evobgp/firewall.conf
 LOG_FILE=/var/log/evobgp-firewall.log
 STATE_DIR=/var/lib/evobgp-firewall
 HASH_FILE="${STATE_DIR}/last_hash"
+PREFIX_FILE="${STATE_DIR}/last_prefixes.txt"
 
 log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" | tee -a "$LOG_FILE"; }
 
@@ -17,36 +18,32 @@ source "$CONF_FILE"
 
 : "${EVOBGP_CP_URL:?}"
 : "${CLIENT_TOKEN:?}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\r'/}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\n'/}"
 
 mkdir -p "$STATE_DIR"
 BACKEND="${KERNEL_BACKEND:-auto}"
 
-curl_get_blocklist() {
+curl_get_blocklist_file() {
   local url="$1"
-  local host
-  host=$(echo "$url" | sed -E 's#https?://([^/]+)/?.*#\1#')
-  local tmp
-  tmp=$(mktemp)
+  local dest="$2"
   local code
-  code=$(curl -sS -o "$tmp" -w "%{http_code}" \
+  code=$(curl -sS -o "$dest" -w "%{http_code}" \
     -H "Authorization: Bearer ${CLIENT_TOKEN}" \
     -H "Accept: application/json" \
     "${url}/v1/firewall/blocklist") || return 1
   if [[ "$code" == "403" ]]; then
     log "pending approval"
-    rm -f "$tmp"
-    exit 0
+    return 2
   fi
   if [[ "$code" != "200" ]]; then
     log "blocklist HTTP $code from $url"
-    rm -f "$tmp"
     return 1
   fi
-  cat "$tmp"
-  rm -f "$tmp"
+  return 0
 }
 
-try_urls() {
+try_fetch_blocklist() {
   local urls=()
   if [[ -n "${EVOBGP_FAILOVER_URLS:-}" ]]; then
     IFS=',' read -r -a urls <<<"$EVOBGP_FAILOVER_URLS"
@@ -57,7 +54,12 @@ try_urls() {
   for u in "${urls[@]}"; do
     u="${u// /}"
     u="${u%/}"
-    if OUT=$(curl_get_blocklist "$u"); then
+    local rc=0
+    curl_get_blocklist_file "$u" "$PREFIX_FILE" || rc=$?
+    if [[ "$rc" == 2 ]]; then
+      exit 0
+    fi
+    if [[ "$rc" == 0 ]]; then
       CP_HIT="$u"
       return 0
     fi
@@ -65,22 +67,87 @@ try_urls() {
   return 1
 }
 
-if ! OUT=$(try_urls); then
+parse_blocklist_file() {
+  local f="$1"
+  if [[ ! -s "$f" ]]; then
+    log "blocklist file empty: $f"
+    return 1
+  fi
+  if command -v jq >/dev/null 2>&1; then
+    HASH=$(jq -r '.hash // empty' "$f")
+    TOTAL=$(jq -r '.total // 0' "$f")
+    mapfile -t PREFIXES < <(jq -r '.prefixes[]? // empty' "$f")
+    return 0
+  fi
+  if command -v python3 >/dev/null 2>&1; then
+    local parsed
+    parsed=$(python3 - "$f" <<'PY'
+import json, sys
+with open(sys.argv[1], encoding="utf-8") as fh:
+    data = json.load(fh)
+print(data.get("hash") or "")
+print(data.get("total") or 0)
+for p in data.get("prefixes") or []:
+    if p:
+        print(p)
+PY
+    )
+    HASH=$(echo "$parsed" | sed -n '1p')
+    TOTAL=$(echo "$parsed" | sed -n '2p')
+    mapfile -t PREFIXES < <(echo "$parsed" | sed -n '3,$p')
+    return 0
+  fi
+  HASH=$(grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' "$f" | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
+  TOTAL=$(grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' "$f" | head -1 | grep -o '[0-9]*$' || true)
+  mapfile -t PREFIXES < <(grep -oE '"[0-9]+(\.[0-9]+){3}/[0-9]+"' "$f" | tr -d '"' || true)
+  return 0
+}
+
+nft_join_elements() {
+  local out="" p
+  for p in "$@"; do
+    if [[ -n "$out" ]]; then
+      out+=", "
+    fi
+    out+="$p"
+  done
+  printf '%s' "$out"
+}
+
+nft_add_v4_chunk() {
+  local table=$1 name=$2
+  shift 2
+  local joined
+  joined=$(nft_join_elements "$@")
+  if nft add element "$table" "$name" v4 "{ ${joined} }" 2>>"$LOG_FILE"; then
+    return 0
+  fi
+  log "nft batch add failed (chunk=$#), retrying one-by-one"
+  local p ok=0
+  for p in "$@"; do
+    if nft add element "$table" "$name" v4 "{ $p }" 2>>"$LOG_FILE"; then
+      ok=$((ok + 1))
+    fi
+  done
+  [[ "$ok" -gt 0 ]]
+}
+
+if ! try_fetch_blocklist; then
   log "all endpoints failed"
   exit 1
 fi
 
-if command -v jq >/dev/null 2>&1; then
-  HASH=$(echo "$OUT" | jq -r '.hash // empty')
-  TOTAL=$(echo "$OUT" | jq -r '.total // 0')
-  mapfile -t PREFIXES < <(echo "$OUT" | jq -r '.prefixes[]?')
-else
-  HASH=$(echo "$OUT" | grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
-  TOTAL=$(echo "$OUT" | grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
-  mapfile -t PREFIXES < <(echo "$OUT" | grep -o '"[0-9a-fA-F:.]*/[0-9]*"' | tr -d '"')
+HASH=""
+TOTAL=0
+PREFIXES=()
+parse_blocklist_file "$PREFIX_FILE"
+log "blocklist bytes=$(wc -c <"$PREFIX_FILE" | tr -d ' ') parsed=${#PREFIXES[@]} api_total=${TOTAL:-0}"
+
+if [[ -z "${TOTAL// }" ]]; then
+  TOTAL=${#PREFIXES[@]}
 fi
 
-if [[ -f "$HASH_FILE" && "$(cat "$HASH_FILE")" == "$HASH" ]]; then
+if [[ -f "$HASH_FILE" && "$(tr -d '\r\n' <"$HASH_FILE")" == "$HASH" && -n "$HASH" ]]; then
   log "unchanged hash $HASH — skip kernel apply"
   exit 0
 fi
@@ -88,48 +155,66 @@ fi
 apply_nft() {
   local table=inet
   local name=evobgp_blocklist
+  local v4=()
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    v4+=("$p")
+  done
+
   nft list table "$table" "$name" >/dev/null 2>&1 || nft add table "$table" "$name"
-  nft list set "$table" "$name" v4 >/dev/null 2>&1 || nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
+  nft list set "$table" "$name" v4 >/dev/null 2>&1 || \
+    nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
   nft flush set "$table" "$name" v4
-  if ((${#PREFIXES[@]})); then
-    local v4=()
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      v4+=("$p")
+
+  if ((${#v4[@]})); then
+    local batch=()
+    local chunk=64
+    for p in "${v4[@]}"; do
+      batch+=("$p")
+      if ((${#batch[@]} >= chunk)); then
+        nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft chunk add partial failure"
+        batch=()
+      fi
     done
-    if ((${#v4[@]})); then
-      nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${v4[*]}") }"
+    if ((${#batch[@]})); then
+      nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft tail chunk add partial failure"
     fi
   fi
+
   nft list chain "$table" "$name" input >/dev/null 2>&1 || {
-    nft add chain "$table" "$name" input '{ type filter hook input priority 0; }'
+    nft add chain "$table" "$name" input '{ type filter hook input priority 0; policy accept; }'
     nft add rule "$table" "$name" input ip saddr @v4 drop
   }
+  APPLIED_V4=${#v4[@]}
 }
 
 apply_ipset() {
   local set=evobgp_blocklist_v4
+  local n=0
   ipset list "$set" >/dev/null 2>&1 || ipset create "$set" hash:net family inet hashsize 4096 maxelem 1048576
   ipset flush "$set"
   local p
   for p in "${PREFIXES[@]}"; do
     [[ "$p" == *:* ]] && continue
     ipset add "$set" "$p" -exist
+    n=$((n + 1))
   done
   iptables -C INPUT -m set --match-set "$set" src -j DROP 2>/dev/null || \
     iptables -I INPUT -m set --match-set "$set" src -j DROP
+  APPLIED_V4=$n
 }
 
 apply_iptables_only() {
   iptables -D INPUT -m comment --comment evobgp-block -j DROP 2>/dev/null || true
-  if ((${#PREFIXES[@]})); then
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
-    done
-  fi
+  local n=0
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
+    n=$((n + 1))
+  done
+  APPLIED_V4=$n
 }
 
 clear_block() {
@@ -141,10 +226,13 @@ clear_block() {
       ;;
     iptables) iptables -S INPUT | grep -i evobgp | sed 's/^-A /-D /' | while read -r line; do iptables $line 2>/dev/null || true; done ;;
   esac
+  APPLIED_V4=0
 }
 
-if [[ "$TOTAL" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
+APPLIED_V4=0
+if [[ "${TOTAL:-0}" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
   clear_block
+  log "cleared blocklist (api total=${TOTAL:-0}) backend=$BACKEND"
 else
   case "$BACKEND" in
     nft|auto) if command -v nft >/dev/null 2>&1; then apply_nft; else apply_ipset; fi ;;
@@ -152,12 +240,12 @@ else
     iptables) apply_iptables_only ;;
     *) apply_ipset ;;
   esac
+  log "applied api_total=${TOTAL} ipv4_in_kernel=${APPLIED_V4} from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND hash=${HASH:-empty}"
 fi
 
 echo "$HASH" >"$HASH_FILE"
-log "applied $TOTAL prefixes from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND"
 
-REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":0,"source":"cp"}' "${TOTAL:-0}")
+REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":%s,"source":"cp"}' "${TOTAL:-0}" "${APPLIED_V4:-0}")
 curl -fsS -X POST "${EVOBGP_CP_URL%/}/v1/firewall/apply-report" \
   -H "Authorization: Bearer ${CLIENT_TOKEN}" \
   -H "Content-Type: application/json" \

scripts/firewall/install.sh

@@ -10,6 +10,16 @@ for cmd in curl bash; do
   command -v "$cmd" >/dev/null 2>&1 || { echo "missing $cmd" >&2; exit 1; }
 done
 
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  if command -v apt-get >/dev/null 2>&1; then
+    apt-get update -qq && apt-get install -y -qq jq
+  fi
+fi
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  echo "evobgp-firewall install: install jq or python3 for blocklist JSON parsing" >&2
+  exit 1
+fi
+
 : "${EVOBGP_CP_URL:?EVOBGP_CP_URL required}"
 : "${EVOBGP_SEED:?EVOBGP_SEED required}"
 : "${EVOBGP_CLIENT_NAME:?EVOBGP_CLIENT_NAME required}"
@@ -110,6 +120,7 @@ WantedBy=timers.target
 UNIT
   systemctl daemon-reload
   systemctl enable --now evobgp-firewall.timer
+  echo "Tip: after UI approve, run: rm -f /var/lib/evobgp-firewall/last_hash && ${SYNC_SCRIPT}"
 else
   echo "*/5 * * * * root ${SYNC_SCRIPT}" >/etc/cron.d/evobgp-firewall
 fi