Compare commits

..
1 Commits
Author SHA1 Message Date
Denozordec 947d1f0cc4 feat(firewall): enhance blocklist handling and installation script
CI / changes (push) Successful in 11s
CI / commitlint (push) Has been skipped
CI / openapi (push) Successful in 29s
CI / web (push) Successful in 58s
CI / go (push) Successful in 1m20s
CI / bird2 (push) Successful in 18s
CI / release (push) Successful in 4m37s
Updated the firewall scripts to improve blocklist handling by introducing a new method for fetching and parsing blocklist data using either `jq` or `python3`. Enhanced the installation script to ensure the presence of required dependencies and provided user guidance for post-approval actions. Additionally, improved logging for applied prefixes and total counts, ensuring better visibility into the firewall's operational status.
2026-07-08 21:45:32 +07:00
5 changed files with 228 additions and 86 deletions
+10
View File
@@ -35,6 +35,16 @@ curl -fsSL https://<api>/v1/firewall/install.sh | \
Файлы: `/etc/evobgp/firewall.conf`, `/usr/local/sbin/evobgp-firewall.sh`, systemd timer `evobgp-firewall.timer`.
После **approve** в UI выполните на сервере (или дождитесь timer):
```bash
sudo rm -f /var/lib/evobgp-firewall/last_hash
sudo /usr/local/sbin/evobgp-firewall.sh
sudo nft list table inet evobgp_blocklist
```
Для парсинга JSON нужен `jq` или `python3` (install.sh ставит `jq` на Debian/Ubuntu при отсутствии).
## Failover через speaker
При `EVOBGP_FIREWALL_FAILOVER_ENABLED=1` на speaker-agent CP реплицирует состояние через `POST /v1/agent/firewall-replicate`. Клиенты используют тот же DNS-домен.
+98 -43
View File
@@ -5,6 +5,7 @@ CONF_FILE=/etc/evobgp/firewall.conf
LOG_FILE=/var/log/evobgp-firewall.log
STATE_DIR=/var/lib/evobgp-firewall
HASH_FILE="${STATE_DIR}/last_hash"
PREFIX_FILE="${STATE_DIR}/last_prefixes.txt"
log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" | tee -a "$LOG_FILE"; }
@@ -17,36 +18,32 @@ source "$CONF_FILE"
: "${EVOBGP_CP_URL:?}"
: "${CLIENT_TOKEN:?}"
CLIENT_TOKEN="${CLIENT_TOKEN//$'\r'/}"
CLIENT_TOKEN="${CLIENT_TOKEN//$'\n'/}"
mkdir -p "$STATE_DIR"
BACKEND="${KERNEL_BACKEND:-auto}"
curl_get_blocklist() {
curl_get_blocklist_file() {
local url="$1"
local host
host=$(echo "$url" | sed -E 's#https?://([^/]+)/?.*#\1#')
local tmp
tmp=$(mktemp)
local dest="$2"
local code
code=$(curl -sS -o "$tmp" -w "%{http_code}" \
code=$(curl -sS -o "$dest" -w "%{http_code}" \
-H "Authorization: Bearer ${CLIENT_TOKEN}" \
-H "Accept: application/json" \
"${url}/v1/firewall/blocklist") || return 1
if [[ "$code" == "403" ]]; then
log "pending approval"
rm -f "$tmp"
exit 0
return 2
fi
if [[ "$code" != "200" ]]; then
log "blocklist HTTP $code from $url"
rm -f "$tmp"
return 1
fi
cat "$tmp"
rm -f "$tmp"
return 0
}
try_urls() {
try_fetch_blocklist() {
local urls=()
if [[ -n "${EVOBGP_FAILOVER_URLS:-}" ]]; then
IFS=',' read -r -a urls <<<"$EVOBGP_FAILOVER_URLS"
@@ -57,7 +54,12 @@ try_urls() {
for u in "${urls[@]}"; do
u="${u// /}"
u="${u%/}"
if OUT=$(curl_get_blocklist "$u"); then
local rc=0
curl_get_blocklist_file "$u" "$PREFIX_FILE" || rc=$?
if [[ "$rc" == 2 ]]; then
exit 0
fi
if [[ "$rc" == 0 ]]; then
CP_HIT="$u"
return 0
fi
@@ -65,22 +67,53 @@ try_urls() {
return 1
}
if ! OUT=$(try_urls); then
parse_blocklist_file() {
local f="$1"
if command -v jq >/dev/null 2>&1; then
HASH=$(jq -r '.hash // empty' "$f")
TOTAL=$(jq -r '.total // 0' "$f")
mapfile -t PREFIXES < <(jq -r '.prefixes[]? // empty' "$f")
return 0
fi
if command -v python3 >/dev/null 2>&1; then
local parsed
parsed=$(python3 - "$f" <<'PY'
import json, sys
with open(sys.argv[1], encoding="utf-8") as fh:
data = json.load(fh)
print(data.get("hash") or "")
print(data.get("total") or 0)
for p in data.get("prefixes") or []:
if p:
print(p)
PY
)
HASH=$(echo "$parsed" | sed -n '1p')
TOTAL=$(echo "$parsed" | sed -n '2p')
mapfile -t PREFIXES < <(echo "$parsed" | sed -n '3,$p')
return 0
fi
HASH=$(grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' "$f" | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
TOTAL=$(grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' "$f" | head -1 | grep -o '[0-9]*$' || true)
mapfile -t PREFIXES < <(grep -oE '"[0-9]+(\.[0-9]+){3}/[0-9]+"' "$f" | tr -d '"' || true)
return 0
}
if ! try_fetch_blocklist; then
log "all endpoints failed"
exit 1
fi
if command -v jq >/dev/null 2>&1; then
HASH=$(echo "$OUT" | jq -r '.hash // empty')
TOTAL=$(echo "$OUT" | jq -r '.total // 0')
mapfile -t PREFIXES < <(echo "$OUT" | jq -r '.prefixes[]?')
else
HASH=$(echo "$OUT" | grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
TOTAL=$(echo "$OUT" | grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
mapfile -t PREFIXES < <(echo "$OUT" | grep -o '"[0-9a-fA-F:.]*/[0-9]*"' | tr -d '"')
HASH=""
TOTAL=0
PREFIXES=()
parse_blocklist_file "$PREFIX_FILE"
if [[ -z "${TOTAL// }" ]]; then
TOTAL=${#PREFIXES[@]}
fi
if [[ -f "$HASH_FILE" && "$(cat "$HASH_FILE")" == "$HASH" ]]; then
if [[ -f "$HASH_FILE" && "$(tr -d '\r\n' <"$HASH_FILE")" == "$HASH" && -n "$HASH" ]]; then
log "unchanged hash $HASH — skip kernel apply"
exit 0
fi
@@ -88,48 +121,67 @@ fi
apply_nft() {
local table=inet
local name=evobgp_blocklist
local v4=()
local p
for p in "${PREFIXES[@]}"; do
[[ "$p" == *:* ]] && continue
v4+=("$p")
done
nft list table "$table" "$name" >/dev/null 2>&1 || nft add table "$table" "$name"
nft list set "$table" "$name" v4 >/dev/null 2>&1 || nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
nft list set "$table" "$name" v4 >/dev/null 2>&1 || \
nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
nft flush set "$table" "$name" v4
if ((${#PREFIXES[@]})); then
local v4=()
local p
for p in "${PREFIXES[@]}"; do
[[ "$p" == *:* ]] && continue
v4+=("$p")
if ((${#v4[@]})); then
local batch=()
local chunk=128
local n
for p in "${v4[@]}"; do
batch+=("$p")
if ((${#batch[@]} >= chunk)); then
nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
batch=()
fi
done
if ((${#v4[@]})); then
nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${v4[*]}") }"
if ((${#batch[@]})); then
nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
fi
fi
nft list chain "$table" "$name" input >/dev/null 2>&1 || {
nft add chain "$table" "$name" input '{ type filter hook input priority 0; }'
nft add chain "$table" "$name" input '{ type filter hook input priority 0; policy accept; }'
nft add rule "$table" "$name" input ip saddr @v4 drop
}
APPLIED_V4=${#v4[@]}
}
apply_ipset() {
local set=evobgp_blocklist_v4
local n=0
ipset list "$set" >/dev/null 2>&1 || ipset create "$set" hash:net family inet hashsize 4096 maxelem 1048576
ipset flush "$set"
local p
for p in "${PREFIXES[@]}"; do
[[ "$p" == *:* ]] && continue
ipset add "$set" "$p" -exist
n=$((n + 1))
done
iptables -C INPUT -m set --match-set "$set" src -j DROP 2>/dev/null || \
iptables -I INPUT -m set --match-set "$set" src -j DROP
APPLIED_V4=$n
}
apply_iptables_only() {
iptables -D INPUT -m comment --comment evobgp-block -j DROP 2>/dev/null || true
if ((${#PREFIXES[@]})); then
local p
for p in "${PREFIXES[@]}"; do
[[ "$p" == *:* ]] && continue
iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
done
fi
local n=0
local p
for p in "${PREFIXES[@]}"; do
[[ "$p" == *:* ]] && continue
iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
n=$((n + 1))
done
APPLIED_V4=$n
}
clear_block() {
@@ -141,10 +193,13 @@ clear_block() {
;;
iptables) iptables -S INPUT | grep -i evobgp | sed 's/^-A /-D /' | while read -r line; do iptables $line 2>/dev/null || true; done ;;
esac
APPLIED_V4=0
}
if [[ "$TOTAL" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
APPLIED_V4=0
if [[ "${TOTAL:-0}" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
clear_block
log "cleared blocklist (api total=${TOTAL:-0}) backend=$BACKEND"
else
case "$BACKEND" in
nft|auto) if command -v nft >/dev/null 2>&1; then apply_nft; else apply_ipset; fi ;;
@@ -152,12 +207,12 @@ else
iptables) apply_iptables_only ;;
*) apply_ipset ;;
esac
log "applied api_total=${TOTAL} ipv4_in_kernel=${APPLIED_V4} from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND hash=${HASH:-empty}"
fi
echo "$HASH" >"$HASH_FILE"
log "applied $TOTAL prefixes from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND"
REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":0,"source":"cp"}' "${TOTAL:-0}")
REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":%s,"source":"cp"}' "${TOTAL:-0}" "${APPLIED_V4:-0}")
curl -fsS -X POST "${EVOBGP_CP_URL%/}/v1/firewall/apply-report" \
-H "Authorization: Bearer ${CLIENT_TOKEN}" \
-H "Content-Type: application/json" \
+11
View File
@@ -10,6 +10,16 @@ for cmd in curl bash; do
command -v "$cmd" >/dev/null 2>&1 || { echo "missing $cmd" >&2; exit 1; }
done
if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
if command -v apt-get >/dev/null 2>&1; then
apt-get update -qq && apt-get install -y -qq jq
fi
fi
if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
echo "evobgp-firewall install: install jq or python3 for blocklist JSON parsing" >&2
exit 1
fi
: "${EVOBGP_CP_URL:?EVOBGP_CP_URL required}"
: "${EVOBGP_SEED:?EVOBGP_SEED required}"
: "${EVOBGP_CLIENT_NAME:?EVOBGP_CLIENT_NAME required}"
@@ -110,6 +120,7 @@ WantedBy=timers.target
UNIT
systemctl daemon-reload
systemctl enable --now evobgp-firewall.timer
echo "Tip: after UI approve, run: rm -f /var/lib/evobgp-firewall/last_hash && ${SYNC_SCRIPT}"
else
echo "*/5 * * * * root ${SYNC_SCRIPT}" >/etc/cron.d/evobgp-firewall
fi
+98 -43
View File
@@ -5,6 +5,7 @@ CONF_FILE=/etc/evobgp/firewall.conf
LOG_FILE=/var/log/evobgp-firewall.log
STATE_DIR=/var/lib/evobgp-firewall
HASH_FILE="${STATE_DIR}/last_hash"
PREFIX_FILE="${STATE_DIR}/last_prefixes.txt"
log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" | tee -a "$LOG_FILE"; }
@@ -17,36 +18,32 @@ source "$CONF_FILE"
: "${EVOBGP_CP_URL:?}"
: "${CLIENT_TOKEN:?}"
CLIENT_TOKEN="${CLIENT_TOKEN//$'\r'/}"
CLIENT_TOKEN="${CLIENT_TOKEN//$'\n'/}"
mkdir -p "$STATE_DIR"
BACKEND="${KERNEL_BACKEND:-auto}"
curl_get_blocklist() {
curl_get_blocklist_file() {
local url="$1"
local host
host=$(echo "$url" | sed -E 's#https?://([^/]+)/?.*#\1#')
local tmp
tmp=$(mktemp)
local dest="$2"
local code
code=$(curl -sS -o "$tmp" -w "%{http_code}" \
code=$(curl -sS -o "$dest" -w "%{http_code}" \
-H "Authorization: Bearer ${CLIENT_TOKEN}" \
-H "Accept: application/json" \
"${url}/v1/firewall/blocklist") || return 1
if [[ "$code" == "403" ]]; then
log "pending approval"
rm -f "$tmp"
exit 0
return 2
fi
if [[ "$code" != "200" ]]; then
log "blocklist HTTP $code from $url"
rm -f "$tmp"
return 1
fi
cat "$tmp"
rm -f "$tmp"
return 0
}
try_urls() {
try_fetch_blocklist() {
local urls=()
if [[ -n "${EVOBGP_FAILOVER_URLS:-}" ]]; then
IFS=',' read -r -a urls <<<"$EVOBGP_FAILOVER_URLS"
@@ -57,7 +54,12 @@ try_urls() {
for u in "${urls[@]}"; do
u="${u// /}"
u="${u%/}"
if OUT=$(curl_get_blocklist "$u"); then
local rc=0
curl_get_blocklist_file "$u" "$PREFIX_FILE" || rc=$?
if [[ "$rc" == 2 ]]; then
exit 0
fi
if [[ "$rc" == 0 ]]; then
CP_HIT="$u"
return 0
fi
@@ -65,22 +67,53 @@ try_urls() {
return 1
}
if ! OUT=$(try_urls); then
parse_blocklist_file() {
local f="$1"
if command -v jq >/dev/null 2>&1; then
HASH=$(jq -r '.hash // empty' "$f")
TOTAL=$(jq -r '.total // 0' "$f")
mapfile -t PREFIXES < <(jq -r '.prefixes[]? // empty' "$f")
return 0
fi
if command -v python3 >/dev/null 2>&1; then
local parsed
parsed=$(python3 - "$f" <<'PY'
import json, sys
with open(sys.argv[1], encoding="utf-8") as fh:
data = json.load(fh)
print(data.get("hash") or "")
print(data.get("total") or 0)
for p in data.get("prefixes") or []:
if p:
print(p)
PY
)
HASH=$(echo "$parsed" | sed -n '1p')
TOTAL=$(echo "$parsed" | sed -n '2p')
mapfile -t PREFIXES < <(echo "$parsed" | sed -n '3,$p')
return 0
fi
HASH=$(grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' "$f" | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
TOTAL=$(grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' "$f" | head -1 | grep -o '[0-9]*$' || true)
mapfile -t PREFIXES < <(grep -oE '"[0-9]+(\.[0-9]+){3}/[0-9]+"' "$f" | tr -d '"' || true)
return 0
}
if ! try_fetch_blocklist; then
log "all endpoints failed"
exit 1
fi
if command -v jq >/dev/null 2>&1; then
HASH=$(echo "$OUT" | jq -r '.hash // empty')
TOTAL=$(echo "$OUT" | jq -r '.total // 0')
mapfile -t PREFIXES < <(echo "$OUT" | jq -r '.prefixes[]?')
else
HASH=$(echo "$OUT" | grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
TOTAL=$(echo "$OUT" | grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
mapfile -t PREFIXES < <(echo "$OUT" | grep -o '"[0-9a-fA-F:.]*/[0-9]*"' | tr -d '"')
HASH=""
TOTAL=0
PREFIXES=()
parse_blocklist_file "$PREFIX_FILE"
if [[ -z "${TOTAL// }" ]]; then
TOTAL=${#PREFIXES[@]}
fi
if [[ -f "$HASH_FILE" && "$(cat "$HASH_FILE")" == "$HASH" ]]; then
if [[ -f "$HASH_FILE" && "$(tr -d '\r\n' <"$HASH_FILE")" == "$HASH" && -n "$HASH" ]]; then
log "unchanged hash $HASH — skip kernel apply"
exit 0
fi
@@ -88,48 +121,67 @@ fi
apply_nft() {
local table=inet
local name=evobgp_blocklist
local v4=()
local p
for p in "${PREFIXES[@]}"; do
[[ "$p" == *:* ]] && continue
v4+=("$p")
done
nft list table "$table" "$name" >/dev/null 2>&1 || nft add table "$table" "$name"
nft list set "$table" "$name" v4 >/dev/null 2>&1 || nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
nft list set "$table" "$name" v4 >/dev/null 2>&1 || \
nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
nft flush set "$table" "$name" v4
if ((${#PREFIXES[@]})); then
local v4=()
local p
for p in "${PREFIXES[@]}"; do
[[ "$p" == *:* ]] && continue
v4+=("$p")
if ((${#v4[@]})); then
local batch=()
local chunk=128
local n
for p in "${v4[@]}"; do
batch+=("$p")
if ((${#batch[@]} >= chunk)); then
nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
batch=()
fi
done
if ((${#v4[@]})); then
nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${v4[*]}") }"
if ((${#batch[@]})); then
nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
fi
fi
nft list chain "$table" "$name" input >/dev/null 2>&1 || {
nft add chain "$table" "$name" input '{ type filter hook input priority 0; }'
nft add chain "$table" "$name" input '{ type filter hook input priority 0; policy accept; }'
nft add rule "$table" "$name" input ip saddr @v4 drop
}
APPLIED_V4=${#v4[@]}
}
apply_ipset() {
local set=evobgp_blocklist_v4
local n=0
ipset list "$set" >/dev/null 2>&1 || ipset create "$set" hash:net family inet hashsize 4096 maxelem 1048576
ipset flush "$set"
local p
for p in "${PREFIXES[@]}"; do
[[ "$p" == *:* ]] && continue
ipset add "$set" "$p" -exist
n=$((n + 1))
done
iptables -C INPUT -m set --match-set "$set" src -j DROP 2>/dev/null || \
iptables -I INPUT -m set --match-set "$set" src -j DROP
APPLIED_V4=$n
}
apply_iptables_only() {
iptables -D INPUT -m comment --comment evobgp-block -j DROP 2>/dev/null || true
if ((${#PREFIXES[@]})); then
local p
for p in "${PREFIXES[@]}"; do
[[ "$p" == *:* ]] && continue
iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
done
fi
local n=0
local p
for p in "${PREFIXES[@]}"; do
[[ "$p" == *:* ]] && continue
iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
n=$((n + 1))
done
APPLIED_V4=$n
}
clear_block() {
@@ -141,10 +193,13 @@ clear_block() {
;;
iptables) iptables -S INPUT | grep -i evobgp | sed 's/^-A /-D /' | while read -r line; do iptables $line 2>/dev/null || true; done ;;
esac
APPLIED_V4=0
}
if [[ "$TOTAL" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
APPLIED_V4=0
if [[ "${TOTAL:-0}" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
clear_block
log "cleared blocklist (api total=${TOTAL:-0}) backend=$BACKEND"
else
case "$BACKEND" in
nft|auto) if command -v nft >/dev/null 2>&1; then apply_nft; else apply_ipset; fi ;;
@@ -152,12 +207,12 @@ else
iptables) apply_iptables_only ;;
*) apply_ipset ;;
esac
log "applied api_total=${TOTAL} ipv4_in_kernel=${APPLIED_V4} from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND hash=${HASH:-empty}"
fi
echo "$HASH" >"$HASH_FILE"
log "applied $TOTAL prefixes from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND"
REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":0,"source":"cp"}' "${TOTAL:-0}")
REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":%s,"source":"cp"}' "${TOTAL:-0}" "${APPLIED_V4:-0}")
curl -fsS -X POST "${EVOBGP_CP_URL%/}/v1/firewall/apply-report" \
-H "Authorization: Bearer ${CLIENT_TOKEN}" \
-H "Content-Type: application/json" \
+11
View File
@@ -10,6 +10,16 @@ for cmd in curl bash; do
command -v "$cmd" >/dev/null 2>&1 || { echo "missing $cmd" >&2; exit 1; }
done
if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
if command -v apt-get >/dev/null 2>&1; then
apt-get update -qq && apt-get install -y -qq jq
fi
fi
if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
echo "evobgp-firewall install: install jq or python3 for blocklist JSON parsing" >&2
exit 1
fi
: "${EVOBGP_CP_URL:?EVOBGP_CP_URL required}"
: "${EVOBGP_SEED:?EVOBGP_SEED required}"
: "${EVOBGP_CLIENT_NAME:?EVOBGP_CLIENT_NAME required}"
@@ -110,6 +120,7 @@ WantedBy=timers.target
UNIT
systemctl daemon-reload
systemctl enable --now evobgp-firewall.timer
echo "Tip: after UI approve, run: rm -f /var/lib/evobgp-firewall/last_hash && ${SYNC_SCRIPT}"
else
echo "*/5 * * * * root ${SYNC_SCRIPT}" >/etc/cron.d/evobgp-firewall
fi