refactor(api): enhance host firewall collection and JSON output

- Improved the `collect_host_firewall` function in `evofw-firewall.sh` to utilize temporary files for better handling of large rule sets, avoiding ARG_MAX limitations.
- Updated the JSON output structure to omit null/empty optional fields, ensuring compatibility with historical data formats.
- Enhanced error handling and logging for the firewall collection process, providing clearer diagnostics in case of failures.
- Adjusted related tests to accommodate changes in the expected output format, ensuring robust validation of the host firewall snapshot functionality.

These changes enhance the reliability and clarity of the host firewall data collection process, improving overall monitoring capabilities.

apps/api/src/agent-scripts/evofw-firewall.sh

@@ -467,32 +467,65 @@ PY
 collect_host_firewall() {
   HOST_FIREWALL_JSON='{"rules":[],"listeners":[]}'
   if ! command -v python3 >/dev/null 2>&1; then
+    log "host_firewall: python3 missing — empty snapshot"
     return 0
   fi
-  local nft_txt="" ipt_txt="" ufw_txt="" fwd_txt="" ss_txt=""
-  nft_txt=$(nft list ruleset 2>/dev/null || true)
-  ipt_txt=$(iptables-save 2>/dev/null || true)
+  local tmpdir nft_f ipt_f ufw_f fwd_f ss_f out_f
+  tmpdir=$(mktemp -d "${STATE_DIR}/hostfw.XXXXXX") || return 0
+  nft_f="$tmpdir/nft.txt"
+  ipt_f="$tmpdir/ipt.txt"
+  ufw_f="$tmpdir/ufw.txt"
+  fwd_f="$tmpdir/fwd.txt"
+  ss_f="$tmpdir/ss.txt"
+  out_f="$tmpdir/out.json"
+  # Dump to files — env vars blow ARG_MAX on large nft rulesets.
+  nft list ruleset >"$nft_f" 2>/dev/null || true
+  iptables-save >"$ipt_f" 2>/dev/null || true
   if command -v ufw >/dev/null 2>&1; then
-    ufw_txt=$(ufw status verbose 2>/dev/null || true)
+    ufw status verbose >"$ufw_f" 2>/dev/null || true
+  else
+    : >"$ufw_f"
   fi
   if command -v firewall-cmd >/dev/null 2>&1; then
-    fwd_txt=$(firewall-cmd --list-all 2>/dev/null || true)
+    firewall-cmd --list-all >"$fwd_f" 2>/dev/null || true
+  else
+    : >"$fwd_f"
   fi
-  ss_txt=$(ss -lntu 2>/dev/null || true)
-  HOST_FIREWALL_JSON=$(NFT_TXT="$nft_txt" IPT_TXT="$ipt_txt" UFW_TXT="$ufw_txt" FWD_TXT="$fwd_txt" SS_TXT="$ss_txt" python3 - <<'PY'
+  ss -lntu >"$ss_f" 2>/dev/null || true
+
+  if NFT_F="$nft_f" IPT_F="$ipt_f" UFW_F="$ufw_f" FWD_F="$fwd_f" SS_F="$ss_f" OUT_F="$out_f" python3 - <<'PY' >/tmp/evofw-hostfw-counts.txt 2>>"$LOG_FILE"
 import json, os, re
 
+def read(path: str) -> str:
+    try:
+        with open(path, "r", encoding="utf-8", errors="replace") as f:
+            return f.read()
+    except OSError:
+        return ""
+
 def ownership_of(text: str) -> str:
     t = text.lower()
-    if "evofw" in t or "evofw-port-" in t:
+    if "evofw" in t:
         return "evofw"
     return "foreign"
 
+def rule(**kwargs):
+    # Omit null/empty optional fields — Zod optional rejects JSON null.
+    out = {
+        "ownership": kwargs["ownership"],
+        "backend": kwargs["backend"],
+        "raw": kwargs["raw"][:512],
+    }
+    for k in ("table", "chain", "action", "protocol", "dport", "sport", "saddr", "daddr", "comment"):
+        v = kwargs.get(k)
+        if v is not None and v != "":
+            out[k] = v if not isinstance(v, str) else v[:128 if k in ("table", "chain", "saddr", "daddr") else (64 if k in ("action", "dport", "sport", "protocol") else 256)]
+    return out
+
 rules = []
 listeners = []
 
-nft = os.environ.get("NFT_TXT") or ""
-# Rough nft rule lines
+nft = read(os.environ["NFT_F"])
 cur_table = ""
 cur_chain = ""
 for line in nft.splitlines():
@@ -526,20 +559,19 @@ for line in nft.splitlines():
         m = re.search(r"saddr\s+(\S+)", ls)
         if m:
             saddr = m.group(1).lstrip("@")
-        raw = ls[:500]
-        rules.append({
-            "ownership": ownership_of(cur_table + " " + cur_chain + " " + raw),
-            "backend": "nft",
-            "table": cur_table[:120],
-            "chain": cur_chain[:120],
-            "action": act,
-            "protocol": proto or None,
-            "dport": dport or None,
-            "saddr": saddr or None,
-            "raw": raw,
-        })
+        rules.append(rule(
+            ownership=ownership_of(cur_table + " " + cur_chain + " " + ls),
+            backend="nft",
+            table=cur_table[:120],
+            chain=cur_chain[:120],
+            action=act,
+            protocol=proto or None,
+            dport=dport or None,
+            saddr=saddr or None,
+            raw=ls[:500],
+        ))
 
-ipt = os.environ.get("IPT_TXT") or ""
+ipt = read(os.environ["IPT_F"])
 cur_chain = ""
 for line in ipt.splitlines():
     if line.startswith(":"):
@@ -549,7 +581,6 @@ for line in ipt.splitlines():
         continue
     parts = line.split(None, 2)
     chain = parts[1] if len(parts) > 1 else ""
-    rest = parts[2] if len(parts) > 2 else line
     act = "DROP" if " -j DROP" in line else (
         "ACCEPT" if " -j ACCEPT" in line else (
             "REJECT" if " -j REJECT" in line else "other"
@@ -567,54 +598,51 @@ for line in ipt.splitlines():
     m = re.search(r"-s\s+(\S+)", line)
     if m:
         saddr = m.group(1)
-    raw = line[:500]
-    rules.append({
-        "ownership": ownership_of(raw),
-        "backend": "iptables",
-        "chain": chain[:120],
-        "action": act.lower() if isinstance(act, str) else act,
-        "protocol": proto or None,
-        "dport": dport or None,
-        "saddr": saddr or None,
-        "raw": raw,
-    })
+    rules.append(rule(
+        ownership=ownership_of(line),
+        backend="iptables",
+        chain=chain[:120],
+        action=act.lower() if isinstance(act, str) else act,
+        protocol=proto or None,
+        dport=dport or None,
+        saddr=saddr or None,
+        raw=line[:500],
+    ))
 
-ufw = os.environ.get("UFW_TXT") or ""
+ufw = read(os.environ["UFW_F"])
 for line in ufw.splitlines():
     ls = line.strip()
     if not ls or ls.startswith("Status") or ls.startswith("Logging") or ls.startswith("Default") or ls.startswith("To") or ls.startswith("--"):
         continue
     if "ALLOW" in ls or "DENY" in ls or "REJECT" in ls:
-        rules.append({
-            "ownership": ownership_of(ls),
-            "backend": "ufw",
-            "action": "allow" if "ALLOW" in ls else ("deny" if "DENY" in ls else "reject"),
-            "raw": ls[:500],
-        })
+        rules.append(rule(
+            ownership=ownership_of(ls),
+            backend="ufw",
+            action="allow" if "ALLOW" in ls else ("deny" if "DENY" in ls else "reject"),
+            raw=ls[:500],
+        ))
 
-fwd = os.environ.get("FWD_TXT") or ""
+fwd = read(os.environ["FWD_F"])
 for line in fwd.splitlines():
     ls = line.strip()
     if not ls:
         continue
     if ls.startswith("ports:") or ls.startswith("services:") or ":" in ls:
-        rules.append({
-            "ownership": "foreign",
-            "backend": "firewalld",
-            "raw": ls[:500],
-        })
+        rules.append(rule(
+            ownership="foreign",
+            backend="firewalld",
+            raw=ls[:500],
+        ))
 
-ss = os.environ.get("SS_TXT") or ""
+ss = read(os.environ["SS_F"])
 for line in ss.splitlines()[1:]:
     parts = line.split()
     if len(parts) < 5:
         continue
     proto = parts[0]
     local = parts[4]
-    # *:22 or 0.0.0.0:22 or [::]:22
     m = re.search(r"([^:]+):(\d+)$", local)
     if not m:
-        # IPv6 [::]:port
         m = re.search(r"\[([^\]]+)\]:(\d+)$", local)
         if not m:
             continue
@@ -631,12 +659,22 @@ for line in ss.splitlines()[1:]:
         "address": addr,
     })
 
-# Cap
 rules = rules[:500]
 listeners = listeners[:200]
-print(json.dumps({"rules": rules, "listeners": listeners}, separators=(",", ":")))
+with open(os.environ["OUT_F"], "w", encoding="utf-8") as f:
+    json.dump({"rules": rules, "listeners": listeners}, f, separators=(",", ":"))
+print(f"{len(rules)} {len(listeners)}")
 PY
-) || HOST_FIREWALL_JSON='{"rules":[],"listeners":[]}'
+  then
+    if [[ -f "$out_f" ]]; then
+      HOST_FIREWALL_JSON=$(cat "$out_f")
+      log "host_firewall collected $(tr -d '\r\n' </tmp/evofw-hostfw-counts.txt 2>/dev/null || echo '?')"
+    fi
+  else
+    log "host_firewall: collect failed — empty snapshot"
+    HOST_FIREWALL_JSON='{"rules":[],"listeners":[]}'
+  fi
+  rm -rf "$tmpdir" 2>/dev/null || true
 }
 
 ensure_ipset_counters() {
@@ -698,34 +736,62 @@ send_report() {
   if [[ -z "${HOST_FW_CAPTURED:-}" ]]; then
     collect_host_firewall
   fi
-  local report
-  # Compose report with python to safely embed host_firewall JSON
+  local report_file http_code counts
+  report_file=$(mktemp "${STATE_DIR}/report.XXXXXX")
+  # Compose report via files — large host_firewall must not go through env ARG_MAX.
   if command -v python3 >/dev/null 2>&1; then
-    report=$(APPLIED="${APPLIED:-0}" DROPPED="${PACKETS_DROPPED:-0}" ACCEPTED="${PACKETS_ACCEPTED:-0}" \
-      METHOD="${KERNEL_METHOD:-$BACKEND}" IP_HITS="${IP_HITS_JSON:-[]}" PORT_HITS="${PORT_HITS_JSON:-[]}" \
-      HOST_FW="${HOST_FIREWALL_JSON}" python3 - <<'PY'
+    local host_fw_file hits_file ports_file
+    host_fw_file=$(mktemp "${STATE_DIR}/hostfwj.XXXXXX")
+    hits_file=$(mktemp "${STATE_DIR}/iphits.XXXXXX")
+    ports_file=$(mktemp "${STATE_DIR}/porthits.XXXXXX")
+    printf '%s' "${HOST_FIREWALL_JSON:-{\"rules\":[],\"listeners\":[]}}" >"$host_fw_file"
+    printf '%s' "${IP_HITS_JSON:-[]}" >"$hits_file"
+    printf '%s' "${PORT_HITS_JSON:-[]}" >"$ports_file"
+    counts=$(APPLIED="${APPLIED:-0}" DROPPED="${PACKETS_DROPPED:-0}" ACCEPTED="${PACKETS_ACCEPTED:-0}" \
+      METHOD="${KERNEL_METHOD:-$BACKEND}" HOST_FW_F="$host_fw_file" IP_HITS_F="$hits_file" \
+      PORT_HITS_F="$ports_file" OUT_F="$report_file" python3 - <<'PY'
 import json, os
-print(json.dumps({
+
+def load(path, default):
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception:
+        return default
+
+payload = {
   "status": "ok",
   "prefix_count": int(os.environ.get("APPLIED") or 0),
   "packets_dropped": int(os.environ.get("DROPPED") or 0),
   "packets_accepted": int(os.environ.get("ACCEPTED") or 0),
   "kernel_method": os.environ.get("METHOD") or "auto",
   "source": "agent",
-  "ip_hits": json.loads(os.environ.get("IP_HITS") or "[]"),
-  "port_hits": json.loads(os.environ.get("PORT_HITS") or "[]"),
-  "host_firewall": json.loads(os.environ.get("HOST_FW") or '{"rules":[],"listeners":[]}'),
-}, separators=(",", ":")))
+  "ip_hits": load(os.environ["IP_HITS_F"], []),
+  "port_hits": load(os.environ["PORT_HITS_F"], []),
+  "host_firewall": load(os.environ["HOST_FW_F"], {"rules": [], "listeners": []}),
+}
+with open(os.environ["OUT_F"], "w", encoding="utf-8") as f:
+    json.dump(payload, f, separators=(",", ":"))
+n_rules = len(payload["host_firewall"].get("rules") or [])
+n_listen = len(payload["host_firewall"].get("listeners") or [])
+print(f"{n_rules} {n_listen}")
 PY
-)
+) || counts="0 0"
+    rm -f "$host_fw_file" "$hits_file" "$ports_file" 2>/dev/null || true
+    log "host_firewall snapshot rules=${counts%% *} listeners=${counts##* }"
   else
-    report=$(printf '{"status":"ok","prefix_count":%s,"packets_dropped":%s,"packets_accepted":%s,"kernel_method":"%s","source":"agent","ip_hits":%s,"port_hits":%s}' \
-      "${APPLIED:-0}" "${PACKETS_DROPPED:-0}" "${PACKETS_ACCEPTED:-0}" "${KERNEL_METHOD:-$BACKEND}" "${IP_HITS_JSON:-[]}" "${PORT_HITS_JSON:-[]}")
+    printf '{"status":"ok","prefix_count":%s,"packets_dropped":%s,"packets_accepted":%s,"kernel_method":"%s","source":"agent","ip_hits":%s,"port_hits":%s}' \
+      "${APPLIED:-0}" "${PACKETS_DROPPED:-0}" "${PACKETS_ACCEPTED:-0}" "${KERNEL_METHOD:-$BACKEND}" "${IP_HITS_JSON:-[]}" "${PORT_HITS_JSON:-[]}" \
+      >"$report_file"
   fi
-  curl -fsS -X POST "${EVOFW_CP_URL%/}/v1/agent/apply-report" \
+  http_code=$(curl -sS -o /tmp/evofw-apply-report.out -w '%{http_code}' -X POST "${EVOFW_CP_URL%/}/v1/agent/apply-report" \
     -H "Authorization: Bearer ${CLIENT_TOKEN}" \
     -H "Content-Type: application/json" \
-    -d "$report" >/dev/null 2>&1 || true
+    --data-binary @"$report_file" 2>/dev/null || echo "000")
+  if [[ "$http_code" != "200" ]]; then
+    log "apply-report failed http=${http_code} body=$(head -c 200 /tmp/evofw-apply-report.out 2>/dev/null || true)"
+  fi
+  rm -f "$report_file" 2>/dev/null || true
   curl -fsS -X POST "${EVOFW_CP_URL%/}/v1/agent/heartbeat" \
     -H "Authorization: Bearer ${CLIENT_TOKEN}" \
     -H "Content-Type: application/json" \

apps/api/src/services/port-acl.test.ts

@@ -247,6 +247,10 @@ describe('port ACL + host firewall snapshot', () => {
               backend: 'iptables',
               chain: 'INPUT',
               action: 'ACCEPT',
+              // Agent historically sent JSON null for missing fields — must accept.
+              protocol: null,
+              dport: null,
+              saddr: null,
               raw: '-A INPUT -p tcp --dport 80 -j ACCEPT',
             },
           ],