Blocking: count Advanced Blocking, Dashboard layout and KPI cards
- Status, sidebar mark and header treat a server that blocks with the Advanced Blocking app as blocking, per server behind the hub. - Overview panel summarising the app's groups; Lists and Rules show and edit the app's list URLs and rules (read, change one field, write back). - Block List Domains adds the app's list domains, counted by the hub (hub/blocking/app-lists/count, public hosts only unless HUB_LISTS_ALLOW_PRIVATE=1) or by the browser where CORS allows. - Header and KPI cards follow the Dashboard: selector in the chrome slot, Compare servers in the header actions, period under the title, StatCard tiles without sparklines. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DenozordecandClaude Opus 5.5 committed
1 parent
b396bfd317
commit
172e06f1ce
40 files changed
+3126
-208
No files matched your search
@@ -0,0 +1,304 @@
|
||||
/*
|
||||
How many domains the block lists of a server's Advanced Blocking app hold (stage 9). The app has
|
||||
no API that says it, so the hub reads the app's config from the server (with the READ token, like
|
||||
any `apps/config/get`), downloads each http(s) block list and counts the distinct domains.
|
||||
|
||||
Counted: a hosts line (`0.0.0.0 example.com`, every name after the address), a plain domain
|
||||
(`example.com`, `*.example.com`) and an Adblock Plus domain rule (`||example.com^`). Not counted:
|
||||
blank lines, comments (`#`, `!`, `[`), Adblock exceptions (`@@`) and regular expressions. The
|
||||
console has the same rules for the no-hub case (`src/lib/list-count.ts`); the two are kept in step
|
||||
by hand and tested on the same sample.
|
||||
|
||||
The hub fetches a URL an administrator typed into a config, so it is careful where it goes:
|
||||
- only `http:` and `https:`, no credentials in the URL;
|
||||
- the host must resolve to public addresses only (no loopback, private, link-local, carrier-grade
|
||||
NAT, multicast or unspecified ones, IPv4 or IPv6). Redirects are followed by hand, up to three,
|
||||
and checked at every step. `HUB_LISTS_ALLOW_PRIVATE=1` lifts this for lists kept on the LAN.
|
||||
The check resolves the name and the fetch resolves it again, so a name that changes between the
|
||||
two is not caught; the hub is not a place an attacker can write DNS for;
|
||||
- a timeout and a size cap per list, a cap on how many lists, and four downloads at a time.
|
||||
|
||||
Results are cached per URL for several hours (a failure for minutes), so opening the Overview
|
||||
does not download a hundred megabytes again.
|
||||
*/
|
||||
import { lookup as dnsLookup } from 'node:dns/promises'
|
||||
import { isIP } from 'node:net'
|
||||
import type { HubServer } from './config.ts'
|
||||
import type { Upstream } from './upstream.ts'
|
||||
|
||||
export interface ListCount {
|
||||
url: string
|
||||
count: number | null
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface ListCounts {
|
||||
lists: ListCount[]
|
||||
total: number
|
||||
}
|
||||
|
||||
export interface AppListsOptions {
|
||||
fetch?: typeof fetch
|
||||
/** Resolves a host to its addresses. */
|
||||
lookup?: (host: string) => Promise<string[]>
|
||||
now?: () => number
|
||||
allowPrivate?: boolean
|
||||
/** How long a counted list is remembered (default 6 hours). */
|
||||
ttlMs?: number
|
||||
/** How long a failure is remembered (default 5 minutes). */
|
||||
failureTtlMs?: number
|
||||
timeoutMs?: number
|
||||
maxBytes?: number
|
||||
maxLists?: number
|
||||
}
|
||||
|
||||
const HOURS = 3_600_000
|
||||
const MAX_CACHED = 500
|
||||
const MAX_REDIRECTS = 3
|
||||
const CONCURRENCY = 4
|
||||
|
||||
/* --- counting --- */
|
||||
|
||||
const ADDRESS = /^(?:\d{1,3}(?:\.\d{1,3}){3}|[0-9a-f:]*:[0-9a-f:]*)$/i
|
||||
const NAME = /^[a-z0-9_]([a-z0-9_.-]*[a-z0-9_])?$/i
|
||||
const OWN = new Set(['localhost', 'localhost.localdomain', 'local', 'broadcasthost', '0.0.0.0'])
|
||||
|
||||
function accept(raw: string, into: Set<string>): void {
|
||||
const name = raw.replace(/^\*\./, '').replace(/\.$/, '').toLowerCase()
|
||||
if (name === '' || OWN.has(name) || name.startsWith('ip6-') || !NAME.test(name)) return
|
||||
into.add(name)
|
||||
}
|
||||
|
||||
function countLine(line: string, into: Set<string>): void {
|
||||
const t = line.trim()
|
||||
if (t === '' || t.startsWith('#') || t.startsWith('!') || t.startsWith('[') || t.startsWith('@@') || t.startsWith('/')) return
|
||||
if (t.startsWith('||')) {
|
||||
const end = t.search(/[\^$/]/)
|
||||
accept(end < 0 ? t.slice(2) : t.slice(2, end), into)
|
||||
return
|
||||
}
|
||||
const tokens = t.split('#')[0].trim().split(/\s+/)
|
||||
if (tokens.length > 1 && ADDRESS.test(tokens[0])) for (const name of tokens.slice(1)) accept(name, into)
|
||||
else if (tokens.length === 1) accept(tokens[0], into)
|
||||
}
|
||||
|
||||
/* Counts as text arrives in pieces, so a big list is never held whole. */
|
||||
export class DomainCounter {
|
||||
#domains = new Set<string>()
|
||||
#rest = ''
|
||||
|
||||
push(chunk: string): void {
|
||||
const lines = (this.#rest + chunk).split(/\r?\n/)
|
||||
this.#rest = lines.pop() ?? ''
|
||||
for (const line of lines) countLine(line, this.#domains)
|
||||
}
|
||||
|
||||
end(): number {
|
||||
countLine(this.#rest, this.#domains)
|
||||
this.#rest = ''
|
||||
return this.#domains.size
|
||||
}
|
||||
}
|
||||
|
||||
export function countDomains(text: string): number {
|
||||
const c = new DomainCounter()
|
||||
c.push(text)
|
||||
return c.end()
|
||||
}
|
||||
|
||||
/* --- where a fetch may go --- */
|
||||
|
||||
function ipv4Allowed(a: number, b: number): boolean {
|
||||
if (a === 0 || a === 10 || a === 127 || a >= 224) return false
|
||||
if (a === 100 && b >= 64 && b <= 127) return false
|
||||
if (a === 169 && b === 254) return false
|
||||
if (a === 172 && b >= 16 && b <= 31) return false
|
||||
if (a === 192 && b === 168) return false
|
||||
if (a === 192 && b === 0) return false
|
||||
if (a === 198 && (b === 18 || b === 19)) return false
|
||||
return true
|
||||
}
|
||||
|
||||
/** Whether an address is one a list may be fetched from: public unicast only. */
|
||||
export function publicAddress(address: string): boolean {
|
||||
const kind = isIP(address)
|
||||
if (kind === 4) {
|
||||
const [a, b] = address.split('.').map(Number)
|
||||
return ipv4Allowed(a, b)
|
||||
}
|
||||
if (kind === 6) {
|
||||
const a = address.toLowerCase()
|
||||
const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(a)
|
||||
if (mapped !== null) return publicAddress(mapped[1])
|
||||
if (a === '::' || a === '::1') return false
|
||||
const first = parseInt(a.split(':')[0] || '0', 16)
|
||||
if ((first & 0xfe00) === 0xfc00) return false // fc00::/7 unique local
|
||||
if ((first & 0xffc0) === 0xfe80) return false // fe80::/10 link local
|
||||
if ((first & 0xff00) === 0xff00) return false // multicast
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
async function defaultLookup(host: string): Promise<string[]> {
|
||||
return (await dnsLookup(host, { all: true })).map((r) => r.address)
|
||||
}
|
||||
|
||||
class Refused extends Error {}
|
||||
|
||||
/* --- the service --- */
|
||||
|
||||
interface Cached {
|
||||
until: number
|
||||
result: ListCount
|
||||
}
|
||||
|
||||
export class AppLists {
|
||||
#upstream: Upstream
|
||||
#fetch: typeof fetch
|
||||
#lookup: (host: string) => Promise<string[]>
|
||||
#now: () => number
|
||||
#allowPrivate: boolean
|
||||
#ttl: number
|
||||
#failureTtl: number
|
||||
#timeout: number
|
||||
#maxBytes: number
|
||||
#maxLists: number
|
||||
#cache = new Map<string, Cached>()
|
||||
#inflight = new Map<string, Promise<ListCount>>()
|
||||
|
||||
constructor(upstream: Upstream, options: AppListsOptions = {}) {
|
||||
this.#upstream = upstream
|
||||
this.#fetch = options.fetch ?? fetch
|
||||
this.#lookup = options.lookup ?? defaultLookup
|
||||
this.#now = options.now ?? Date.now
|
||||
this.#allowPrivate = options.allowPrivate ?? false
|
||||
this.#ttl = options.ttlMs ?? 6 * HOURS
|
||||
this.#failureTtl = options.failureTtlMs ?? 5 * 60_000
|
||||
this.#timeout = options.timeoutMs ?? 20_000
|
||||
this.#maxBytes = options.maxBytes ?? 64 * 1024 * 1024
|
||||
this.#maxLists = options.maxLists ?? 100
|
||||
}
|
||||
|
||||
/** The distinct block list URLs (plain and Adblock) of the app's config on one server. */
|
||||
async urlsOf(server: HubServer, appName: string): Promise<string[]> {
|
||||
const response = (await this.#upstream.call(server, 'apps/config/get', { name: appName })) as { config?: string | null } | null
|
||||
const text = response?.config
|
||||
if (typeof text !== 'string' || text.trim() === '') return []
|
||||
let root: unknown
|
||||
try {
|
||||
root = JSON.parse(text)
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
const groups = root !== null && typeof root === 'object' && Array.isArray((root as { groups?: unknown }).groups) ? ((root as { groups: unknown[] }).groups) : []
|
||||
const urls = new Set<string>()
|
||||
for (const g of groups) {
|
||||
if (g === null || typeof g !== 'object') continue
|
||||
for (const field of ['blockListUrls', 'adblockListUrls']) {
|
||||
const list = (g as Record<string, unknown>)[field]
|
||||
if (!Array.isArray(list)) continue
|
||||
for (const e of list) {
|
||||
const url = typeof e === 'string' ? e : e !== null && typeof e === 'object' && typeof (e as { url?: unknown }).url === 'string' ? (e as { url: string }).url : ''
|
||||
if (/^https?:\/\//i.test(url.trim())) urls.add(url.trim())
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...urls]
|
||||
}
|
||||
|
||||
async count(server: HubServer, appName: string): Promise<ListCounts> {
|
||||
const urls = (await this.urlsOf(server, appName)).slice(0, this.#maxLists)
|
||||
const lists: ListCount[] = new Array(urls.length)
|
||||
let next = 0
|
||||
const worker = async () => {
|
||||
while (next < urls.length) {
|
||||
const i = next++
|
||||
lists[i] = await this.countUrl(urls[i])
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, urls.length) }, worker))
|
||||
return { lists, total: lists.reduce((n, l) => n + (l.count ?? 0), 0) }
|
||||
}
|
||||
|
||||
countUrl(url: string): Promise<ListCount> {
|
||||
const hit = this.#cache.get(url)
|
||||
if (hit !== undefined && hit.until > this.#now()) return Promise.resolve(hit.result)
|
||||
const running = this.#inflight.get(url)
|
||||
if (running !== undefined) return running
|
||||
const job = this.#download(url).then((result) => {
|
||||
if (this.#cache.size >= MAX_CACHED) this.#prune()
|
||||
this.#cache.set(url, { until: this.#now() + (result.count === null ? this.#failureTtl : this.#ttl), result })
|
||||
return result
|
||||
})
|
||||
this.#inflight.set(url, job)
|
||||
void job.finally(() => this.#inflight.delete(url))
|
||||
return job
|
||||
}
|
||||
|
||||
#prune(): void {
|
||||
const now = this.#now()
|
||||
for (const [key, entry] of this.#cache) if (entry.until <= now) this.#cache.delete(key)
|
||||
while (this.#cache.size >= MAX_CACHED) this.#cache.delete(this.#cache.keys().next().value as string)
|
||||
}
|
||||
|
||||
async #allowed(url: URL): Promise<void> {
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') throw new Refused('only http and https lists are read')
|
||||
if (url.username !== '' || url.password !== '') throw new Refused('a list address may not carry credentials')
|
||||
if (this.#allowPrivate) return
|
||||
const host = url.hostname.replace(/^\[|\]$/g, '')
|
||||
const addresses = isIP(host) !== 0 ? [host] : await this.#lookup(host)
|
||||
if (addresses.length === 0 || !addresses.every(publicAddress)) throw new Refused('the address of this list is not a public one')
|
||||
}
|
||||
|
||||
async #download(start: string): Promise<ListCount> {
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), this.#timeout)
|
||||
try {
|
||||
let url = new URL(start)
|
||||
for (let hop = 0; ; hop++) {
|
||||
await this.#allowed(url)
|
||||
const res = await this.#fetch(url, { redirect: 'manual', signal: controller.signal, headers: { accept: 'text/plain, */*' } })
|
||||
if (res.status >= 300 && res.status < 400 && res.headers.get('location') !== null) {
|
||||
void res.body?.cancel()
|
||||
if (hop >= MAX_REDIRECTS) throw new Refused('too many redirects')
|
||||
url = new URL(res.headers.get('location') as string, url)
|
||||
continue
|
||||
}
|
||||
if (!res.ok) {
|
||||
void res.body?.cancel()
|
||||
throw new Refused(`the list server answered ${res.status}`)
|
||||
}
|
||||
const declared = Number(res.headers.get('content-length') ?? 0)
|
||||
if (declared > this.#maxBytes) {
|
||||
void res.body?.cancel()
|
||||
throw new Refused('the list is larger than the limit')
|
||||
}
|
||||
const counter = new DomainCounter()
|
||||
const decoder = new TextDecoder()
|
||||
let bytes = 0
|
||||
if (res.body !== null) {
|
||||
const reader = res.body.getReader()
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
bytes += value.byteLength
|
||||
if (bytes > this.#maxBytes) {
|
||||
void reader.cancel()
|
||||
throw new Refused('the list is larger than the limit')
|
||||
}
|
||||
counter.push(decoder.decode(value, { stream: true }))
|
||||
}
|
||||
counter.push(decoder.decode())
|
||||
}
|
||||
return { url: start, count: counter.end() }
|
||||
}
|
||||
} catch (e) {
|
||||
const message =
|
||||
e instanceof Refused ? e.message : e instanceof Error && e.name === 'AbortError' ? 'timed out' : 'could not be downloaded'
|
||||
return { url: start, count: null, error: message }
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user