From 655d7507e54beda90828896f62784f8c9810f052 Mon Sep 17 00:00:00 2001 From: Denozordec Date: Mon, 6 Apr 2026 17:59:52 +0700 Subject: [PATCH] feat: update Go module dependencies and enhance DNS resolution in refresh pipeline. Upgrade Go version to 1.24.0, add support for DNS over HTTPS using RFC8484 format, and improve error handling for domain resolution. Update UI components for better log entry presentation in Svelte. --- go.mod | 14 +-- go.sum | 28 +++--- internal/pipeline/refresh.go | 120 +++++++++++++++++++++++-- web/src/routes/operations/+page.svelte | 44 ++++++--- 4 files changed, 167 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index f6523f9..2181546 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ module evobgp -go 1.22 +go 1.24.0 require ( github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.7.2 + github.com/miekg/dns v1.1.72 github.com/prometheus/client_golang v1.20.5 modernc.org/sqlite v1.34.5 ) @@ -24,10 +25,13 @@ require ( github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/mod v0.31.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + golang.org/x/tools v0.40.0 // indirect google.golang.org/protobuf v1.34.2 // indirect modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/go.sum b/go.sum index f81e26f..eae2bcf 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= +github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= @@ -49,19 +51,21 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/pipeline/refresh.go b/internal/pipeline/refresh.go index 06aecdb..5a5e0fd 100644 --- a/internal/pipeline/refresh.go +++ b/internal/pipeline/refresh.go @@ -1,13 +1,16 @@ package pipeline import ( + "bytes" "context" "crypto/sha256" + "encoding/base64" "encoding/json" "fmt" "io" - "net/netip" + "net" "net/http" + "net/netip" "net/url" "os" "sort" @@ -20,6 +23,7 @@ import ( "evobgp/internal/store" "github.com/google/uuid" + "github.com/miekg/dns" ) const ( @@ -264,7 +268,7 @@ func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohPr return nil, nil } if profile == nil || strings.TrimSpace(profile.URL) == "" { - return nil, nil + return resolveDomainWithSystemDNS(ctx, host) } timeout := 10 * time.Second @@ -274,13 +278,113 @@ func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohPr dctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - // RFC8484 endpoint with JSON mode: ?name=&type=A/AAAA - v4, err4 := resolveDomainWithDOHJSON(dctx, hc, strings.TrimSpace(profile.URL), host, "A") - v6, err6 := resolveDomainWithDOHJSON(dctx, hc, strings.TrimSpace(profile.URL), host, "AAAA") - if err4 != nil && err6 != nil { - return nil, fmt.Errorf("doh failed for A and AAAA: %v; %v", err4, err6) + baseURL := strings.TrimSpace(profile.URL) + // Prefer RFC8484 dns-message transport. Some providers don't support dns-json. + v4, err4 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeA) + v6, err6 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeAAAA) + if err4 != nil { + // Fallback to JSON mode for providers that only expose dns-json. + v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A") } - return uniqAddrs(append(v4, v6...)), nil + if err6 != nil { + v6, err6 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "AAAA") + } + if err4 != nil && err6 != nil { + // Some DoH providers return non-JSON responses (RFC8484 dns-message, HTML error pages, etc.). + // Fall back to system resolver to avoid failing the whole module refresh. + ips, fallbackErr := resolveDomainWithSystemDNS(dctx, host) + if fallbackErr != nil { + return nil, fmt.Errorf("doh failed for A and AAAA: %v; %v; fallback dns failed: %w", err4, err6, fallbackErr) + } + return ips, nil + } + out := uniqAddrs(append(v4, v6...)) + if len(out) > 0 { + return out, nil + } + // If DoH succeeds but returns no A/AAAA records, attempt system resolver as best-effort fallback. + ips, err := resolveDomainWithSystemDNS(dctx, host) + if err != nil { + return nil, err + } + return ips, nil +} + +func resolveDomainWithDOHMessage(ctx context.Context, hc *http.Client, baseURL, host string, qtype uint16) ([]netip.Addr, error) { + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), qtype) + wire, err := msg.Pack() + if err != nil { + return nil, err + } + + u, err := url.Parse(baseURL) + if err != nil { + return nil, err + } + q := u.Query() + q.Set("dns", base64.RawURLEncoding.EncodeToString(wire)) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/dns-message") + resp, err := hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return nil, fmt.Errorf("doh dns-message status %s: %s", resp.Status, strings.TrimSpace(string(body))) + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if err != nil { + return nil, err + } + parsed := new(dns.Msg) + if err := parsed.Unpack(raw); err != nil { + return nil, err + } + if parsed.Rcode != dns.RcodeSuccess { + return nil, fmt.Errorf("doh rcode=%s", dns.RcodeToString[parsed.Rcode]) + } + var out []netip.Addr + for _, rr := range parsed.Answer { + switch x := rr.(type) { + case *dns.A: + if qtype == dns.TypeA { + if ip, ok := netip.AddrFromSlice(x.A.To4()); ok { + out = append(out, ip.Unmap()) + } + } + case *dns.AAAA: + if qtype == dns.TypeAAAA { + if ip, ok := netip.AddrFromSlice(x.AAAA.To16()); ok { + out = append(out, ip.Unmap()) + } + } + } + } + // Some providers may return JSON even on dns-message request. + if len(out) == 0 && bytes.Contains(bytes.ToLower(raw), []byte(`"answer"`)) { + qs := "A" + if qtype == dns.TypeAAAA { + qs = "AAAA" + } + return resolveDomainWithDOHJSON(ctx, hc, baseURL, host, qs) + } + return uniqAddrs(out), nil +} + +func resolveDomainWithSystemDNS(ctx context.Context, host string) ([]netip.Addr, error) { + ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host) + if err != nil { + return nil, err + } + return uniqAddrs(ips), nil } func resolveDomainWithDOHJSON(ctx context.Context, hc *http.Client, baseURL, host, qtype string) ([]netip.Addr, error) { diff --git a/web/src/routes/operations/+page.svelte b/web/src/routes/operations/+page.svelte index c3101c3..cc8983b 100644 --- a/web/src/routes/operations/+page.svelte +++ b/web/src/routes/operations/+page.svelte @@ -756,21 +756,37 @@
{#each logEntries as entry, idx (`${job.job_id}-${idx}`)} -
-

{entry.message}

-

- source: {entry.source} - · - kind: {entry.kind} - · - community: {entry.community} - · - prefixes: {entry.prefix_count} -

+
+

{entry.message}

+
+
+ source + {entry.source} +
+
+ kind + {entry.kind} +
+
+ community + {entry.community} +
+
+ prefixes + {entry.prefix_count} +
+
{#if entry.sample && entry.sample.length > 0} -

- {entry.sample.join(', ')} -

+
+

sample

+ +
+ {#each entry.sample as sampleValue, sampleIdx (`${job.job_id}-${idx}-sample-${sampleIdx}`)} +

{sampleValue}

+ {/each} +
+ +
{/if}
{/each}