Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f39df7c4bf |
@@ -1212,6 +1212,70 @@ components:
|
||||
format: date-time
|
||||
additionalProperties: true
|
||||
|
||||
RevisionPruneEstimate:
|
||||
type: object
|
||||
required:
|
||||
- retention_minutes
|
||||
- cutoff_at
|
||||
- revision_count
|
||||
- prefix_row_count
|
||||
- orphan_snapshot_count
|
||||
- bytes_estimate
|
||||
properties:
|
||||
retention_minutes:
|
||||
type: integer
|
||||
minimum: 15
|
||||
maximum: 43200
|
||||
cutoff_at:
|
||||
type: string
|
||||
format: date-time
|
||||
revision_count:
|
||||
type: integer
|
||||
minimum: 0
|
||||
prefix_row_count:
|
||||
type: integer
|
||||
minimum: 0
|
||||
description: Строки prefix_snapshot_row в освобождаемых снимках.
|
||||
orphan_snapshot_count:
|
||||
type: integer
|
||||
minimum: 0
|
||||
bytes_estimate:
|
||||
type: integer
|
||||
format: int64
|
||||
minimum: 0
|
||||
description: Ориентировочный логический объём данных (байты).
|
||||
|
||||
RevisionPruneResult:
|
||||
type: object
|
||||
required:
|
||||
- deleted_revisions
|
||||
- deleted_prefix_snapshots
|
||||
- deleted_prefix_rows
|
||||
- bytes_estimate
|
||||
properties:
|
||||
deleted_revisions:
|
||||
type: integer
|
||||
minimum: 0
|
||||
deleted_prefix_snapshots:
|
||||
type: integer
|
||||
minimum: 0
|
||||
deleted_prefix_rows:
|
||||
type: integer
|
||||
minimum: 0
|
||||
bytes_estimate:
|
||||
type: integer
|
||||
format: int64
|
||||
minimum: 0
|
||||
|
||||
RevisionPruneRequest:
|
||||
type: object
|
||||
properties:
|
||||
retention_minutes:
|
||||
type: integer
|
||||
minimum: 15
|
||||
maximum: 43200
|
||||
description: TTL в минутах; если не задан — из revision_retention_minutes tenant settings.
|
||||
|
||||
PrefixSnapshotItem:
|
||||
type: object
|
||||
description: >
|
||||
@@ -2823,6 +2887,66 @@ paths:
|
||||
default:
|
||||
$ref: "#/components/responses/DefaultProblem"
|
||||
|
||||
/v1/revisions/prune-estimate:
|
||||
get:
|
||||
tags: [Revisions]
|
||||
summary: Оценка очистки ревизий по retention
|
||||
description: >
|
||||
Считает ревизии и ориентировочный объём данных, которые будут удалены при prune
|
||||
(те же правила, что applyRevisionRetention: последняя ревизия tenant и раскатанные на спикерах сохраняются).
|
||||
operationId: getRevisionPruneEstimate
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/TenantId"
|
||||
- name: retention_minutes
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: integer
|
||||
minimum: 15
|
||||
maximum: 43200
|
||||
description: TTL в минутах; если не задан — из tenant settings (default 30d).
|
||||
responses:
|
||||
"200":
|
||||
description: Оценка.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/RevisionPruneEstimate"
|
||||
"422":
|
||||
$ref: "#/components/responses/UnprocessableEntity"
|
||||
default:
|
||||
$ref: "#/components/responses/DefaultProblem"
|
||||
|
||||
/v1/revisions/prune:
|
||||
post:
|
||||
tags: [Revisions]
|
||||
summary: Очистить старые ревизии (синхронно)
|
||||
description: >
|
||||
Удаляет ревизии старше cutoff по retention и GC неиспользуемых prefix_snapshot.
|
||||
Operator-only.
|
||||
operationId: pruneRevisions
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/TenantId"
|
||||
requestBody:
|
||||
required: false
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/RevisionPruneRequest"
|
||||
responses:
|
||||
"200":
|
||||
description: Результат очистки.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/RevisionPruneResult"
|
||||
"403":
|
||||
$ref: "#/components/responses/Forbidden"
|
||||
"422":
|
||||
$ref: "#/components/responses/UnprocessableEntity"
|
||||
default:
|
||||
$ref: "#/components/responses/DefaultProblem"
|
||||
|
||||
/v1/revisions/{revision_id}:
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/TenantId"
|
||||
|
||||
@@ -59,6 +59,8 @@ func (s *Server) registerV1(m *http.ServeMux) {
|
||||
m.HandleFunc("POST /modules/{module_id}/refresh", s.handleModuleRefresh)
|
||||
m.HandleFunc("POST /tenant/refresh", s.handleTenantRefresh)
|
||||
m.HandleFunc("GET /revisions", s.handleListRevisions)
|
||||
m.HandleFunc("GET /revisions/prune-estimate", s.handleRevisionPruneEstimate)
|
||||
m.HandleFunc("POST /revisions/prune", s.handleRevisionPrune)
|
||||
m.HandleFunc("GET /revisions/{revision_id}", s.handleGetRevision)
|
||||
m.HandleFunc("GET /revisions/{revision_id}/preview", s.handleRevisionPreview)
|
||||
m.HandleFunc("GET /revisions/{revision_id}/diagnostic-log", s.handleRevisionDiagnosticLog)
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/pipeline"
|
||||
)
|
||||
|
||||
func (s *Server) resolveRevisionRetentionMinutesQuery(r *http.Request, tenantID string) (minutes int, ok bool) {
|
||||
if raw := r.URL.Query().Get("retention_minutes"); raw != "" {
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
if n < pipeline.RevisionMinTTLMin || n > pipeline.RevisionMaxTTLMin {
|
||||
return 0, false
|
||||
}
|
||||
return n, true
|
||||
}
|
||||
return s.defaultRevisionRetentionMinutes(tenantID)
|
||||
}
|
||||
|
||||
func (s *Server) defaultRevisionRetentionMinutes(tenantID string) (int, bool) {
|
||||
settings, err := s.store.ListGlobalSettings(tenantID)
|
||||
if err != nil {
|
||||
return pipeline.ClampRevisionRetentionMinutes(0), true
|
||||
}
|
||||
return pipeline.ClampRevisionRetentionMinutes(pipeline.RevisionRetentionMinutesFromSettings(settings)), true
|
||||
}
|
||||
|
||||
func (s *Server) resolveRevisionRetentionMinutesBody(r *http.Request, tenantID string) (minutes int, ok bool) {
|
||||
var body struct {
|
||||
RetentionMinutes *int `json:"retention_minutes"`
|
||||
}
|
||||
if r.Body != nil && r.ContentLength != 0 {
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
if body.RetentionMinutes != nil {
|
||||
m := *body.RetentionMinutes
|
||||
if m < pipeline.RevisionMinTTLMin || m > pipeline.RevisionMaxTTLMin {
|
||||
return 0, false
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
}
|
||||
return s.defaultRevisionRetentionMinutes(tenantID)
|
||||
}
|
||||
|
||||
func (s *Server) handleRevisionPruneEstimate(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok || !s.requireAtLeast(w, a, "viewer") {
|
||||
return
|
||||
}
|
||||
minutes, valid := s.resolveRevisionRetentionMinutesQuery(r, a.TenantID)
|
||||
if !valid {
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "retention_minutes must be an integer in range 15..43200")
|
||||
return
|
||||
}
|
||||
cutoff := pipeline.RevisionCutoffFromMinutes(minutes)
|
||||
est, err := s.store.EstimateRevisionPrune(a.TenantID, cutoff, minutes)
|
||||
if err != nil {
|
||||
writeInternalError(w, "revision prune estimate", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"retention_minutes": est.RetentionMinutes,
|
||||
"cutoff_at": est.CutoffAt.UTC().Format(time.RFC3339Nano),
|
||||
"revision_count": est.RevisionCount,
|
||||
"prefix_row_count": est.PrefixRowCount,
|
||||
"orphan_snapshot_count": est.OrphanSnapshotCount,
|
||||
"bytes_estimate": est.BytesEstimate,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) handleRevisionPrune(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok || !s.requireAtLeast(w, a, "operator") {
|
||||
return
|
||||
}
|
||||
minutes, valid := s.resolveRevisionRetentionMinutesBody(r, a.TenantID)
|
||||
if !valid {
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "retention_minutes must be an integer in range 15..43200")
|
||||
return
|
||||
}
|
||||
cutoff := pipeline.RevisionCutoffFromMinutes(minutes)
|
||||
res, err := s.store.PruneRevisionsWithStats(a.TenantID, cutoff)
|
||||
if err != nil {
|
||||
writeInternalError(w, "revision prune", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"deleted_revisions": res.DeletedRevisions,
|
||||
"deleted_prefix_snapshots": res.DeletedPrefixSnapshots,
|
||||
"deleted_prefix_rows": res.DeletedPrefixRows,
|
||||
"bytes_estimate": res.BytesEstimate,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRevisionPruneEstimateViewerOK(t *testing.T) {
|
||||
srv, err := New(Options{InsecureDev: true, SeedDemo: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer srv.Close()
|
||||
tenant, _, _, _, _ := srv.Store().DemoIDs()
|
||||
mustSetTestAPIKeys(t, srv, "vwkey|"+tenant+"|viewer")
|
||||
|
||||
ts := httptest.NewServer(srv.Handler())
|
||||
defer ts.Close()
|
||||
|
||||
req, _ := http.NewRequest(http.MethodGet, ts.URL+"/v1/revisions/prune-estimate?retention_minutes=15", nil)
|
||||
req.Header.Set("Authorization", "Bearer vwkey")
|
||||
resp, err := ts.Client().Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := body["revision_count"]; !ok {
|
||||
t.Fatalf("missing revision_count: %v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRevisionPruneViewerForbidden(t *testing.T) {
|
||||
srv, err := New(Options{InsecureDev: true, SeedDemo: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer srv.Close()
|
||||
tenant, _, _, _, _ := srv.Store().DemoIDs()
|
||||
mustSetTestAPIKeys(t, srv, "vwkey|"+tenant+"|viewer")
|
||||
|
||||
ts := httptest.NewServer(srv.Handler())
|
||||
defer ts.Close()
|
||||
|
||||
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/revisions/prune", strings.NewReader(`{"retention_minutes":43200}`))
|
||||
req.Header.Set("Authorization", "Bearer vwkey")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := ts.Client().Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusForbidden {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRevisionPruneOperatorOK(t *testing.T) {
|
||||
srv, err := New(Options{InsecureDev: true, SeedDemo: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer srv.Close()
|
||||
tenant, _, _, _, _ := srv.Store().DemoIDs()
|
||||
mustSetTestAPIKeys(t, srv, "opkey|"+tenant+"|operator")
|
||||
|
||||
ts := httptest.NewServer(srv.Handler())
|
||||
defer ts.Close()
|
||||
|
||||
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/revisions/prune", strings.NewReader(`{"retention_minutes":15}`))
|
||||
req.Header.Set("Authorization", "Bearer opkey")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := ts.Client().Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
}
|
||||
@@ -31,10 +31,7 @@ const (
|
||||
birdFilterNameV4 = "evobgp_export_v4"
|
||||
birdFilterNameV6 = "evobgp_export_v6"
|
||||
auxBirdFullExpanded = "_bird_full_expanded.conf"
|
||||
revisionTTLKey = "revision_retention_minutes"
|
||||
revisionMinTTLMin = 15
|
||||
revisionMaxTTLMin = 30 * 24 * 60
|
||||
revisionDefaultTTL = 30 * 24 * time.Hour
|
||||
revisionTTLKey = RevisionRetentionKey
|
||||
)
|
||||
|
||||
// AuxBirdFullExpandedKey returns the preview map key for the expanded BIRD config (generated on demand).
|
||||
@@ -969,17 +966,7 @@ func applyRevisionRetention(st store.Backend, tenantID string) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ttl := revisionDefaultTTL
|
||||
if minutes := intFromSettingsMap(settings, revisionTTLKey); minutes > 0 {
|
||||
if minutes < revisionMinTTLMin {
|
||||
minutes = revisionMinTTLMin
|
||||
}
|
||||
if minutes > revisionMaxTTLMin {
|
||||
minutes = revisionMaxTTLMin
|
||||
}
|
||||
ttl = time.Duration(minutes) * time.Minute
|
||||
}
|
||||
cutoff := time.Now().UTC().Add(-ttl)
|
||||
cutoff := RevisionRetentionCutoff(settings)
|
||||
_, _ = st.PruneRevisionsBefore(tenantID, cutoff)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// RevisionRetentionKey is the global_settings KV for revision TTL (minutes).
|
||||
RevisionRetentionKey = "revision_retention_minutes"
|
||||
RevisionMinTTLMin = 15
|
||||
RevisionMaxTTLMin = 30 * 24 * 60
|
||||
RevisionDefaultTTL = 30 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// RevisionCutoffFromMinutes returns created_at cutoff for revision prune (UTC now minus clamped TTL).
|
||||
func RevisionCutoffFromMinutes(minutes int) time.Time {
|
||||
ttl := RevisionDefaultTTL
|
||||
if minutes > 0 {
|
||||
m := minutes
|
||||
if m < RevisionMinTTLMin {
|
||||
m = RevisionMinTTLMin
|
||||
}
|
||||
if m > RevisionMaxTTLMin {
|
||||
m = RevisionMaxTTLMin
|
||||
}
|
||||
ttl = time.Duration(m) * time.Minute
|
||||
}
|
||||
return time.Now().UTC().Add(-ttl)
|
||||
}
|
||||
|
||||
// RevisionRetentionMinutesFromSettings reads revision_retention_minutes from tenant KV (0 if unset).
|
||||
func RevisionRetentionMinutesFromSettings(settings map[string]any) int {
|
||||
if settings == nil {
|
||||
return 0
|
||||
}
|
||||
v, ok := settings[RevisionRetentionKey]
|
||||
if !ok || v == nil {
|
||||
return 0
|
||||
}
|
||||
switch x := v.(type) {
|
||||
case float64:
|
||||
return int(x)
|
||||
case int:
|
||||
return x
|
||||
case int64:
|
||||
return int(x)
|
||||
case string:
|
||||
n, err := strconv.Atoi(strings.TrimSpace(x))
|
||||
if err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// RevisionRetentionCutoff resolves tenant revision_retention_minutes from settings (default 30d).
|
||||
func RevisionRetentionCutoff(settings map[string]any) time.Time {
|
||||
return RevisionCutoffFromMinutes(RevisionRetentionMinutesFromSettings(settings))
|
||||
}
|
||||
|
||||
// ClampRevisionRetentionMinutes normalizes user input to the allowed range.
|
||||
func ClampRevisionRetentionMinutes(minutes int) int {
|
||||
if minutes <= 0 {
|
||||
return int(RevisionDefaultTTL / time.Minute)
|
||||
}
|
||||
if minutes < RevisionMinTTLMin {
|
||||
return RevisionMinTTLMin
|
||||
}
|
||||
if minutes > RevisionMaxTTLMin {
|
||||
return RevisionMaxTTLMin
|
||||
}
|
||||
return minutes
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestClampRevisionRetentionMinutes(t *testing.T) {
|
||||
if got := ClampRevisionRetentionMinutes(0); got != int(RevisionDefaultTTL/time.Minute) {
|
||||
t.Fatalf("default: got %d", got)
|
||||
}
|
||||
if got := ClampRevisionRetentionMinutes(5); got != RevisionMinTTLMin {
|
||||
t.Fatalf("min clamp: got %d", got)
|
||||
}
|
||||
if got := ClampRevisionRetentionMinutes(999999); got != RevisionMaxTTLMin {
|
||||
t.Fatalf("max clamp: got %d", got)
|
||||
}
|
||||
if got := ClampRevisionRetentionMinutes(120); got != 120 {
|
||||
t.Fatalf("unchanged: got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRevisionCutoffFromMinutes(t *testing.T) {
|
||||
before := time.Now().UTC()
|
||||
cutoff := RevisionCutoffFromMinutes(60)
|
||||
after := time.Now().UTC().Add(-59 * time.Minute)
|
||||
if cutoff.After(before.Add(-59*time.Minute)) || cutoff.Before(after.Add(-2*time.Minute)) {
|
||||
t.Fatalf("cutoff out of range: %v", cutoff)
|
||||
}
|
||||
}
|
||||
@@ -970,43 +970,6 @@ func (p *Postgres) RevisionDiff(tenantID, aID, bID string) (map[string]any, erro
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) PruneRevisionsBefore(tenantID string, cutoff time.Time) (int, error) {
|
||||
ctx := context.Background()
|
||||
total := 0
|
||||
const batchSize = 50
|
||||
for {
|
||||
cmd, err := p.pool.Exec(ctx, `
|
||||
DELETE FROM config_revision AS cr
|
||||
WHERE cr.id IN (
|
||||
SELECT id FROM config_revision
|
||||
WHERE tenant_id = $1
|
||||
AND created_at < $2
|
||||
AND id <> (
|
||||
SELECT id FROM config_revision
|
||||
WHERE tenant_id = $1
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM bgp_speaker AS sp
|
||||
WHERE sp.tenant_id = $1
|
||||
AND (sp.last_applied_revision_id = config_revision.id OR sp.published_revision_id = config_revision.id)
|
||||
)
|
||||
ORDER BY created_at ASC
|
||||
LIMIT $3
|
||||
)`, tenantID, cutoff.UTC(), batchSize)
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
n := int(cmd.RowsAffected())
|
||||
total += n
|
||||
if n < batchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) SetLastAppliedRevision(tenantID, speakerID, revisionID string) error {
|
||||
ctx := context.Background()
|
||||
tag, err := p.pool.Exec(ctx, `
|
||||
|
||||
@@ -53,21 +53,8 @@ func (p *Postgres) ensurePrefixSnapshot(ctx context.Context, db execQuerier, con
|
||||
return "", nil
|
||||
}
|
||||
hash := normalizeSnapshotHash(contentHash)
|
||||
var existing string
|
||||
err := db.QueryRow(ctx, `SELECT id::text FROM prefix_snapshot WHERE content_hash = $1`, hash).Scan(&existing)
|
||||
if err == nil && existing != "" {
|
||||
return existing, nil
|
||||
}
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", err
|
||||
}
|
||||
snapID := uuid.NewString()
|
||||
if _, err := db.Exec(ctx, `
|
||||
INSERT INTO prefix_snapshot (id, content_hash) VALUES ($1::uuid, $2)
|
||||
ON CONFLICT (content_hash) DO NOTHING`, snapID, hash); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := db.QueryRow(ctx, `SELECT id::text FROM prefix_snapshot WHERE content_hash = $1`, hash).Scan(&snapID); err != nil {
|
||||
snapID, err := p.resolvePrefixSnapshotID(ctx, db, hash)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var rowCount int
|
||||
@@ -94,6 +81,27 @@ func (p *Postgres) ensurePrefixSnapshot(ctx context.Context, db execQuerier, con
|
||||
return snapID, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) resolvePrefixSnapshotID(ctx context.Context, db execQuerier, hash string) (string, error) {
|
||||
var existing string
|
||||
err := db.QueryRow(ctx, `SELECT id::text FROM prefix_snapshot WHERE content_hash = $1`, hash).Scan(&existing)
|
||||
if err == nil && existing != "" {
|
||||
return existing, nil
|
||||
}
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", err
|
||||
}
|
||||
snapID := uuid.NewString()
|
||||
if _, err := db.Exec(ctx, `
|
||||
INSERT INTO prefix_snapshot (id, content_hash) VALUES ($1::uuid, $2)
|
||||
ON CONFLICT (content_hash) DO NOTHING`, snapID, hash); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := db.QueryRow(ctx, `SELECT id::text FROM prefix_snapshot WHERE content_hash = $1`, hash).Scan(&snapID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return snapID, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) listSnapshotPrefixes(ctx context.Context, snapshotID, cursor string, limit int) ([]store.PrefixRow, string, bool) {
|
||||
afterOrd, off, useOffset := store.ParsePrefixPageCursor(cursor)
|
||||
var rows pgx.Rows
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/db"
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestEnsurePrefixSnapshotFillsEmptyExistingIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("EVOBGP_TEST_DATABASE_URL not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.OpenPostgresPool(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
pg, err := NewPostgres(ctx, pool, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
hash := normalizeSnapshotHash("sha256:empty-fill-" + uuid.NewString())
|
||||
snapID := uuid.NewString()
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO prefix_snapshot (id, content_hash) VALUES ($1::uuid, $2)`, snapID, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM prefix_snapshot WHERE id = $1::uuid`, snapID)
|
||||
})
|
||||
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
got, err := pg.ensurePrefixSnapshot(ctx, tx, "sha256:"+hash, []store.PrefixRow{
|
||||
{Prefix: "203.0.113.1/32", Source: "test"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != snapID {
|
||||
t.Fatalf("snap id: got %q want %q", got, snapID)
|
||||
}
|
||||
var rowCount int
|
||||
if err := tx.QueryRow(ctx, `SELECT COUNT(*)::int FROM prefix_snapshot_row WHERE snapshot_id = $1::uuid`, snapID).Scan(&rowCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rowCount != 1 {
|
||||
t.Fatalf("row count: got %d", rowCount)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsurePrefixSnapshotIdempotentIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("EVOBGP_TEST_DATABASE_URL not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.OpenPostgresPool(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
pg, err := NewPostgres(ctx, pool, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
contentHash := "sha256:idempotent-" + uuid.NewString()
|
||||
prefixes := []store.PrefixRow{{Prefix: "198.51.100.0/24", Source: "test"}}
|
||||
|
||||
tx1, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id1, err := pg.ensurePrefixSnapshot(ctx, tx1, contentHash, prefixes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx1.Commit(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM prefix_snapshot WHERE id = $1::uuid`, id1)
|
||||
})
|
||||
|
||||
tx2, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = tx2.Rollback(ctx) }()
|
||||
id2, err := pg.ensurePrefixSnapshot(ctx, tx2, contentHash, prefixes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id1 != id2 {
|
||||
t.Fatalf("ids differ: %q vs %q", id1, id2)
|
||||
}
|
||||
var rowCount int
|
||||
if err := tx2.QueryRow(ctx, `SELECT COUNT(*)::int FROM prefix_snapshot_row WHERE snapshot_id = $1::uuid`, id1).Scan(&rowCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rowCount != 1 {
|
||||
t.Fatalf("expected 1 row, got %d", rowCount)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
const revisionPruneBatchSize = 50
|
||||
|
||||
// sqlPrunableRevisionsWhere appends prunable revision predicates (tenant + cutoff params).
|
||||
func sqlPrunableRevisionsWhere(tenantParam, cutoffParam string) string {
|
||||
return `tenant_id = ` + tenantParam + `
|
||||
AND created_at < ` + cutoffParam + `
|
||||
AND id <> (
|
||||
SELECT id FROM config_revision
|
||||
WHERE tenant_id = ` + tenantParam + `
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM bgp_speaker AS sp
|
||||
WHERE sp.tenant_id = ` + tenantParam + `
|
||||
AND (sp.last_applied_revision_id = config_revision.id OR sp.published_revision_id = config_revision.id)
|
||||
)`
|
||||
}
|
||||
|
||||
func (p *Postgres) EstimateRevisionPrune(tenantID string, cutoff time.Time, retentionMinutes int) (store.RevisionPruneEstimate, error) {
|
||||
ctx := context.Background()
|
||||
out := store.RevisionPruneEstimate{
|
||||
RetentionMinutes: retentionMinutes,
|
||||
CutoffAt: cutoff.UTC(),
|
||||
}
|
||||
where := sqlPrunableRevisionsWhere("$1", "$2")
|
||||
var revBytes, previewBytes, rmpBytes, snapRowBytes int64
|
||||
err := p.pool.QueryRow(ctx, `
|
||||
WITH prunable AS (
|
||||
SELECT id, prefix_snapshot_id, content_hash, meta_json
|
||||
FROM config_revision
|
||||
WHERE `+where+`
|
||||
),
|
||||
freed_snaps AS (
|
||||
SELECT DISTINCT pr.prefix_snapshot_id AS snap_id
|
||||
FROM prunable pr
|
||||
WHERE pr.prefix_snapshot_id IS NOT NULL
|
||||
EXCEPT
|
||||
SELECT DISTINCT cr.prefix_snapshot_id
|
||||
FROM config_revision cr
|
||||
WHERE cr.prefix_snapshot_id IS NOT NULL
|
||||
AND cr.id NOT IN (SELECT id FROM prunable)
|
||||
)
|
||||
SELECT
|
||||
(SELECT COUNT(*)::int FROM prunable),
|
||||
COALESCE((SELECT SUM(octet_length(content_hash) + octet_length(meta_json::text))::bigint FROM prunable), 0),
|
||||
COALESCE((
|
||||
SELECT SUM(octet_length(value))::bigint
|
||||
FROM config_revision_preview p
|
||||
JOIN prunable pr ON pr.id = p.revision_id
|
||||
CROSS JOIN LATERAL jsonb_each_text(COALESCE(p.fragments, '{}'::jsonb))
|
||||
), 0),
|
||||
COALESCE((
|
||||
SELECT SUM(octet_length(rmp.prefix) + octet_length(COALESCE(rmp.source, '')))::bigint
|
||||
FROM revision_materialized_prefix rmp
|
||||
WHERE rmp.revision_id IN (SELECT id FROM prunable)
|
||||
), 0),
|
||||
(SELECT COUNT(*)::int FROM freed_snaps),
|
||||
COALESCE((SELECT COUNT(*)::int FROM prefix_snapshot_row psr WHERE psr.snapshot_id IN (SELECT snap_id FROM freed_snaps)), 0),
|
||||
COALESCE((
|
||||
SELECT SUM(octet_length(psr.prefix) + octet_length(COALESCE(psr.source, '')))::bigint
|
||||
FROM prefix_snapshot_row psr
|
||||
WHERE psr.snapshot_id IN (SELECT snap_id FROM freed_snaps)
|
||||
), 0)`,
|
||||
tenantID, cutoff.UTC()).Scan(
|
||||
&out.RevisionCount,
|
||||
&revBytes,
|
||||
&previewBytes,
|
||||
&rmpBytes,
|
||||
&out.OrphanSnapshotCount,
|
||||
&out.PrefixRowCount,
|
||||
&snapRowBytes,
|
||||
)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.BytesEstimate = revBytes + previewBytes + rmpBytes + snapRowBytes
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (store.RevisionPruneResult, error) {
|
||||
ctx := context.Background()
|
||||
est, err := p.EstimateRevisionPrune(tenantID, cutoff, 0)
|
||||
if err != nil {
|
||||
return store.RevisionPruneResult{}, err
|
||||
}
|
||||
out := store.RevisionPruneResult{BytesEstimate: est.BytesEstimate}
|
||||
|
||||
where := sqlPrunableRevisionsWhere("$1", "$2")
|
||||
for {
|
||||
cmd, err := p.pool.Exec(ctx, `
|
||||
DELETE FROM config_revision AS cr
|
||||
WHERE cr.id IN (
|
||||
SELECT id FROM config_revision
|
||||
WHERE `+where+`
|
||||
ORDER BY created_at ASC
|
||||
LIMIT $3
|
||||
)`, tenantID, cutoff.UTC(), revisionPruneBatchSize)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
n := int(cmd.RowsAffected())
|
||||
out.DeletedRevisions += n
|
||||
if n < revisionPruneBatchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
snaps, rows, err := p.pruneUnreferencedPrefixSnapshots(ctx, revisionPruneBatchSize)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.DeletedPrefixSnapshots += snaps
|
||||
out.DeletedPrefixRows += rows
|
||||
if snaps < revisionPruneBatchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) PruneRevisionsBefore(tenantID string, cutoff time.Time) (int, error) {
|
||||
res, err := p.PruneRevisionsWithStats(tenantID, cutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.DeletedRevisions, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) pruneUnreferencedPrefixSnapshots(ctx context.Context, batchSize int) (deletedSnapshots int, deletedRows int, err error) {
|
||||
if !prefixSnapshotTableExists(ctx, p.pool) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
var snapCount, rowCount int
|
||||
err = p.pool.QueryRow(ctx, `
|
||||
WITH doomed AS (
|
||||
SELECT ps.id FROM prefix_snapshot ps
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM config_revision cr WHERE cr.prefix_snapshot_id = ps.id
|
||||
)
|
||||
LIMIT $1
|
||||
)
|
||||
SELECT
|
||||
(SELECT COUNT(*)::int FROM doomed),
|
||||
(SELECT COUNT(*)::int FROM prefix_snapshot_row psr WHERE psr.snapshot_id IN (SELECT id FROM doomed))`,
|
||||
batchSize).Scan(&snapCount, &rowCount)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if snapCount == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
_, err = p.pool.Exec(ctx, `
|
||||
DELETE FROM prefix_snapshot
|
||||
WHERE id IN (
|
||||
SELECT ps.id FROM prefix_snapshot ps
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM config_revision cr WHERE cr.prefix_snapshot_id = ps.id
|
||||
)
|
||||
LIMIT $1
|
||||
)`, batchSize)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return snapCount, rowCount, nil
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/db"
|
||||
"evobgp/internal/pipeline"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestPostgresPruneUnreferencedPrefixSnapshotsIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("EVOBGP_TEST_DATABASE_URL not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.OpenPostgresPool(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
pg, err := NewPostgres(ctx, pool, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
snapID := uuid.NewString()
|
||||
hash := normalizeSnapshotHash("sha256:test-orphan-" + snapID)
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO prefix_snapshot (id, content_hash) VALUES ($1::uuid, $2)`, snapID, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO prefix_snapshot_row (snapshot_id, ord, prefix, source)
|
||||
VALUES ($1::uuid, 0, '203.0.113.0/24', 'test')`, snapID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
snaps, rows, err := pg.pruneUnreferencedPrefixSnapshots(ctx, 50)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snaps < 1 || rows < 1 {
|
||||
t.Fatalf("expected orphan cleanup, got snaps=%d rows=%d", snaps, rows)
|
||||
}
|
||||
|
||||
var n int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*)::int FROM prefix_snapshot WHERE id = $1::uuid`, snapID).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Fatalf("snapshot still exists")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresEstimateRevisionPruneIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("EVOBGP_TEST_DATABASE_URL not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.OpenPostgresPool(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
pg, err := NewPostgres(ctx, pool, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tenant := uuid.NewString()
|
||||
cutoff := pipeline.RevisionCutoffFromMinutes(15)
|
||||
est, err := pg.EstimateRevisionPrune(tenant, cutoff, 15)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if est.RevisionCount != 0 {
|
||||
t.Fatalf("expected 0 revisions for empty tenant, got %d", est.RevisionCount)
|
||||
}
|
||||
}
|
||||
@@ -84,6 +84,8 @@ type Backend interface {
|
||||
// CreateRenderRevision inserts a new config_revision (revID must be unique) with materialized prefixes and preview fragments.
|
||||
CreateRenderRevision(revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []PrefixRow) error
|
||||
RevisionDiff(tenantID, aID, bID string) (map[string]any, error)
|
||||
EstimateRevisionPrune(tenantID string, cutoff time.Time, retentionMinutes int) (RevisionPruneEstimate, error)
|
||||
PruneRevisionsWithStats(tenantID string, cutoff time.Time) (RevisionPruneResult, error)
|
||||
PruneRevisionsBefore(tenantID string, cutoff time.Time) (deleted int, err error)
|
||||
SetLastAppliedRevision(tenantID, speakerID, revisionID string) error
|
||||
PublishRevisionForSpeaker(speakerID, revisionID string) error
|
||||
|
||||
@@ -616,62 +616,6 @@ func (m *Memory) RevisionDiff(tenantID, aID, bID string) (map[string]any, error)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Memory) PruneRevisionsBefore(tenantID string, cutoff time.Time) (int, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
var newest *Revision
|
||||
for _, rev := range m.revisions {
|
||||
if rev.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if newest == nil || rev.CreatedAt.After(newest.CreatedAt) {
|
||||
newest = rev
|
||||
}
|
||||
}
|
||||
if newest == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
protected := map[string]struct{}{
|
||||
newest.ID: {},
|
||||
}
|
||||
for _, sp := range m.speakers {
|
||||
if sp == nil || sp.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if sp.LastAppliedRevisionID != nil && strings.TrimSpace(*sp.LastAppliedRevisionID) != "" {
|
||||
protected[strings.TrimSpace(*sp.LastAppliedRevisionID)] = struct{}{}
|
||||
}
|
||||
}
|
||||
for speakerID, info := range m.publishedRevision {
|
||||
sp, ok := m.speakers[speakerID]
|
||||
if !ok || sp == nil || sp.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(info.RevisionID) != "" {
|
||||
protected[strings.TrimSpace(info.RevisionID)] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
for id, rev := range m.revisions {
|
||||
if rev == nil || rev.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if !rev.CreatedAt.Before(cutoff) {
|
||||
continue
|
||||
}
|
||||
if _, keep := protected[id]; keep {
|
||||
continue
|
||||
}
|
||||
delete(m.revisions, id)
|
||||
delete(m.revPrefixes, id)
|
||||
deleted++
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (m *Memory) getRevisionLocked(tenantID, revisionID string) (*Revision, error) {
|
||||
rev, ok := m.revisions[revisionID]
|
||||
if !ok {
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (m *Memory) prunableRevisionIDsLocked(tenantID string, cutoff time.Time) []string {
|
||||
var newest *Revision
|
||||
for _, rev := range m.revisions {
|
||||
if rev == nil || rev.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if newest == nil || rev.CreatedAt.After(newest.CreatedAt) {
|
||||
newest = rev
|
||||
}
|
||||
}
|
||||
if newest == nil {
|
||||
return nil
|
||||
}
|
||||
protected := map[string]struct{}{newest.ID: {}}
|
||||
for _, sp := range m.speakers {
|
||||
if sp == nil || sp.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if sp.LastAppliedRevisionID != nil && strings.TrimSpace(*sp.LastAppliedRevisionID) != "" {
|
||||
protected[strings.TrimSpace(*sp.LastAppliedRevisionID)] = struct{}{}
|
||||
}
|
||||
}
|
||||
for speakerID, info := range m.publishedRevision {
|
||||
sp, ok := m.speakers[speakerID]
|
||||
if !ok || sp == nil || sp.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(info.RevisionID) != "" {
|
||||
protected[strings.TrimSpace(info.RevisionID)] = struct{}{}
|
||||
}
|
||||
}
|
||||
var ids []string
|
||||
for id, rev := range m.revisions {
|
||||
if rev == nil || rev.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if !rev.CreatedAt.Before(cutoff) {
|
||||
continue
|
||||
}
|
||||
if _, keep := protected[id]; keep {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func revisionBytesEstimate(rev *Revision, prefixes []PrefixRow) int64 {
|
||||
if rev == nil {
|
||||
return 0
|
||||
}
|
||||
var n int64
|
||||
n += int64(len(rev.ContentHash))
|
||||
for _, v := range rev.PreviewFragments {
|
||||
n += int64(len(v))
|
||||
}
|
||||
for _, pr := range prefixes {
|
||||
n += int64(len(pr.Prefix) + len(pr.Source))
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (m *Memory) EstimateRevisionPrune(tenantID string, cutoff time.Time, retentionMinutes int) (RevisionPruneEstimate, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
ids := m.prunableRevisionIDsLocked(tenantID, cutoff)
|
||||
var bytes int64
|
||||
for _, id := range ids {
|
||||
bytes += revisionBytesEstimate(m.revisions[id], m.revPrefixes[id])
|
||||
}
|
||||
prefixRows := 0
|
||||
for _, id := range ids {
|
||||
prefixRows += len(m.revPrefixes[id])
|
||||
}
|
||||
return RevisionPruneEstimate{
|
||||
RetentionMinutes: retentionMinutes,
|
||||
CutoffAt: cutoff.UTC(),
|
||||
RevisionCount: len(ids),
|
||||
PrefixRowCount: prefixRows,
|
||||
OrphanSnapshotCount: 0,
|
||||
BytesEstimate: bytes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Memory) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (RevisionPruneResult, error) {
|
||||
est, err := m.EstimateRevisionPrune(tenantID, cutoff, 0)
|
||||
if err != nil {
|
||||
return RevisionPruneResult{}, err
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
ids := m.prunableRevisionIDsLocked(tenantID, cutoff)
|
||||
for _, id := range ids {
|
||||
delete(m.revisions, id)
|
||||
delete(m.revPrefixes, id)
|
||||
}
|
||||
return RevisionPruneResult{
|
||||
DeletedRevisions: len(ids),
|
||||
DeletedPrefixSnapshots: 0,
|
||||
DeletedPrefixRows: est.PrefixRowCount,
|
||||
BytesEstimate: est.BytesEstimate,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Memory) PruneRevisionsBefore(tenantID string, cutoff time.Time) (int, error) {
|
||||
res, err := m.PruneRevisionsWithStats(tenantID, cutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.DeletedRevisions, nil
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package store
|
||||
|
||||
import "time"
|
||||
|
||||
// RevisionPruneEstimate describes revisions and storage that would be removed by prune.
|
||||
type RevisionPruneEstimate struct {
|
||||
RetentionMinutes int `json:"retention_minutes"`
|
||||
CutoffAt time.Time `json:"cutoff_at"`
|
||||
RevisionCount int `json:"revision_count"`
|
||||
PrefixRowCount int `json:"prefix_row_count"`
|
||||
OrphanSnapshotCount int `json:"orphan_snapshot_count"`
|
||||
BytesEstimate int64 `json:"bytes_estimate"`
|
||||
}
|
||||
|
||||
// RevisionPruneResult is the outcome of a revision prune run.
|
||||
type RevisionPruneResult struct {
|
||||
DeletedRevisions int `json:"deleted_revisions"`
|
||||
DeletedPrefixSnapshots int `json:"deleted_prefix_snapshots"`
|
||||
DeletedPrefixRows int `json:"deleted_prefix_rows"`
|
||||
BytesEstimate int64 `json:"bytes_estimate"`
|
||||
}
|
||||
@@ -2,10 +2,17 @@
|
||||
import { onMount } from 'svelte';
|
||||
import { defaults, superForm } from 'sveltekit-superforms';
|
||||
import { zod4 } from 'sveltekit-superforms/adapters';
|
||||
import type { AuthSession } from '$lib/api/types.js';
|
||||
import { apiJSON } from '$lib/api/client.js';
|
||||
import {
|
||||
emptyRevisionSettingsForm,
|
||||
revisionSettingsSchema
|
||||
} from '$lib/settings/revision-settings.schema.js';
|
||||
import {
|
||||
fetchRevisionPruneEstimate,
|
||||
pruneRevisionsNow,
|
||||
type RevisionPruneEstimate
|
||||
} from '$lib/settings/revision-prune-api.js';
|
||||
import {
|
||||
buildPayloadFromFormFields,
|
||||
loadSettings,
|
||||
@@ -13,6 +20,7 @@
|
||||
patchSettings
|
||||
} from '$lib/settings/settings-api.js';
|
||||
import { REVISION_SETTING_KEYS } from '$lib/settings/settings-known-keys.js';
|
||||
import { formatBytes } from '$lib/monitoring/postgres.js';
|
||||
import { Button } from '$lib/ui/core/button/index.js';
|
||||
import {
|
||||
Card,
|
||||
@@ -23,12 +31,18 @@
|
||||
} from '$lib/ui/core/card/index.js';
|
||||
import { Input } from '$lib/ui/core/input/index.js';
|
||||
import FormField from '$lib/ui/patterns/form/form-field.svelte';
|
||||
import { confirm } from '$lib/ui/patterns/confirm/confirm-state.svelte.js';
|
||||
import { notify, notifyApiError } from '$lib/ui/app/toast.js';
|
||||
import Save from '@lucide/svelte/icons/save';
|
||||
import Trash2 from '@lucide/svelte/icons/trash-2';
|
||||
|
||||
let loading = $state(false);
|
||||
let saving = $state(false);
|
||||
let pruning = $state(false);
|
||||
let loaded = $state(false);
|
||||
let session = $state<AuthSession | null>(null);
|
||||
let estimateLoading = $state(false);
|
||||
let estimate = $state<RevisionPruneEstimate | null>(null);
|
||||
|
||||
const { form, errors, reset, validateForm } = superForm(
|
||||
defaults(emptyRevisionSettingsForm(), zod4(revisionSettingsSchema)),
|
||||
@@ -39,13 +53,60 @@
|
||||
}
|
||||
);
|
||||
|
||||
let hasValidationErrors = $derived(Boolean($errors.revision_retention_minutes?.length));
|
||||
const isOperator = $derived(session?.role === 'operator');
|
||||
const hasValidationErrors = $derived(Boolean($errors.revision_retention_minutes?.length));
|
||||
|
||||
let canSave = $derived.by(() => {
|
||||
const parsedRetentionMinutes = $derived.by(() => {
|
||||
const s = String($form.revision_retention_minutes ?? '').trim();
|
||||
if (s === '' || !/^\d+$/.test(s)) return null;
|
||||
const n = Number(s);
|
||||
if (!Number.isInteger(n) || n < 15 || n > 43200) return null;
|
||||
return n;
|
||||
});
|
||||
|
||||
const canSave = $derived.by(() => {
|
||||
if (loading || saving || hasValidationErrors || !loaded) return false;
|
||||
return String($form.revision_retention_minutes ?? '').trim() !== '';
|
||||
});
|
||||
|
||||
const canPruneNow = $derived.by(() => {
|
||||
if (!isOperator || !loaded || pruning || saving || parsedRetentionMinutes === null)
|
||||
return false;
|
||||
return (estimate?.revision_count ?? 0) > 0;
|
||||
});
|
||||
|
||||
async function loadSession() {
|
||||
try {
|
||||
session = await apiJSON<AuthSession>('/v1/auth/session');
|
||||
} catch {
|
||||
session = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshEstimate(minutes: number) {
|
||||
estimateLoading = true;
|
||||
try {
|
||||
estimate = await fetchRevisionPruneEstimate(minutes);
|
||||
} catch (e) {
|
||||
estimate = null;
|
||||
notifyApiError(e, 'Не удалось рассчитать оценку очистки');
|
||||
} finally {
|
||||
estimateLoading = false;
|
||||
}
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
const minutes = parsedRetentionMinutes;
|
||||
if (!loaded || minutes === null) {
|
||||
estimate = null;
|
||||
return;
|
||||
}
|
||||
const handle = setTimeout(() => {
|
||||
void refreshEstimate(minutes);
|
||||
}, 400);
|
||||
return () => clearTimeout(handle);
|
||||
});
|
||||
|
||||
async function load() {
|
||||
loading = true;
|
||||
try {
|
||||
@@ -89,7 +150,46 @@
|
||||
}
|
||||
}
|
||||
|
||||
async function requestPruneNow() {
|
||||
const minutes = parsedRetentionMinutes;
|
||||
if (minutes === null) return;
|
||||
let est: RevisionPruneEstimate;
|
||||
try {
|
||||
est = await fetchRevisionPruneEstimate(minutes);
|
||||
} catch (e) {
|
||||
notifyApiError(e);
|
||||
return;
|
||||
}
|
||||
estimate = est;
|
||||
if (est.revision_count === 0) {
|
||||
notify.info('Нет ревизий для удаления по выбранному retention');
|
||||
return;
|
||||
}
|
||||
void confirm({
|
||||
title: 'Очистить старые ревизии?',
|
||||
description: `Будет удалено ${est.revision_count} ревизий. Ориентировочно освободится ~${formatBytes(est.bytes_estimate)}. Действие необратимо.`,
|
||||
confirmLabel: 'Очистить',
|
||||
destructive: true,
|
||||
onConfirm: async () => {
|
||||
pruning = true;
|
||||
try {
|
||||
const res = await pruneRevisionsNow(minutes);
|
||||
notify.success(
|
||||
`Удалено ревизий: ${res.deleted_revisions}, освобождено ~${formatBytes(res.bytes_estimate)}`
|
||||
);
|
||||
await refreshEstimate(minutes);
|
||||
} catch (e) {
|
||||
notifyApiError(e);
|
||||
throw e;
|
||||
} finally {
|
||||
pruning = false;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
void loadSession();
|
||||
void load();
|
||||
});
|
||||
</script>
|
||||
@@ -98,7 +198,8 @@
|
||||
<CardHeader>
|
||||
<CardTitle>Хранение ревизий</CardTitle>
|
||||
<CardDescription>
|
||||
Автоматическая очистка старых ревизий. Последняя раскатанная ревизия не удаляется.
|
||||
Автоматическая очистка старых ревизий. Последняя ревизия и раскатанные на спикерах не
|
||||
удаляются.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent class="space-y-4">
|
||||
@@ -123,16 +224,54 @@
|
||||
/>
|
||||
</FormField>
|
||||
|
||||
{#if parsedRetentionMinutes !== null}
|
||||
<div class="space-y-1 rounded-md border bg-muted/30 p-4 text-sm">
|
||||
<p class="font-medium">Оценка очистки по введённому retention</p>
|
||||
{#if estimateLoading}
|
||||
<p class="text-muted-foreground">Расчёт…</p>
|
||||
{:else if estimate}
|
||||
<p>
|
||||
Будет удалено ревизий: <strong>{estimate.revision_count}</strong>
|
||||
</p>
|
||||
<p>
|
||||
Освободится ориентировочно: <strong>~{formatBytes(estimate.bytes_estimate)}</strong>
|
||||
</p>
|
||||
{#if estimate.prefix_row_count > 0}
|
||||
<p class="text-xs text-muted-foreground">
|
||||
Строк префиксов в снимках: {estimate.prefix_row_count}
|
||||
{#if estimate.orphan_snapshot_count > 0}
|
||||
· снимков: {estimate.orphan_snapshot_count}
|
||||
{/if}
|
||||
</p>
|
||||
{/if}
|
||||
{:else}
|
||||
<p class="text-muted-foreground">Оценка недоступна</p>
|
||||
{/if}
|
||||
<p class="pt-1 text-xs text-muted-foreground">
|
||||
Учитываются те же правила, что при автоочистке: последняя ревизия и раскатанные на
|
||||
спикерах не удаляются.
|
||||
</p>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if hasValidationErrors}
|
||||
<p class="text-sm text-destructive">
|
||||
Есть ошибки в полях. Исправьте их, чтобы сохранить изменения.
|
||||
</p>
|
||||
{/if}
|
||||
|
||||
<Button onclick={save} disabled={!canSave}>
|
||||
<Save />
|
||||
{saving ? 'Сохранение…' : 'Применить параметры ревизий'}
|
||||
</Button>
|
||||
<div class="flex flex-wrap gap-2">
|
||||
<Button onclick={save} disabled={!canSave}>
|
||||
<Save />
|
||||
{saving ? 'Сохранение…' : 'Применить параметры ревизий'}
|
||||
</Button>
|
||||
{#if isOperator}
|
||||
<Button variant="destructive" disabled={!canPruneNow} onclick={requestPruneNow}>
|
||||
<Trash2 />
|
||||
{pruning ? 'Очистка…' : 'Очистить сейчас'}
|
||||
</Button>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { apiJSON, apiMutate } from '$lib/api/client.js';
|
||||
|
||||
export type RevisionPruneEstimate = {
|
||||
retention_minutes: number;
|
||||
cutoff_at: string;
|
||||
revision_count: number;
|
||||
prefix_row_count: number;
|
||||
orphan_snapshot_count: number;
|
||||
bytes_estimate: number;
|
||||
};
|
||||
|
||||
export type RevisionPruneResult = {
|
||||
deleted_revisions: number;
|
||||
deleted_prefix_snapshots: number;
|
||||
deleted_prefix_rows: number;
|
||||
bytes_estimate: number;
|
||||
};
|
||||
|
||||
export async function fetchRevisionPruneEstimate(
|
||||
retentionMinutes: number
|
||||
): Promise<RevisionPruneEstimate> {
|
||||
const q = new URLSearchParams({ retention_minutes: String(retentionMinutes) });
|
||||
return apiJSON<RevisionPruneEstimate>(`/v1/revisions/prune-estimate?${q}`);
|
||||
}
|
||||
|
||||
export async function pruneRevisionsNow(retentionMinutes: number): Promise<RevisionPruneResult> {
|
||||
return apiMutate<RevisionPruneResult>('/v1/revisions/prune', 'POST', {
|
||||
retention_minutes: retentionMinutes
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user