Hub stage 2: install, update and uninstall apps on several servers

Writes use only <ID>_ADMIN_TOKEN and need the apps.write capability.
Store URLs are resolved from each server's own listing. The config is saved
to history before an uninstall. Zip uploads are fanned out to the servers.
Every target's result is audited. The console gets a row menu on the
matrix, a target picker, a results dialog with Retry failed, the store and
install from file in All servers mode, and single-server actions for writers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

hub/upstream.ts

@@ -28,6 +28,9 @@ interface Cached {
 
 const MAX_CACHED = 500
 
+/* An app download or upload can take a while. */
+const WRITE_TIMEOUT_MS = 60_000
+
 export class Upstream {
   #config: HubConfig
   #fetch: typeof fetch
@@ -47,20 +50,50 @@ export class Upstream {
     const hit = this.#cache.get(key)
     if (hit !== undefined && hit.until > this.#now()) return hit.response
 
-    const query = new URLSearchParams(sorted)
+    const response = await this.#exec(server, path, new URLSearchParams(sorted), token, { method: 'GET' }, this.#config.timeoutMs)
+    if (this.#config.cacheSeconds > 0) {
+      if (this.#cache.size >= MAX_CACHED) this.#prune()
+      this.#cache.set(key, { until: this.#now() + this.#config.cacheSeconds * 1000, response })
+    }
+    return response
+  }
+
+  /*
+  A write: POST with the server's ADMIN token and never the read one, never cached. Whatever
+  the outcome, what the hub remembers of this server is dropped afterwards, so the next read
+  shows the state the write left. `body` is a multipart upload (the app zip).
+  */
+  async write(server: HubServer, path: string, params: Params = {}, body?: FormData): Promise<unknown> {
+    if (server.adminToken === undefined) throw new UpstreamError('rejected', 'no admin token configured')
+    try {
+      const init: RequestInit = body === undefined ? { method: 'POST' } : { method: 'POST', body }
+      return await this.#exec(server, path, new URLSearchParams(params), server.adminToken, init, Math.max(this.#config.timeoutMs, WRITE_TIMEOUT_MS))
+    } finally {
+      this.invalidate(server.id)
+    }
+  }
+
+  /** Forget every cached answer of one server. */
+  invalidate(serverId: string): void {
+    const prefix = `${serverId}\n`
+    for (const key of this.#cache.keys()) if (key.startsWith(prefix)) this.#cache.delete(key)
+  }
+
+  async #exec(server: HubServer, path: string, query: URLSearchParams, token: string, init: RequestInit, timeoutMs: number): Promise<unknown> {
     query.set('token', token)
     const controller = new AbortController()
-    const timer = setTimeout(() => controller.abort(), this.#config.timeoutMs)
+    const timer = setTimeout(() => controller.abort(), timeoutMs)
 
     let envelope: { status?: string; response?: unknown; errorMessage?: string }
     try {
       const res = await this.#fetch(`${server.url}/api/${path}?${query.toString()}`, {
+        ...init,
         signal: controller.signal,
         headers: { accept: 'application/json' },
       })
       envelope = (await res.json()) as typeof envelope
     } catch (e) {
-      throw new UpstreamError('unreachable', this.#describe(e, [server.token, token]))
+      throw new UpstreamError('unreachable', this.#describe(e, [server.token, token], timeoutMs))
     } finally {
       clearTimeout(timer)
     }
@@ -72,11 +105,6 @@ export class Upstream {
           : (envelope?.errorMessage ?? `status ${String(envelope?.status)}`)
       throw new UpstreamError('rejected', why)
     }
-
-    if (this.#config.cacheSeconds > 0) {
-      if (this.#cache.size >= MAX_CACHED) this.#prune()
-      this.#cache.set(key, { until: this.#now() + this.#config.cacheSeconds * 1000, response: envelope.response })
-    }
     return envelope.response
   }
 
@@ -88,8 +116,8 @@ export class Upstream {
   }
 
   /* A short reason without anything that could carry the token. */
-  #describe(e: unknown, secrets: string[]): string {
-    if (e instanceof Error && e.name === 'AbortError') return `timed out after ${this.#config.timeoutMs} ms`
+  #describe(e: unknown, secrets: string[], timeoutMs: number): string {
+    if (e instanceof Error && e.name === 'AbortError') return `timed out after ${timeoutMs} ms`
     const cause = e instanceof Error ? (e.cause as { code?: unknown } | undefined) : undefined
     const text = typeof cause?.code === 'string' ? cause.code : e instanceof Error ? e.message : 'request failed'
     return secrets.reduce((out, secret) => (secret === '' ? out : out.split(secret).join('***')), text)