Hub stage 2: install, update and uninstall apps on several servers

Writes use only <ID>_ADMIN_TOKEN and need the apps.write capability. Store URLs are resolved from each server's own listing. The config is saved to history before an uninstall. Zip uploads are fanned out to the servers. Every target's result is audited. The console gets a row menu on the matrix, a target picker, a results dialog with Retry failed, the store and install from file in All servers mode, and single-server actions for writers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-31 16:11:40 +07:00
1 parent 3d8df08fa0
commit 6acd2d656d
@@ -202,11 +202,18 @@ const appConfigs = {
  'Query Logs (Sqlite)': { enableLogging: true, maxLogDays: 365, sqliteDbPath: VARIANT === 'b' ? '/var/lib/dns/ndns/querylogs.db' : '/var/lib/dns/mdns/querylogs.db' },
  'Split Horizon': { domainGroupMap: { 'lan.example.net': 'internal' }, networkGroupMap: { '10.0.0.0/8': 'internal' } },
 }
-const store = [
+const catalogRows = [
  ['Failover', 'Fails over to a backup address when the primary is down', '8.0', '58 KB'], ['Geo Country', 'Answers by the country of the client', '7.2', '1.2 MB'],
  ['Wild IP', 'Answers with the IP address in the name', '5.1', '22 KB'], ['Auto PTR', 'Answers PTR queries from the forward zones', '6.0', '31 KB'],
  ['Drop Requests', 'Drops requests from the listed networks', '7.0', '19 KB'], ['NX Domain', 'Answers NXDOMAIN for the listed domains', '6.1', '17 KB'],
-].map(([name, description, version, size]) => ({ name, description, version, size, url: `https://download.technitium.com/dns/apps/${name.replace(/ /g, '')}App-v${version}.zip`, installed: false }))
+  ['Advanced Blocking', 'Group based blocking with regex', '10.2', '240 KB'], ['Query Logs (Sqlite)', 'Logs DNS queries into a SQLite database', '7.2', '310 KB'],
+  ['Split Horizon', 'Different answers for different networks', '9.0', '120 KB'],
+].map(([name, description, version, size]) => ({ name, description, version, size, url: `https://download.technitium.com/dns/apps/${name.replace(/[ ()]/g, '')}App-v${version}.zip` }))
+const catalog = () => catalogRows
+const storeListing = () => catalogRows.map((c) => {
+  const have = apps.find((a) => a.name === c.name)
+  return have ? { ...c, installed: true, installedVersion: have.version, updateAvailable: have.version !== c.version } : { ...c, installed: false }
+})

 // ── Query logs ─────────────────────────────────────────────────────────────
 const RTYPES = ['Recursive', 'Cached', 'Authoritative', 'Blocked', 'Cached', 'Recursive']
@@ -279,7 +286,27 @@ function route(path, q) {
    case 'dhcp/scopes/enable': case 'dhcp/scopes/disable': scopes = scopes.map((s) => (s.name === q.get('name') ? { ...s, enabled: path === 'dhcp/scopes/enable' } : s)); return ok()
    case 'dhcp/scopes/delete': scopes = scopes.filter((s) => s.name !== q.get('name')); return ok()
    case 'apps/list': return ok({ apps })
-    case 'apps/listStoreApps': return ok({ storeApps: store })
+    case 'apps/listStoreApps': return ok({ storeApps: storeListing() })
+    case 'apps/downloadAndInstall': case 'apps/downloadAndUpdate': {
+      const entry = catalog().find((c) => c.name === q.get('name') && c.url === q.get('url'))
+      if (!entry) return { status: 'error', errorMessage: 'The app was not found in the store.' }
+      const update = path === 'apps/downloadAndUpdate'
+      const have = apps.find((a) => a.name === entry.name)
+      if (update !== Boolean(have)) return { status: 'error', errorMessage: update ? `DNS application '${entry.name}' was not found.` : `DNS application '${entry.name}' already exists.` }
+      const installed = { ...(have ?? { name: entry.name, description: entry.description, dnsApps: [app(false)] }), version: entry.version }
+      delete installed.updateAvailable; delete installed.updateVersion; delete installed.updateUrl
+      apps = update ? apps.map((a) => (a.name === entry.name ? installed : a)) : [...apps, installed]
+      return ok(update ? { updatedApp: installed } : { installedApp: installed })
+    }
+    case 'apps/install': case 'apps/update': {
+      const name = q.get('name') ?? ''
+      const have = apps.find((a) => a.name === name)
+      const update = path === 'apps/update'
+      if (update !== Boolean(have)) return { status: 'error', errorMessage: update ? `DNS application '${name}' was not found.` : `DNS application '${name}' already exists.` }
+      const installed = { ...(have ?? { name, description: 'Installed from a zip file', dnsApps: [app(false)] }), version: update ? `${have.version}.1` : '1.0' }
+      apps = update ? apps.map((a) => (a.name === name ? installed : a)) : [...apps, installed]
+      return ok(update ? { updatedApp: installed } : { installedApp: installed })
+    }
    case 'apps/config/get': return ok({ config: appConfigs[q.get('name')] ? JSON.stringify(appConfigs[q.get('name')], null, 2) : JSON.stringify({ enabled: true }, null, 2) })
    case 'apps/uninstall': apps = apps.filter((a) => a.name !== q.get('name')); return ok()
    case 'logs/list': return ok({ logFiles: [{ fileName: '2026-10-26', size: '412.8 KB' }, { fileName: '2026-10-25', size: '1.2 MB' }, { fileName: '2026-10-24', size: '980.4 KB' }] })
@@ -11,6 +11,8 @@ version: "3.8"
 # Токены кладутся в .env рядом с этим файлом (не в git):
 #   MDNS_TOKEN=...
 #   NDNS_TOKEN=...
+#   MDNS_ADMIN_TOKEN=...   # токены hub-admin (Modify и Delete только на Apps): нужны, чтобы хаб менял приложения
+#   NDNS_ADMIN_TOKEN=...
 #   HUB_USER=...        # логин в сам хаб
 #   HUB_PASSWORD=...    # первый вход; затем пользователи в /data/users.json (docs/hub-deploy.md, шаг 4a)
 #
@@ -30,6 +32,8 @@ services:
      - HUB_SERVERS=mdns=https://mdns.shnt.top,ndns=https://ndns.shnt.top
      - MDNS_TOKEN=${MDNS_TOKEN}
      - NDNS_TOKEN=${NDNS_TOKEN}
+      - MDNS_ADMIN_TOKEN=${MDNS_ADMIN_TOKEN}
+      - NDNS_ADMIN_TOKEN=${NDNS_ADMIN_TOKEN}
      # Только пока нет /data/users.json; потом пользователи создаются через users-cli (docs/hub-deploy.md)
      - HUB_USER=${HUB_USER}
      - HUB_PASSWORD=${HUB_PASSWORD}
@@ -189,6 +189,9 @@ services:
      - HUB_SERVERS=mdns=https://mdns.shnt.top,ndns=https://ndns.shnt.top
      - MDNS_TOKEN=${MDNS_TOKEN}
      - NDNS_TOKEN=${NDNS_TOKEN}
+      # Токены hub-admin: без них хаб только читает. Нужны для установки и обновления приложений
+      - MDNS_ADMIN_TOKEN=${MDNS_ADMIN_TOKEN}
+      - NDNS_ADMIN_TOKEN=${NDNS_ADMIN_TOKEN}
      # Только пока нет /data/users.json; потом пользователи создаются через users-cli (docs/hub-deploy.md)
      - HUB_USER=${HUB_USER}
      - HUB_PASSWORD=${HUB_PASSWORD}
@@ -4,7 +4,7 @@

 - запрос «All servers» хаб отправляет на оба сервера и складывает ответы;
 - запрос к одному серверу хаб пересылает только ему;
- запись хаб пока не выполняет: любой запрос на изменение получает отказ и попадает в журнал.
+- менять хаб умеет только приложения (установка, обновление, удаление), и только для пользователей с ролью `apps-admin` или `admin`. Остальные запросы на изменение получают отказ и попадают в журнал.

 Серверы Technitium не меняются, кластер не нужен. В инструкции хаб ставится на хост, где уже работают Traefik и `technitium-smart` (`docs/compose.shnt.example.yaml`). Адрес хаба: `https://dnss.shnt.top`.

@@ -72,6 +72,19 @@ git tag hub-v1.0.0 && git push origin hub-v1.0.0

 Результат шага: два токена, `MDNS_TOKEN` (с `mdns.shnt.top`) и `NDNS_TOKEN` (с `ndns.shnt.top`).

+### Шаг 1а. Пользователь для записи (нужен, чтобы менять приложения)
+
+Без этого шага хаб только читает. Токены записи отдельные, у отдельного пользователя. Сделайте это **на обоих серверах**:
+
+1. **Группа** `hub-admins`. Права группы (Administration › Permissions):
+   - **Apps**: View, Modify и Delete;
+   - **Dashboard**, **Logs**, **Settings**: только View, и то лишь если хотите, чтобы этот токен мог читать то же, что `hub-reader`. Для записи приложений достаточно Apps.
+   - остальные разделы без прав.
+2. **Пользователь** `hub-admin` в этой группе, длинный случайный пароль.
+3. **API-токен** для `hub-admin` (так же, как в шаге 1.4). Это `MDNS_ADMIN_TOKEN` и `NDNS_ADMIN_TOKEN`.
+
+Хаб использует эти токены только для записи приложений и для чтения конфига приложения перед удалением. Если для сервера токена нет, хаб показывает его «только для чтения» и не вызывает для записи.
+
 ---

 ## Шаг 2. DNS-запись для dnss.shnt.top
@@ -91,6 +104,10 @@ git tag hub-v1.0.0 && git push origin hub-v1.0.0
 MDNS_TOKEN=вставьте_токен_mdns
 NDNS_TOKEN=вставьте_токен_ndns

+# Токены hub-admin с шага 1а (без них хаб не меняет приложения)
+MDNS_ADMIN_TOKEN=вставьте_токен_записи_mdns
+NDNS_ADMIN_TOKEN=вставьте_токен_записи_ndns
+
 # Первый вход в хаб (это НЕ логин Technitium). Нужен, пока не создан файл пользователей (шаг 4a)
 HUB_USER=admin
 HUB_PASSWORD=длинный_случайный_пароль
@@ -124,6 +141,8 @@ chmod 600 .env
      - HUB_SERVERS=mdns=https://mdns.shnt.top,ndns=https://ndns.shnt.top
      - MDNS_TOKEN=${MDNS_TOKEN}
      - NDNS_TOKEN=${NDNS_TOKEN}
+      - MDNS_ADMIN_TOKEN=${MDNS_ADMIN_TOKEN}
+      - NDNS_ADMIN_TOKEN=${NDNS_ADMIN_TOKEN}
      - HUB_USER=${HUB_USER}
      - HUB_PASSWORD=${HUB_PASSWORD}
      - HUB_DATA=/data
@@ -168,7 +187,7 @@ volumes:
 |---|---|
 | `viewer` | смотреть |
 | `operator` | смотреть, блокировка и кеш (после появления записи) |
-| `apps-admin` | смотреть, управлять приложениями (после появления записи) |
+| `apps-admin` | смотреть, устанавливать, обновлять и удалять приложения на выбранных серверах |
 | `admin` | всё, включая журнал действий |

 Пока файла нет, в хаб входит один `admin` из `HUB_USER` / `HUB_PASSWORD`. Когда файл появился, эти две переменные игнорируются (в логе будет строка об этом). Если нет ни файла, ни переменных, хаб не запустится.
@@ -258,7 +277,7 @@ docker compose rm -f technitium-hub     # удалить контейнер
 | `HUB_USER`, `HUB_PASSWORD` | нужны, пока нет `users.json` | Первый `admin`; при наличии файла игнорируются |
 | `HUB_DATA` | `/data` | Папка `users.json` и `audit.log` (на ней том) |
 | `HUB_TRUST_PROXY` | выключено | `1`: адрес клиента из `X-Forwarded-For` (только за Traefik) |
-| `<ID>_ADMIN_TOKEN` | нет | Токен записи сервера (`MDNS_ADMIN_TOKEN`). Пока не используется, в логи не пишется |
+| `<ID>_ADMIN_TOKEN` | нет | Токен записи сервера (`MDNS_ADMIN_TOKEN`). Им хаб меняет приложения; без него сервер только для чтения. В логи не пишется |
 | `HUB_DOMAIN` | `hub` | Имя, которое консоль показывает как имя сервера |
 | `HUB_CACHE_SECONDS` | `15` | Сколько секунд хранить ответы серверов (0 выключает кеш) |
 | `HUB_TIMEOUT_MS` | `8000` | Сколько ждать ответа сервера, мс |
@@ -288,10 +307,28 @@ docker compose rm -f technitium-hub     # удалить контейнер
 | В подсказке `ndns: ...` с сообщением об отказе в доступе | У группы `hub-readers` на этом сервере не хватает View в каком-то разделе (шаг 1.2). |
 | Query Logs пустой или «нет приложения» | На сервере не установлено приложение логирования (Query Logs (Sqlite) и подобные) или нет права View на Apps. |
 | Цифры «All servers» больше суммы на 1-2 клиента | Clients считается по каждому серверу отдельно: клиент, который ходит на оба сервера, учтён дважды. Это ожидаемо. |
-| Нажатие на что-то пишет «The hub is read-only.» | Хаб только читает. Изменения делаются в консоли нужного сервера. |
+| Нажатие на что-то пишет «The hub is read-only.» | Хаб меняет только приложения. Остальные изменения делаются в консоли нужного сервера. |
+
+## Приложения: установка, обновление, удаление
+
+Роль `apps-admin` (или `admin`) видит в разделе Apps при «All servers» меню действий в каждой строке и кнопки **App Store** и **Install from file**:
+
+- **Update on all**: обновить с сервера, где есть обновление;
+- **Install on missing servers**: поставить из магазина там, где приложения нет;
+- **Align versions**: подтянуть отстающие серверы до самой новой установленной версии (если магазин её предлагает);
+- **Uninstall from…**: удалить с выбранных серверов.
+
+Каждое действие сначала открывает выбор серверов (сервер без токена записи недоступен, с причиной), потом показывает итог по каждому серверу. Если на каком-то сервере не вышло, кнопка **Retry failed** повторит только его.
+
+Что хаб делает для безопасности:
+- ссылку на скачивание берёт из магазина самого сервера, произвольный адрес не принимает;
+- перед удалением сохраняет конфиг приложения в `/data/apps/<имя>/history/` (50 последних на приложение) и не удаляет, если сохранить не удалось. Откат: вставить сохранённый текст в Config приложения на сервере;
+- каждое действие по каждому серверу пишется в журнал (`apps.update-store`, `apps.uninstall` и т. д.).
+
+Права на запись у `viewer` и `operator` нет: они получают «Your role does not allow this.».

 ## Безопасность

- Хаб только читает: любая запись получает отказ ещё в хабе (и запись в журнале), а токены выданы пользователю с одними правами View.
+- Читает хаб токенами с одними правами View. Менять он умеет только приложения, и только токенами `*_ADMIN_TOKEN` (права Apps) по запросу пользователя с ролью `apps-admin` или `admin`. Всё остальное получает отказ в хабе и запись в журнале.
 - Токены лежат только в `.env` на хосте и в памяти контейнера. В браузер они не попадают, в ответы и логи хаба не пишутся.
 - У каждого человека свой логин и роль. Вход защищён блокировкой после 5 неудач. Нужны длинные пароли, а если хаб не должен быть виден из интернета, добавьте к роутеру `hub` в Traefik `ipallowlist` или `basicAuth`.
@@ -8,7 +8,7 @@ read-only, Technitium-compatible `/api/*`:
 - No `node` (or `node=cluster`) asks every server in parallel and merges the answers
  (statistics are summed, top lists and query logs are merged). If a server does not
  answer, the others are shown and the response carries `hubWarnings`.
- Anything that would change data is refused with `403` (and written to the audit log). The hub has its own accounts with roles, a login rate limit and an audit log.
+- Reads go with each server's read token. The only writes are the app changes below, which use a separate admin token and a role with `apps.write`. Anything else that would change data is refused with `403` (and written to the audit log). The hub has its own accounts with roles, a login rate limit and an audit log.

 The servers themselves are not changed: each only needs a read-only API token. Create it
 for a user whose group has View, and nothing else, on Dashboard, Logs, Apps (the logging
@@ -23,7 +23,7 @@ apps Query Logs reads from) and Settings (the blocking state).
 | `HUB_USER`, `HUB_PASSWORD` | see Users | Bootstrap: one `admin`, used only while `${HUB_DATA}/users.json` does not exist. Set both or neither. |
 | `HUB_DATA` | `/data` | Folder of `users.json` and `audit.log`. Mount a volume on it. |
 | `HUB_TRUST_PROXY` | off | `1` believes the first `X-Forwarded-For` entry as the client address (set it behind Traefik). Leave it off when clients reach the hub directly, or anyone can fake their address. |
-| `<ID>_ADMIN_TOKEN` | optional | A write-capable token per server (`MDNS_ADMIN_TOKEN`). Read into the config for the coming write stages; not used yet and never logged. |
+| `<ID>_ADMIN_TOKEN` | optional | A write-capable token per server (`MDNS_ADMIN_TOKEN`). Only app writes use it, and only it: a server without one is read-only for the hub (the matrix and the session list the writable servers). Never logged or returned. |
 | `HUB_PORT` | `8080` | Listening port. |
 | `HUB_CACHE_SECONDS` | `15` | How long successful answers of the servers are cached. |
 | `HUB_TIMEOUT_MS` | `8000` | Per-request timeout towards a server. |
@@ -57,7 +57,7 @@ atomically with mode 0600. The running hub notices a change on the next request
 | `apps-admin` | read, apps.write |
 | `admin` | everything, including audit.read |

-Only reads exist today; the write capabilities are the machinery for the next stages. Every response also carries
+`apps.write` is what the app writes below need; `operate` and the rest are still unused. Every response also carries
 `hubRole` (and `info.hubRole`).

 ## Login protection
@@ -83,7 +83,27 @@ Public: `status`, `user/login`. After signing in: `user/session/get`, `user/logo
 `logs/query`, `logs/list`, `settings/get`, `admin/cluster/state` (synthesized), and the hub's own `hub/audit/list` (role with
 `audit.read`). They are listed in `routes.ts` with a kind (`read`, `write-one`, `write-all`) and the capability they need.

-Everything else answers `403 The hub is read-only.` and is audited.
+Everything else answers `403 The hub is read-only.` and is audited. `apps/config/get` and `apps/config/set` are not available yet.
+
+## App writes
+
+Needs the `apps.write` role (`apps-admin`, `admin`); others get `403 Your role does not allow this.` and a `denied-write` audit line.
+Every write goes to a server with its `<ID>_ADMIN_TOKEN`; a server without one answers "no admin token configured" and is not called.
+
+- `POST hub/apps/run` (form or JSON): `action` = `install-store` | `update-store` | `uninstall`, `name`, `targets` (list or
+  comma-separated server ids). Runs on every target in parallel and answers `{ results: [{ server, ok, message? }] }`.
+  The download URL is taken from each target's own `apps/listStoreApps`; a `url` field is only accepted when it matches.
+- `POST hub/apps/upload` (multipart, at most 20 MB, kept in memory only): `action` = `install` | `update`, `name`, `targets`
+  (comma-separated), file field `fileAppZip`. The same zip is forwarded to every target.
+- `apps/downloadAndInstall`, `apps/downloadAndUpdate`, `apps/uninstall`, `apps/install`, `apps/update` with `node=<id>`:
+  the console's own single-server calls, forwarded to that server (store URLs checked as above). Refused without a node, and
+  with `node=cluster`.
+- `apps/listStoreApps`: the first server's store, each app tagged with `installedOn`.
+
+Before an uninstall the app's config is saved to `${HUB_DATA}/apps/<app>/history/<time>__<server>__before-uninstall.txt`
+(50 files per app are kept). If that cannot be done the uninstall does not happen. Every target's result is written to the
+audit log as `apps.<action>`; the history files hold configs, so protect the data folder like the users file.
+After any write the hub forgets its cached answers of that server.

 ## Run locally

@@ -183,7 +183,7 @@ describe('hub/apps/matrix', () => {
    const { body } = await matrix()
    expect(body.status).toBe('ok')
    expect(body.hubWarnings).toEqual([])
-    expect(body.response.servers).toEqual([{ id: 'a', ok: true }, { id: 'b', ok: true }])
+    expect(body.response.servers).toEqual([{ id: 'a', ok: true, writable: false }, { id: 'b', ok: true, writable: false }])
    expect((body.response.apps as any[]).map((x) => x.name)).toEqual(['Differs', 'Empty', 'Only A', 'Plain', 'Same'])

    const same = byName(body, 'Same')
@@ -246,7 +246,7 @@ describe('hub/apps/matrix', () => {
    b.down = true
    const { body } = await matrix()
    expect(body.status).toBe('ok')
-    expect(body.response.servers[0]).toEqual({ id: 'a', ok: true })
+    expect(body.response.servers[0]).toEqual({ id: 'a', ok: true, writable: false })
    expect(body.response.servers[1]).toMatchObject({ id: 'b', ok: false })
    expect(body.response.servers[1].error).toBeTruthy()
    expect(body.hubWarnings).toHaveLength(1)
@@ -272,9 +272,9 @@ describe('hub/apps/matrix', () => {
 })

 describe('session permissions', () => {
-  it('lets every role view Apps without modifying it', async () => {
+  it('lets the admin view and modify Apps (the other roles are in apps-write.test.ts)', async () => {
    const res = await fetch(`${base}/api/user/session/get`, { headers: { authorization: `Bearer ${token}` } })
    const info = ((await res.json()) as any).info
-    expect(info.permissions.Apps).toEqual({ canView: true, canModify: false, canDelete: false })
+    expect(info.permissions.Apps).toEqual({ canView: true, canModify: true, canDelete: true })
  })
 })
@@ -29,7 +29,7 @@ export interface MatrixApp {
 }

 export interface Matrix {
-  servers: { id: string; ok: boolean; error?: string }[]
+  servers: { id: string; ok: boolean; writable: boolean; error?: string }[]
  apps: MatrixApp[]
 }

@@ -128,7 +128,9 @@ export async function buildMatrix(config: HubConfig, upstream: Upstream): Promis
    }),
  )

-  const servers = perServer.map((p) => (p.error === undefined ? { id: p.server.id, ok: true } : { id: p.server.id, ok: false, error: p.error }))
+  const servers = perServer.map((p) => (p.error === undefined
+      ? { id: p.server.id, ok: true, writable: p.server.adminToken !== undefined }
+      : { id: p.server.id, ok: false, writable: p.server.adminToken !== undefined, error: p.error }))
  const warnings = perServer.filter((p) => p.error !== undefined).map((p) => `${p.server.id}: ${p.error}`)
  if (warnings.length === perServer.length) throw new Error(`No server answered. ${warnings.join('; ')}`.trim())

@@ -0,0 +1,513 @@
+// @vitest-environment node
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { parseConfig } from './config.ts'
+import { createHub } from './server.ts'
+import { boundaryOf, parseMultipart } from './multipart.ts'
+import { safeName } from './apps-write.ts'
+import { addUser, writeUsersFile } from './users.ts'
+
+interface Call {
+  path: string
+  token: string
+  method: string
+  query: URLSearchParams
+  body: Buffer
+  contentType: string
+}
+
+interface Fake {
+  server: http.Server
+  url: string
+  apps: Record<string, { version: string; config: string | null }>
+  store: { name: string; version: string; url: string; description: string; size: string }[]
+  calls: Call[]
+  failWrites: boolean
+  denyConfig: boolean
+}
+
+function fake(): Promise<Fake> {
+  return new Promise((resolve) => {
+    const f: Fake = { server: http.createServer(), url: '', apps: {}, store: [], calls: [], failWrites: false, denyConfig: false }
+    f.server.on('request', (req, res) => {
+      const chunks: Buffer[] = []
+      req.on('data', (c: Buffer) => chunks.push(c))
+      req.on('end', () => {
+        const url = new URL(req.url ?? '/', 'http://fake')
+        const apiPath = url.pathname.replace(/^\/api\//, '')
+        const token = url.searchParams.get('token') ?? ''
+        const name = url.searchParams.get('name') ?? ''
+        f.calls.push({ path: apiPath, token, method: req.method ?? '', query: url.searchParams, body: Buffer.concat(chunks), contentType: String(req.headers['content-type'] ?? '') })
+        res.setHeader('content-type', 'application/json')
+        const ok = (response: unknown = {}) => res.end(JSON.stringify({ status: 'ok', response }))
+        const error = (m: string) => res.end(JSON.stringify({ status: 'error', errorMessage: m }))
+        const isWrite = ['apps/downloadAndInstall', 'apps/downloadAndUpdate', 'apps/uninstall', 'apps/install', 'apps/update'].includes(apiPath)
+        if (isWrite && !token.startsWith('admin-')) return error("Access was denied to user 'reader' for apps.")
+        if (isWrite && f.failWrites) return error('Disk full.')
+        switch (apiPath) {
+          case 'apps/list':
+            return ok({ apps: Object.entries(f.apps).map(([n, a]) => ({ name: n, description: `${n} d`, version: a.version })) })
+          case 'apps/listStoreApps':
+            return ok({ storeApps: f.store })
+          case 'apps/config/get':
+            if (f.denyConfig) return error('Access was denied.')
+            return f.apps[name] ? ok({ config: f.apps[name].config }) : error('not found')
+          case 'apps/downloadAndInstall':
+          case 'apps/downloadAndUpdate': {
+            const entry = f.store.find((s) => s.name === name && s.url === url.searchParams.get('url'))
+            if (!entry) return error('not in store')
+            f.apps[name] = { version: entry.version, config: f.apps[name]?.config ?? null }
+            return ok()
+          }
+          case 'apps/uninstall':
+            delete f.apps[name]
+            return ok()
+          case 'apps/install':
+          case 'apps/update':
+            f.apps[name] = { version: 'zip', config: null }
+            return ok()
+          default:
+            return ok()
+        }
+      })
+    })
+    f.server.listen(0, '127.0.0.1', () => {
+      f.url = `http://127.0.0.1:${(f.server.address() as AddressInfo).port}`
+      resolve(f)
+    })
+  })
+}
+
+let a: Fake
+let b: Fake
+let dataDir: string
+let staticDir: string
+let hub: http.Server | undefined
+let base: string
+const tokens: Record<string, string> = {}
+
+const STORE = (v: string) => [{ name: 'Blocker', version: v, url: `https://store.example/Blocker-v${v}.zip`, description: 'd', size: '1 KB' }]
+
+function reset(): void {
+  for (const f of [a, b]) {
+    f.calls = []
+    f.failWrites = false
+    f.denyConfig = false
+    f.store = STORE('2.0')
+    f.apps = { Blocker: { version: '1.0', config: '{"secret":"cfg"}' } }
+  }
+}
+
+async function start(extra: Record<string, string> = {}): Promise<void> {
+  hub?.close()
+  hub = createHub(
+    parseConfig({
+      HUB_SERVERS: `a=${a.url},b=${b.url}`,
+      A_TOKEN: 'read-a',
+      B_TOKEN: 'read-b',
+      A_ADMIN_TOKEN: 'admin-a',
+      B_ADMIN_TOKEN: 'admin-b',
+      HUB_DATA: dataDir,
+      HUB_STATIC: staticDir,
+      HUB_CACHE_SECONDS: '0',
+      HUB_TIMEOUT_MS: '2000',
+      ...extra,
+    }),
+  )
+  await new Promise<void>((resolve) => hub!.listen(0, '127.0.0.1', resolve))
+  base = `http://127.0.0.1:${(hub.address() as AddressInfo).port}`
+  for (const role of ['viewer', 'operator', 'apps-admin', 'admin']) {
+    const res = await fetch(`${base}/api/user/login`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: `user=${role}&pass=password-${role}`,
+    })
+    tokens[role] = ((await res.json()) as { token: string }).token
+  }
+}
+
+async function api(role: string, apiPath: string, init: { params?: Record<string, string>; method?: string; json?: unknown; body?: string | FormData; headers?: Record<string, string> } = {}) {
+  const query = new URLSearchParams(init.params ?? {})
+  const headers: Record<string, string> = { authorization: `Bearer ${tokens[role]}`, ...(init.headers ?? {}) }
+  let body = init.body
+  if (init.json !== undefined) {
+    headers['content-type'] = 'application/json'
+    body = JSON.stringify(init.json)
+  }
+  const res = await fetch(`${base}/api/${apiPath}?${query}`, { method: init.method ?? (body === undefined ? 'GET' : 'POST'), headers, body })
+  return { code: res.status, body: (await res.json()) as Record<string, any> }
+}
+
+const run = (role: string, json: Record<string, unknown>) => api(role, 'hub/apps/run', { json })
+
+function zipForm(extra: Record<string, string>, bytes: Uint8Array): FormData {
+  const form = new FormData()
+  for (const [k, v] of Object.entries(extra)) form.append(k, v)
+  form.append('fileAppZip', new Blob([bytes], { type: 'application/zip' }), 'Blocker.zip')
+  return form
+}
+
+const writes = (f: Fake) => f.calls.filter((c) => c.path.startsWith('apps/') && c.method === 'POST')
+
+beforeAll(async () => {
+  a = await fake()
+  b = await fake()
+  dataDir = mkdtempSync(path.join(tmpdir(), 'hub-write-data-'))
+  staticDir = mkdtempSync(path.join(tmpdir(), 'hub-write-static-'))
+  let users = addUser([], 'viewer', 'viewer', 'password-viewer')
+  users = addUser(users, 'operator', 'operator', 'password-operator')
+  users = addUser(users, 'apps-admin', 'apps-admin', 'password-apps-admin')
+  users = addUser(users, 'admin', 'admin', 'password-admin')
+  writeUsersFile(dataDir, users)
+})
+
+afterAll(() => {
+  hub?.close()
+  a.server.close()
+  b.server.close()
+  rmSync(dataDir, { recursive: true, force: true })
+  rmSync(staticDir, { recursive: true, force: true })
+})
+
+beforeEach(async () => {
+  reset()
+  rmSync(path.join(dataDir, 'apps'), { recursive: true, force: true })
+  rmSync(path.join(dataDir, 'audit.log'), { force: true })
+  await start()
+})
+
+afterEach(() => {
+  hub?.close()
+})
+
+describe('multipart parser', () => {
+  const boundary = '----testBoundary123'
+  const type = `multipart/form-data; boundary=${boundary}`
+
+  it('reads fields and a binary file, whatever bytes it holds', () => {
+    const binary = Buffer.alloc(256)
+    for (let i = 0; i < 256; i++) binary[i] = i
+    const tricky = Buffer.concat([binary, Buffer.from(`\r\n--${boundary.slice(0, 5)}\r\n\r\n--`), binary])
+    const body = Buffer.concat([
+      Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="name"\r\n\r\nBlocker\r\n`),
+      Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="fileAppZip"; filename="a b.zip"\r\nContent-Type: application/zip\r\n\r\n`),
+      tricky,
+      Buffer.from(`\r\n--${boundary}--\r\n`),
+    ])
+    const parsed = parseMultipart(body, type)
+    expect(parsed.fields).toEqual({ name: 'Blocker' })
+    expect(parsed.files).toHaveLength(1)
+    expect(parsed.files[0]).toMatchObject({ field: 'fileAppZip', filename: 'a b.zip', type: 'application/zip' })
+    expect(parsed.files[0].data.equals(tricky)).toBe(true)
+  })
+
+  it('reads a quoted boundary and an empty file', () => {
+    expect(boundaryOf('multipart/form-data; boundary="x y"')).toBe('x y')
+    const body = Buffer.from(`--q\r\nContent-Disposition: form-data; name="f"; filename="e.zip"\r\n\r\n\r\n--q--`)
+    expect(parseMultipart(body, 'multipart/form-data; boundary=q').files[0].data.length).toBe(0)
+  })
+
+  it('refuses what is not multipart', () => {
+    expect(() => parseMultipart(Buffer.from('x'), 'text/plain')).toThrow()
+    expect(() => parseMultipart(Buffer.from('nothing here'), type)).toThrow(/boundary/)
+    expect(() => parseMultipart(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="a"\r\n\r\nno end`), type)).toThrow(/terminated/)
+  })
+
+  it('names folders safely', () => {
+    expect(safeName('Blocker')).toBe('Blocker')
+    expect(safeName('Query Logs (Sqlite)')).toMatch(/^Query_Logs__Sqlite_-[0-9a-f]{6}$/)
+    expect(safeName('../../etc')).not.toContain('/')
+    expect(safeName('..')).not.toMatch(/^\./)
+  })
+})
+
+describe('roles', () => {
+  it('refuses viewer and operator on every write and audits it', async () => {
+    for (const role of ['viewer', 'operator']) {
+      const r = await run(role, { action: 'uninstall', name: 'Blocker', targets: ['a'] })
+      expect(r.code).toBe(403)
+      expect(r.body).toEqual({ status: 'error', errorMessage: 'Your role does not allow this.' })
+      expect((await api(role, 'apps/uninstall', { params: { name: 'Blocker', node: 'a' } })).code).toBe(403)
+      expect((await api(role, 'hub/apps/upload', { body: zipForm({ action: 'install', name: 'X', targets: 'a' }, new Uint8Array([1])) })).code).toBe(403)
+    }
+    expect(writes(a)).toHaveLength(0)
+    const entries = (await api('admin', 'hub/audit/list', { params: { limit: '100' } })).body.response.entries as Record<string, string>[]
+    expect(entries.filter((e) => e.action === 'denied-write' && e.user === 'viewer').length).toBeGreaterThanOrEqual(3)
+    expect(entries.some((e) => e.action === 'denied-write' && e.user === 'operator' && e.target === 'hub/apps/run')).toBe(true)
+  })
+
+  it('lets apps-admin and admin write', async () => {
+    for (const role of ['apps-admin', 'admin']) {
+      b.apps = a.apps = { Blocker: { version: '1.0', config: null } }
+      const r = await run(role, { action: 'update-store', name: 'Blocker', targets: ['a'] })
+      expect(r.body.status).toBe('ok')
+      expect(r.body.response.results).toEqual([{ server: 'a', ok: true, message: 'Updated from the store.' }])
+    }
+  })
+
+  it('does not let a write endpoint be called with GET', async () => {
+    expect((await api('admin', 'hub/apps/run', { method: 'GET', params: { action: 'uninstall', name: 'Blocker', targets: 'a' } })).code).toBe(405)
+    expect(writes(a)).toHaveLength(0)
+  })
+
+  it('tells the session who may write what', async () => {
+    const viewer = (await api('viewer', 'user/session/get')).body.info
+    const admin = (await api('apps-admin', 'user/session/get')).body.info
+    expect(viewer.permissions.Apps).toEqual({ canView: true, canModify: false, canDelete: false })
+    expect(admin.permissions.Apps).toEqual({ canView: true, canModify: true, canDelete: true })
+    expect(admin.permissions.Zones.canModify).toBe(false)
+    expect(admin.hubWritable).toEqual(['a', 'b'])
+    await start({ B_ADMIN_TOKEN: '' })
+    expect((await api('viewer', 'user/session/get')).body.info.hubWritable).toEqual(['a'])
+  })
+})
+
+describe('hub/apps/run', () => {
+  it('uses only the admin token and the URL of the target own store', async () => {
+    a.store = STORE('2.0')
+    b.store = [{ ...STORE('2.5')[0], url: 'https://other.example/b.zip' }]
+    const r = await run('admin', { action: 'update-store', name: 'Blocker', targets: ['a', 'b'] })
+    expect(r.body.response.results.map((x: any) => x.ok)).toEqual([true, true])
+    const [wa] = writes(a)
+    const [wb] = writes(b)
+    expect(wa.token).toBe('admin-a')
+    expect(wa.query.get('url')).toBe('https://store.example/Blocker-v2.0.zip')
+    expect(wb.token).toBe('admin-b')
+    expect(wb.query.get('url')).toBe('https://other.example/b.zip')
+    expect(a.calls.filter((c) => c.token === 'read-a').every((c) => !c.path.includes('downloadAnd'))).toBe(true)
+  })
+
+  it('refuses a url that is not from the store listing, and an app the store does not have', async () => {
+    const r = await run('admin', { action: 'install-store', name: 'Blocker', url: 'https://evil.example/x.zip', targets: ['a'] })
+    expect(r.code).toBe(400)
+    expect(r.body.errorMessage).toMatch(/store listing/)
+    expect(writes(a)).toHaveLength(0)
+
+    const same = await run('admin', { action: 'update-store', name: 'Blocker', url: STORE('2.0')[0].url, targets: ['a'] })
+    expect(same.body.status).toBe('ok')
+
+    const missing = await run('admin', { action: 'install-store', name: 'Nope', targets: ['a'] })
+    expect(missing.body.response.results[0]).toMatchObject({ server: 'a', ok: false })
+    expect(missing.body.response.results[0].message).toMatch(/not in this server/)
+  })
+
+  it('refuses unknown servers, no targets and bad actions', async () => {
+    expect((await run('admin', { action: 'update-store', name: 'Blocker', targets: ['zzz'] })).code).toBe(400)
+    expect((await run('admin', { action: 'update-store', name: 'Blocker', targets: [] })).code).toBe(400)
+    expect((await run('admin', { action: 'format', name: 'Blocker', targets: ['a'] })).code).toBe(400)
+    expect((await run('admin', { action: 'uninstall', name: '', targets: ['a'] })).code).toBe(400)
+  })
+
+  it('accepts a form body with comma separated targets', async () => {
+    const r = await api('admin', 'hub/apps/run', {
+      body: new URLSearchParams({ action: 'update-store', name: 'Blocker', targets: 'a,b' }).toString(),
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+    })
+    expect(r.body.response.results).toHaveLength(2)
+  })
+
+  it('refuses a server without an admin token, without calling it', async () => {
+    await start({ B_ADMIN_TOKEN: '' })
+    const r = await run('admin', { action: 'update-store', name: 'Blocker', targets: ['a', 'b'] })
+    const results = r.body.response.results
+    expect(results.find((x: any) => x.server === 'a').ok).toBe(true)
+    expect(results.find((x: any) => x.server === 'b')).toEqual({ server: 'b', ok: false, message: 'no admin token configured' })
+    expect(writes(b)).toHaveLength(0)
+    expect(b.calls.every((c) => c.token === 'read-b')).toBe(true)
+    const m = await api('admin', 'hub/apps/matrix')
+    expect(m.body.response.servers).toEqual([{ id: 'a', ok: true, writable: true }, { id: 'b', ok: true, writable: false }])
+  })
+
+  it('reports each target on its own when one fails, and audits every result', async () => {
+    b.failWrites = true
+    const r = await run('apps-admin', { action: 'update-store', name: 'Blocker', targets: ['a', 'b'] })
+    expect(r.body.response.results).toEqual([
+      { server: 'a', ok: true, message: 'Updated from the store.' },
+      { server: 'b', ok: false, message: 'Disk full.' },
+    ])
+    expect(a.apps.Blocker.version).toBe('2.0')
+    expect(b.apps.Blocker.version).toBe('1.0')
+    const entries = (await api('admin', 'hub/audit/list', { params: { limit: '20' } })).body.response.entries as Record<string, string>[]
+    const mine = entries.filter((e) => e.action === 'apps.update-store')
+    expect(mine).toHaveLength(2)
+    expect(mine.find((e) => e.server === 'a')).toMatchObject({ user: 'apps-admin', role: 'apps-admin', target: 'Blocker', outcome: 'ok' })
+    expect(mine.find((e) => e.server === 'b')).toMatchObject({ outcome: 'failed', detail: 'Disk full.' })
+    const raw = readFileSync(path.join(dataDir, 'audit.log'), 'utf8')
+    for (const secret of ['admin-a', 'admin-b', 'read-a', 'password-apps-admin', '"secret":"cfg"']) expect(raw).not.toContain(secret)
+  })
+
+  it('shows the new version in the matrix after an update (the cache is dropped)', async () => {
+    await start({ HUB_CACHE_SECONDS: '60' })
+    const before = (await api('viewer', 'hub/apps/matrix')).body.response.apps[0]
+    expect(before.servers.a.version).toBe('1.0')
+    await run('admin', { action: 'update-store', name: 'Blocker', targets: ['a'] })
+    const after = (await api('viewer', 'hub/apps/matrix')).body.response.apps[0]
+    expect(after.servers.a.version).toBe('2.0')
+    expect(after.servers.b.version).toBe('1.0')
+    expect(after.versions).toBe('different')
+  })
+})
+
+describe('uninstall and its history', () => {
+  const history = () => path.join(dataDir, 'apps', 'Blocker', 'history')
+
+  it('saves the config first, then uninstalls', async () => {
+    const r = await run('admin', { action: 'uninstall', name: 'Blocker', targets: ['a', 'b'] })
+    expect(r.body.response.results.map((x: any) => x.ok)).toEqual([true, true])
+    expect(a.apps.Blocker).toBeUndefined()
+    const files = readdirSync(history()).sort()
+    expect(files).toHaveLength(2)
+    expect(files.some((f) => /^\d{4}-\d\d-\d\dT\d\d-\d\d-\d\d\.\d{3}Z__a__before-uninstall\.txt$/.test(f))).toBe(true)
+    expect(readFileSync(path.join(history(), files[0]), 'utf8')).toBe('{"secret":"cfg"}')
+    expect(readdirSync(history()).some((f) => f.endsWith('.tmp'))).toBe(false)
+  })
+
+  it('keeps at most 50 files per app, dropping the oldest', async () => {
+    mkdirSync(history(), { recursive: true })
+    for (let i = 0; i < 55; i++) writeFileSync(path.join(history(), `2020-01-01T00-00-${String(i).padStart(2, '0')}.000Z__a__before-uninstall.txt`), 'old')
+    await run('admin', { action: 'uninstall', name: 'Blocker', targets: ['a'] })
+    const files = readdirSync(history()).sort()
+    expect(files).toHaveLength(50)
+    expect(files.some((f) => f.startsWith('2020-01-01T00-00-00.'))).toBe(false)
+    expect(files.some((f) => f.includes('__a__') && f.startsWith('20') && !f.startsWith('2020-'))).toBe(true)
+  })
+
+  it('does not uninstall when the config cannot be saved', async () => {
+    a.denyConfig = true
+    const r = await run('admin', { action: 'uninstall', name: 'Blocker', targets: ['a'] })
+    expect(r.body.response.results[0].ok).toBe(false)
+    expect(r.body.response.results[0].message).toMatch(/nothing was uninstalled/)
+    expect(a.apps.Blocker).toBeDefined()
+    expect(writes(a)).toHaveLength(0)
+  })
+
+  it('keeps an odd app name inside the data folder', async () => {
+    a.apps['../../evil'] = { version: '1', config: 'x' }
+    await run('admin', { action: 'uninstall', name: '../../evil', targets: ['a'] })
+    expect(existsSync(path.join(dataDir, '..', 'evil'))).toBe(false)
+    const dirs = readdirSync(path.join(dataDir, 'apps'))
+    expect(dirs.every((d) => d !== '.' && d !== '..' && !/[\/]/.test(d) && !d.includes('..'))).toBe(true)
+  })
+})
+
+describe('hub/apps/upload', () => {
+  const bytes = new Uint8Array(70_000).map((_, i) => (i * 7 + 3) % 256)
+
+  it('forwards the same zip to every target, byte for byte', async () => {
+    b.apps = {}
+    a.apps = {}
+    const r = await api('apps-admin', 'hub/apps/upload', { body: zipForm({ action: 'install', name: 'Blocker', targets: 'a,b' }, bytes) })
+    expect(r.body.response.results).toEqual([
+      { server: 'a', ok: true, message: 'Installed from the file.' },
+      { server: 'b', ok: true, message: 'Installed from the file.' },
+    ])
+    for (const [f, admin] of [[a, 'admin-a'], [b, 'admin-b']] as const) {
+      const [w] = writes(f)
+      expect(w.path).toBe('apps/install')
+      expect(w.token).toBe(admin)
+      expect(w.query.get('name')).toBe('Blocker')
+      const parsed = parseMultipart(w.body, w.contentType)
+      expect(Buffer.from(parsed.files[0].data).equals(Buffer.from(bytes))).toBe(true)
+    }
+    const audit = (await api('admin', 'hub/audit/list', { params: { limit: '10' } })).body.response.entries as Record<string, string>[]
+    expect(audit.filter((e) => e.action === 'apps.install-file' && e.outcome === 'ok')).toHaveLength(2)
+  })
+
+  it('updates, and reports an app that is not there as that target failure', async () => {
+    delete b.apps.Blocker
+    const r = await api('admin', 'hub/apps/upload', { body: zipForm({ action: 'update', name: 'Blocker', targets: 'a,b' }, bytes) })
+    expect(writes(a)[0].path).toBe('apps/update')
+    expect(r.body.response.results.map((x: any) => x.ok)).toEqual([true, true])
+  })
+
+  it('refuses a missing file, a bad action and a body above 20 MB', async () => {
+    const noFile = new FormData()
+    noFile.append('action', 'install')
+    noFile.append('name', 'X')
+    noFile.append('targets', 'a')
+    expect((await api('admin', 'hub/apps/upload', { body: noFile })).code).toBe(400)
+    expect((await api('admin', 'hub/apps/upload', { body: zipForm({ action: 'wipe', name: 'X', targets: 'a' }, bytes) })).code).toBe(400)
+
+    const big = new Uint8Array(21 * 1024 * 1024)
+    const r = await api('admin', 'hub/apps/upload', { body: zipForm({ action: 'install', name: 'X', targets: 'a' }, big) })
+    expect(r.code).toBe(413)
+    expect(r.body.errorMessage).toMatch(/20 MB/)
+    expect(writes(a)).toHaveLength(0)
+  })
+
+  it('does not read a big body from someone who is not allowed to upload', async () => {
+    const r = await api('viewer', 'hub/apps/upload', { body: zipForm({ action: 'install', name: 'X', targets: 'a' }, bytes) })
+    expect(r.code).toBe(403)
+  })
+})
+
+describe('single-server writes', () => {
+  it('forwards with the admin token', async () => {
+    const r = await api('apps-admin', 'apps/downloadAndUpdate', { params: { node: 'a', name: 'Blocker', url: STORE('2.0')[0].url } })
+    expect(r.body.status).toBe('ok')
+    const [w] = writes(a)
+    expect(w).toMatchObject({ path: 'apps/downloadAndUpdate', token: 'admin-a' })
+    expect(w.query.get('node')).toBeNull()
+    expect(writes(b)).toHaveLength(0)
+    expect(a.apps.Blocker.version).toBe('2.0')
+  })
+
+  it('works with a form POST too, and uninstalls with the history', async () => {
+    const r = await api('admin', 'apps/uninstall', {
+      body: new URLSearchParams({ node: 'b', name: 'Blocker' }).toString(),
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+    })
+    expect(r.body.status).toBe('ok')
+    expect(b.apps.Blocker).toBeUndefined()
+    expect(readdirSync(path.join(dataDir, 'apps', 'Blocker', 'history'))).toHaveLength(1)
+  })
+
+  it('is refused without a node or with the aggregate', async () => {
+    for (const params of [{ name: 'Blocker' } as Record<string, string>, { name: 'Blocker', node: 'cluster' }, { name: 'Blocker', node: 'zzz' }]) {
+      const r = await api('admin', 'apps/uninstall', { params })
+      expect(r.code).toBe(400)
+    }
+    expect(writes(a)).toHaveLength(0)
+    expect(writes(b)).toHaveLength(0)
+  })
+
+  it('does not forward a download url the store did not offer', async () => {
+    const r = await api('admin', 'apps/downloadAndInstall', { params: { node: 'a', name: 'Blocker', url: 'https://evil.example/x.zip' } })
+    expect(r.body.status).toBe('error')
+    expect(r.body.errorMessage).toMatch(/store listing/)
+    expect(writes(a)).toHaveLength(0)
+  })
+
+  it('reports a server without an admin token', async () => {
+    await start({ A_ADMIN_TOKEN: '' })
+    const r = await api('admin', 'apps/uninstall', { params: { node: 'a', name: 'Blocker' } })
+    expect(r.body).toEqual({ status: 'error', errorMessage: 'a: no admin token configured' })
+    expect(a.calls.filter((c) => c.method === 'POST')).toHaveLength(0)
+  })
+
+  it('forwards a multipart install to the node', async () => {
+    delete a.apps.Blocker
+    const bytes = new Uint8Array([80, 75, 3, 4, 13, 10, 45, 45, 0, 255])
+    const r = await api('admin', 'apps/install', { params: { node: 'a', name: 'Blocker' }, body: zipForm({}, bytes) })
+    expect(r.body.status).toBe('ok')
+    const [w] = writes(a)
+    expect(w.token).toBe('admin-a')
+    expect(Buffer.from(parseMultipart(w.body, w.contentType).files[0].data).equals(Buffer.from(bytes))).toBe(true)
+  })
+
+  it('keeps config get and set refused for now', async () => {
+    for (const p of ['apps/config/get', 'apps/config/set']) {
+      expect((await api('admin', p, { params: { node: 'a', name: 'Blocker' } })).code).toBe(403)
+    }
+  })
+
+  it('lists the store with the servers each app is installed on', async () => {
+    delete b.apps.Blocker
+    const r = await api('viewer', 'apps/listStoreApps')
+    expect(r.body.response.storeApps[0]).toMatchObject({ name: 'Blocker', installedOn: ['a'] })
+  })
+})
@@ -0,0 +1,185 @@
+/*
+App writes: install, update and uninstall on one or several servers. The first part of the
+hub that changes anything, so the rules are strict:
+
+  * only a server's ADMIN token writes (`<ID>_ADMIN_TOKEN`); a server without one is refused;
+  * a store download URL is never taken from the client: it is looked up in the target
+    server's own `apps/listStoreApps`, and a URL the client sends must match it;
+  * before an uninstall the app's config is saved to `${HUB_DATA}/apps/<app>/history/`, and
+    the uninstall is refused if that cannot be done;
+  * every target's result is audited (the caller does that with `Audit`).
+*/
+import { createHash } from 'node:crypto'
+import { mkdirSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'
+import path from 'node:path'
+import type { HubConfig, HubServer } from './config.ts'
+import type { UploadedFile } from './multipart.ts'
+import { Upstream } from './upstream.ts'
+
+export type StoreAction = 'install-store' | 'update-store' | 'uninstall'
+export type FileAction = 'install-file' | 'update-file'
+export type AppAction = StoreAction | FileAction
+
+export interface TargetResult {
+  server: string
+  ok: boolean
+  message?: string
+}
+
+export const KEEP_HISTORY = 50
+
+const API: Record<AppAction, string> = {
+  'install-store': 'apps/downloadAndInstall',
+  'update-store': 'apps/downloadAndUpdate',
+  uninstall: 'apps/uninstall',
+  'install-file': 'apps/install',
+  'update-file': 'apps/update',
+}
+
+const DONE: Record<AppAction, string> = {
+  'install-store': 'Installed from the store.',
+  'update-store': 'Updated from the store.',
+  uninstall: 'Uninstalled.',
+  'install-file': 'Installed from the file.',
+  'update-file': 'Updated from the file.',
+}
+
+/** The audit/api action for an upstream write path, or null when it is not an app write. */
+export function actionForPath(apiPath: string): AppAction | null {
+  for (const [action, p] of Object.entries(API)) if (p === apiPath) return action as AppAction
+  return null
+}
+
+/* A folder name for an app: letters, digits and . _ - only, with a short hash when anything had to change. */
+export function safeName(name: string): string {
+  const cleaned = name.replace(/[^A-Za-z0-9._-]/g, '_').replace(/\.{2,}/g, '_').replace(/^\.+/, '_').slice(0, 80)
+  if (cleaned === name && cleaned !== '') return cleaned
+  return `${cleaned === '' ? '_' : cleaned}-${createHash('sha256').update(name).digest('hex').slice(0, 6)}`
+}
+
+const reason = (e: unknown): string => (e instanceof Error ? e.message : 'failed')
+
+export class AppsWriter {
+  #config: HubConfig
+  #upstream: Upstream
+  #now: () => number
+
+  constructor(config: HubConfig, upstream: Upstream, now: () => number = Date.now) {
+    this.#config = config
+    this.#upstream = upstream
+    this.#now = now
+  }
+
+  /** The ids of the servers that have an admin token. */
+  writableIds(): string[] {
+    return this.#config.servers.filter((s) => s.adminToken !== undefined).map((s) => s.id)
+  }
+
+  /* The URL the server's own store gives for this app, or null. */
+  async storeUrl(server: HubServer, name: string): Promise<string | null> {
+    const response = (await this.#upstream.call(server, 'apps/listStoreApps')) as { storeApps?: { name: string; url?: string }[] } | null
+    const found = (response?.storeApps ?? []).find((a) => a.name === name)
+    return typeof found?.url === 'string' && found.url !== '' ? found.url : null
+  }
+
+  /* Saves the app's current config next to the hub's data; throws if it cannot. */
+  async saveHistory(server: HubServer, name: string): Promise<string> {
+    if (server.adminToken === undefined) throw new Error('no admin token configured')
+    this.#upstream.invalidate(server.id)
+    const response = (await this.#upstream.call(server, 'apps/config/get', { name }, server.adminToken)) as { config?: unknown } | null
+    const text = typeof response?.config === 'string' ? response.config : ''
+    const dir = path.join(this.#config.dataDir, 'apps', safeName(name), 'history')
+    mkdirSync(dir, { recursive: true })
+    const stamp = new Date(this.#now()).toISOString().replace(/:/g, '-')
+    const file = path.join(dir, `${stamp}__${safeName(server.id)}__before-uninstall.txt`)
+    const temp = `${file}.${process.pid}.tmp`
+    writeFileSync(temp, text, { mode: 0o600 })
+    renameSync(temp, file)
+    // Oldest first by name (the name starts with the time); keep the newest KEEP_HISTORY.
+    const files = readdirSync(dir).filter((f) => f.endsWith('.txt')).sort()
+    for (const old of files.slice(0, Math.max(0, files.length - KEEP_HISTORY))) rmSync(path.join(dir, old), { force: true })
+    return file
+  }
+
+  /*
+  One write on one server. `url` must already be checked against the store; `file` is the zip
+  for the two file actions. Returns the server's response, or throws.
+  */
+  async perform(server: HubServer, action: AppAction, name: string, url?: string, file?: UploadedFile): Promise<unknown> {
+    if (server.adminToken === undefined) throw new Error('no admin token configured')
+    switch (action) {
+      case 'install-store':
+      case 'update-store':
+        return await this.#upstream.write(server, API[action], { name, url: url ?? '' })
+      case 'uninstall':
+        try {
+          await this.saveHistory(server, name)
+        } catch (e) {
+          throw new Error(`the config could not be saved before the uninstall (${reason(e)}); nothing was uninstalled`)
+        }
+        return await this.#upstream.write(server, API[action], { name })
+      case 'install-file':
+      case 'update-file': {
+        if (file === undefined) throw new Error('no zip file')
+        const form = new FormData()
+        form.append('fileAppZip', new Blob([file.data], { type: file.type }), file.filename)
+        return await this.#upstream.write(server, API[action], { name }, form)
+      }
+    }
+  }
+
+  /*
+  The same write on several servers, in parallel. A store action first resolves each target's
+  URL from its own store; `suppliedUrl` (if the client sent one) has to match every one of them.
+  Returns `{ error }` when the request itself is refused, else one result per target.
+  */
+  async runMany(
+    action: AppAction,
+    name: string,
+    targets: string[],
+    options: { suppliedUrl?: string; file?: UploadedFile } = {},
+  ): Promise<{ error: string } | { results: TargetResult[] }> {
+    const known = new Map(this.#config.servers.map((s) => [s.id, s]))
+    const unique = [...new Set(targets)]
+    if (name === '' || name.length > 200) return { error: 'An app name is required.' }
+    if (unique.length === 0) return { error: 'Choose at least one server.' }
+    const unknown = unique.filter((t) => !known.has(t))
+    if (unknown.length > 0) return { error: `Unknown server: ${unknown.join(', ')}.` }
+
+    const urls = new Map<string, string>()
+    const early = new Map<string, string>()
+    if (action === 'install-store' || action === 'update-store') {
+      await Promise.all(
+        unique.map(async (id) => {
+          const server = known.get(id) as HubServer
+          if (server.adminToken === undefined) return early.set(id, 'no admin token configured')
+          try {
+            const url = await this.storeUrl(server, name)
+            if (url === null) early.set(id, `"${name}" is not in this server's store listing`)
+            else urls.set(id, url)
+          } catch (e) {
+            early.set(id, `the store listing could not be read (${reason(e)})`)
+          }
+        }),
+      )
+      if (options.suppliedUrl !== undefined && options.suppliedUrl !== '' && [...urls.values()].some((u) => u !== options.suppliedUrl)) {
+        return { error: 'The url is not from the store listing; the hub only installs what a server offers.' }
+      }
+    }
+
+    const results = await Promise.all(
+      unique.map(async (id): Promise<TargetResult> => {
+        const server = known.get(id) as HubServer
+        const refused = early.get(id) ?? (server.adminToken === undefined ? 'no admin token configured' : undefined)
+        if (refused !== undefined) return { server: id, ok: false, message: refused }
+        try {
+          await this.perform(server, action, name, urls.get(id), options.file)
+          return { server: id, ok: true, message: DONE[action] }
+        } catch (e) {
+          return { server: id, ok: false, message: reason(e) }
+        }
+      }),
+    )
+    return { results }
+  }
+}
@@ -0,0 +1,91 @@
+/*
+A small multipart/form-data reader for the app zip uploads, and a body reader with a size
+limit. No dependencies; the whole body stays in memory (the limit is 20 MB).
+*/
+import type http from 'node:http'
+
+export const MAX_UPLOAD = 20 * 1024 * 1024
+
+export class TooLarge extends Error {
+  constructor(limit: number) {
+    super(`The upload is larger than ${Math.floor(limit / (1024 * 1024))} MB.`)
+    this.name = 'TooLarge'
+  }
+}
+
+/*
+Reads the request body, counting bytes as they arrive; throws `TooLarge` above `limit`.
+Past the limit nothing more is kept, but the rest is still read (up to DRAIN more bytes), so the
+client gets its 413 instead of a reset connection.
+*/
+const DRAIN = 32 * 1024 * 1024
+
+export async function readLimited(req: http.IncomingMessage, limit: number = MAX_UPLOAD): Promise<Buffer> {
+  const declared = Number(req.headers['content-length'])
+  if (Number.isFinite(declared) && declared > limit + DRAIN) throw new TooLarge(limit)
+  const chunks: Buffer[] = []
+  let size = 0
+  for await (const chunk of req) {
+    size += (chunk as Buffer).length
+    if (size > limit + DRAIN) throw new TooLarge(limit)
+    if (size <= limit) chunks.push(chunk as Buffer)
+  }
+  if (size > limit) throw new TooLarge(limit)
+  return Buffer.concat(chunks)
+}
+
+export interface UploadedFile {
+  field: string
+  filename: string
+  type: string
+  data: Buffer
+}
+
+export interface Multipart {
+  fields: Record<string, string>
+  files: UploadedFile[]
+}
+
+export function boundaryOf(contentType: string): string | null {
+  const match = /boundary=(?:"([^"]+)"|([^;\s]+))/i.exec(contentType)
+  return match === null ? null : (match[1] ?? match[2])
+}
+
+const CRLF2 = Buffer.from('\r\n\r\n')
+
+/* Throws on anything that is not a well-formed multipart body. */
+export function parseMultipart(body: Buffer, contentType: string): Multipart {
+  const boundary = boundaryOf(contentType)
+  if (boundary === null || !/^multipart\/form-data/i.test(contentType)) throw new Error('Expected multipart/form-data with a boundary.')
+  const delimiter = Buffer.from(`--${boundary}`)
+  const result: Multipart = { fields: {}, files: [] }
+
+  let at = body.indexOf(delimiter)
+  if (at < 0) throw new Error('The multipart body has no boundary.')
+  for (;;) {
+    at += delimiter.length
+    // "--" after a delimiter closes the body.
+    if (body[at] === 0x2d && body[at + 1] === 0x2d) return result
+    if (body[at] !== 0x0d || body[at + 1] !== 0x0a) throw new Error('Malformed multipart body.')
+    at += 2
+    const headerEnd = body.indexOf(CRLF2, at)
+    if (headerEnd < 0) throw new Error('Malformed multipart part.')
+    const headers = body.subarray(at, headerEnd).toString('utf8')
+    const start = headerEnd + CRLF2.length
+    // The part ends at the next CRLF + delimiter, so binary content can hold anything else.
+    const next = body.indexOf(Buffer.concat([Buffer.from('\r\n'), delimiter]), start)
+    if (next < 0) throw new Error('The multipart body is not terminated.')
+    const data = body.subarray(start, next)
+
+    const disposition = /^content-disposition:\s*form-data;(.*)$/im.exec(headers)?.[1] ?? ''
+    const name = /\bname="([^"]*)"/i.exec(disposition)?.[1]
+    const filename = /\bfilename="([^"]*)"/i.exec(disposition)?.[1]
+    if (name === undefined) throw new Error('A multipart part has no name.')
+    if (filename === undefined) result.fields[name] = data.toString('utf8')
+    else {
+      const type = /^content-type:\s*(.+)$/im.exec(headers)?.[1].trim() ?? 'application/octet-stream'
+      result.files.push({ field: name, filename, type, data: Buffer.from(data) })
+    }
+    at = next + 2
+  }
+}
@@ -1,7 +1,7 @@
 /*
 What each `/api/<path>` does and who may call it. `kind` says what it touches:
 `read` (no change), `write-one` (changes one server) and `write-all` (changes every server).
-A path that is not listed is refused. Today every entry is a read.
+A path that is not listed is refused. The app writes need `apps.write`.

 Not in the table: `status`, `user/login`, `user/session/get` and `user/logout`, which the
 server answers itself before the table is consulted.
@@ -16,6 +16,8 @@ export interface Route {
 }

 const read = (capability: Capability = 'read'): Route => ({ kind: 'read', capability })
+const writeOne = (capability: Capability): Route => ({ kind: 'write-one', capability })
+const writeAll = (capability: Capability): Route => ({ kind: 'write-all', capability })

 export const ROUTES: Record<string, Route> = {
  'dashboard/stats/get': read(),
@@ -30,6 +32,15 @@ export const ROUTES: Record<string, Route> = {
  // The hub's own namespace; never forwarded to a server.
  'hub/audit/list': read('audit.read'),
  'hub/apps/matrix': read(),
+  'apps/listStoreApps': read(),
+  // Writes. They go to a server's ADMIN token only; see apps-write.ts. apps/config/set waits for a later stage.
+  'apps/downloadAndInstall': writeOne('apps.write'),
+  'apps/downloadAndUpdate': writeOne('apps.write'),
+  'apps/uninstall': writeOne('apps.write'),
+  'apps/install': writeOne('apps.write'),
+  'apps/update': writeOne('apps.write'),
+  'hub/apps/run': writeAll('apps.write'),
+  'hub/apps/upload': writeAll('apps.write'),
 }

 export function routeFor(apiPath: string): Route | undefined {
@@ -22,6 +22,8 @@ import { LoginLimiter, lockedMessage } from './ratelimit.ts'
 import { Audit } from './audit.ts'
 import { routeFor } from './routes.ts'
 import { buildMatrix } from './apps-matrix.ts'
+import { AppsWriter, actionForPath, type FileAction, type TargetResult } from './apps-write.ts'
+import { TooLarge, parseMultipart, readLimited, type UploadedFile } from './multipart.ts'
 import {
  mergeQueryLogs,
  mergeSettled,
@@ -80,10 +82,30 @@ async function readForm(req: http.IncomingMessage): Promise<URLSearchParams> {
    chunks.push(chunk as Buffer)
  }
  const type = String(req.headers['content-type'] ?? '')
-  if (size > MAX_BODY || !type.includes('application/x-www-form-urlencoded')) return new URLSearchParams()
-  return new URLSearchParams(Buffer.concat(chunks).toString('utf8'))
+  if (size > MAX_BODY) return new URLSearchParams()
+  const text = Buffer.concat(chunks).toString('utf8')
+  if (type.includes('application/x-www-form-urlencoded')) return new URLSearchParams(text)
+  if (type.includes('application/json')) {
+    // A flat object; an array becomes a comma-separated list, as the form fields are.
+    const out = new URLSearchParams()
+    try {
+      const data = JSON.parse(text) as Record<string, unknown>
+      for (const [k, v] of Object.entries(data)) {
+        if (Array.isArray(v)) out.set(k, v.map(String).join(','))
+        else if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') out.set(k, String(v))
+      }
+    } catch {
+      // Not JSON: no parameters.
+    }
+    return out
+  }
+  return new URLSearchParams()
 }

+const isMultipart = (req: http.IncomingMessage): boolean => String(req.headers['content-type'] ?? '').toLowerCase().startsWith('multipart/form-data')
+
+const csv = (value: string | undefined): string[] => (value ?? '').split(',').map((v) => v.trim()).filter((v) => v !== '')
+
 function bearer(req: http.IncomingMessage): string | null {
  const header = String(req.headers.authorization ?? '')
  return header.startsWith('Bearer ') ? header.slice(7).trim() : null
@@ -108,6 +130,7 @@ export function createHub(config: HubConfig, users: Users = Users.load(config.da
  const sessions = new Sessions(config, users)
  const limiter = new LoginLimiter()
  const audit = new Audit(config.dataDir)
+  const writer = new AppsWriter(config, upstream)
  const byId = new Map(config.servers.map((s) => [s.id, s]))

  /* Ask every server; the ones that fail become warnings (see mergeSettled). */
@@ -162,6 +185,16 @@ export function createHub(config: HubConfig, users: Users = Users.load(config.da
          const pages = data.map((page, i) => ({ node: answered[i], page }))
          return ok(mergeQueryLogs(pages, { pageNumber, entriesPerPage, descending: params.descendingOrder !== 'false' }), warnings)
        }
+        case 'apps/listStoreApps': {
+          // The first answering server's listing, each app tagged with the servers it is installed on.
+          const { data, warnings } = mergeSettled(await everywhere<{ storeApps?: { name: string }[] }>(apiPath, params))
+          const installedOn = new Map<string, string[]>()
+          for (const r of await everywhere<{ apps?: { name: string }[] }>('apps/list', {})) {
+            if (r.ok) for (const a of r.data.apps ?? []) installedOn.set(a.name, [...(installedOn.get(a.name) ?? []), r.node])
+          }
+          const storeApps = (data[0].storeApps ?? []).map((a) => ({ ...a, installedOn: installedOn.get(a.name) ?? [] }))
+          return ok({ ...data[0], storeApps }, warnings)
+        }
        case 'apps/list': {
          const { data, warnings } = mergeSettled(await everywhere<{ apps?: { name: string }[] }>(apiPath, params))
          const seen = new Map<string, { name: string }>()
@@ -178,8 +211,131 @@ export function createHub(config: HubConfig, users: Users = Users.load(config.da
    }
  }

+  type Who = { user: string; role: string }
+
+  /* One audit line per target: who, which server, which app, what happened. */
+  function auditResults(who: Who, ip: string, action: string, name: string, results: TargetResult[]): void {
+    for (const r of results) {
+      audit.log({
+        user: who.user, role: who.role, ip, action: `apps.${action}`, server: r.server, target: name,
+        outcome: r.ok ? 'ok' : 'failed', detail: r.message,
+      })
+    }
+  }
+
+  async function uploadOf(req: http.IncomingMessage, res: http.ServerResponse, params: Params): Promise<UploadedFile | null> {
+    let parsed
+    try {
+      parsed = parseMultipart(await readLimited(req), String(req.headers['content-type'] ?? ''))
+    } catch (e) {
+      if (e instanceof TooLarge) {
+        res.setHeader('connection', 'close')
+        send(res, 413, fail(e.message))
+      } else send(res, 400, fail('The upload could not be read.'))
+      return null
+    }
+    Object.assign(params, parsed.fields)
+    const file = parsed.files[0]
+    if (file === undefined || file.data.length === 0) {
+      send(res, 400, fail('The zip file is missing.'))
+      return null
+    }
+    return file
+  }
+
+  /* The write endpoints. The role has already been checked against the route's capability. */
+  async function write(req: http.IncomingMessage, res: http.ServerResponse, apiPath: string, kind: string, params: Params, who: Who, ip: string): Promise<void> {
+    if (kind === 'write-all') {
+      if (req.method !== 'POST') {
+        send(res, 405, fail('Use POST.'))
+        return
+      }
+      if (apiPath === 'hub/apps/run') {
+        const action = params.action
+        if (action !== 'install-store' && action !== 'update-store' && action !== 'uninstall') {
+          send(res, 400, fail('action must be install-store, update-store or uninstall.'))
+          return
+        }
+        const outcome = await writer.runMany(action, params.name ?? '', csv(params.targets), { suppliedUrl: params.url })
+        if ('error' in outcome) {
+          audit.log({ user: who.user, role: who.role, ip, action: `apps.${action}`, target: params.name, outcome: 'refused', detail: outcome.error })
+          send(res, 400, fail(outcome.error))
+          return
+        }
+        auditResults(who, ip, action, params.name ?? '', outcome.results)
+        send(res, 200, ok({ results: outcome.results }))
+        return
+      }
+      if (apiPath === 'hub/apps/upload') {
+        if (!isMultipart(req)) {
+          send(res, 400, fail('Send multipart/form-data.'))
+          return
+        }
+        const file = await uploadOf(req, res, params)
+        if (file === null) return
+        if (params.action !== 'install' && params.action !== 'update') {
+          send(res, 400, fail('action must be install or update.'))
+          return
+        }
+        const action: FileAction = params.action === 'install' ? 'install-file' : 'update-file'
+        const outcome = await writer.runMany(action, params.name ?? '', csv(params.targets), { file })
+        if ('error' in outcome) {
+          audit.log({ user: who.user, role: who.role, ip, action: `apps.${action}`, target: params.name, outcome: 'refused', detail: outcome.error })
+          send(res, 400, fail(outcome.error))
+          return
+        }
+        auditResults(who, ip, action, params.name ?? '', outcome.results)
+        send(res, 200, ok({ results: outcome.results }))
+        return
+      }
+    }
+
+    // write-one: an upstream endpoint aimed at ONE server (`node=<id>`).
+    const action = actionForPath(apiPath)
+    const node = params.node ?? ''
+    const server = byId.get(node)
+    if (action === null) {
+      send(res, 403, fail('The hub is read-only.'))
+      return
+    }
+    if (server === undefined) {
+      send(res, 400, fail(node === '' || node === 'cluster' ? 'Choose one server: this change cannot be made on every server at once here.' : `Unknown node: ${node}`))
+      return
+    }
+    const name = params.name ?? ''
+    let file: UploadedFile | undefined
+    if (action === 'install-file' || action === 'update-file') {
+      if (!isMultipart(req)) {
+        send(res, 400, fail('Send multipart/form-data.'))
+        return
+      }
+      const uploaded = await uploadOf(req, res, params)
+      if (uploaded === null) return
+      file = uploaded
+    }
+    let result: TargetResult
+    let response: unknown = {}
+    try {
+      let url: string | undefined
+      if (action === 'install-store' || action === 'update-store') {
+        const fromStore = server.adminToken === undefined ? null : await writer.storeUrl(server, name).catch(() => null)
+        if (server.adminToken !== undefined && (fromStore === null || params.url !== fromStore)) {
+          throw new Error("the url is not from this server's store listing")
+        }
+        url = fromStore ?? undefined
+      }
+      response = await writer.perform(server, action, name, url, file)
+      result = { server: server.id, ok: true }
+    } catch (e) {
+      result = { server: server.id, ok: false, message: e instanceof Error ? e.message : 'failed' }
+    }
+    auditResults(who, ip, action, name, [result])
+    send(res, 200, result.ok ? ok(response) : fail(`${server.id}: ${result.message}`))
+  }
+
  async function api(req: http.IncomingMessage, res: http.ServerResponse, apiPath: string, url: URL): Promise<void> {
-    const form = await readForm(req)
+    // A multipart body (an app zip) is read later, once the caller is known to be allowed to send it.
+    const form = isMultipart(req) ? new URLSearchParams() : await readForm(req)
    const params: Params = {}
    for (const [k, v] of url.searchParams) params[k] = v
    for (const [k, v] of form) params[k] = v
@@ -249,6 +405,10 @@ export function createHub(config: HubConfig, users: Users = Users.load(config.da
      send(res, 200, ok({ entries: audit.list(Number(params.limit ?? 200), params.before) }))
      return
    }
+    if (route.kind !== 'read') {
+      await write(req, res, apiPath, route.kind, params, who, ip)
+      return
+    }
    if (apiPath === 'hub/apps/matrix') {
      try {
        const { matrix, warnings } = await buildMatrix(config, upstream)
@@ -6,7 +6,7 @@ the console reads from `user/login` and `user/session/get` (flat, as the real se
 */
 import { randomBytes } from 'node:crypto'
 import { HUB_VERSION, type HubConfig } from './config.ts'
-import { signature, type Role, type UserRecord, type Users } from './users.ts'
+import { can, signature, type Role, type UserRecord, type Users } from './users.ts'

 export const IDLE_MS = 30 * 60_000

@@ -23,8 +23,8 @@ interface Session extends Identity {
 /*
 Every key `src/app/sections.ts` reads, spelled out: the console hides a section only
 when `canView === false`, so a missing key would leave it visible. The hub shows the
-Dashboard, the Logs and the Apps; About has no permission and is always there. The hub is
-read-only, so nothing can be modified or deleted.
+Dashboard, the Logs and the Apps; About has no permission and is always there. Only the Apps can be
+modified or deleted, and only by a role with `apps.write`.
 */
 const PERMISSION_KEYS = [
  'Dashboard', 'Zones', 'Cache', 'Allowed', 'Blocked', 'Apps', 'DnsClient', 'Settings', 'DhcpServer', 'Administration', 'Logs',
@@ -84,7 +84,10 @@ export class Sessions {
      uptimestamp: this.#started,
      dnsServerDomain: domain,
      permissions: Object.fromEntries(
-        PERMISSION_KEYS.map((key) => [key, { canView: VIEWABLE.has(key), canModify: false, canDelete: false }]),
+        PERMISSION_KEYS.map((key) => {
+          const writes = key === 'Apps' && can(role, 'apps.write')
+          return [key, { canView: VIEWABLE.has(key), canModify: writes, canDelete: writes }]
+        }),
      ),
      /* The console offers its node selector when the cluster is initialised. The
         first server plays the Primary (`primaryNodeName` looks for one). */
@@ -98,6 +101,8 @@ export class Sessions {
      dnssecValidation: true,
      hub: true,
      hubRole: role,
+      /* The servers the hub holds an admin token for: the only ones a write can reach. */
+      hubWritable: servers.filter((s) => s.adminToken !== undefined).map((s) => s.id),
    }
  }

@@ -28,6 +28,9 @@ interface Cached {

 const MAX_CACHED = 500

+/* An app download or upload can take a while. */
+const WRITE_TIMEOUT_MS = 60_000
+
 export class Upstream {
  #config: HubConfig
  #fetch: typeof fetch
@@ -47,20 +50,50 @@ export class Upstream {
    const hit = this.#cache.get(key)
    if (hit !== undefined && hit.until > this.#now()) return hit.response

-    const query = new URLSearchParams(sorted)
+    const response = await this.#exec(server, path, new URLSearchParams(sorted), token, { method: 'GET' }, this.#config.timeoutMs)
+    if (this.#config.cacheSeconds > 0) {
+      if (this.#cache.size >= MAX_CACHED) this.#prune()
+      this.#cache.set(key, { until: this.#now() + this.#config.cacheSeconds * 1000, response })
+    }
+    return response
+  }
+
+  /*
+  A write: POST with the server's ADMIN token and never the read one, never cached. Whatever
+  the outcome, what the hub remembers of this server is dropped afterwards, so the next read
+  shows the state the write left. `body` is a multipart upload (the app zip).
+  */
+  async write(server: HubServer, path: string, params: Params = {}, body?: FormData): Promise<unknown> {
+    if (server.adminToken === undefined) throw new UpstreamError('rejected', 'no admin token configured')
+    try {
+      const init: RequestInit = body === undefined ? { method: 'POST' } : { method: 'POST', body }
+      return await this.#exec(server, path, new URLSearchParams(params), server.adminToken, init, Math.max(this.#config.timeoutMs, WRITE_TIMEOUT_MS))
+    } finally {
+      this.invalidate(server.id)
+    }
+  }
+
+  /** Forget every cached answer of one server. */
+  invalidate(serverId: string): void {
+    const prefix = `${serverId}\n`
+    for (const key of this.#cache.keys()) if (key.startsWith(prefix)) this.#cache.delete(key)
+  }
+
+  async #exec(server: HubServer, path: string, query: URLSearchParams, token: string, init: RequestInit, timeoutMs: number): Promise<unknown> {
    query.set('token', token)
    const controller = new AbortController()
-    const timer = setTimeout(() => controller.abort(), this.#config.timeoutMs)
+    const timer = setTimeout(() => controller.abort(), timeoutMs)

    let envelope: { status?: string; response?: unknown; errorMessage?: string }
    try {
      const res = await this.#fetch(`${server.url}/api/${path}?${query.toString()}`, {
+        ...init,
        signal: controller.signal,
        headers: { accept: 'application/json' },
      })
      envelope = (await res.json()) as typeof envelope
    } catch (e) {
-      throw new UpstreamError('unreachable', this.#describe(e, [server.token, token]))
+      throw new UpstreamError('unreachable', this.#describe(e, [server.token, token], timeoutMs))
    } finally {
      clearTimeout(timer)
    }
@@ -72,11 +105,6 @@ export class Upstream {
          : (envelope?.errorMessage ?? `status ${String(envelope?.status)}`)
      throw new UpstreamError('rejected', why)
    }
-
-    if (this.#config.cacheSeconds > 0) {
-      if (this.#cache.size >= MAX_CACHED) this.#prune()
-      this.#cache.set(key, { until: this.#now() + this.#config.cacheSeconds * 1000, response: envelope.response })
-    }
    return envelope.response
  }

@@ -88,8 +116,8 @@ export class Upstream {
  }

  /* A short reason without anything that could carry the token. */
-  #describe(e: unknown, secrets: string[]): string {
-    if (e instanceof Error && e.name === 'AbortError') return `timed out after ${this.#config.timeoutMs} ms`
+  #describe(e: unknown, secrets: string[], timeoutMs: number): string {
+    if (e instanceof Error && e.name === 'AbortError') return `timed out after ${timeoutMs} ms`
    const cause = e instanceof Error ? (e.cause as { code?: unknown } | undefined) : undefined
    const text = typeof cause?.code === 'string' ? cause.code : e instanceof Error ? e.message : 'request failed'
    return secrets.reduce((out, secret) => (secret === '' ? out : out.split(secret).join('***')), text)
@@ -61,6 +61,8 @@ export interface StoreApp {
  installedVersion?: string
  /** Only when `installed` is true. */
  updateAvailable?: boolean
+  /** Only behind a hub, in the All servers listing: the servers that have it installed. */
+  installedOn?: string[]
 }

 /*
@@ -88,28 +90,32 @@ export function listApps(

 export function listStoreApps(
  token: string | null,
+  node = '',
 ): Promise<ApiOutcome<{ response: { storeApps: StoreApp[] } }>> {
-  return apiRequest('apps/listStoreApps', { token })
+  return apiRequest('apps/listStoreApps', { token, node })
 }

+/* Behind a hub, `node` is the one server the call is for; elsewhere it stays empty and is not sent. */
 export function downloadAndInstall(
  token: string | null,
  name: string,
  url: string,
+  node = '',
 ): Promise<ApiOutcome<{ response: { installedApp: InstalledApp } }>> {
-  return apiRequest('apps/downloadAndInstall', { token, body: { name, url } })
+  return apiRequest('apps/downloadAndInstall', { token, body: { name, url }, node })
 }

 export function downloadAndUpdate(
  token: string | null,
  name: string,
  url: string,
+  node = '',
 ): Promise<ApiOutcome<{ response: { updatedApp: InstalledApp } }>> {
-  return apiRequest('apps/downloadAndUpdate', { token, body: { name, url } })
+  return apiRequest('apps/downloadAndUpdate', { token, body: { name, url }, node })
 }

-export function uninstallApp(token: string | null, name: string): Promise<ApiOutcome> {
-  return apiRequest('apps/uninstall', { token, body: { name } })
+export function uninstallApp(token: string | null, name: string, node = ''): Promise<ApiOutcome> {
+  return apiRequest('apps/uninstall', { token, body: { name }, node })
 }

 /*
@@ -174,18 +180,20 @@ export function installApp(
  token: string | null,
  name: string,
  file: File,
+  node = '',
 ): Promise<ApiOutcome<{ response: { installedApp: InstalledApp } }>> {
  const { path, form } = buildUpload('apps/install', name, file)
-  return apiRequest(path, { token, form })
+  return apiRequest(path, { token, form, node })
 }

 export function updateApp(
  token: string | null,
  name: string,
  file: File,
+  node = '',
 ): Promise<ApiOutcome<{ response: { updatedApp: InstalledApp } }>> {
  const { path, form } = buildUpload('apps/update', name, file)
-  return apiRequest(path, { token, form })
+  return apiRequest(path, { token, form, node })
 }

 /*
@@ -212,10 +220,44 @@ export interface MatrixApp {
 }

 export interface AppsMatrixData {
-  servers: { id: string; ok: boolean; error?: string }[]
+  servers: { id: string; ok: boolean; writable: boolean; error?: string }[]
  apps: MatrixApp[]
 }

 export function getAppsMatrix(token: string | null): Promise<ApiOutcome<{ response: AppsMatrixData }>> {
  return apiRequest('hub/apps/matrix', { token })
 }
+
+/*
+App writes across servers (hub/apps-write.ts). The hub resolves a store URL from each
+server's own store; the client never sends one.
+*/
+export type HubRunAction = 'install-store' | 'update-store' | 'uninstall'
+
+export interface TargetResult {
+  server: string
+  ok: boolean
+  message?: string
+}
+
+export type RunOutcome = ApiOutcome<{ response: { results: TargetResult[] } }>
+
+export function runHubApps(token: string | null, action: HubRunAction, name: string, targets: string[]): Promise<RunOutcome> {
+  return apiRequest('hub/apps/run', { token, method: 'POST', body: { action, name, targets: targets.join(',') } })
+}
+
+/* The same zip to several servers: multipart, as the hub's `hub/apps/upload` expects. */
+export function uploadHubApp(
+  token: string | null,
+  action: 'install' | 'update',
+  name: string,
+  file: File,
+  targets: string[],
+): Promise<RunOutcome> {
+  const form = new FormData()
+  form.append('action', action)
+  form.append('name', name)
+  form.append('targets', targets.join(','))
+  form.append('fileAppZip', file)
+  return apiRequest('hub/apps/upload', { token, form })
+}
@@ -105,6 +105,8 @@ export interface ShellSession {
    clusterInitialized?: boolean
    /** Only on the hub (`hub/`): a read-only console over several servers. */
    hub?: boolean
+    /** Only on the hub: the servers it holds an admin token for. */
+    hubWritable?: string[]
    /** Only on an initialised cluster (WebServiceAuthApi.cs:104-106). */
    clusterDomain?: string
    clusterNodes?: { name: string; type: string }[]
@@ -649,7 +651,7 @@ export function Shell({
        ) : current?.id === 'about' ? (
          <About info={info} />
        ) : current?.id === 'apps' ? (
-          <Apps token={session.token} nodes={session.info?.clusterNodes ?? []} clusterInitialised={session.info?.clusterInitialized === true} />
+          <Apps token={session.token} nodes={session.info?.clusterNodes ?? []} clusterInitialised={session.info?.clusterInitialized === true} canWrite={permissions?.Apps?.canModify === true} writable={session.info?.hubWritable ?? []} />
        ) : current?.id === 'cache' ? (
          <Cache token={session.token} nodes={session.info?.clusterNodes ?? []} clusterInitialised={session.info?.clusterInitialized === true} />
        ) : current?.id === 'blocking' ? (
@@ -22,6 +22,7 @@ export function AppCard({
  onStoreUpdate,
  onUninstall,
  readOnly = false,
+  hideConfig = false,
 }: {
  app: InstalledApp
  busy: boolean
@@ -31,6 +32,8 @@ export function AppCard({
  onUninstall: () => void
  /** Behind a hub nothing can be changed yet: no action buttons. */
  readOnly?: boolean
+  /** Behind a hub the config is not edited from here yet. */
+  hideConfig?: boolean
 }) {
  const hasUpdate = app.updateAvailable === true && Boolean(app.updateUrl)

@@ -61,9 +64,11 @@ export function AppCard({
        <div />
      ) : (
      <div className={styles.rowActs}>
-        <Button disabled={busy} onClick={onConfig}>
-          Config
-        </Button>
+        {!hideConfig && (
+          <Button disabled={busy} onClick={onConfig}>
+            Config
+          </Button>
+        )}
        <Button disabled={busy} onClick={onUpdate}>
          Update
        </Button>
@@ -28,6 +28,9 @@ import { Confirm } from '../../ui/Confirm'
 import { InChrome } from '../../app/ChromeSlot'
 import { AGGREGATE, ClusterNodeSelect, HubMode, type ClusterNode } from '../../ui/ClusterNodeSelect'
 import { AppsMatrix } from './AppsMatrix'
+import { HubStore } from './HubStore'
+import { HubInstallFile } from './HubInstallFile'
+import { RunFlow, type RunRequest } from './RunFlow'

 /*
 A replica of the Apps tab (apps.js + index.html:807-835).
@@ -63,11 +66,17 @@ export function Apps({
  token,
  nodes = [],
  clusterInitialised = false,
+  canWrite = false,
+  writable = [],
 }: {
  token: string | null
  /** Only used behind a hub: its servers. */
  nodes?: ClusterNode[]
  clusterInitialised?: boolean
+  /** Behind a hub: the role may change apps (`Apps.canModify`). */
+  canWrite?: boolean
+  /** Behind a hub: the servers the hub holds an admin token for. */
+  writable?: string[]
 }) {
  /*
  Behind a hub (`info.hub`) this screen is read-only and has a server selector: "All
@@ -85,6 +94,13 @@ export function Apps({
  const chosen = node === AGGREGATE || nodes.some((n) => n.name === node) ? node : AGGREGATE
  const matrixMode = hub && chosen === AGGREGATE
  const requestNode = hub && !matrixMode ? chosen : ''
+  /* Behind a hub, changes are for writers and only on a server the hub can write to. Elsewhere always. */
+  const mayChange = !hub || (!matrixMode && canWrite && writable.includes(chosen))
+  const hubServers = nodes.map((n) => ({ id: n.name, writable: writable.includes(n.name) }))
+  const [hubView, setHubView] = useState<'matrix' | 'store'>('matrix')
+  const [fileDialog, setFileDialog] = useState(false)
+  const [fileRequest, setFileRequest] = useState<RunRequest | null>(null)
+  const [refresh, setRefresh] = useState(0)
  useEffect(() => {
    if (!hub) return
    try {
@@ -140,7 +156,7 @@ export function Apps({
  */
  async function uninstall(name: string) {
    setBusy(name)
-    const outcome = await uninstallApp(token, name)
+    const outcome = await uninstallApp(token, name, requestNode)
    setBusy(null)

    if (outcome.kind !== 'ok') {
@@ -160,7 +176,7 @@ export function Apps({
    if (!app.updateUrl) return

    setBusy(app.name)
-    const outcome = await downloadAndUpdate(token, app.name, app.updateUrl)
+    const outcome = await downloadAndUpdate(token, app.name, app.updateUrl, requestNode)
    setBusy(null)

    if (outcome.kind !== 'ok') {
@@ -216,8 +232,35 @@ export function Apps({
    return (
      <>
        {selector}
-        <SectionHeader title="Apps" subtitle="DNS apps on every server" />
-        <AppsMatrix token={token} />
+        <SectionHeader
+          title="Apps"
+          subtitle="DNS apps on every server"
+          actions={
+            canWrite ? (
+              <>
+                <Button variant="primary" onClick={() => setHubView((v) => (v === 'store' ? 'matrix' : 'store'))}>
+                  {hubView === 'store' ? 'Back to apps' : 'App Store'}
+                </Button>
+                <Button onClick={() => setFileDialog(true)}>Install from file</Button>
+              </>
+            ) : undefined
+          }
+        />
+        {hubView === 'store' && canWrite ? (
+          <HubStore token={token} servers={hubServers} canWrite={canWrite} onChanged={() => setRefresh((n) => n + 1)} />
+        ) : (
+          <AppsMatrix token={token} canWrite={canWrite} refresh={refresh} />
+        )}
+        <HubInstallFile open={fileDialog} onOpenChange={setFileDialog} token={token} servers={hubServers} onRequest={setFileRequest} />
+        {fileRequest !== null && (
+          <RunFlow
+            request={fileRequest}
+            onClose={(changed) => {
+              setFileRequest(null)
+              if (changed) setRefresh((n) => n + 1)
+            }}
+          />
+        )}
      </>
    )
  }
@@ -238,7 +281,7 @@ export function Apps({
          ) : undefined
        }
        actions={
-          hub ? undefined : (
+          !mayChange ? undefined : (
            <>
              <Button variant="primary" onClick={() => setTab('store')}>
                App Store
@@ -260,7 +303,7 @@ export function Apps({
        />
      )}

-      {!hub && (
+      {mayChange && (
        <PageTabs
          label="Apps"
          tabs={[
@@ -272,8 +315,8 @@ export function Apps({
        />
      )}

-      {tab === 'store' && !hub ? (
-        <StoreApps token={token} onChanged={() => void reload()} />
+      {tab === 'store' && mayChange ? (
+        <StoreApps token={token} node={requestNode} onChanged={() => void reload()} />
      ) : apps === null ? (
        <Loading />
      ) : failure ? (
@@ -285,7 +328,7 @@ export function Apps({
        <Empty
          title="No apps installed"
          actions={
-            hub ? undefined : (
+            !mayChange ? undefined : (
              <Button variant="primary" onClick={() => setTab('store')}>
                Open App Store
              </Button>
@@ -294,7 +337,7 @@ export function Apps({
        >
          DNS Apps add behaviour to the server (query logging, advanced blocking, split horizon)
          without touching the base configuration.
-          {hub ? ' Install apps in the console of this server.' : ' Open the store to see what is available.'}
+          {!mayChange && hub ? ' Install apps in the console of this server.' : ' Open the store to see what is available.'}
        </Empty>
      ) : (
        <Panel title="Installed" description="Version, DNS apps and status">
@@ -319,7 +362,8 @@ export function Apps({
                key={app.name}
                app={app}
                busy={busy === app.name}
-                readOnly={hub}
+                readOnly={!mayChange}
+                hideConfig={hub}
                onConfig={() => void openConfig(app.name)}
                onUpdate={() => setModal({ kind: 'update', name: app.name })}
                onStoreUpdate={() => void updateFromStore(app)}
@@ -343,6 +387,7 @@ export function Apps({
        open={modal?.kind === 'install'}
        onOpenChange={(o) => setModal(o ? { kind: 'install' } : null)}
        token={token}
+        node={requestNode}
        onInstalled={(name) => {
          setModal(null)
          setAlert({
@@ -358,6 +403,7 @@ export function Apps({
        onOpenChange={(o) => setModal(o && modal?.kind === 'update' ? modal : null)}
        token={token}
        name={modal?.kind === 'update' ? modal.name : ''}
+        node={requestNode}
        onUpdated={(name) => {
          setModal(null)
          setAlert({
@@ -15,8 +15,8 @@ afterEach(() => {

 const DATA: AppsMatrixData = {
  servers: [
-    { id: 'mdns', ok: true },
-    { id: 'ndns', ok: true },
+    { id: 'mdns', ok: true, writable: false },
+    { id: 'ndns', ok: true, writable: false },
  ],
  apps: [
    {
@@ -121,7 +121,7 @@ describe('AppsMatrix', () => {

  it('marks a server that did not answer as unknown', async () => {
    mockMatrix({
-      servers: [{ id: 'mdns', ok: true }, { id: 'ndns', ok: false, error: 'timed out' }],
+      servers: [{ id: 'mdns', ok: true, writable: false }, { id: 'ndns', ok: false, writable: false, error: 'timed out' }],
      apps: [
        {
          name: 'Split Horizon',
@@ -1,12 +1,14 @@
 import { useCallback, useEffect, useMemo, useState } from 'react'
-import { getAppsMatrix, type AppsMatrixData, type MatrixApp } from '../../api/apps'
+import { getAppsMatrix, listStoreApps, runHubApps, type AppsMatrixData, type HubRunAction, type MatrixApp } from '../../api/apps'
 import { Badge } from '@/components/reui/badge'
 import { Button } from '../../ui/Button'
 import { DataTable, type Col } from '../../ui/DataTable'
 import { Empty, Failure, Loading } from '../../ui/Empty'
 import { Panel } from '../../ui/Panel'
 import { Segmented } from '../../ui/Segmented'
+import { Menu } from '../../ui/Menu'
 import { Tag } from '../../ui/Tag'
+import { RunFlow, pickerServers, type RunRequest } from './RunFlow'
 import { Tooltip } from '../../ui/Tooltip'
 import { head } from '../../ui/Table'

@@ -18,6 +20,16 @@ hash and the comparison, so nothing here can show a secret.

 type Filter = 'all' | 'version' | 'config' | 'update'

+/* 9.10 is newer than 9.9: compared part by part, numerically. */
+export function compareVersions(a: string, b: string): number {
+  const [x, y] = [a, b].map((v) => v.split('.').map((p) => Number.parseInt(p, 10) || 0))
+  for (let i = 0; i < Math.max(x.length, y.length); i++) {
+    const d = (x[i] ?? 0) - (y[i] ?? 0)
+    if (d !== 0) return d
+  }
+  return 0
+}
+
 const hasUpdate = (app: MatrixApp): boolean => Object.values(app.servers).some((c) => c.installed && c.updateAvailable === true)

 const FILTERS: Record<Filter, (app: MatrixApp) => boolean> = {
@@ -70,8 +82,11 @@ function ServerCell({ app, id, data }: { app: MatrixApp; id: string; data: AppsM
  )
 }

-export function AppsMatrix({ token }: { token: string | null }) {
+export function AppsMatrix({ token, canWrite = false, refresh = 0 }: { token: string | null; canWrite?: boolean; refresh?: number }) {
  const [data, setData] = useState<AppsMatrixData | null>(null)
+  const [request, setRequest] = useState<RunRequest | null>(null)
+  /** What the store offers, by app: needed to know whether "Align versions" can be done. */
+  const [store, setStore] = useState<Record<string, string> | null>(null)
  const [failure, setFailure] = useState<string | null>(null)
  const [filter, setFilter] = useState<Filter>('all')

@@ -87,7 +102,14 @@ export function AppsMatrix({ token }: { token: string | null }) {

  useEffect(() => {
    void load()
-  }, [load])
+  }, [load, refresh])
+
+  useEffect(() => {
+    if (!canWrite) return
+    void listStoreApps(token).then((outcome) => {
+      if (outcome.kind === 'ok') setStore(Object.fromEntries(outcome.data.response.storeApps.map((s) => [s.name, s.version])))
+    })
+  }, [canWrite, token])

  const counts = useMemo(() => {
    const apps = data?.apps ?? []
@@ -138,6 +160,65 @@ export function AppsMatrix({ token }: { token: string | null }) {
  }

  const rows = data.apps.filter(FILTERS[filter])
+
+  /* The change menu of one app. Each item opens the server picker with the servers it makes sense on. */
+  function actions(app: MatrixApp, d: AppsMatrixData) {
+    const has = (id: string) => app.servers[id]?.installed === true
+    const installed = d.servers.filter((s) => has(s.id)).map((s) => s.id)
+    const outdated = installed.filter((id) => app.servers[id].updateAvailable === true)
+    const missing = d.servers.filter((s) => s.ok && !has(s.id)).map((s) => s.id)
+    const versions = installed.map((id) => app.servers[id].version ?? '0')
+    const newest = versions.reduce((best, v) => (compareVersions(v, best) > 0 ? v : best), versions[0] ?? '0')
+    const behind = installed.filter((id) => compareVersions(app.servers[id].version ?? '0', newest) < 0)
+    const canAlign = behind.length > 0 && store?.[app.name] === newest
+
+    const open = (
+      action: HubRunAction,
+      title: string,
+      confirmLabel: string,
+      preselected: string[],
+      allowed: string[],
+      why: string,
+      intro?: string,
+    ) =>
+      setRequest({
+        title,
+        intro,
+        confirmLabel,
+        servers: pickerServers(d.servers, Object.fromEntries(d.servers.filter((s) => !allowed.includes(s.id)).map((s) => [s.id, why]))),
+        preselected,
+        run: (targets) => runHubApps(token, action, app.name, targets),
+      })
+
+    return (
+      <Menu label={`Actions: ${app.name}`}>
+        {(close) => (
+          <>
+            <Button size="sm" disabled={outdated.length === 0} onClick={() => { close(); open('update-store', `Update ${app.name}`, 'Update', outdated, installed, 'Not installed') }}>
+              Update on all
+            </Button>
+            <Button size="sm" disabled={missing.length === 0} onClick={() => { close(); open('install-store', `Install ${app.name}`, 'Install', missing, missing, 'Already installed') }}>
+              Install on missing servers
+            </Button>
+            <Button size="sm" disabled={!canAlign} onClick={() => { close(); open('update-store', `Align ${app.name} to v${newest}`, 'Update', behind, behind, 'Already at the newest version') }}>
+              Align versions
+            </Button>
+            <Button
+              size="sm"
+              variant="danger"
+              disabled={installed.length === 0}
+              onClick={() => {
+                close()
+                open('uninstall', `Uninstall ${app.name}`, 'Uninstall', [], installed, 'Not installed', "Each server's config of this app is saved to the hub's history before it is uninstalled.")
+              }}
+            >
+              Uninstall from…
+            </Button>
+          </>
+        )}
+      </Menu>
+    )
+  }
  const columns: Col<MatrixApp>[] = [
    {
      id: 'name',
@@ -157,6 +238,7 @@ export function AppsMatrix({ token }: { token: string | null }) {
      }),
    ),
    { id: 'config', ...head({ children: 'Config' }), cell: (a) => <ConfigState app={a} data={data} /> },
+    ...(canWrite ? [{ id: 'actions', ...head({ children: <span className="sr-only">Actions</span> }), cell: (a: MatrixApp) => actions(a, data) }] : []),
  ]

  return (
@@ -188,6 +270,15 @@ export function AppsMatrix({ token }: { token: string | null }) {
          <DataTable data={rows} columns={columns} getRowId={(a) => a.name} />
        )}
      </Panel>
+      {request !== null && (
+        <RunFlow
+          request={request}
+          onClose={(changed) => {
+            setRequest(null)
+            if (changed) void load()
+          }}
+        />
+      )}
    </div>
  )
 }
@@ -0,0 +1,288 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { render, screen, waitFor, within } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { AppsMatrix } from './AppsMatrix'
+import { Apps } from './Apps'
+import { SlotProvider } from '../../app/ChromeSlot'
+import { HubMode } from '../../ui/ClusterNodeSelect'
+import * as api from '../../api/apps'
+import type { AppsMatrixData, InstalledApp } from '../../api/apps'
+
+afterEach(() => {
+  vi.restoreAllMocks()
+  localStorage.clear()
+})
+
+/* mdns has Blocker v1.0 with an update; ndns has v2.0; Wild IP is only on mdns. ndns has no admin token. */
+const DATA: AppsMatrixData = {
+  servers: [
+    { id: 'mdns', ok: true, writable: true },
+    { id: 'ndns', ok: true, writable: false },
+  ],
+  apps: [
+    {
+      name: 'Blocker',
+      description: '',
+      configs: 'same',
+      versions: 'different',
+      servers: {
+        mdns: { installed: true, version: '1.0', updateAvailable: true, updateVersion: '2.0', configStatus: 'read' },
+        ndns: { installed: true, version: '2.0', configStatus: 'read' },
+      },
+    },
+    {
+      name: 'Wild IP',
+      description: '',
+      configs: 'single',
+      versions: 'single',
+      servers: {
+        mdns: { installed: true, version: '5.1', configStatus: 'read' },
+        ndns: { installed: false, configStatus: 'none' },
+      },
+    },
+  ],
+}
+
+const ok = (results: api.TargetResult[]) => ({ kind: 'ok', data: { response: { results } } }) as never
+
+function mockReads(data: AppsMatrixData = DATA, store = [{ name: 'Blocker', version: '2.0' }]) {
+  const matrix = vi.spyOn(api, 'getAppsMatrix').mockResolvedValue({ kind: 'ok', data: { response: data } } as never)
+  vi.spyOn(api, 'listStoreApps').mockResolvedValue({ kind: 'ok', data: { response: { storeApps: store } } } as never)
+  return matrix
+}
+
+async function openMenu(app: string) {
+  await userEvent.click(await screen.findByRole('button', { name: `Actions: ${app}` }))
+}
+
+describe('matrix actions', () => {
+  it('has no menu for a reader and one per row for a writer', async () => {
+    mockReads()
+    const { unmount } = render(<AppsMatrix token="t" />)
+    await screen.findByText('Blocker')
+    expect(screen.queryByRole('button', { name: /^Actions:/ })).not.toBeInTheDocument()
+    unmount()
+
+    render(<AppsMatrix token="t" canWrite />)
+    expect(await screen.findByRole('button', { name: 'Actions: Blocker' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Actions: Wild IP' })).toBeInTheDocument()
+  })
+
+  it('enables only the actions that make sense for the row', async () => {
+    mockReads()
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Blocker')
+    expect(screen.getByRole('menuitem', { name: 'Update on all' })).toBeEnabled()
+    expect(screen.getByRole('button', { name: 'Install on missing servers' })).toBeDisabled()
+    expect(screen.getByRole('menuitem', { name: 'Align versions' })).toBeEnabled()
+    expect(screen.getByRole('menuitem', { name: 'Uninstall from…' })).toBeEnabled()
+  })
+
+  it('offers Align versions only when the store has the newest installed version', async () => {
+    mockReads(DATA, [{ name: 'Blocker', version: '1.5' }])
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Blocker')
+    await waitFor(() => expect(api.listStoreApps).toHaveBeenCalled())
+    expect(screen.getByRole('button', { name: 'Align versions' })).toBeDisabled()
+  })
+
+  it('offers Install on missing servers for an app that is not everywhere', async () => {
+    mockReads()
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Wild IP')
+    expect(screen.getByRole('menuitem', { name: 'Install on missing servers' })).toBeEnabled()
+    expect(screen.getByRole('button', { name: 'Update on all' })).toBeDisabled()
+  })
+
+  it('shows the picker with a server the hub cannot write to disabled, with the reason', async () => {
+    mockReads()
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Wild IP')
+    await userEvent.click(screen.getByRole('menuitem', { name: 'Install on missing servers' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Install Wild IP' })
+    // mdns already has it, ndns has no admin token: neither can be chosen.
+    expect(within(dialog).getByRole('checkbox', { name: 'mdns' })).toBeDisabled()
+    expect(within(dialog).getByRole('checkbox', { name: 'ndns' })).toBeDisabled()
+    expect(within(dialog).getByText('Already installed')).toBeInTheDocument()
+    expect(within(dialog).getByText('No admin token configured on the hub')).toBeInTheDocument()
+    expect(within(dialog).getByRole('button', { name: 'Install' })).toBeDisabled()
+  })
+
+  it('runs the change on the ticked servers, shows each result and refreshes', async () => {
+    const matrix = mockReads()
+    const run = vi.spyOn(api, 'runHubApps').mockResolvedValue(ok([{ server: 'mdns', ok: true, message: 'Updated from the store.' }]))
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Blocker')
+    await userEvent.click(screen.getByRole('menuitem', { name: 'Update on all' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Update Blocker' })
+    expect(within(dialog).getByRole('checkbox', { name: 'mdns' })).toBeChecked()
+    await userEvent.click(within(dialog).getByRole('button', { name: 'Update' }))
+    expect(run).toHaveBeenCalledWith('t', 'update-store', 'Blocker', ['mdns'])
+
+    const results = await screen.findByRole('dialog', { name: 'Update Blocker: results' })
+    expect(within(results).getByText('Updated from the store.')).toBeInTheDocument()
+    expect(within(results).getByText('Done')).toBeInTheDocument()
+    expect(within(results).queryByRole('button', { name: 'Retry failed' })).not.toBeInTheDocument()
+    const before = matrix.mock.calls.length
+    await userEvent.click(within(results).getAllByRole('button', { name: 'Close' }).at(-1) as HTMLElement)
+    await waitFor(() => expect(matrix.mock.calls.length).toBeGreaterThan(before))
+  })
+
+  it('retries only the servers that failed', async () => {
+    const wide: AppsMatrixData = {
+      servers: [
+        { id: 'mdns', ok: true, writable: true },
+        { id: 'ndns', ok: true, writable: true },
+      ],
+      apps: [{ ...DATA.apps[0], servers: { ...DATA.apps[0].servers, ndns: { installed: true, version: '1.0', updateAvailable: true, configStatus: 'read' } } }],
+    }
+    mockReads(wide)
+    const run = vi
+      .spyOn(api, 'runHubApps')
+      .mockResolvedValueOnce(ok([{ server: 'mdns', ok: true, message: 'Updated from the store.' }, { server: 'ndns', ok: false, message: 'Disk full.' }]))
+      .mockResolvedValueOnce(ok([{ server: 'ndns', ok: true, message: 'Updated from the store.' }]))
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Blocker')
+    await userEvent.click(screen.getByRole('menuitem', { name: 'Update on all' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Update Blocker' })
+    await userEvent.click(within(dialog).getByRole('button', { name: 'Update' }))
+    expect(run).toHaveBeenLastCalledWith('t', 'update-store', 'Blocker', ['mdns', 'ndns'])
+
+    const results = await screen.findByRole('dialog', { name: 'Update Blocker: results' })
+    expect(within(results).getByText('Disk full.')).toBeInTheDocument()
+    await userEvent.click(within(results).getByRole('button', { name: 'Retry failed' }))
+    expect(run).toHaveBeenLastCalledWith('t', 'update-store', 'Blocker', ['ndns'])
+    await waitFor(() => expect(within(screen.getByRole('dialog')).queryByText('Disk full.')).not.toBeInTheDocument())
+    expect(within(screen.getByRole('dialog')).getAllByText('Done')).toHaveLength(2)
+  })
+
+  it('says the config is saved before an uninstall and starts with nothing ticked', async () => {
+    mockReads()
+    const run = vi.spyOn(api, 'runHubApps').mockResolvedValue(ok([{ server: 'mdns', ok: true }]))
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Wild IP')
+    await userEvent.click(screen.getByRole('menuitem', { name: 'Uninstall from…' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Uninstall Wild IP' })
+    expect(within(dialog).getByText(/saved to the hub's history/)).toBeInTheDocument()
+    expect(within(dialog).getByRole('button', { name: 'Uninstall' })).toBeDisabled()
+    await userEvent.click(within(dialog).getByRole('checkbox', { name: 'mdns' }))
+    await userEvent.click(within(dialog).getByRole('button', { name: 'Uninstall' }))
+    expect(run).toHaveBeenCalledWith('t', 'uninstall', 'Wild IP', ['mdns'])
+  })
+
+  it('shows a failed call as a failure with Retry', async () => {
+    mockReads()
+    const run = vi
+      .spyOn(api, 'runHubApps')
+      .mockResolvedValueOnce({ kind: 'error', message: 'Your role does not allow this.' } as never)
+      .mockResolvedValueOnce(ok([{ server: 'mdns', ok: true }]))
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Blocker')
+    await userEvent.click(screen.getByRole('menuitem', { name: 'Update on all' }))
+    await userEvent.click(within(await screen.findByRole('dialog', { name: 'Update Blocker' })).getByRole('button', { name: 'Update' }))
+    const results = await screen.findByRole('dialog', { name: 'Update Blocker: results' })
+    expect(within(results).getByText('Your role does not allow this.')).toBeInTheDocument()
+    await userEvent.click(within(results).getByRole('button', { name: 'Retry' }))
+    await waitFor(() => expect(run).toHaveBeenCalledTimes(2))
+    expect(await within(screen.getByRole('dialog')).findByText('Done')).toBeInTheDocument()
+  })
+})
+
+describe('Apps behind a hub, for writers', () => {
+  const INSTALLED: InstalledApp = { name: 'Split Horizon', description: 'd', version: '9.0', updateAvailable: true, updateVersion: '9.1', updateUrl: 'https://x/s.zip', dnsApps: [] }
+  const NODES = [{ name: 'mdns', type: 'Primary' }, { name: 'ndns', type: 'Secondary' }]
+
+  function mount(props: { canWrite: boolean; writable: string[] }) {
+    return render(
+      <HubMode.Provider value>
+        <SlotProvider>
+          {(ref) => (
+            <>
+              <div ref={ref} />
+              <Apps token="t" nodes={NODES} clusterInitialised {...props} />
+            </>
+          )}
+        </SlotProvider>
+      </HubMode.Provider>,
+    )
+  }
+
+  it('shows App Store and Install from file on All servers to writers only', async () => {
+    mockReads()
+    const { unmount } = mount({ canWrite: false, writable: ['mdns'] })
+    await screen.findByText('Blocker')
+    expect(screen.queryByRole('button', { name: 'App Store' })).not.toBeInTheDocument()
+    expect(screen.queryByRole('button', { name: 'Install from file' })).not.toBeInTheDocument()
+    unmount()
+    mount({ canWrite: true, writable: ['mdns'] })
+    expect(await screen.findByRole('button', { name: 'App Store' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Install from file' })).toBeInTheDocument()
+  })
+
+  it('opens the hub store and sends an install through the picker', async () => {
+    mockReads()
+    vi.spyOn(api, 'listStoreApps').mockResolvedValue({
+      kind: 'ok',
+      data: { response: { storeApps: [{ name: 'Failover', description: 'Backup', version: '8.0', url: 'u', size: '1', installed: false, installedOn: [] }] } },
+    } as never)
+    const run = vi.spyOn(api, 'runHubApps').mockResolvedValue(ok([{ server: 'mdns', ok: true }]))
+    mount({ canWrite: true, writable: ['mdns'] })
+    await userEvent.click(await screen.findByRole('button', { name: 'App Store' }))
+    expect(await screen.findByText('Failover')).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Install…' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Install Failover' })
+    expect(within(dialog).getByRole('checkbox', { name: 'ndns' })).toBeDisabled()
+    await userEvent.click(within(dialog).getByRole('button', { name: 'Install' }))
+    expect(run).toHaveBeenCalledWith('t', 'install-store', 'Failover', ['mdns'])
+  })
+
+  it('uploads a zip to the chosen servers', async () => {
+    mockReads()
+    const upload = vi.spyOn(api, 'uploadHubApp').mockResolvedValue(ok([{ server: 'mdns', ok: true }]))
+    mount({ canWrite: true, writable: ['mdns'] })
+    await userEvent.click(await screen.findByRole('button', { name: 'Install from file' }))
+    const form = await screen.findByRole('dialog', { name: 'Install App' })
+    await userEvent.type(within(form).getByLabelText('App Name'), 'Mine')
+    const file = new File([new Uint8Array([1, 2, 3])], 'Mine.zip', { type: 'application/zip' })
+    await userEvent.upload(within(form).getByLabelText('App Zip File'), file)
+    await userEvent.click(within(form).getByRole('button', { name: 'Choose servers' }))
+    const picker = await screen.findByRole('dialog', { name: 'Install Mine from file' })
+    await userEvent.click(within(picker).getByRole('button', { name: 'Install' }))
+    expect(upload).toHaveBeenCalledWith('t', 'install', 'Mine', file, ['mdns'])
+  })
+
+  it('brings the single-server actions back for a writer, without Config, and sends them to that node', async () => {
+    mockReads()
+    localStorage.setItem('hubAppsNode', 'mdns')
+    vi.spyOn(api, 'listApps').mockResolvedValue({ kind: 'ok', data: { response: { apps: [INSTALLED] } } } as never)
+    const store = vi.spyOn(api, 'downloadAndUpdate').mockResolvedValue({ kind: 'ok', data: { response: {} } } as never)
+    mount({ canWrite: true, writable: ['mdns'] })
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Update' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'App Store' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Install from file' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: /Actions: Split Horizon/ })).toBeInTheDocument()
+    expect(screen.queryByRole('button', { name: 'Config' })).not.toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Store Update' }))
+    expect(store).toHaveBeenCalledWith('t', 'Split Horizon', 'https://x/s.zip', 'mdns')
+  })
+
+  it('keeps a viewer, and a server without an admin token, read-only', async () => {
+    mockReads()
+    vi.spyOn(api, 'listApps').mockResolvedValue({ kind: 'ok', data: { response: { apps: [INSTALLED] } } } as never)
+    localStorage.setItem('hubAppsNode', 'ndns')
+    const { unmount } = mount({ canWrite: true, writable: ['mdns'] })
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    for (const name of ['Update', 'Store Update', 'App Store', 'Install from file']) {
+      expect(screen.queryByRole('button', { name })).not.toBeInTheDocument()
+    }
+    unmount()
+
+    localStorage.setItem('hubAppsNode', 'mdns')
+    mount({ canWrite: false, writable: ['mdns'] })
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    for (const name of ['Update', 'Store Update', 'App Store', 'Install from file']) {
+      expect(screen.queryByRole('button', { name })).not.toBeInTheDocument()
+    }
+  })
+})
@@ -0,0 +1,76 @@
+import { useEffect, useRef, useState } from 'react'
+import { uploadHubApp } from '../../api/apps'
+import { Button } from '../../ui/Button'
+import { Dialog } from '../../ui/Dialog'
+import { LabeledInput } from '../../ui/Field'
+import { Notifier } from '../../ui/Notifier'
+import { pickerServers, type RunRequest } from './RunFlow'
+import type { AlertState } from './Apps'
+
+/*
+"Install from file" in the All servers view: the same two fields as the single-server dialog
+(the validation order and wording are the same), then the server picker. The zip goes to the
+hub once and the hub sends it to every chosen server.
+*/
+export function HubInstallFile({
+  open,
+  onOpenChange,
+  token,
+  servers,
+  onRequest,
+}: {
+  open: boolean
+  onOpenChange: (o: boolean) => void
+  token: string | null
+  servers: { id: string; writable: boolean }[]
+  onRequest: (request: RunRequest) => void
+}) {
+  const [name, setName] = useState('')
+  const [alert, setAlert] = useState<AlertState | null>(null)
+  const fileRef = useRef<HTMLInputElement>(null)
+
+  useEffect(() => {
+    if (open) {
+      setName('')
+      setAlert(null)
+    }
+  }, [open])
+
+  function next() {
+    if (name === '') {
+      setAlert({ type: 'warning', title: 'Missing!', text: 'Please enter an application name.' })
+      return
+    }
+    const file = fileRef.current?.files?.[0]
+    if (!file) {
+      setAlert({ type: 'warning', title: 'Missing!', text: 'Please select an application zip file to install.' })
+      return
+    }
+    onRequest({
+      title: `Install ${name} from file`,
+      intro: 'The zip is sent to every server you choose.',
+      confirmLabel: 'Install',
+      servers: pickerServers(servers),
+      preselected: servers.filter((s) => s.writable).map((s) => s.id),
+      run: (targets) => uploadHubApp(token, 'install', name, file, targets),
+    })
+    onOpenChange(false)
+  }
+
+  return (
+    <Dialog
+      open={open}
+      onOpenChange={onOpenChange}
+      title="Install App"
+      actions={
+        <Button variant="primary" onClick={next}>
+          Choose servers
+        </Button>
+      }
+    >
+      <Notifier notice={alert} onClose={() => setAlert(null)} />
+      <LabeledInput label="App Name" value={name} onChange={(e) => setName(e.target.value)} />
+      <LabeledInput label="App Zip File" type="file" ref={fileRef} />
+    </Dialog>
+  )
+}
@@ -0,0 +1,133 @@
+import { useCallback, useEffect, useState } from 'react'
+import { listStoreApps, runHubApps, type HubRunAction, type StoreApp } from '../../api/apps'
+import { Button } from '../../ui/Button'
+import { Empty, Failure, Loading } from '../../ui/Empty'
+import { Panel } from '../../ui/Panel'
+import { Tag } from '../../ui/Tag'
+import { RunFlow, pickerServers, type RunRequest } from './RunFlow'
+
+/*
+The App Store behind a hub, in the All servers view. It lists what the first server's store
+offers and where each app is installed. Install and Update go to the servers the user picks;
+the hub takes each download URL from that server's own store, so none is sent from here.
+*/
+export function HubStore({
+  token,
+  servers,
+  canWrite,
+  onChanged,
+}: {
+  token: string | null
+  servers: { id: string; writable: boolean }[]
+  canWrite: boolean
+  onChanged: () => void
+}) {
+  const [apps, setApps] = useState<StoreApp[] | null>(null)
+  const [failure, setFailure] = useState<string | null>(null)
+  const [request, setRequest] = useState<RunRequest | null>(null)
+
+  const load = useCallback(async () => {
+    const outcome = await listStoreApps(token)
+    if (outcome.kind === 'ok') {
+      setApps(outcome.data.response.storeApps)
+      setFailure(null)
+    } else setFailure(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+  }, [token])
+
+  useEffect(() => {
+    void load()
+  }, [load])
+
+  function open(app: StoreApp, action: HubRunAction, verb: string) {
+    const on = app.installedOn ?? []
+    const allowed = action === 'install-store' ? servers.map((s) => s.id).filter((id) => !on.includes(id)) : on
+    setRequest({
+      title: `${verb} ${app.name}`,
+      confirmLabel: verb,
+      servers: pickerServers(
+        servers,
+        Object.fromEntries(servers.filter((s) => !allowed.includes(s.id)).map((s) => [s.id, action === 'install-store' ? 'Already installed' : 'Not installed'])),
+      ),
+      preselected: allowed,
+      run: (targets) => runHubApps(token, action, app.name, targets),
+    })
+  }
+
+  if (failure !== null && apps === null) {
+    return (
+      <Failure>
+        <span className="flex flex-col items-center gap-3">
+          <span>Could not load the App Store. {failure}</span>
+          <Button variant="primary" onClick={() => void load()}>
+            Retry
+          </Button>
+        </span>
+      </Failure>
+    )
+  }
+  if (apps === null) return <Loading>Loading the App Store…</Loading>
+  if (apps.length === 0) {
+    return (
+      <Empty
+        title="No apps in the store"
+        actions={
+          <Button variant="primary" onClick={() => void load()}>
+            Refresh
+          </Button>
+        }
+      >
+        The store did not list any app.
+      </Empty>
+    )
+  }
+
+  return (
+    <Panel title="DNS App Store" description="What the store offers and where it is installed" groups2>
+      <ul className="m-0 flex list-none flex-col p-0" aria-label="Store apps">
+        {apps.map((app) => {
+          const on = app.installedOn ?? []
+          return (
+            <li key={app.name} aria-label={app.name} className="flex flex-wrap items-start justify-between gap-4 border-b border-border px-4 py-3 last:border-b-0">
+              <div className="flex min-w-0 flex-1 flex-col gap-2">
+                <span className="flex flex-wrap items-center gap-2 text-sm font-medium">
+                  {app.name}
+                  <Tag>Version {app.version}</Tag>
+                  {on.map((id) => (
+                    <Tag key={id} tone="info">
+                      {id}
+                    </Tag>
+                  ))}
+                </span>
+                <span className="text-xs text-muted-foreground">{app.description}</span>
+              </div>
+              {canWrite && (
+                <div className="flex items-center gap-2">
+                  <Button size="sm" variant="primary" onClick={() => open(app, 'install-store', 'Install')}>
+                    Install…
+                  </Button>
+                  {on.length > 0 && (
+                    <Button size="sm" onClick={() => open(app, 'update-store', 'Update')}>
+                      Update…
+                    </Button>
+                  )}
+                </div>
+              )}
+            </li>
+          )
+        })}
+      </ul>
+      {request !== null && (
+        <RunFlow
+          request={request}
+          onClose={(changed) => {
+            setRequest(null)
+            if (changed) {
+              void load()
+              onChanged()
+            }
+          }}
+        />
+      )}
+    </Panel>
+  )
+}
@@ -17,11 +17,14 @@ export function InstallApp({
  onOpenChange,
  token,
  onInstalled,
+  node = '',
 }: {
  open: boolean
  onOpenChange: (o: boolean) => void
  token: string | null
  onInstalled: (name: string) => void
+  /** Behind a hub: the one server it goes to. */
+  node?: string
 }) {
  const [name, setName] = useState('')
  const [alert, setAlert] = useState<AlertState | null>(null)
@@ -53,7 +56,7 @@ export function InstallApp({
    }

    setBusy(true)
-    const outcome = await installApp(token, name, file)
+    const outcome = await installApp(token, name, file, node)
    setBusy(false)

    if (outcome.kind !== 'ok') {
@@ -0,0 +1,142 @@
+import { useState, type ReactNode } from 'react'
+import type { RunOutcome, TargetResult } from '../../api/apps'
+import { Button } from '../../ui/Button'
+import { Check } from '../../ui/Check'
+import { Dialog } from '../../ui/Dialog'
+import { Failure, Loading } from '../../ui/Empty'
+import { Tag } from '../../ui/Tag'
+
+/*
+Behind a hub, a change to the apps of several servers goes through three steps in one place:
+pick the servers, wait, read what happened on each. A server that cannot be chosen is still
+listed, disabled, with the reason, so nobody wonders where it went. "Retry failed" runs the
+same change again on the servers that failed and nowhere else.
+*/
+
+export interface PickerServer {
+  id: string
+  /** Why this server cannot be chosen; absent when it can. */
+  reason?: string
+}
+
+export interface RunRequest {
+  title: string
+  intro?: ReactNode
+  confirmLabel: string
+  servers: PickerServer[]
+  /** The servers ticked at the start. Only those without a reason count. */
+  preselected: string[]
+  run: (targets: string[]) => Promise<RunOutcome>
+}
+
+/* What stops a server from being a target: it did not answer, or the hub has no admin token for it. */
+export function pickerServers(
+  servers: { id: string; writable: boolean; ok?: boolean }[],
+  extra: Record<string, string> = {},
+): PickerServer[] {
+  return servers.map((s) => {
+    const reason = s.ok === false ? 'Did not answer' : !s.writable ? 'No admin token configured on the hub' : extra[s.id]
+    return reason === undefined ? { id: s.id } : { id: s.id, reason }
+  })
+}
+
+type Stage = 'pick' | 'running' | 'results'
+
+export function RunFlow({ request, onClose }: { request: RunRequest; onClose: (changed: boolean) => void }) {
+  const [stage, setStage] = useState<Stage>('pick')
+  const [picked, setPicked] = useState<string[]>(() =>
+    request.preselected.filter((id) => request.servers.some((s) => s.id === id && s.reason === undefined)),
+  )
+  const [results, setResults] = useState<TargetResult[]>([])
+  const [failure, setFailure] = useState<string | null>(null)
+  const [lastTargets, setLastTargets] = useState<string[]>([])
+
+  async function go(targets: string[]) {
+    setStage('running')
+    setFailure(null)
+    setLastTargets(targets)
+    const outcome = await request.run(targets)
+    if (outcome.kind === 'ok') {
+      const fresh = outcome.data.response.results
+      // A retry replaces only the rows it re-ran.
+      setResults((old) => [...old.filter((r) => !fresh.some((f) => f.server === r.server)), ...fresh])
+    } else {
+      setFailure(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+    }
+    setStage('results')
+  }
+
+  const failed = results.filter((r) => !r.ok).map((r) => r.server)
+  const changed = results.some((r) => r.ok)
+
+  if (stage === 'pick') {
+    return (
+      <Dialog
+        open
+        onOpenChange={() => onClose(false)}
+        title={request.title}
+        close="Cancel"
+        actions={
+          <Button variant="primary" disabled={picked.length === 0} onClick={() => void go(picked)}>
+            {request.confirmLabel}
+          </Button>
+        }
+      >
+        {request.intro != null && <p className="m-0 text-sm text-muted-foreground">{request.intro}</p>}
+        <div className="flex flex-col gap-3" role="group" aria-label="Servers">
+          {request.servers.map((s) => (
+            <Check
+              key={s.id}
+              label={s.id}
+              checked={picked.includes(s.id)}
+              disabled={s.reason !== undefined}
+              help={s.reason}
+              onChange={(on) => setPicked((old) => (on ? [...old, s.id] : old.filter((id) => id !== s.id)))}
+            />
+          ))}
+        </div>
+      </Dialog>
+    )
+  }
+
+  if (stage === 'running') {
+    return (
+      <Dialog open onOpenChange={() => {}} title={request.title} close="Close">
+        <Loading>Working on {lastTargets.join(', ')}…</Loading>
+      </Dialog>
+    )
+  }
+
+  return (
+    <Dialog
+      open
+      onOpenChange={() => onClose(changed)}
+      title={`${request.title}: results`}
+      actions={
+        failure !== null ? (
+          <Button variant="primary" onClick={() => void go(lastTargets)}>
+            Retry
+          </Button>
+        ) : failed.length > 0 ? (
+          <Button variant="primary" onClick={() => void go(failed)}>
+            Retry failed
+          </Button>
+        ) : undefined
+      }
+    >
+      {failure !== null ? (
+        <Failure>{failure}</Failure>
+      ) : (
+        <ul className="m-0 flex list-none flex-col gap-3 p-0" aria-label="Results">
+          {results.map((r) => (
+            <li key={r.server} className="flex flex-wrap items-center gap-2 text-sm">
+              <span className="font-medium">{r.server}</span>
+              <Tag tone={r.ok ? 'ok' : 'dan'}>{r.ok ? 'Done' : 'Failed'}</Tag>
+              {r.message !== undefined && <span className="text-muted-foreground">{r.message}</span>}
+            </li>
+          ))}
+        </ul>
+      )}
+    </Dialog>
+  )
+}
@@ -30,8 +30,11 @@ and the freshly placed row is left without its "Store Update" button.
 export function StoreApps({
  token,
  onChanged,
+  node = '',
 }: {
  token: string | null
+  /** Behind a hub: the one server it acts on. */
+  node?: string
  onChanged: () => void
 }) {
  const [storeApps, setStoreApps] = useState<StoreApp[] | null>(null)
@@ -50,7 +53,7 @@ export function StoreApps({
  */

  const load = useCallback(async () => {
-    const outcome = await listStoreApps(token)
+    const outcome = await listStoreApps(token, node)
    if (outcome.kind === 'ok') {
      setStoreApps(outcome.data.response.storeApps)
      return
@@ -85,7 +88,7 @@ export function StoreApps({
        title: 'Store App Installed!',
        text: `DNS application '${app.name}' was installed successfully from DNS App Store.`,
      },
-      downloadAndInstall(token, app.name, app.url),
+      downloadAndInstall(token, app.name, app.url, node),
    )
  }

@@ -97,7 +100,7 @@ export function StoreApps({
        title: 'Store App Updated!',
        text: `DNS application '${app.name}' was updated successfully from DNS App Store.`,
      },
-      downloadAndUpdate(token, app.name, app.url),
+      downloadAndUpdate(token, app.name, app.url, node),
    )
  }

@@ -116,7 +119,7 @@ export function StoreApps({
        title: 'Store App Uninstalled!',
        text: `DNS application '${app.name}' was uninstalled successfully.`,
      },
-      uninstallApp(token, app.name),
+      uninstallApp(token, app.name, node),
    )
  }

@@ -19,12 +19,15 @@ export function UpdateApp({
  token,
  name,
  onUpdated,
+  node = '',
 }: {
  open: boolean
  onOpenChange: (o: boolean) => void
  token: string | null
  name: string
  onUpdated: (name: string) => void
+  /** Behind a hub: the one server it goes to. */
+  node?: string
 }) {
  const [alert, setAlert] = useState<AlertState | null>(null)
  const [busy, setBusy] = useState(false)
@@ -49,7 +52,7 @@ export function UpdateApp({
    }

    setBusy(true)
-    const outcome = await updateApp(token, name, file)
+    const outcome = await updateApp(token, name, file, node)
    setBusy(false)

    if (outcome.kind !== 'ok') {