Denozordec
4b8cf83ee9
Docker images / prepare-release (push) Successful in 5s
Docker images / backend-image (push) Successful in 1m34s
Docker images / frontend-image (push) Successful in 1m41s
Docker images / updater-image (push) Successful in 45s
Docker images / notify-webhook (push) Has been skipped
Docker images / publish-release (push) Successful in 7s
525 lines
15 KiB
Bash
525 lines
15 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
TARGETS_FILE="${TARGETS_FILE:-/etc/updater/targets.json}"
|
|
STATE_FILE="${STATE_FILE:-/state/updater-state.json}"
|
|
POLL_INTERVAL_SECONDS="${POLL_INTERVAL_SECONDS:-300}"
|
|
HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-120}"
|
|
STOP_TIMEOUT_SECONDS="${STOP_TIMEOUT_SECONDS:-30}"
|
|
REGISTRY="${REGISTRY:-git.shts.su}"
|
|
MANAGED_LABEL="mmapp.updater.managed"
|
|
TARGET_LABEL="mmapp.updater.target"
|
|
IMAGE_LABEL="mmapp.updater.image"
|
|
|
|
log() {
|
|
local level="$1"
|
|
shift
|
|
printf '%s level=%s %s\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$level" "$*"
|
|
}
|
|
|
|
require_command() {
|
|
command -v "$1" >/dev/null 2>&1 || {
|
|
log error "missing_command=$1"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
registry_login() {
|
|
if [[ -n "${REGISTRY_USERNAME:-}" && -n "${REGISTRY_PASSWORD:-}" ]]; then
|
|
log info "action=registry_login registry=${REGISTRY}"
|
|
printf '%s' "$REGISTRY_PASSWORD" | docker login "$REGISTRY" -u "$REGISTRY_USERNAME" --password-stdin >/dev/null
|
|
fi
|
|
}
|
|
|
|
validate_targets_file() {
|
|
if [[ ! -f "$TARGETS_FILE" ]]; then
|
|
log error "targets_file_missing path=${TARGETS_FILE}"
|
|
exit 1
|
|
fi
|
|
jq -e '.targets | type == "array" and length > 0' "$TARGETS_FILE" >/dev/null
|
|
}
|
|
|
|
state_init() {
|
|
mkdir -p "$(dirname "$STATE_FILE")"
|
|
if [[ ! -f "$STATE_FILE" ]]; then
|
|
printf '{}\n' >"$STATE_FILE"
|
|
fi
|
|
}
|
|
|
|
state_get() {
|
|
local target_id="$1"
|
|
local field="$2"
|
|
jq -r --arg id "$target_id" --arg field "$field" '.[$id][$field] // empty' "$STATE_FILE"
|
|
}
|
|
|
|
state_set() {
|
|
local target_id="$1"
|
|
local field="$2"
|
|
local value="$3"
|
|
local tmp
|
|
tmp="$(mktemp)"
|
|
jq --arg id "$target_id" --arg field "$field" --arg value "$value" \
|
|
'.[$id] = (.[$id] // {}) | .[$id][$field] = $value' "$STATE_FILE" >"$tmp"
|
|
mv "$tmp" "$STATE_FILE"
|
|
}
|
|
|
|
state_clear_error() {
|
|
local target_id="$1"
|
|
local tmp
|
|
tmp="$(mktemp)"
|
|
jq --arg id "$target_id" '.[$id].last_error = null' "$STATE_FILE" >"$tmp"
|
|
mv "$tmp" "$STATE_FILE"
|
|
}
|
|
|
|
state_set_error() {
|
|
local target_id="$1"
|
|
local message="$2"
|
|
local tmp
|
|
tmp="$(mktemp)"
|
|
jq --arg id "$target_id" --arg message "$message" '.[$id].last_error = $message' "$STATE_FILE" >"$tmp"
|
|
mv "$tmp" "$STATE_FILE"
|
|
}
|
|
|
|
cleanup_legacy_lock_dirs() {
|
|
local locks_dir="/state/locks"
|
|
[[ -d "$locks_dir" ]] || return 0
|
|
find "$locks_dir" -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} + 2>/dev/null || true
|
|
}
|
|
|
|
with_target_lock() {
|
|
local target_id="$1"
|
|
shift
|
|
local lock_file="/state/locks/${target_id}.lock"
|
|
mkdir -p "$(dirname "$lock_file")"
|
|
|
|
if [[ -f "$lock_file" ]]; then
|
|
local lock_pid=""
|
|
lock_pid="$(tr -d '[:space:]' <"$lock_file" 2>/dev/null || true)"
|
|
if [[ -n "$lock_pid" ]] && kill -0 "$lock_pid" 2>/dev/null; then
|
|
log warn "target=${target_id} action=skip reason=lock_busy pid=${lock_pid}"
|
|
return 0
|
|
fi
|
|
|
|
log warn "target=${target_id} action=stale_lock_removed previous_pid=${lock_pid:-unknown}"
|
|
rm -f "$lock_file"
|
|
fi
|
|
|
|
printf '%s\n' "$$" >"$lock_file"
|
|
trap 'rm -f "'"$lock_file"'"' RETURN
|
|
"$@"
|
|
}
|
|
|
|
remote_digest() {
|
|
local image="$1"
|
|
docker buildx imagetools inspect "$image" --format '{{json .Manifest}}' 2>/dev/null | jq -er '.digest // empty'
|
|
}
|
|
|
|
running_image_digest() {
|
|
local container_name="$1"
|
|
local image_id
|
|
image_id="$(docker inspect --format '{{.Image}}' "$container_name" 2>/dev/null || true)"
|
|
if [[ -z "$image_id" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
local repo_digest
|
|
repo_digest="$(docker image inspect "$image_id" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)"
|
|
if [[ -n "$repo_digest" && "$repo_digest" == *@* ]]; then
|
|
printf '%s' "${repo_digest#*@}"
|
|
return 0
|
|
fi
|
|
|
|
docker image inspect "$image_id" --format '{{.Id}}'
|
|
}
|
|
|
|
validate_managed_container() {
|
|
local target_id="$1"
|
|
local container_name="$2"
|
|
local expected_image="$3"
|
|
|
|
if ! docker inspect "$container_name" >/dev/null 2>&1; then
|
|
log warn "target=${target_id} action=skip reason=container_missing name=${container_name}"
|
|
return 1
|
|
fi
|
|
|
|
local managed target_label image_label
|
|
managed="$(docker inspect --format "{{ index .Config.Labels \"${MANAGED_LABEL}\" }}" "$container_name")"
|
|
target_label="$(docker inspect --format "{{ index .Config.Labels \"${TARGET_LABEL}\" }}" "$container_name")"
|
|
image_label="$(docker inspect --format "{{ index .Config.Labels \"${IMAGE_LABEL}\" }}" "$container_name")"
|
|
|
|
if [[ "$managed" != "true" ]]; then
|
|
log warn "target=${target_id} action=skip reason=not_managed name=${container_name}"
|
|
return 1
|
|
fi
|
|
|
|
if [[ -n "$target_label" && "$target_label" != "$target_id" ]]; then
|
|
log warn "target=${target_id} action=skip reason=target_label_mismatch label=${target_label}"
|
|
return 1
|
|
fi
|
|
|
|
if [[ -n "$image_label" && "$image_label" != "$expected_image" ]]; then
|
|
log warn "target=${target_id} action=skip reason=image_label_mismatch label=${image_label}"
|
|
return 1
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
snapshot_container() {
|
|
local container_name="$1"
|
|
local snapshot_file="$2"
|
|
docker inspect "$container_name" | jq '.[0]' >"$snapshot_file"
|
|
}
|
|
|
|
image_with_digest() {
|
|
local image="$1"
|
|
local digest="$2"
|
|
local base="${image%@*}"
|
|
|
|
if docker image inspect "$digest" >/dev/null 2>&1; then
|
|
printf '%s' "$digest"
|
|
return 0
|
|
fi
|
|
|
|
local repo="$base"
|
|
if [[ "$base" == *:* ]]; then
|
|
repo="${base%:*}"
|
|
fi
|
|
|
|
local ref="${repo}@${digest}"
|
|
if docker image inspect "$ref" >/dev/null 2>&1; then
|
|
printf '%s' "$ref"
|
|
return 0
|
|
fi
|
|
|
|
printf '%s' "$ref"
|
|
}
|
|
|
|
resolve_image_ref() {
|
|
local image="$1"
|
|
local digest="$2"
|
|
local ref
|
|
ref="$(image_with_digest "$image" "$digest")"
|
|
|
|
if docker image inspect "$ref" >/dev/null 2>&1; then
|
|
printf '%s' "$ref"
|
|
return 0
|
|
fi
|
|
|
|
docker pull "$ref" >/dev/null
|
|
printf '%s' "$ref"
|
|
}
|
|
|
|
build_run_args_from_snapshot() {
|
|
local snapshot_file="$1"
|
|
local args=()
|
|
|
|
while IFS= read -r env_line; do
|
|
args+=(--env "$env_line")
|
|
done < <(jq -r '.Config.Env[]?' "$snapshot_file")
|
|
|
|
while IFS= read -r mount_line; do
|
|
args+=(--mount "$mount_line")
|
|
done < <(jq -r '
|
|
.Mounts[]?
|
|
| if .Type == "bind" then
|
|
"type=bind,source=\(.Source),target=\(.Destination)\(if .RW == false then ",readonly" else "" end)"
|
|
elif .Type == "volume" then
|
|
"type=volume,source=\(.Name),target=\(.Destination)"
|
|
else empty end
|
|
' "$snapshot_file")
|
|
|
|
while IFS= read -r publish_line; do
|
|
args+=(-p "$publish_line")
|
|
done < <(jq -r '
|
|
.HostConfig.PortBindings // {}
|
|
| to_entries[]
|
|
| .key as $containerPort
|
|
| .value[0] as $binding
|
|
| if ($binding.HostIp // "") != "" and $binding.HostIp != "0.0.0.0" then
|
|
"\($binding.HostIp):\($binding.HostPort):\($containerPort | sub("/tcp$"; "") | sub("/udp$"; ""))"
|
|
else
|
|
"\($binding.HostPort):\($containerPort | sub("/tcp$"; "") | sub("/udp$"; ""))"
|
|
end
|
|
' "$snapshot_file")
|
|
|
|
local restart_policy
|
|
restart_policy="$(jq -r '.HostConfig.RestartPolicy.Name // empty' "$snapshot_file")"
|
|
if [[ -n "$restart_policy" && "$restart_policy" != "no" ]]; then
|
|
args+=(--restart "$restart_policy")
|
|
fi
|
|
|
|
local user
|
|
user="$(jq -r '.Config.User // empty' "$snapshot_file")"
|
|
if [[ -n "$user" ]]; then
|
|
args+=(--user "$user")
|
|
fi
|
|
|
|
while IFS= read -r label_line; do
|
|
args+=(--label "$label_line")
|
|
done < <(jq -r '
|
|
.Config.Labels // {}
|
|
| to_entries[]
|
|
| "\(.key)=\(.value)"
|
|
' "$snapshot_file")
|
|
|
|
local network_mode
|
|
local network_name=""
|
|
network_mode="$(jq -r '.HostConfig.NetworkMode // empty' "$snapshot_file")"
|
|
if [[ "$network_mode" == "host" ]]; then
|
|
args+=(--network host)
|
|
elif [[ -n "$network_mode" && "$network_mode" != "default" && "$network_mode" != "bridge" ]]; then
|
|
network_name="$network_mode"
|
|
args+=(--network "$network_name")
|
|
else
|
|
network_name="$(jq -r '.NetworkSettings.Networks // {} | keys[0] // empty' "$snapshot_file")"
|
|
if [[ -n "$network_name" && "$network_name" != "bridge" ]]; then
|
|
args+=(--network "$network_name")
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "$network_name" ]]; then
|
|
while IFS= read -r alias; do
|
|
[[ -n "$alias" ]] || continue
|
|
args+=(--network-alias "$alias")
|
|
done < <(jq -r --arg net "$network_name" '
|
|
.NetworkSettings.Networks[$net].Aliases // []
|
|
| .[]
|
|
' "$snapshot_file")
|
|
fi
|
|
|
|
if [[ "$(jq -r '.HostConfig.Privileged // false' "$snapshot_file")" == "true" ]]; then
|
|
args+=(--privileged)
|
|
fi
|
|
|
|
if [[ "$(jq -r '.HostConfig.ReadonlyRootfs // false' "$snapshot_file")" == "true" ]]; then
|
|
args+=(--read-only)
|
|
fi
|
|
|
|
printf '%s\0' "${args[@]}"
|
|
}
|
|
|
|
run_container_from_snapshot() {
|
|
local container_name="$1"
|
|
local snapshot_file="$2"
|
|
local image="$3"
|
|
|
|
local -a run_args=()
|
|
mapfile -d '' -t run_args < <(build_run_args_from_snapshot "$snapshot_file")
|
|
|
|
docker run -d --name "$container_name" "${run_args[@]}" "$image"
|
|
}
|
|
|
|
wait_for_health() {
|
|
local target_id="$1"
|
|
local health_type="$2"
|
|
local health_url="$3"
|
|
local expect_status="$4"
|
|
local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS))
|
|
local last_status=""
|
|
|
|
while (( SECONDS < deadline )); do
|
|
if [[ "$health_type" == "http" ]]; then
|
|
local status
|
|
status="$(curl --silent --output /dev/null --write-out '%{http_code}' --max-time 5 "$health_url" || true)"
|
|
last_status="$status"
|
|
if [[ "$status" == "$expect_status" ]]; then
|
|
log info "target=${target_id} action=health_ok status=${status}"
|
|
return 0
|
|
fi
|
|
else
|
|
log error "target=${target_id} action=health_fail reason=unsupported_type type=${health_type}"
|
|
return 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
log error "target=${target_id} action=health_fail reason=timeout seconds=${HEALTH_TIMEOUT_SECONDS} url=${health_url} last_status=${last_status:-none}"
|
|
return 1
|
|
}
|
|
|
|
ensure_baseline_state() {
|
|
local target_id="$1"
|
|
local container_name="$2"
|
|
local image="$3"
|
|
|
|
local applied
|
|
applied="$(state_get "$target_id" "last_applied_digest")"
|
|
if [[ -n "$applied" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local digest=""
|
|
if digest="$(running_image_digest "$container_name" 2>/dev/null)"; then
|
|
:
|
|
elif digest="$(remote_digest "$image" 2>/dev/null)"; then
|
|
:
|
|
fi
|
|
|
|
if [[ -n "$digest" ]]; then
|
|
state_set "$target_id" "image" "$image"
|
|
state_set "$target_id" "last_applied_digest" "$digest"
|
|
log info "target=${target_id} action=baseline_state digest=${digest}"
|
|
fi
|
|
}
|
|
|
|
restore_container() {
|
|
local target_id="$1"
|
|
local container_name="$2"
|
|
local snapshot_file="$3"
|
|
local image="$4"
|
|
local previous_digest="$5"
|
|
|
|
docker rm -f "$container_name" >/dev/null 2>&1 || true
|
|
if [[ -n "$previous_digest" ]]; then
|
|
local rollback_image
|
|
if ! rollback_image="$(resolve_image_ref "$image" "$previous_digest")"; then
|
|
log error "target=${target_id} action=rollback_failed reason=image_unavailable digest=${previous_digest}"
|
|
return 1
|
|
fi
|
|
run_container_from_snapshot "$container_name" "$snapshot_file" "$rollback_image" || {
|
|
log error "target=${target_id} action=rollback_failed"
|
|
return 1
|
|
}
|
|
log info "target=${target_id} action=rollback_success digest=${previous_digest}"
|
|
else
|
|
log error "target=${target_id} action=rollback_skipped reason=no_previous_digest"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
update_target() {
|
|
local target_id="$1"
|
|
local container_name="$2"
|
|
local image="$3"
|
|
local health_type="$4"
|
|
local health_url="$5"
|
|
local expect_status="$6"
|
|
|
|
validate_managed_container "$target_id" "$container_name" "$image" || return 0
|
|
ensure_baseline_state "$target_id" "$container_name" "$image"
|
|
|
|
local remote
|
|
if ! remote="$(remote_digest "$image")"; then
|
|
state_set_error "$target_id" "remote_digest_unavailable"
|
|
log error "target=${target_id} action=remote_digest_failed image=${image}"
|
|
return 1
|
|
fi
|
|
|
|
local applied
|
|
applied="$(state_get "$target_id" "last_applied_digest")"
|
|
if [[ "$remote" == "$applied" ]]; then
|
|
log info "target=${target_id} action=noop digest=${remote}"
|
|
state_clear_error "$target_id"
|
|
return 0
|
|
fi
|
|
|
|
log info "target=${target_id} action=update_start remote_digest=${remote} applied_digest=${applied:-none}"
|
|
|
|
if ! docker pull "$image"; then
|
|
state_set_error "$target_id" "pull_failed"
|
|
log error "target=${target_id} action=pull_failed image=${image}"
|
|
return 1
|
|
fi
|
|
|
|
local snapshot_file
|
|
snapshot_file="$(mktemp)"
|
|
snapshot_container "$container_name" "$snapshot_file"
|
|
|
|
local previous_digest
|
|
previous_digest="$(running_image_digest "$container_name" || true)"
|
|
|
|
docker stop -t "$STOP_TIMEOUT_SECONDS" "$container_name" >/dev/null
|
|
docker rm "$container_name" >/dev/null
|
|
|
|
if ! run_container_from_snapshot "$container_name" "$snapshot_file" "$image"; then
|
|
state_set_error "$target_id" "recreate_failed"
|
|
log error "target=${target_id} action=recreate_failed"
|
|
restore_container "$target_id" "$container_name" "$snapshot_file" "$image" "$previous_digest" || true
|
|
rm -f "$snapshot_file"
|
|
return 1
|
|
fi
|
|
|
|
if wait_for_health "$target_id" "$health_type" "$health_url" "$expect_status"; then
|
|
state_set "$target_id" "image" "$image"
|
|
state_set "$target_id" "last_applied_digest" "$remote"
|
|
if [[ -n "$previous_digest" ]]; then
|
|
state_set "$target_id" "previous_digest" "$previous_digest"
|
|
fi
|
|
state_set "$target_id" "last_success_at" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
|
state_clear_error "$target_id"
|
|
log info "target=${target_id} action=update_success digest=${remote}"
|
|
rm -f "$snapshot_file"
|
|
return 0
|
|
fi
|
|
|
|
state_set_error "$target_id" "health_failed"
|
|
log error "target=${target_id} action=rollback_start"
|
|
restore_container "$target_id" "$container_name" "$snapshot_file" "$image" "$previous_digest" || true
|
|
rm -f "$snapshot_file"
|
|
return 1
|
|
}
|
|
|
|
process_targets() {
|
|
local had_error=0
|
|
local target_count
|
|
target_count="$(jq '.targets | length' "$TARGETS_FILE")"
|
|
|
|
for (( index=0; index<target_count; index++ )); do
|
|
local target_id container_name image health_type health_url expect_status
|
|
target_id="$(jq -r ".targets[$index].id" "$TARGETS_FILE")"
|
|
container_name="$(jq -r ".targets[$index].container_name" "$TARGETS_FILE")"
|
|
image="$(jq -r ".targets[$index].image" "$TARGETS_FILE")"
|
|
health_type="$(jq -r ".targets[$index].health.type" "$TARGETS_FILE")"
|
|
health_url="$(jq -r ".targets[$index].health.url" "$TARGETS_FILE")"
|
|
expect_status="$(jq -r ".targets[$index].health.expect_status" "$TARGETS_FILE")"
|
|
|
|
with_target_lock "$target_id" update_target \
|
|
"$target_id" "$container_name" "$image" "$health_type" "$health_url" "$expect_status" \
|
|
|| had_error=1
|
|
done
|
|
|
|
return "$had_error"
|
|
}
|
|
|
|
sleep_with_jitter() {
|
|
local base="$1"
|
|
local jitter_max=$((base / 10))
|
|
local jitter=0
|
|
if (( jitter_max > 0 )); then
|
|
jitter=$((RANDOM % (jitter_max + 1)))
|
|
fi
|
|
sleep $((base + jitter))
|
|
}
|
|
|
|
main() {
|
|
require_command docker
|
|
require_command jq
|
|
require_command curl
|
|
|
|
validate_targets_file
|
|
state_init
|
|
cleanup_legacy_lock_dirs
|
|
registry_login
|
|
|
|
log info "action=startup targets_file=${TARGETS_FILE} state_file=${STATE_FILE} poll_interval=${POLL_INTERVAL_SECONDS}"
|
|
|
|
local backoff=0
|
|
while true; do
|
|
if process_targets; then
|
|
backoff=0
|
|
else
|
|
backoff=$((POLL_INTERVAL_SECONDS * 2))
|
|
log warn "action=cycle_failed backoff_seconds=${backoff}"
|
|
fi
|
|
|
|
local sleep_for="$POLL_INTERVAL_SECONDS"
|
|
if (( backoff > 0 )); then
|
|
sleep_for="$backoff"
|
|
fi
|
|
sleep_with_jitter "$sleep_for"
|
|
done
|
|
}
|
|
|
|
main "$@"
|