From 0148d4ca37b36d58592ef4bb177631375bc31e10 Mon Sep 17 00:00:00 2001 From: Denozordec Date: Tue, 18 Aug 2026 18:22:26 +0700 Subject: [PATCH] feat(docker): enhance base image mirroring process and update documentation - Updated `mirror-base-images.sh` to skip existing tags in Gitea and retry on 429 errors, improving efficiency and reducing unnecessary requests. - Modified CI workflow to utilize a temporary environment file for mirrored base images, ensuring better handling of existing images. - Enhanced README documentation to reflect changes in the mirroring process and clarify the behavior when tags already exist. --- .gitea/README.md | 2 +- .gitea/workflows/cd.yaml | 17 +++--- deploy/docker/README.md | 2 +- deploy/docker/mirror-base-images.sh | 81 ++++++++++++++++++++++++----- 4 files changed, 76 insertions(+), 26 deletions(-) diff --git a/.gitea/README.md b/.gitea/README.md index f66b0e2..a4f298e 100644 --- a/.gitea/README.md +++ b/.gitea/README.md @@ -24,7 +24,7 @@ Runner: `ubuntu-latest`, **bird2** из apt, Docker для **docker-check** (PR) 1. `pnpm exec semantic-release` — тег `vX.Y.Z` на **текущий commit** (без дополнительного commit в main). 2. Gitea Release + `CHANGELOG.md` как attachment (не в git). -3. Зеркало base-образов в `evobgp-buildcache:base-*` (`deploy/docker/mirror-base-images.sh`). +3. Зеркало base-образов в `evobgp-buildcache:base-*` (`deploy/docker/mirror-base-images.sh`; skip существующих тегов, `linux/amd64`, retry при 429). 4. `docker buildx bake default --push` с `VERSION=X.Y.Z`, `pull=false`, named builder `evobgp` (`cleanup: false`). Если releasable-коммитов нет — semantic-release no-op, образы не публикуются. diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index f11bef0..82b0e7a 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -108,10 +108,9 @@ jobs: password: ${{ secrets.ACTIONS_PAT }} - name: Mirror base images into buildcache if: steps.rel.outputs.released == 'true' - id: mirror - continue-on-error: true env: REGISTRY: git.shx.one/${{ steps.meta.outputs.owner_lc }} + MIRROR_ENV_FILE: ${{ runner.temp }}/mirror-base.env run: sh deploy/docker/mirror-base-images.sh - name: Build and push images (bake) if: steps.rel.outputs.released == 'true' @@ -126,16 +125,14 @@ jobs: CACHE_REF_WEB: git.shx.one/${{ steps.meta.outputs.owner_lc }}/evobgp-buildcache:web-buildcache CACHE_REF_BIRDC: git.shx.one/${{ steps.meta.outputs.owner_lc }}/evobgp-buildcache:birdc-buildcache BUILDX_BAKE_ENTITLEMENTS_FS: "0" + MIRROR_ENV_FILE: ${{ runner.temp }}/mirror-base.env run: | set -euxo pipefail - cache="git.shx.one/${{ steps.meta.outputs.owner_lc }}/evobgp-buildcache" - if [ "${{ steps.mirror.outcome }}" = "success" ]; then - export BASE_GOLANG="${cache}:base-golang-1.24-alpine" - export BASE_DEBIAN="${cache}:base-debian-bookworm-slim" - export BASE_NODE="${cache}:base-node-22-alpine" - export BASE_NGINX="${cache}:base-nginx-1.27-alpine" - export BASE_UBUNTU="${cache}:base-ubuntu-noble" - export BASE_DISTROLESS="${cache}:base-distroless-static-debian12-nonroot" + if [ -f "${MIRROR_ENV_FILE}" ]; then + set -a + # shellcheck disable=SC1090 + . "${MIRROR_ENV_FILE}" + set +a fi docker buildx bake --allow=fs.read="${{ github.workspace }}" \ -f deploy/docker/docker-bake.hcl default --push diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 8c6435d..54357e9 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -27,7 +27,7 @@ BuildKit кэширует `/go/pkg/mod`, `~/.cache/go-build` и pnpm store че - `git.shx.one//evobgp-buildcache:go-buildcache` — **запись** только из `go-build-all` - `git.shx.one//evobgp-buildcache:web-buildcache` — **запись** только из `web-build` - `git.shx.one//evobgp-buildcache:birdc-buildcache` — **запись** только из `go-birdc` -- `git.shx.one//evobgp-buildcache:base-*` — зеркало FROM-образов (`mirror-base-images.sh`), без rebuild +- `git.shx.one//evobgp-buildcache:base-*` — зеркало FROM (`mirror-base-images.sh`): только `linux/amd64`, skip если тег уже в Gitea (без повторного pull с ECR / 429). Неуспешный copy не валит CD — bake берёт public FROM для этой базы. `pull = false` в bake: не перекачивать FROM, если слой уже в builder. Не запускайте `docker system prune -a` на runner. diff --git a/deploy/docker/mirror-base-images.sh b/deploy/docker/mirror-base-images.sh index 1cf8d4f..a199150 100644 --- a/deploy/docker/mirror-base-images.sh +++ b/deploy/docker/mirror-base-images.sh @@ -2,27 +2,80 @@ # Copy-by-digest зеркало базовых образов в Gitea Container Registry (без rebuild). # Требует: docker login в REGISTRY host, docker buildx. # REGISTRY=git.shx.one/ (без суффикса /evobgp-buildcache) +# +# Не ходим в public.ecr.aws, если тег уже есть в Gitea (иначе 429 Too Many Requests). +# Копируем только linux/amd64 — bake platforms совпадает, multi-arch индекс ECR не нужен. +# Шаг CD не должен падать: неуспешный copy оставляет bake на public FROM для этой базы. set -eu REGISTRY="${REGISTRY:?REGISTRY required (git.shx.one/)}" CACHE_REPO="${REGISTRY}/evobgp-buildcache" +ENV_FILE="${MIRROR_ENV_FILE:-}" +PLATFORM="${MIRROR_PLATFORM:-linux/amd64}" +RETRIES="${MIRROR_RETRIES:-4}" + +tmp="$(mktemp)" +cleanup() { rm -f "$tmp"; } +trap cleanup EXIT + +dest_exists() { + docker buildx imagetools inspect "$1" >/dev/null 2>&1 +} + +copy_retry() { + src="$1" + dest="$2" + n=0 + while [ "$n" -lt "$RETRIES" ]; do + n=$((n + 1)) + if docker buildx imagetools create --platform "$PLATFORM" --tag "$dest" "$src"; then + return 0 + fi + delay=$((n * 25)) + echo "mirror retry ${n}/${RETRIES}, sleep ${delay}s: ${dest}" + sleep "$delay" + done + return 1 +} mirror() { src="$1" tag="$2" - echo "mirror ${src} -> ${CACHE_REPO}:${tag}" - docker buildx imagetools create --tag "${CACHE_REPO}:${tag}" "${src}" + var="$3" + dest="${CACHE_REPO}:${tag}" + if dest_exists "$dest"; then + echo "skip (already in registry): ${dest}" + printf '%s=%s\n' "$var" "$dest" >>"$tmp" + return 0 + fi + echo "mirror ${src} -> ${dest} (${PLATFORM})" + if copy_retry "$src" "$dest"; then + printf '%s=%s\n' "$var" "$dest" >>"$tmp" + return 0 + fi + echo "warn: ${dest} not mirrored — bake uses public FROM for ${var}" + return 0 } -mirror "public.ecr.aws/docker/library/golang:1.24-alpine" "base-golang-1.24-alpine" -mirror "public.ecr.aws/docker/library/debian:bookworm-slim" "base-debian-bookworm-slim" -mirror "public.ecr.aws/docker/library/node:22-alpine" "base-node-22-alpine" -mirror "public.ecr.aws/docker/library/nginx:1.27-alpine" "base-nginx-1.27-alpine" -mirror "public.ecr.aws/docker/library/ubuntu:noble" "base-ubuntu-noble" -mirror "gcr.io/distroless/static-debian12:nonroot" "base-distroless-static-debian12-nonroot" +mirror "public.ecr.aws/docker/library/golang:1.24-alpine" "base-golang-1.24-alpine" BASE_GOLANG +sleep 8 +mirror "public.ecr.aws/docker/library/debian:bookworm-slim" "base-debian-bookworm-slim" BASE_DEBIAN +sleep 8 +mirror "public.ecr.aws/docker/library/node:22-alpine" "base-node-22-alpine" BASE_NODE +sleep 8 +mirror "public.ecr.aws/docker/library/nginx:1.27-alpine" "base-nginx-1.27-alpine" BASE_NGINX +sleep 8 +mirror "public.ecr.aws/docker/library/ubuntu:noble" "base-ubuntu-noble" BASE_UBUNTU +sleep 8 +mirror "gcr.io/distroless/static-debian12:nonroot" "base-distroless-static-debian12-nonroot" BASE_DISTROLESS -echo "BASE_GOLANG=${CACHE_REPO}:base-golang-1.24-alpine" -echo "BASE_DEBIAN=${CACHE_REPO}:base-debian-bookworm-slim" -echo "BASE_NODE=${CACHE_REPO}:base-node-22-alpine" -echo "BASE_NGINX=${CACHE_REPO}:base-nginx-1.27-alpine" -echo "BASE_UBUNTU=${CACHE_REPO}:base-ubuntu-noble" -echo "BASE_DISTROLESS=${CACHE_REPO}:base-distroless-static-debian12-nonroot" +echo "----- mirrored BASE_* -----" +cat "$tmp" +if [ -n "$ENV_FILE" ]; then + env_dir="$(dirname "$ENV_FILE")" + if [ -d "$env_dir" ]; then + cp "$tmp" "$ENV_FILE" + echo "wrote ${ENV_FILE}" + else + echo "warn: MIRROR_ENV_FILE dir missing (${env_dir}) — skip write" + fi +fi