From 3a44941d2c5fcd8822398af171d5e9fbbd786b10 Mon Sep 17 00:00:00 2001 From: Tobias Brunner Date: Fri, 17 Apr 2026 15:18:11 +0200 Subject: [PATCH] github: Remove commit ID from cache keys and manually evict old entries Let's try this again :) Since cache entries with the same key are not updated/replaced and there is no option to do so, we manually delete the previous entry for the current branch. This reduces the cache storage for active branches, which can cause caches of the master branch to get evicted, which in turn will slow down builds of not only master but also new branches as they can't fall back on those caches. Permission has to be explicitly granted in order to delete the cache entries when not using the legacy all-write tokens that are the default for old repositories. The continue-on-error option is set for the step that deletes the old cache entry as it's expected that cache-hit will be true for a new feature branch when restoring the cache from the master branch. However, because there won't be anything to delete for this branch yet, the command will fail. The --succeed-on-no-caches option of the command unfortunately only works with --all. For the Linux tests, several jobs use the same cache key. So there is a chance that two jobs try to store a new entry concurrently, which will fail (it works if there was a cache hit and they are slightly off as previous entries are first deleted). To avoid that, we store the cache only for one particular config. Also made sure that the "openssl" test does not remove "openssl-3/4" caches by adding a suffix to the former. For alpine, the repository had to be set explicitly as gh wasn't able to determine it (didn't detect the Git working dir). --- .../{openssl => openssl-sys} | 0 .github/workflows/android.yml | 22 ++++- .github/workflows/codeql.yml | 25 +++-- .github/workflows/linux.yml | 96 +++++++++++++++---- .github/workflows/macos.yml | 22 ++++- .github/workflows/sonarcloud.yml | 25 +++-- .github/workflows/tkm.yml | 22 ++++- .github/workflows/windows.yml | 42 ++++++-- 8 files changed, 194 insertions(+), 60 deletions(-) rename .github/active-transforms/{openssl => openssl-sys} (100%) diff --git a/.github/active-transforms/openssl b/.github/active-transforms/openssl-sys similarity index 100% rename from .github/active-transforms/openssl rename to .github/active-transforms/openssl-sys diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index a36ebb93d..79c91c3f7 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -6,6 +6,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + actions: write + env: CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: true @@ -42,12 +45,11 @@ jobs: echo Using NDK ${NDK_VERSION} yes | ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager --install "ndk;${NDK_VERSION}" echo "ANDROID_NDK_ROOT=${ANDROID_HOME}/ndk/${NDK_VERSION}" >> "$GITHUB_OUTPUT" - - uses: actions/cache@v5 + - uses: actions/cache/restore@v5 + id: cache-restore with: path: ~/.cache/ccache - key: ccache-android-${{ github.sha }} - restore-keys: | - ccache-android- + key: ccache-android # necessary for newer versions of the Gradle plugin - uses: actions/setup-java@v5 with: @@ -61,8 +63,18 @@ jobs: - uses: ./.github/actions/default env: ANDROID_NDK_ROOT: ${{ steps.ndk-install.outputs.ANDROID_NDK_ROOT }} - - run: ccache -s + - run: ccache -sv - uses: actions/upload-artifact@v6 with: name: Lint Results path: src/frontends/android/app/build/reports/lint-results*.xml + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.cache-restore.outputs.cache-hit + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} + - uses: actions/cache/save@v5 + with: + path: ~/.cache/ccache + key: ${{ steps.cache-restore.outputs.cache-primary-key }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6343818d2..f2007a5b1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -30,8 +30,7 @@ jobs: if: ${{ needs.pre-check.outputs.should_skip != 'true' }} runs-on: ubuntu-latest permissions: - actions: read - contents: read + actions: write security-events: write strategy: fail-fast: false @@ -51,14 +50,13 @@ jobs: # this follows the steps of the Linux workflow - if: matrix.language == 'cpp' - uses: actions/cache@v5 + uses: actions/cache/restore@v5 + id: cache-restore with: path: ~/.cache/ccache - key: ccache-ubuntu-latest-gcc-codeql-${{ github.sha }} + key: ccache-ubuntu-latest-gcc-codeql restore-keys: | - ccache-ubuntu-latest-gcc-codeql - ccache-ubuntu-latest-gcc-all-${{ github.sha }} - ccache-ubuntu-latest-gcc-all- + ccache-ubuntu-latest-gcc-all ccache-ubuntu-latest-gcc- - if: matrix.language == 'cpp' run: | @@ -70,7 +68,18 @@ jobs: TEST: codeql uses: ./.github/actions/default - if: matrix.language == 'cpp' - run: ccache -s + run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - if: matrix.language == 'cpp' && steps.cache-restore.outputs.cache-hit + env: + GH_TOKEN: ${{ github.token }} + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} + - if: matrix.language == 'cpp' + uses: actions/cache/save@v5 + with: + path: ~/.cache/ccache + key: ${{ steps.cache-restore.outputs.cache-primary-key }} - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index d13e8dc14..66a4d588f 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -6,6 +6,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + actions: write + env: # this test case does not actually test anything but tries to access system # directories that might be inaccessible on build hosts @@ -64,25 +67,44 @@ jobs: MONOLITHIC: ${{ matrix.monolithic || 'no' }} CC: ${{ matrix.compiler || 'gcc' }} TEST: ${{ matrix.test }} + # as several jobs use the same key, make sure we only store the cache for + # one specific config in case there is a race + STORE_CACHE: >- + ${{ + !contains(fromJSON('["all", "default", "printf-builtin"]'), + matrix.test) || + (matrix.leak-detective == 'no' && matrix.monolithic == 'no') + }} steps: - uses: actions/checkout@v6 - - uses: actions/cache@v5 + - uses: actions/cache/restore@v5 + id: cache-restore with: path: ~/.cache/ccache # with regards to ccache, monolithic builds don't differ from regular # builds and, similarly, builds with leak-detective only differ in two # files (LD itself and library.c); but different tests build different # dependencies, so different caches are needed - key: ccache-ubuntu-latest-${{ env.CC }}-${{ matrix.test }}-${{ github.sha }} + key: ccache-ubuntu-latest-${{ env.CC }}-${{ matrix.test }} restore-keys: | - ccache-ubuntu-latest-${{ env.CC }}-${{ matrix.test }}- ccache-ubuntu-latest-${{ env.CC }}- - run: | sudo apt-get install -qq ccache echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV ccache -z - uses: ./.github/actions/default - - run: ccache -s + - run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.cache-restore.outputs.cache-hit && fromJSON(env.STORE_CACHE) + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} + - if: fromJSON(env.STORE_CACHE) + uses: actions/cache/save@v5 + with: + path: ~/.cache/ccache + key: ${{ steps.cache-restore.outputs.cache-primary-key }} - if: ${{ success() && matrix.test == 'coverage' }} uses: codecov/codecov-action@v4 with: @@ -105,7 +127,7 @@ jobs: strategy: fail-fast: false matrix: - test: [ botan, wolfssl, openssl, openssl-3, openssl-4, openssl-awslc, gcrypt ] + test: [ botan, wolfssl, openssl-sys, openssl-3, openssl-4, openssl-awslc, gcrypt ] os: [ ubuntu-latest, ubuntu-22.04 ] leak-detective: [ no, yes ] exclude: @@ -125,19 +147,19 @@ jobs: CC: ${{ matrix.compiler || 'gcc' }} TEST: ${{ matrix.test }} ACTIVE_TRANSFORMS_REF: .github/active-transforms/${{ matrix.test }} + STORE_CACHE: ${{ !matrix.leak-detective || matrix.leak-detective == 'no' }} steps: - uses: actions/checkout@v6 - - uses: actions/cache@v5 + - uses: actions/cache/restore@v5 + id: cache-restore with: # path is different on newer systems path: | ~/.cache/ccache ~/.ccache - key: ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }}-${{ github.sha }} + key: ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }} restore-keys: | - ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }}- - ccache-${{ matrix.os }}-${{ env.CC }}-all-${{ github.sha }} - ccache-${{ matrix.os }}-${{ env.CC }}-all- + ccache-${{ matrix.os }}-${{ env.CC }}-all ccache-${{ matrix.os }}-${{ env.CC }}- - run: | sudo apt-get install -qq ccache @@ -154,7 +176,20 @@ jobs: - name: Verify active transforms run: | test ! -f $ACTIVE_TRANSFORMS_REF || diff -u --color=always $ACTIVE_TRANSFORMS_REF $TESTS_ACTIVE_TRANSFORMS - - run: ccache -s + - run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.cache-restore.outputs.cache-hit && fromJSON(env.STORE_CACHE) + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} + - if: fromJSON(env.STORE_CACHE) + uses: actions/cache/save@v5 + with: + path: | + ~/.cache/ccache + ~/.ccache + key: ${{ steps.cache-restore.outputs.cache-primary-key }} - if: ${{ failure() }} uses: actions/upload-artifact@v6 with: @@ -180,22 +215,34 @@ jobs: TEST: ${{ matrix.test }} steps: - uses: actions/checkout@v6 - - uses: actions/cache@v5 + - uses: actions/cache/restore@v5 + id: cache-restore with: # path is different on newer systems path: | ~/.cache/ccache ~/.ccache - key: ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }}-${{ github.sha }} + key: ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }} restore-keys: | - ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }}- ccache-${{ matrix.os }}-${{ env.CC }}- - run: | sudo apt-get install -qq ccache echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV ccache -z - uses: ./.github/actions/default - - run: ccache -s + - run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.cache-restore.outputs.cache-hit + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} + - uses: actions/cache/save@v5 + with: + path: | + ~/.cache/ccache + ~/.ccache + key: ${{ steps.cache-restore.outputs.cache-primary-key }} - if: ${{ failure() }} uses: actions/upload-artifact@v6 with: @@ -216,19 +263,28 @@ jobs: - uses: actions/checkout@v6 # install tar and zstd before the cache action that requires them - run: | - apk add ccache tar zstd + apk add ccache tar zstd github-cli echo "PATH=/usr/lib/ccache/bin:$PATH" >> $GITHUB_ENV - - uses: actions/cache@v5 + - uses: actions/cache/restore@v5 + id: cache-restore with: path: ~/.cache/ccache - key: ccache-alpine-${{ github.sha }} - restore-keys: | - ccache-alpine- + key: ccache-alpine - run: ccache -z # don't use the default action as we don't want to build dependencies or install bash - run: ./scripts/test.sh deps - run: ./scripts/test.sh - run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.cache-restore.outputs.cache-hit + continue-on-error: true + run: gh cache delete --repo ${{ github.repository }} -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} + - uses: actions/cache/save@v5 + with: + path: ~/.cache/ccache + key: ${{ steps.cache-restore.outputs.cache-primary-key }} - if: ${{ failure() }} uses: actions/upload-artifact@v6 with: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2bd155bfc..4644f1983 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -6,6 +6,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + actions: write + env: TESTS_REDUCED_KEYLENGTHS: yes CCACHE_BASEDIR: ${{ github.workspace }} @@ -36,12 +39,11 @@ jobs: TEST: macos steps: - uses: actions/checkout@v6 - - uses: actions/cache@v5 + - uses: actions/cache/restore@v5 + id: cache-restore with: path: ~/Library/Caches/ccache - key: ccache-${{ runner.os }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}- + key: ccache-${{ runner.os }} # workaround for conflict between Python installed in the image and via brew - run: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete -print - run: | @@ -49,7 +51,17 @@ jobs: echo "PATH=$(brew --prefix)/opt/ccache/libexec:$PATH" >> $GITHUB_ENV ccache -z - uses: ./.github/actions/default - - run: ccache -s + - run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.cache-restore.outputs.cache-hit + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} + - uses: actions/cache/save@v5 + with: + path: ~/Library/Caches/ccache + key: ${{ steps.cache-restore.outputs.cache-primary-key }} - if: ${{ failure() }} uses: actions/upload-artifact@v6 with: diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 47d9e8be2..17340d468 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -6,6 +6,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + actions: write + env: CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: true @@ -33,13 +36,11 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 - - uses: actions/cache@v5 + - uses: actions/cache/restore@v5 + id: cache-restore with: - path: | - ~/.cache/ccache - key: ccache-sonarcloud-${{ github.sha }} - restore-keys: | - ccache-sonarcloud- + path: ~/.cache/ccache + key: ccache-sonarcloud - run: | sudo apt-get install -qq ccache echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV @@ -57,4 +58,14 @@ jobs: -Dsonar.organization=${{ secrets.SONAR_ORGANIZATION }} -Dsonar.cfamily.threads=2 -Dsonar.cfamily.compile-commands=${{ env.BUILD_WRAPPER_OUT_DIR }}/compile_commands.json - - run: ccache -s + - run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.cache-restore.outputs.cache-hit + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} + - uses: actions/cache/save@v5 + with: + path: ~/.cache/ccache + key: ${{ steps.cache-restore.outputs.cache-primary-key }} diff --git a/.github/workflows/tkm.yml b/.github/workflows/tkm.yml index c6be0af31..d57cb4437 100644 --- a/.github/workflows/tkm.yml +++ b/.github/workflows/tkm.yml @@ -6,6 +6,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + actions: write + env: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_CONTAINER: /root/.ccache @@ -32,12 +35,11 @@ jobs: TEST: tkm steps: - uses: actions/checkout@v6 - - uses: actions/cache@v5 + - uses: actions/cache/restore@v5 + id: cache-restore with: path: ${{ env.CCACHE_DIR }} - key: ccache-tkm-${{ github.sha }} - restore-keys: | - ccache-tkm- + key: ccache-tkm - name: Build Docker Image run: docker build -t strongswan-tkm -f testing/tkm/Dockerfile testing - name: Run Tests in Container @@ -72,5 +74,15 @@ jobs: echo '### run the tests against TKM and get TKM log'; \ make -j check TESTS_RUNNERS=tkm TESTS_TKM=1 || exit 1; \ cat /tmp/tkm.log; \ - ccache -s; \ + ccache -sv; \ " + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.cache-restore.outputs.cache-hit + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} + - uses: actions/cache/save@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ steps.cache-restore.outputs.cache-primary-key }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 958a75e1c..6cf331a02 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -6,6 +6,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + actions: write + env: TESTS_REDUCED_KEYLENGTHS: yes CCACHE_COMPRESS: true @@ -39,18 +42,27 @@ jobs: TEST: ${{ matrix.test }} steps: - uses: actions/checkout@v6 - - uses: actions/cache@v5 + - uses: actions/cache/restore@v5 + id: cache-restore with: path: ~/.cache/ccache - key: ccache-${{ runner.os }}-${{ matrix.test }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ matrix.test }}- + key: ccache-${{ runner.os }}-${{ matrix.test }} - run: | sudo apt-get install -qq ccache echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV ccache -sz - uses: ./.github/actions/default - - run: ccache -s + - run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.cache-restore.outputs.cache-hit + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} + - uses: actions/cache/save@v5 + with: + path: ~/.cache/ccache + key: ${{ steps.cache-restore.outputs.cache-primary-key }} - if: ${{ failure() }} uses: actions/upload-artifact@v6 with: @@ -84,12 +96,11 @@ jobs: - run: git config --global core.autocrlf input shell: bash - uses: actions/checkout@v6 - - uses: actions/cache@v5 + - uses: actions/cache/restore@v5 + id: cache-restore with: path: ~\AppData\Local\ccache - key: ccache-${{ runner.os }}-${{ matrix.test }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ matrix.test }}- + key: ccache-${{ runner.os }}-${{ matrix.test }} - run: | pacman --noconfirm -S --needed mingw-w64-${{ matrix.arch }}-ccache ccache -sz @@ -101,7 +112,18 @@ jobs: ./scripts/test.sh # sometimes streaming/threading tests hang completely timeout-minutes: 30 - - run: ccache -s + - run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.cache-restore.outputs.cache-hit + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} + shell: bash + - uses: actions/cache/save@v5 + with: + path: ~\AppData\Local\ccache + key: ${{ steps.cache-restore.outputs.cache-primary-key }} - if: ${{ failure() }} uses: actions/upload-artifact@v6 with: