From 3b902ea59b86a8ac097e27a34db85feb877a00f9 Mon Sep 17 00:00:00 2001 From: Tobias Brunner Date: Tue, 12 May 2026 10:46:29 +0200 Subject: [PATCH] github: Use separate caches for custom-built dependencies These are shared by many tests, in particular the "all", "coverage", "no-dbg" and "no-testable-ke" tests, which each would otherwise require their own large cache. Similarly, the "codeql" and "sonarcloud" tests rely on the same dependencies but only the latter uses ccache for the strongSwan build. Also reduce the maximum size per cache for all workflows to keep them in check over time (some could even be set lower, we'll have to see how this develops). --- .github/workflows/android.yml | 2 +- .github/workflows/codeql.yml | 48 ++++++++-- .github/workflows/linux.yml | 148 ++++++++++++++++++++++--------- .github/workflows/sonarcloud.yml | 21 +++-- .github/workflows/tkm.yml | 2 +- .github/workflows/windows.yml | 2 +- 6 files changed, 166 insertions(+), 57 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 4dff2256c..a29c8da7b 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -12,7 +12,7 @@ permissions: env: CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: true - CCACHE_MAXSIZE: 400M + CCACHE_MAXSIZE: 150M CC: gcc OS_NAME: linux diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 5e3b39a3e..b01a9e2c9 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -7,6 +7,7 @@ concurrency: cancel-in-progress: true env: + CCACHE_BASEDIR: ${{ github.workspace }} OS_NAME: linux jobs: @@ -30,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'cpp', 'python', 'ruby' ] + language: [ 'python', 'ruby' ] steps: - uses: actions/checkout@v6 - name: Initialize CodeQL @@ -40,16 +41,47 @@ jobs: config-file: ./.github/codeql/config.yml trap-caching: false - - if: matrix.language == 'python' || matrix.language == 'ruby' - name: Autobuild + - name: Autobuild uses: github/codeql-action/autobuild@v4 - - if: matrix.language == 'cpp' - env: - TEST: codeql - uses: ./.github/actions/default - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" + + analyze-cpp: + needs: pre-check + if: ${{ needs.pre-check.outputs.should_skip != 'true' }} + runs-on: ubuntu-latest + permissions: + actions: write + security-events: write + env: + TEST: codeql + steps: + - uses: actions/checkout@v6 + - run: ./scripts/test.sh deps + - uses: actions/cache/restore@v5 + with: + path: ~/.cache/ccache + key: ccache-deps-ubuntu-latest-gcc-all + - run: | + sudo apt-get install -qq ccache + echo "OLD_PATH=$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + ccache -z + - run: ./scripts/test.sh build-deps + - run: ccache -sv + - run: echo "PATH=$OLD_PATH" >> $GITHUB_ENV + + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: cpp + config-file: ./.github/codeql/config.yml + trap-caching: false + - run: ./scripts/test.sh + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + category: "/language:cpp" diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 0c0bd90e7..2a21f0c4e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -16,7 +16,7 @@ env: TESTS_REDUCED_KEYLENGTHS: yes CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: true - CCACHE_MAXSIZE: 200M + CCACHE_MAXSIZE: 100M OS_NAME: linux jobs: @@ -67,18 +67,21 @@ jobs: TEST: ${{ matrix.test }} # as several jobs use the same key, make sure we only store the cache for # one specific config in case there is a race + STORE_DEPS_CACHE: >- + ${{ + github.event_name == 'push' && + matrix.test == 'all' && + matrix.monolithic == 'no' + }} STORE_CACHE: >- ${{ github.event_name == 'push' && !contains(fromJSON('["apidoc"]'), matrix.test) && (!contains(fromJSON('["all", "default", "printf-builtin"]'), - matrix.test) || - ((!matrix.leak-detective || matrix.leak-detective == 'no') && - matrix.monolithic == 'no')) + matrix.test) || matrix.monolithic == 'no') }} - # with regards to ccache, monolithic builds don't differ from regular - # builds; but some tests build different dependencies or use different - # compiler flags, so we use different caches for these + # with regards to ccache, monolithic builds don't differ from regular builds. + # but most tests use different compiler flags, so we use separate caches CACHE_KEY: >- ${{ case(contains(fromJSON('["apidoc"]'), matrix.test), 'ccache-ubuntu-latest-gcc-default', @@ -86,16 +89,37 @@ jobs: matrix.test)) }} steps: - uses: actions/checkout@v6 + - run: ./scripts/test.sh deps + - uses: actions/cache/restore@v5 + id: deps-cache-restore + with: + path: ~/.cache/ccache + key: ccache-deps-ubuntu-latest-${{ env.CC }}-all + - run: | + sudo apt-get install -qq ccache + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + ccache -z + - run: ./scripts/test.sh build-deps + - run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.deps-cache-restore.outputs.cache-hit && fromJSON(env.STORE_DEPS_CACHE) + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.deps-cache-restore.outputs.cache-primary-key }} + - if: fromJSON(env.STORE_DEPS_CACHE) + uses: actions/cache/save@v5 + with: + path: ~/.cache/ccache + key: ${{ steps.deps-cache-restore.outputs.cache-primary-key }} + - run: rm -rf ~/.cache/ccache - uses: actions/cache/restore@v5 id: cache-restore with: path: ~/.cache/ccache key: ${{ env.CACHE_KEY }} - - run: | - sudo apt-get install -qq ccache - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - ccache -z - - uses: ./.github/actions/default + - run: ccache -z + - run: ./scripts/test.sh - run: ccache -sv # delete old cache entry as we currently can't update it any other way - env: @@ -151,29 +175,58 @@ jobs: ACTIVE_TRANSFORMS_REF: .github/active-transforms/${{ matrix.test }} # only store a cache for some tests as the others have a minimal diff (if # any) compared to the 'all' build - STORE_CACHE: >- - ${{ github.event_name == 'push' && - contains(fromJSON('["openssl-4", "openssl-awslc"]'), matrix.test) && - (!matrix.leak-detective || matrix.leak-detective == 'no') }} + STORE_DEPS_CACHE: >- + ${{ + github.event_name == 'push' && + contains(fromJSON('["openssl-4", "openssl-awslc"]'), matrix.test) + }} + DEPS_CACHE_KEY: >- + ${{ + case(contains(fromJSON('["openssl-4", "openssl-awslc"]'), matrix.test), + format('ccache-deps-{0}-gcc-{1}', matrix.os, matrix.test), + format('ccache-deps-{0}-gcc-all', matrix.os)) + }} CACHE_KEY: >- - ${{ case(contains(fromJSON('["openssl-4", "openssl-awslc"]'), matrix.test), - format('ccache-{0}-gcc-{1}', matrix.os, matrix.test), - matrix.os == 'ubuntu-latest' && matrix.test == 'openssl-sys', - format('ccache-{0}-gcc-default', matrix.os), - format('ccache-{0}-gcc-all', matrix.os)) }} + ${{ + case(matrix.os == 'ubuntu-latest' && matrix.test == 'openssl-sys', + format('ccache-{0}-gcc-default', matrix.os), + format('ccache-{0}-gcc-all', matrix.os)) + }} steps: - uses: actions/checkout@v6 + - run: ./scripts/test.sh deps - uses: actions/cache/restore@v5 - id: cache-restore + id: deps-cache-restore with: path: ~/.cache/ccache - key: ${{ env.CACHE_KEY }} + key: ${{ env.DEPS_CACHE_KEY }} - run: | sudo apt-get install -qq ccache echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV ccache -z echo "TESTS_ACTIVE_TRANSFORMS=$HOME/active-transforms.log" >> $GITHUB_ENV - - uses: ./.github/actions/default + - run: ./scripts/test.sh build-deps + - run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.deps-cache-restore.outputs.cache-hit && fromJSON(env.STORE_DEPS_CACHE) + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.deps-cache-restore.outputs.cache-primary-key }} + - if: fromJSON(env.STORE_DEPS_CACHE) + uses: actions/cache/save@v5 + with: + path: ~/.cache/ccache + key: ${{ steps.deps-cache-restore.outputs.cache-primary-key }} + - run: rm -rf ~/.cache/ccache + - uses: actions/cache/restore@v5 + id: cache-restore + with: + path: ~/.cache/ccache + key: ${{ env.CACHE_KEY }} + - run: ccache -z + - run: ./scripts/test.sh + - run: ccache -sv - name: Upload active transforms uses: actions/upload-artifact@v6 with: @@ -184,17 +237,6 @@ jobs: run: | test ! -f $ACTIVE_TRANSFORMS_REF || diff -us --color=always $ACTIVE_TRANSFORMS_REF $TESTS_ACTIVE_TRANSFORMS - run: ccache -sv - # delete old cache entry as we currently can't update it any other way - - env: - GH_TOKEN: ${{ github.token }} - if: steps.cache-restore.outputs.cache-hit && fromJSON(env.STORE_CACHE) - continue-on-error: true - run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }} - - if: fromJSON(env.STORE_CACHE) - uses: actions/cache/save@v5 - with: - path: ~/.cache/ccache - key: ${{ steps.cache-restore.outputs.cache-primary-key }} - if: ${{ failure() }} uses: actions/upload-artifact@v6 with: @@ -218,18 +260,44 @@ jobs: LEAK_DETECTIVE: ${{ matrix.leak-detective || 'no' }} CC: ${{ matrix.compiler || 'gcc' }} TEST: ${{ matrix.test }} + STORE_DEPS_CACHE: >- + ${{ + github.event_name == 'push' && + matrix.test == 'all' + }} steps: - uses: actions/checkout@v6 + - run: ./scripts/test.sh deps + - uses: actions/cache/restore@v5 + id: deps-cache-restore + with: + path: ~/.cache/ccache + key: ccache-deps-${{ matrix.os }}-${{ env.CC }}-all + - run: | + sudo apt-get install -qq ccache + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + ccache -z + - run: ./scripts/test.sh build-deps + - run: ccache -sv + # delete old cache entry as we currently can't update it any other way + - env: + GH_TOKEN: ${{ github.token }} + if: steps.deps-cache-restore.outputs.cache-hit && fromJSON(env.STORE_DEPS_CACHE) + continue-on-error: true + run: gh cache delete -r ${{ github.ref }} ${{ steps.deps-cache-restore.outputs.cache-primary-key }} + - if: fromJSON(env.STORE_DEPS_CACHE) + uses: actions/cache/save@v5 + with: + path: ~/.cache/ccache + key: ${{ steps.deps-cache-restore.outputs.cache-primary-key }} + - run: rm -rf ~/.cache/ccache - uses: actions/cache/restore@v5 id: cache-restore with: path: ~/.cache/ccache key: ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }} - - run: | - sudo apt-get install -qq ccache - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - ccache -z - - uses: ./.github/actions/default + - run: ccache -z + - run: ./scripts/test.sh - run: ccache -sv # delete old cache entry as we currently can't update it any other way - env: diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 9b8cb2688..d6c02698b 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -12,7 +12,7 @@ permissions: env: CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: true - CCACHE_MAXSIZE: 200M + CCACHE_MAXSIZE: 100M OS_NAME: linux jobs: @@ -36,19 +36,28 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 + - run: ./scripts/test.sh deps + - uses: actions/cache/restore@v5 + with: + path: ~/.cache/ccache + key: ccache-deps-ubuntu-latest-gcc-all + - run: | + sudo apt-get install -qq ccache + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + ccache -z + - run: ./scripts/test.sh build-deps + - run: ccache -sv + - run: rm -rf ~/.cache/ccache - uses: actions/cache/restore@v5 id: cache-restore with: path: ~/.cache/ccache key: ccache-sonarcloud - - run: | - sudo apt-get install -qq ccache - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - ccache -z + - run: ccache -z - uses: SonarSource/sonarqube-scan-action/install-build-wrapper@v7.1.0 - run: | echo "BUILD_WRAPPER_OUT_DIR=$HOME/bw-output" >> $GITHUB_ENV - - uses: ./.github/actions/default + - run: ./scripts/test.sh - uses: SonarSource/sonarqube-scan-action@v7.1.0 env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/tkm.yml b/.github/workflows/tkm.yml index 9d10376a6..7eb2d6b1a 100644 --- a/.github/workflows/tkm.yml +++ b/.github/workflows/tkm.yml @@ -14,7 +14,7 @@ env: CCACHE_CONTAINER: /root/.ccache CCACHE_COMPILERCHECK: content CCACHE_COMPRESS: true - CCACHE_MAXSIZE: 200M + CCACHE_MAXSIZE: 100M jobs: pre-check: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e8cb20861..177ce7c8a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -12,7 +12,7 @@ permissions: env: TESTS_REDUCED_KEYLENGTHS: yes CCACHE_COMPRESS: true - CCACHE_MAXSIZE: 200M + CCACHE_MAXSIZE: 100M # since the compilers are newly installed every time, we have to use this to # avoid cache misses CCACHE_COMPILERCHECK: content