diff --git a/.github/actions/cache-restore/action.yml b/.github/actions/cache-restore/action.yml new file mode 100644 index 00000000000..b0b53842fc7 --- /dev/null +++ b/.github/actions/cache-restore/action.yml @@ -0,0 +1,173 @@ +name: 'Cache Restore Action' +description: 'Restores caches of downloaded files and build artifacts.' +author: 'Andrew Kaster ' + +inputs: + os: + description: 'Operating System to restore caches for' + required: true + default: 'Linux' + arch: + description: 'Target Architecture to restore caches for' + required: false + default: 'x86_64' + toolchain: + description: 'Toolchain to restore caches for (GNU or Clang)' + required: false + default: 'GNU' + cache_key_extra: + description: 'Code coverage setting, ON or OFF, or debug setting, ALL or NORMAL' + required: false + default: 'OFF' + ccache_version: + description: 'Increment this number if CI has trouble with ccache.' + required: false + default: '0' + serenity_ccache_path: + description: 'Path to the SerenityOS ccache directory' + required: false + default: '' + toolchain_ccache_path: + description: 'Path to the toolchain ccache directory' + required: false + default: '' + download_cache_path: + description: 'Path to the download cache directory' + required: false + default: 'caches' + +outputs: + qemu_cache_primary_key: + description: 'Primary key for the AArch64 QEMU cache' + value: ${{ steps.qemu-cache.outputs.cache-primary-key }} + qemu_cache_hit: + description: 'Whether the AArch64 QEMU cache was hit' + value: ${{ steps.qemu-cache.outputs.cache-hit }} + serenity_ccache_primary_key: + description: 'Primary key for the Serenity ccache' + value: ${{ steps.serenity-ccache.outputs.cache-primary-key }} + toolchain_ccache_primary_key: + description: 'Primary key for the Toolchain ccache' + value: ${{ steps.toolchain-ccache.outputs.cache-hit }} + toolchain_prebuilt_primary_key: + description: 'Primary key for the toolchain cache' + value: ${{ steps.toolchain-prebuilt.outputs.cache-primary-key }} + toolchain_prebuilt_path: + description: 'Path to the prebuilt toolchain' + value: ${{ steps.toolchain-stamps.outputs.toolchain_path }} + toolchain_prebuilt_hit: + description: 'Whether the toolchain prebuilt cache was hit' + value: ${{ steps.toolchain-prebuilt.outputs.cache-hit }} + +runs: + using: "composite" + steps: + - name: 'Date Stamp' + shell: bash + id: 'date-stamp' + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: 'Toolchain Stamps' + shell: bash + id: 'toolchain-stamps' + if: ${{ inputs.arch != 'Lagom' }} + run: | + if ${{ inputs.toolchain == 'Clang' }} ; then + echo "toolchain_path=${{ github.workspace }}/Toolchain/Local/clang" >> "$GITHUB_OUTPUT" + + hashed_files='${{ hashFiles('Toolchain/BuildClang.sh', 'Toolchain/Patches/llvm/*.patch', 'Toolchain/CMake/*.cmake', 'Userland/Libraries/LibC/**/*.h') }}' + echo "toolchain_stamp=${hashed_files}" >> "$GITHUB_OUTPUT" + + source $GITHUB_WORKSPACE/Ports/llvm/package.sh + echo "toolchain_version=$(echo ${version} | cut -d'.' -f1)" >> "$GITHUB_OUTPUT" + elif ${{ inputs.toolchain == 'GNU' }} ; then + echo "toolchain_path=${{ github.workspace }}/Toolchain/Local/${{ inputs.arch }}" >> $GITHUB_OUTPUT + + hashed_files='${{ hashFiles('Toolchain/BuildGNU.sh', 'Toolchain/Patches/binutils/*.patch', 'Toolchain/Patches/gcc/*.patch', 'Userland/Libraries/LibC/**/*.h') }}' + echo "toolchain_stamp=${hashed_files}" >> $GITHUB_OUTPUT + + source $GITHUB_WORKSPACE/Ports/gcc/package.sh + echo "toolchain_version=$(echo ${version} | cut -d'.' -f1)" >> $GITHUB_OUTPUT + fi + + - name: 'Toolchain Prebuilt Cache' + uses: actions/cache/restore@v4 + id: 'toolchain-prebuilt' + if: ${{ inputs.arch != 'Lagom' }} + with: + path: ${{ steps.toolchain-stamps.outputs.toolchain_path }} + key: '"toolchain" | "${{ inputs.arch }}" | "${{ inputs.toolchain }}" | "${{ steps.toolchain-stamps.outputs.toolchain_stamp }}"' + + # FIXME: Remove manually built QEMU when we bump QEMU to >=8.1.x + - name: 'AArch64 QEMU Cache' + id: 'qemu-cache' + uses: actions/cache/restore@v4 + if: ${{ inputs.arch == 'aarch64' }} + with: + path: ${{ github.workspace }}/Toolchain/Local/qemu + key: ${{ runner.os }}-qemu-${{ hashFiles('Ports/qemu/version.sh', 'Toolchain/BuildQemu.sh') }} + + - name: 'Toolchain Compiler Cache' + uses: actions/cache/restore@v4 + id: 'toolchain-ccache' + if: ${{ inputs.toolchain_ccache_path != '' }} + with: + path: ${{ inputs.toolchain_ccache_path }} + key: '"toolchain ccache" | "${{ inputs.arch }}" | "${{ inputs.toolchain }}" | "${{ steps.toolchain-stamps.outputs.toolchain_version }}" | "${{ inputs.ccache_version }}" | ${{ steps.date-stamp.outputs.timestamp }}' + restore-keys: | + "toolchain ccache" | "${{ inputs.arch }}" | "${{ inputs.toolchain }}" | "${{ steps.toolchain-stamps.outputs.toolchain_version }}" | "${{ inputs.ccache_version }}" + + - name: 'Configure Toolchain ccache' + if: ${{ inputs.toolchain_ccache_path != '' }} + shell: bash + run: | + CCACHE_DIR=${{ inputs.toolchain_ccache_path }} ccache -M 0 + CCACHE_DIR=${{ inputs.toolchain_ccache_path }} ccache -s + CCACHE_DIR=${{ inputs.toolchain_ccache_path }} ccache -z + + - name: 'Serenity Compiler Cache' + uses: actions/cache/restore@v4 + id: 'serenity-ccache' + if: ${{ inputs.serenity_ccache_path != '' }} + with: + path: ${{ inputs.serenity_ccache_path }} + key: '"ccache" | "${{ inputs.os }}" | "${{ inputs.arch }}" | "${{ inputs.toolchain }}" | "${{ inputs.cache_key_extra }}" | "${{ inputs.ccache_version }}" | ${{ steps.date-stamp.outputs.timestamp }}' + restore-keys: | + "ccache" | "${{ inputs.os }}" | "${{ inputs.arch }}" | "${{ inputs.toolchain }}" | "${{ inputs.cache_key_extra }}" | "${{ inputs.ccache_version }}" + + - name: 'Configure Serenity ccache' + if: ${{ inputs.serenity_ccache_path != '' }} + shell: bash + run: | + CCACHE_DIR=${{ inputs.serenity_ccache_path }} ccache -M 0 + + # Reset all ccache modification dates to a known epoch. This provides a baseline that we can prune against. + find ${{ inputs.serenity_ccache_path }} | tac | xargs touch -a -m -d "2018-10-10T09:53:07Z" + + CCACHE_DIR=${{ inputs.serenity_ccache_path }} ccache -s + CCACHE_DIR=${{ inputs.serenity_ccache_path }} ccache -z + + - name: 'TimeZoneData cache' + uses: actions/cache@v4 + with: + path: ${{ inputs.download_cache_path }}/TZDB + key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }}-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: | + TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} + + - name: 'UnicodeData cache' + uses: actions/cache@v4 + with: + path: ${{ inputs.download_cache_path }}/UCD + key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }}-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: | + UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} + + - name: 'UnicodeLocale cache' + uses: actions/cache@v4 + with: + path: ${{ inputs.download_cache_path }}/CLDR + key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }}-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: | + UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} diff --git a/.github/actions/cache-save/action.yml b/.github/actions/cache-save/action.yml new file mode 100644 index 00000000000..78ee86b8f05 --- /dev/null +++ b/.github/actions/cache-save/action.yml @@ -0,0 +1,94 @@ +name: 'Cache Save Action' +description: 'Saves caches of build artifacts.' +author: 'Andrew Kaster ' + +inputs: + arch: + description: 'Target Architecture to restore caches for' + required: false + default: 'x86_64' + qemu_cache_primary_key: + description: 'Primary key for the AArch64 QEMU cache' + required: false + default: '' + qemu_cache_hit: + description: 'Whether the AArch64 QEMU cache was hit' + required: false + default: false + serenity_ccache_path: + description: 'Path to the SerenityOS ccache directory' + required: false + default: '' + serenity_ccache_primary_key: + description: 'Primary key for the Serenity ccache' + required: false + default: '' + toolchain_ccache_path: + description: 'Path to the toolchain ccache directory' + required: false + default: '' + toolchain_ccache_primary_key: + description: 'Primary key for the Toolchain ccache' + required: false + default: '' + toolchain_prebuilt_path: + description: 'Path to the prebuilt toolchain' + required: false + default: '' + toolchain_prebuilt_primary_key: + description: 'Primary key for the toolchain cache' + required: false + default: '' + toolchain_prebuilt_hit: + description: 'Whether the toolchain prebuilt cache was hit' + required: false + default: false + +runs: + using: "composite" + steps: + - name: 'Toolchain Prebuilt Cache' + uses: actions/cache/save@v4 + # Do not waste time and storage space by updating the toolchain cache from a PR, + # as it would be discarded after being merged anyway. + if: ${{ github.event_name != 'pull_request' && !inputs.toolchain_prebuilt_hit && inputs.arch != 'Lagom' }} + with: + path: ${{ inputs.toolchain_prebuilt_path }} + key: ${{ inputs.toolchain_prebuilt_primary_key }} + + # FIXME: Remove manually built QEMU when we bump QEMU to >=8.1.x + - name: 'AArch64 QEMU Cache' + uses: actions/cache/save@v4 + if: ${{ github.event_name != 'pull_request' && !inputs.qemu_cache_hit && inputs.arch == 'aarch64' }} + with: + path: ${{ github.workspace }}/Toolchain/Local/qemu + key: ${{ inputs.qemu_cache_primary_key }} + + - name: 'Toolchain Compiler Cache' + uses: actions/cache/save@v4 + if: ${{ github.event_name != 'pull_request' && inputs.toolchain_ccache_path != '' }} + with: + path: ${{ inputs.toolchain_ccache_path }} + key: ${{ inputs.toolchain_ccache_primary_key }} + + - name: 'Prune obsolete ccache files' + shell: bash + if: ${{ inputs.serenity_ccache_path != '' }} + run: | + CCACHE_DIR=${{ inputs.serenity_ccache_path }} ccache --evict-older-than=1d + + - name: 'Serenity Compiler Cache' + uses: actions/cache/save@v4 + if: ${{ inputs.serenity_ccache_path != '' }} + with: + path: ${{ inputs.serenity_ccache_path }} + key: ${{ inputs.serenity_ccache_primary_key }} + + - name: 'Cache Stats' + shell: bash + run: | + echo "Toolchain Compiler Cache" + CCACHE_DIR=${{ inputs.toolchain_ccache_path }} ccache -s + + echo "Serenity Compiler Cache" + CCACHE_DIR=${{ inputs.serenity_ccache_path }} ccache -s diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 4ad1a84c1c1..c6d06e04a93 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -7,7 +7,8 @@ env: # runner.workspace = /home/runner/work/serenity # github.workspace = /home/runner/work/serenity/serenity SERENITY_SOURCE_DIR: ${{ github.workspace }} - CCACHE_DIR: ${{ github.workspace }}/.ccache + SERENITY_CCACHE_DIR: ${{ github.workspace }}/.ccache + TOOLCHAIN_CCACHE_DIR: ${{ github.workspace }}/Toolchain/.ccache concurrency: group: ${{ github.head_ref || format('{0}-{1}', github.ref, github.run_number) }} @@ -52,107 +53,31 @@ jobs: - name: Lint (Phase 1/2) run: ${{ github.workspace }}/Meta/lint-ci.sh - - name: Prepare useful stamps - id: stamps - shell: cmake -P {0} - run: | - string(TIMESTAMP current_date "%Y_%m_%d_%H_%M_%S" UTC) - # Output everything twice to make it visible both in the logs - # *and* as actual output variable, in this order. - message(" set-output name=time::${current_date}") - message("::set-output name=time::${current_date}") - message(" set-output name=libc_headers::${{ hashFiles('Userland/Libraries/LibC/**/*.h', 'Userland/Libraries/LibPthread/**/*.h', 'Toolchain/Patches/*.patch', 'Toolchain/Patches/gcc/*.patch', 'Toolchain/BuildGNU.sh') }}") - message("::set-output name=libc_headers::${{ hashFiles('Userland/Libraries/LibC/**/*.h', 'Userland/Libraries/LibPthread/**/*.h', 'Toolchain/Patches/*.patch', 'Toolchain/Patches/gcc/*.patch', 'Toolchain/BuildGNU.sh') }}") - - name: Toolchain cache - uses: actions/cache/restore@v4 - id: toolchain-cache + - name: Restore Caches + uses: ./.github/actions/cache-restore + id: 'cache-restore' with: - path: ${{ github.workspace }}/Toolchain/Local/${{ matrix.arch }} - # This assumes that *ALL* LibC and LibPthread headers have an impact on the Toolchain. - # This is wrong, and causes more Toolchain rebuilds than necessary. - # However, we want to avoid false cache hits at all costs. - key: ${{ runner.os }}-toolchain-${{ matrix.arch }}-${{ steps.stamps.outputs.libc_headers }} + os: 'Serenity' + arch: ${{ matrix.arch }} + toolchain: 'GNU' + cache_key_extra: ${{ matrix.debug-options }} + serenity_ccache_path: ${{ env.SERENITY_CCACHE_DIR }} + toolchain_ccache_path: ${{ env.TOOLCHAIN_CCACHE_DIR }} + download_cache_path: ${{ github.workspace }}/Build/caches - name: Build toolchain - if: ${{ !steps.toolchain-cache.outputs.cache-hit }} + if: ${{ !steps.cache-restore.outputs.toolchain_prebuilt_hit }} run: ARCH="${{ matrix.arch }}" ${{ github.workspace }}/Toolchain/BuildGNU.sh - - - name: Update toolchain cache - uses: actions/cache/save@v4 - # Do not waste time and storage space by updating the toolchain cache from a PR, - # as it would be discarded after being merged anyway. - if: ${{ github.event_name != 'pull_request' && !steps.toolchain-cache.outputs.cache-hit }} - with: - path: ${{ github.workspace }}/Toolchain/Local/${{ matrix.arch }} - key: ${{ steps.toolchain-cache.outputs.cache-primary-key }} - - # FIXME: Qemu currently needs a local patch for AArch64 testing. It is included in Qemu 8.1; remove this when upgrading! - - name: AArch64 Qemu cache - id: qemu-cache - uses: actions/cache/restore@v4 - if: ${{ matrix.arch == 'aarch64' }} - with: - path: ${{ github.workspace }}/Toolchain/Local/qemu - key: ${{ runner.os }}-qemu-${{ hashFiles('Ports/qemu/version.sh', 'Toolchain/BuildQemu.sh') }} + env: + CCACHE_DIR: ${{ env.TOOLCHAIN_CCACHE_DIR }} - name: Build AArch64 Qemu - if: ${{ matrix.arch == 'aarch64' && !steps.qemu-cache.outputs.cache-hit }} + if: ${{ matrix.arch == 'aarch64' && !steps.cache-restore.outputs.qemu_cache_hit }} run: ${{ github.workspace }}/Toolchain/BuildQemu.sh + env: + CCACHE_DIR: ${{ env.TOOLCHAIN_CCACHE_DIR }} - - name: Update AArch64 Qemu cache - uses: actions/cache/save@v4 - if: ${{ github.event_name != 'pull_request' && matrix.arch == 'aarch64' && !steps.qemu-cache.outputs.cache-hit }} - with: - path: ${{ github.workspace }}/Toolchain/Local/qemu - key: ${{ steps.qemu-cache.outputs.cache-primary-key }} - - - name: ccache(1) cache - # Pull the ccache *after* building the toolchain, in case building the Toolchain somehow interferes. - uses: actions/cache/restore@v4 - id: ccache - with: - path: ${{ github.workspace }}/.ccache - # If you're here because ccache broke (it never should), increment matrix.ccache-mark. - # We want to always reuse the last cache, but upload a new one. - # This is achieved by using the "prefix-timestamp" format, - # and permitting the restore-key "prefix-" without specifying a timestamp. - # For this trick to work, the timestamp *must* come last, and it *must* be missing in 'restore-keys'. - key: ${{ runner.os }}-ccache-${{ matrix.arch }}-v${{ matrix.ccache-mark }}-D${{ matrix.debug-options }}-toolchain_${{steps.stamps.outputs.libc_headers}}-time${{ steps.stamps.outputs.time }} - restore-keys: | - ${{ runner.os }}-ccache-${{ matrix.arch }}-v${{ matrix.ccache-mark }}-D${{ matrix.debug-options }}-toolchain_${{steps.stamps.outputs.libc_headers}}- - - - name: Show ccache stats before build and configure - run: | - ccache -M 0 - - # Reset all ccache modification dates to a known epoch. This provides a baseline that we can prune against. - find ${{ github.workspace }}/.ccache | tac | xargs touch -a -m -d "2018-10-10T09:53:07Z" - - ccache -s - ccache -z - - - name: Create build directory - run: | - mkdir -p ${{ github.workspace }}/Build/${{ matrix.arch }} - mkdir -p ${{ github.workspace }}/Build/caches/TZDB - mkdir -p ${{ github.workspace }}/Build/caches/UCD - mkdir -p ${{ github.workspace }}/Build/caches/CLDR - - name: TimeZoneData cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/TZDB - key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} - - name: UnicodeData cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/UCD - key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} - - name: UnicodeLocale Cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/CLDR - key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} - name: Create build environment with extra debug options # Build the entire project with all available debug options turned on, to prevent code rot. # However, it is unwieldy and slow to run tests with them enabled, so we will build twice. @@ -167,6 +92,8 @@ jobs: -DENABLE_PCI_IDS_DOWNLOAD=OFF \ -DENABLE_USB_IDS_DOWNLOAD=OFF if: ${{ matrix.debug-options == 'ALL_DEBUG' }} + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} - name: Create build environment working-directory: ${{ github.workspace }} # Note that we do not set BUILD_LAGOM for the normal debug build @@ -183,25 +110,30 @@ jobs: -DENABLE_PCI_IDS_DOWNLOAD=OFF \ -DENABLE_USB_IDS_DOWNLOAD=OFF if: ${{ matrix.debug-options == 'NORMAL_DEBUG' }} + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} # === ACTUALLY BUILD === - name: Build Serenity and Tests working-directory: ${{ github.workspace }}/Build/superbuild run: cmake --build . + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} - - name: Prune obsolete ccache files - run: ccache --evict-older-than 1d - - - name: Show ccache stats after build - run: ccache -s - - - name: Update ccache(1) cache - uses: actions/cache/save@v4 - if: ${{ github.event_name != 'pull_request' }} + - name: Save Caches + uses: ./.github/actions/cache-save with: - path: ${{ github.workspace }}/.ccache - key: ${{ steps.ccache.outputs.cache-primary-key }} + arch: ${{ matrix.arch }} + qemu_cache_primary_key: ${{ steps.cache-restore.outputs.qemu_cache_primary_key }} + qemu_cache_hit: ${{ steps.cache-restore.outputs.qemu_cache_hit }} + serenity_ccache_path: ${{ env.SERENITY_CCACHE_DIR }} + serenity_ccache_primary_key: ${{ steps.cache-restore.outputs.serenity_ccache_primary_key }} + toolchain_ccache_path: ${{ env.TOOLCHAIN_CCACHE_DIR }} + toolchain_ccache_primary_key: ${{ steps.cache-restore.outputs.toolchain_ccache_primary_key }} + toolchain_prebuilt_path: ${{ steps.cache-restore.outputs.toolchain_prebuilt_path }} + toolchain_prebuilt_primary_key: ${{ steps.cache-restore.outputs.toolchain_prebuilt_primary_key }} + toolchain_prebuilt_hit: ${{ steps.cache-restore.outputs.toolchain_prebuilt_hit }} - name: Lint (Phase 2/2) working-directory: ${{ github.workspace }}/Meta diff --git a/.github/workflows/libjs-test262.yml b/.github/workflows/libjs-test262.yml index bd4b57fc7da..81e06fdf855 100644 --- a/.github/workflows/libjs-test262.yml +++ b/.github/workflows/libjs-test262.yml @@ -70,23 +70,12 @@ jobs: - name: Check versions run: set +e; g++ --version; g++-13 --version; python --version; python3 --version; ninja --version - - name: TimeZoneData cache - uses: actions/cache@v4 + - name: Restore Caches + uses: ./.github/actions/cache-restore with: - path: ${{ github.workspace }}/libjs-test262/Build/caches/TZDB - key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} - - - name: UnicodeData cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/libjs-test262/Build/caches/UCD - key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} - - - name: UnicodeLocale cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/libjs-test262/Build/caches/CLDR - key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} + os: 'Linux' + arch: 'Lagom' + download_cache_path: ${{ github.workspace }}/libjs-test262/Build/caches - name: Get previous results run: | diff --git a/.github/workflows/pvs-studio-static-analysis.yml b/.github/workflows/pvs-studio-static-analysis.yml index b0fa1245efa..266fcec2eb6 100644 --- a/.github/workflows/pvs-studio-static-analysis.yml +++ b/.github/workflows/pvs-studio-static-analysis.yml @@ -4,6 +4,10 @@ on: schedule: - cron: '0 0 * * *' +env: + SERENITY_CCACHE_DIR: ${{ github.workspace }}/.ccache + TOOLCHAIN_CCACHE_DIR: ${{ github.workspace }}/Toolchain/.ccache + jobs: build: name: Static Analysis @@ -33,55 +37,23 @@ jobs: - name: Check versions run: set +e; g++ --version; g++-13 --version; ninja --version; - - name: Prepare useful stamps - id: stamps - shell: cmake -P {0} - run: | - string(TIMESTAMP current_date "%Y_%m_%d_%H_%M_%S" UTC) - # Output everything twice to make it visible both in the logs - # *and* as actual output variable, in this order. - message(" set-output name=time::${current_date}") - message("::set-output name=time::${current_date}") - message(" set-output name=libc_headers::${{ hashFiles('Userland/Libraries/LibC/**/*.h', 'Userland/Libraries/LibPthread/**/*.h', 'Toolchain/Patches/*.patch', 'Toolchain/Patches/gcc/*.patch', 'Toolchain/BuildGNU.sh') }}") - message("::set-output name=libc_headers::${{ hashFiles('Userland/Libraries/LibC/**/*.h', 'Userland/Libraries/LibPthread/**/*.h', 'Toolchain/Patches/*.patch', 'Toolchain/Patches/gcc/*.patch', 'Toolchain/BuildGNU.sh') }}") - - - name: Toolchain cache - # This job should always read the cache, never populate it. - uses: actions/cache/restore@v4 - id: toolchain-cache + - name: Restore Caches + uses: ./.github/actions/cache-restore + id: 'cache-restore' with: - path: ${{ github.workspace }}/Toolchain/Local/${{ env.PVS_STUDIO_ANALYSIS_ARCH }} - # This assumes that *ALL* LibC and LibPthread headers have an impact on the Toolchain. - # This is wrong, and causes more Toolchain rebuilds than necessary. - # However, we want to avoid false cache hits at all costs. - key: ${{ runner.os }}-toolchain-${{ env.PVS_STUDIO_ANALYSIS_ARCH }}-${{ steps.stamps.outputs.libc_headers }} + os: 'Serenity' + arch: ${{ env.PVS_STUDIO_ANALYSIS_ARCH }} + toolchain: 'GNU' + cache_key_extra: 'Static Analysis' + serenity_ccache_path: ${{ env.SERENITY_CCACHE_DIR }} + toolchain_ccache_path: ${{ env.TOOLCHAIN_CCACHE_DIR }} + download_cache_path: ${{ github.workspace }}/Build/caches - name: Build toolchain if: ${{ !steps.toolchain-cache.outputs.cache-hit }} run: ARCH="${{ env.PVS_STUDIO_ANALYSIS_ARCH }}" ${{ github.workspace }}/Toolchain/BuildGNU.sh - - - name: Create build directory - run: | - mkdir -p ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }} - mkdir -p ${{ github.workspace }}/Build/caches/TZDB - mkdir -p ${{ github.workspace }}/Build/caches/UCD - mkdir -p ${{ github.workspace }}/Build/caches/CLDR - - - name: TimeZoneData cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/TZDB - key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} - - name: UnicodeData cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/UCD - key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} - - name: UnicodeLocale Cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/CLDR - key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} + env: + CCACHE_DIR: ${{ env.TOOLCHAIN_CCACHE_DIR }} - name: Create build environment working-directory: ${{ github.workspace }} @@ -92,7 +64,10 @@ jobs: -DCMAKE_C_COMPILER=gcc-13 \ -DCMAKE_CXX_COMPILER=g++-13 \ -DENABLE_PCI_IDS_DOWNLOAD=OFF \ - -DENABLE_USB_IDS_DOWNLOAD=OFF + -DENABLE_USB_IDS_DOWNLOAD=OFF \ + -DSERENITY_CACHE_DIR=${{ github.workspace }}/Build/caches + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} - name: Build generated sources so they are available for analysis. working-directory: ${{ github.workspace }} @@ -103,6 +78,20 @@ jobs: ninja -C Build/superbuild serenity-configure cmake -B Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ninja -C Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }} all_generated + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} + + - name: Save Caches + uses: ./.github/actions/cache-save + with: + arch: ${{ matrix.arch }} + serenity_ccache_path: ${{ env.SERENITY_CCACHE_DIR }} + serenity_ccache_primary_key: ${{ steps.cache-restore.outputs.serenity_ccache_primary_key }} + toolchain_ccache_path: ${{ env.TOOLCHAIN_CCACHE_DIR }} + toolchain_ccache_primary_key: ${{ steps.cache-restore.outputs.toolchain_ccache_primary_key }} + toolchain_prebuilt_path: ${{ steps.cache-restore.outputs.toolchain_prebuilt_path }} + toolchain_prebuilt_primary_key: ${{ steps.cache-restore.outputs.toolchain_prebuilt_primary_key }} + toolchain_prebuilt_hit: ${{ steps.cache-restore.outputs.toolchain_prebuilt_hit }} - name: Configure PVS-Studio License env: diff --git a/.github/workflows/serenity-js-artifacts.yml b/.github/workflows/serenity-js-artifacts.yml index 7047eb6ca35..35ca9d49712 100644 --- a/.github/workflows/serenity-js-artifacts.yml +++ b/.github/workflows/serenity-js-artifacts.yml @@ -4,6 +4,7 @@ on: [push] env: SERENITY_SOURCE_DIR: ${{ github.workspace }} + SERENITY_CCACHE_DIR: ${{ github.workspace }}/.ccache jobs: build-and-package: @@ -34,37 +35,26 @@ jobs: os: ${{ matrix.os_name }} arch: 'Lagom' - - name: Create build directory - run: | - mkdir -p Build/TZDB - mkdir -p Build/UCD - mkdir -p Build/CLDR - - - name: TimeZoneData cache - uses: actions/cache@v4 + - name: Restore Caches + uses: ./.github/actions/cache-restore + id: 'cache-restore' with: - path: ${{ github.workspace }}/libjs-test262/Build/TZDB - key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} - - - name: UnicodeData cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/libjs-test262/Build/UCD - key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} - - - name: UnicodeLocale cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/libjs-test262/Build/CLDR - key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} + os: ${{ matrix.os_name }} + arch: 'Lagom' + cache_key_extra: 'LibJS Artifacts' + serenity_ccache_path: ${{ env.SERENITY_CCACHE_DIR }} + download_cache_path: ${{ github.workspace }}/Build/caches - name: Create build directory Ubuntu run: | cmake -S Meta/Lagom -B Build -G Ninja \ -DCMAKE_C_COMPILER=gcc-13 \ -DCMAKE_CXX_COMPILER=g++-13 \ - -DBUILD_LAGOM=ON + -DBUILD_LAGOM=ON \ + -DSERENITY_CACHE_DIR=${{ github.workspace }}/Build/caches if: ${{ matrix.os == 'ubuntu-22.04' }} + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} - name: Create build directory macOS run: | @@ -73,14 +63,26 @@ jobs: -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64" \ -DCMAKE_OSX_DEPLOYMENT_TARGET="11.0" \ - -DBUILD_LAGOM=ON + -DBUILD_LAGOM=ON \ + -DSERENITY_CACHE_DIR=${{ github.workspace }}/Build/caches if: ${{ matrix.os == 'macos-14' }} + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} - name: Build and package js working-directory: Build run: | ninja js cpack + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} + + - name: Save Caches + uses: ./.github/actions/cache-save + with: + arch: ${{ matrix.arch }} + serenity_ccache_path: ${{ env.SERENITY_CCACHE_DIR }} + serenity_ccache_primary_key: ${{ steps.cache-restore.outputs.serenity_ccache_primary_key }} - name: Upload js package uses: actions/upload-artifact@v4 diff --git a/.github/workflows/sonar-cloud-static-analysis.yml b/.github/workflows/sonar-cloud-static-analysis.yml index 3540113129e..9a5d9b5ccd6 100644 --- a/.github/workflows/sonar-cloud-static-analysis.yml +++ b/.github/workflows/sonar-cloud-static-analysis.yml @@ -4,6 +4,10 @@ on: schedule: - cron: '0 0 * * *' +env: + SERENITY_CCACHE_DIR: ${{ github.workspace }}/.ccache + TOOLCHAIN_CCACHE_DIR: ${{ github.workspace }}/Toolchain/.ccache + jobs: build: name: Static Analysis @@ -57,55 +61,23 @@ jobs: os: 'Serenity' arch: ${{ env.SONAR_ANALYSIS_ARCH }} - - name: Prepare useful stamps - id: stamps - shell: cmake -P {0} - run: | - string(TIMESTAMP current_date "%Y_%m_%d_%H_%M_%S" UTC) - # Output everything twice to make it visible both in the logs - # *and* as actual output variable, in this order. - message(" set-output name=time::${current_date}") - message("::set-output name=time::${current_date}") - message(" set-output name=libc_headers::${{ hashFiles('Userland/Libraries/LibC/**/*.h', 'Userland/Libraries/LibPthread/**/*.h', 'Toolchain/Patches/*.patch', 'Toolchain/Patches/gcc/*.patch', 'Toolchain/BuildGNU.sh') }}") - message("::set-output name=libc_headers::${{ hashFiles('Userland/Libraries/LibC/**/*.h', 'Userland/Libraries/LibPthread/**/*.h', 'Toolchain/Patches/*.patch', 'Toolchain/Patches/gcc/*.patch', 'Toolchain/BuildGNU.sh') }}") - - - name: Toolchain cache - # This job should always read the cache, never populate it. - uses: actions/cache/restore@v4 - id: toolchain-cache + - name: Restore Caches + uses: ./.github/actions/cache-restore + id: 'cache-restore' with: - path: ${{ github.workspace }}/Toolchain/Local/${{ env.SONAR_ANALYSIS_ARCH }} - # This assumes that *ALL* LibC and LibPthread headers have an impact on the Toolchain. - # This is wrong, and causes more Toolchain rebuilds than necessary. - # However, we want to avoid false cache hits at all costs. - key: ${{ runner.os }}-toolchain-${{ env.SONAR_ANALYSIS_ARCH }}-${{ steps.stamps.outputs.libc_headers }} + os: 'Serenity' + arch: ${{ env.SONAR_ANALYSIS_ARCH }} + toolchain: 'GNU' + cache_key_extra: 'Static Analysis' + serenity_ccache_path: ${{ env.SERENITY_CCACHE_DIR }} + toolchain_ccache_path: ${{ env.TOOLCHAIN_CCACHE_DIR }} + download_cache_path: ${{ github.workspace }}/Build/caches - name: Build toolchain if: ${{ !steps.toolchain-cache.outputs.cache-hit }} run: ARCH="${{ env.SONAR_ANALYSIS_ARCH }}" ${{ github.workspace }}/Toolchain/BuildGNU.sh - - - name: Create build directory - run: | - mkdir -p ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }} - mkdir -p ${{ github.workspace }}/Build/caches/TZDB - mkdir -p ${{ github.workspace }}/Build/caches/UCD - mkdir -p ${{ github.workspace }}/Build/caches/CLDR - - - name: TimeZoneData cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/TZDB - key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} - - name: UnicodeData cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/UCD - key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} - - name: UnicodeLocale Cache - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/CLDR - key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} + env: + CCACHE_DIR: ${{ env.TOOLCHAIN_CCACHE_DIR }} - name: Create build environment working-directory: ${{ github.workspace }} @@ -116,7 +88,10 @@ jobs: -DCMAKE_C_COMPILER=gcc-13 \ -DCMAKE_CXX_COMPILER=g++-13 \ -DENABLE_PCI_IDS_DOWNLOAD=OFF \ - -DENABLE_USB_IDS_DOWNLOAD=OFF + -DENABLE_USB_IDS_DOWNLOAD=OFF \ + -DSERENITY_CACHE_DIR=${{ github.workspace }}/Build/caches + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} - name: Build generated sources so they are available for analysis. working-directory: ${{ github.workspace }} @@ -127,6 +102,20 @@ jobs: ninja -C Build/superbuild serenity-configure cmake -B Build/${{ env.SONAR_ANALYSIS_ARCH }} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ninja -C Build/${{ env.SONAR_ANALYSIS_ARCH }} all_generated + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} + + - name: Save Caches + uses: ./.github/actions/cache-save + with: + arch: ${{ matrix.arch }} + serenity_ccache_path: ${{ env.SERENITY_CCACHE_DIR }} + serenity_ccache_primary_key: ${{ steps.cache-restore.outputs.serenity_ccache_primary_key }} + toolchain_ccache_path: ${{ env.TOOLCHAIN_CCACHE_DIR }} + toolchain_ccache_primary_key: ${{ steps.cache-restore.outputs.toolchain_ccache_primary_key }} + toolchain_prebuilt_path: ${{ steps.cache-restore.outputs.toolchain_prebuilt_path }} + toolchain_prebuilt_primary_key: ${{ steps.cache-restore.outputs.toolchain_prebuilt_primary_key }} + toolchain_prebuilt_hit: ${{ steps.cache-restore.outputs.toolchain_prebuilt_hit }} - name: Run sonar-scanner, upload results env: diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index c9da0e33a67..cab3db4205b 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -3,6 +3,7 @@ on: [ push, pull_request ] env: SERENITY_SOURCE_DIR: ${{ github.workspace }} + SERENITY_CCACHE_DIR: ${{ github.workspace }}/.ccache concurrency: wasm @@ -34,27 +35,16 @@ jobs: with: version: 3.1.25 - - name: "Create build directories" - run: | - mkdir -p ${{ github.workspace }}/Build/caches/TZDB - mkdir -p ${{ github.workspace }}/Build/caches/UCD - mkdir -p ${{ github.workspace }}/Build/caches/CLDR + - name: Restore Caches + uses: ./.github/actions/cache-restore + id: 'cache-restore' + with: + os: 'Linux' + arch: 'Lagom' + cache_key_extra: 'WASM' + serenity_ccache_path: ${{ env.SERENITY_CCACHE_DIR }} + download_cache_path: ${{ github.workspace }}/Build/caches - - name: "TimeZoneData cache" - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/TZDB - key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} - - name: "UnicodeData cache" - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/UCD - key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} - - name: "UnicodeLocale cache" - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/Build/caches/CLDR - key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} - name: "Build host lagom tools" run: | cmake -GNinja \ @@ -66,8 +56,10 @@ jobs: -DCMAKE_CXX_COMPILER=g++-13 \ -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/Build/lagom-tools \ -Dpackage=LagomTools - ninja -C ${{ github.workspace }}/Build/lagom-tools install + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} + - name: "Create wasm build environment" run: | emcmake cmake -GNinja \ @@ -77,12 +69,21 @@ jobs: -DBUILD_LAGOM=ON \ -DSERENITY_CACHE_DIR=${{ github.workspace }}/Build/caches \ -DBUILD_SHARED_LIBS=OFF + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} + - name: "Build libjs.{js,wasm}" run: | ninja -C ${{ github.workspace }}/Build/wasm libjs.js - - name: "Show me what you built :)" - run: | - ls -l ${{ github.workspace }}/Build/wasm + env: + CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} + + - name: Save Caches + uses: ./.github/actions/cache-save + with: + arch: ${{ matrix.arch }} + serenity_ccache_path: ${{ env.SERENITY_CCACHE_DIR }} + serenity_ccache_primary_key: ${{ steps.cache-restore.outputs.serenity_ccache_primary_key }} - name: "Prepare files" run: |