CI: Build the Clang toolchain a single time for both onboard pipelines

As of the Clang 13 upgrade, we only need to build the toolchain once and
can use that toolchain for both x86_64 and i686. To do this, this breaks
the main Azure configuration into 3 "stages" (Lagom, Toolchain, and
Serenity), where the Serenity stage depends on the Toolchain stage.

This has the added benefit of uploading a new prebuilt toolchain cache
sooner than before, which should help alleviate pressure from PRs.
This commit is contained in:
Timothy Flynn 2021-10-17 12:45:04 -04:00 committed by Brian Gianforcaro
parent 957f98805a
commit b11a34330a
Notes: sideshowbarker 2024-07-18 02:11:43 +09:00
4 changed files with 95 additions and 51 deletions

View file

@ -4,8 +4,8 @@ parameters:
toolchain: 'gcc'
build_directory: ''
ccache_version: 1 # Increment this number if CI has trouble with ccache.
serenity_ccache_path: $(CCACHE_DIR)
toolchain_ccache_path: $(CCACHE_DIR)
serenity_ccache_path: ''
toolchain_ccache_path: ''
toolchain_ccache_size: $(CCACHE_MAXSIZE)
with_unicode_caches: true
@ -18,7 +18,7 @@ steps:
- ${{ if eq(parameters.toolchain, 'clang') }}:
- task: Cache@2
inputs:
key: '"toolchain" | "${{ parameters.arch }}" | Toolchain/BuildClang.sh | Toolchain/Patches/*[!gcc].patch | Userland/Libraries/LibC/**/*.h | Userland/Libraries/LibPthread/**/*.h'
key: '"toolchain" | "x86_64" | Toolchain/BuildClang.sh | Toolchain/Patches/*[!gcc].patch | Userland/Libraries/LibC/**/*.h | Userland/Libraries/LibPthread/**/*.h'
path: $(Build.SourcesDirectory)/Toolchain/Cache
displayName: 'Toolchain Prebuilt Cache'
- ${{ if eq(parameters.toolchain, 'gcc') }}:
@ -28,26 +28,33 @@ steps:
path: $(Build.SourcesDirectory)/Toolchain/Cache
displayName: 'Toolchain Prebuilt Cache'
- ${{ if ne(parameters.toolchain_ccache_path, '') }}:
- task: Cache@2
inputs:
key: '"toolchain ccache" | "x86_64" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}" | "$(timestamp)"'
restoreKeys: |
"toolchain ccache" | "x86_64" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}"
path: ${{ parameters.toolchain_ccache_path }}
displayName: 'Toolchain Compiler Cache'
- script: |
CCACHE_DIR=${{ parameters.toolchain_ccache_path }} ccache -M ${{ parameters.toolchain_ccache_size }}
CCACHE_DIR=${{ parameters.toolchain_ccache_path }} ccache -s
displayName: 'Configure Toolchain ccache'
- ${{ if ne(parameters.serenity_ccache_path, '') }}:
- task: Cache@2
inputs:
key: '"toolchain ccache" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}" | "$(timestamp)"'
key: '"ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}" | "$(timestamp)"'
restoreKeys: |
"toolchain ccache" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}"
path: ${{ parameters.toolchain_ccache_path }}
displayName: 'Toolchain Compiler Cache'
"ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}"
path: ${{ parameters.serenity_ccache_path }}
displayName: 'Serenity Compiler Cache'
- script: |
CCACHE_DIR=${{ parameters.toolchain_ccache_path }} ccache -M ${{ parameters.toolchain_ccache_size }}
CCACHE_DIR=${{ parameters.toolchain_ccache_path }} ccache -s
displayName: 'Configure Toolchain ccache'
- task: Cache@2
inputs:
key: '"ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}" | "$(timestamp)"'
restoreKeys: |
"ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}"
path: ${{ parameters.serenity_ccache_path }}
displayName: 'Serenity Compiler Cache'
CCACHE_DIR=${{ parameters.serenity_ccache_path }} ccache -M 5G
CCACHE_DIR=${{ parameters.serenity_ccache_path }} ccache -s
displayName: 'Configure Serenity ccache'
- ${{ if eq(parameters.with_unicode_caches, true) }}:
- task: Cache@2
@ -61,8 +68,3 @@ steps:
key: '"unicode_locale" | Meta/CMake/unicode_data.cmake'
path: $(Build.SourcesDirectory)/${{ parameters.build_directory }}/CLDR
displayName: 'UnicodeLocale Cache'
- script: |
CCACHE_DIR=${{ parameters.serenity_ccache_path }} ccache -M 5G
CCACHE_DIR=${{ parameters.serenity_ccache_path }} ccache -s
displayName: 'Configure Serenity ccache'

View file

@ -8,10 +8,6 @@ jobs:
variables:
- name: SERENITY_CCACHE_DIR
value: $(Build.SourcesDirectory)/.ccache
- name: LLVM_CCACHE_DIR
value: $(Build.SourcesDirectory)/Toolchain/.ccache
- name: LLVM_CCACHE_MAXSIZE
value: 20GB
pool:
vmImage: ubuntu-20.04
@ -27,14 +23,11 @@ jobs:
toolchain: 'clang'
build_directory: 'Build/${{ parameters.arch }}clang'
serenity_ccache_path: '$(SERENITY_CCACHE_DIR)'
toolchain_ccache_path: '$(LLVM_CCACHE_DIR)'
toolchain_ccache_size: '$(LLVM_CCACHE_MAXSIZE)'
- script: ./Toolchain/BuildClang.sh --ci
displayName: Build Toolchain
- script: ./Toolchain/BuildClang.sh
displayName: Extract Toolchain
env:
TRY_USE_LOCAL_TOOLCHAIN: 'y'
ARCH: '${{ parameters.arch }}'
- script: |
mkdir -p Build/superbuild

34
Meta/Azure/Toolchain.yml Normal file
View file

@ -0,0 +1,34 @@
jobs:
- job: 'Serenity_Clang_Toolchain'
timeoutInMinutes: 0 # Setting to 0 means the maximum allowed timeout is used.
variables:
- name: LLVM_CCACHE_DIR
value: $(Build.SourcesDirectory)/Toolchain/.ccache
- name: LLVM_CCACHE_MAXSIZE
value: 20GB
pool:
vmImage: ubuntu-20.04
steps:
- template: Setup.yml
parameters:
os: 'Serenity'
- template: Caches.yml
parameters:
toolchain: 'clang'
toolchain_ccache_path: '$(LLVM_CCACHE_DIR)'
toolchain_ccache_size: '$(LLVM_CCACHE_MAXSIZE)'
with_unicode_caches: false
- script: ./Toolchain/BuildClang.sh --ci
displayName: Build Toolchain
env:
TRY_USE_LOCAL_TOOLCHAIN: 'y'
- script: |
echo "##[section]Toolchain Cache"
CCACHE_DIR='$(LLVM_CCACHE_DIR)' ccache -s
displayName: 'Cache Stats'

View file

@ -4,26 +4,41 @@ trigger:
include:
- master
jobs:
- template: Meta/Azure/Lagom.yml
parameters:
os: 'Linux'
lagom_lints: true
stages:
- stage: Lagom
dependsOn: []
- template: Meta/Azure/Lagom.yml
parameters:
os: 'Linux'
fuzzer: 'Fuzz'
jobs:
- template: Meta/Azure/Lagom.yml
parameters:
os: 'Linux'
lagom_lints: true
- template: Meta/Azure/Lagom.yml
parameters:
os: 'macOS'
allow_test_failures: true
- template: Meta/Azure/Lagom.yml
parameters:
os: 'Linux'
fuzzer: 'Fuzz'
- template: Meta/Azure/Serenity.yml
parameters:
arch: 'i686'
- template: Meta/Azure/Lagom.yml
parameters:
os: 'macOS'
allow_test_failures: true
- template: Meta/Azure/Serenity.yml
parameters:
arch: 'x86_64'
- stage: Toolchain
dependsOn: []
jobs:
- template: Meta/Azure/Toolchain.yml
- stage: SerenityOS
dependsOn: Toolchain
jobs:
- template: Meta/Azure/Serenity.yml
parameters:
arch: 'i686'
- template: Meta/Azure/Serenity.yml
parameters:
arch: 'x86_64'