pkg: fedora: Create containers manually, reclaim space from the runner

The GitHub runners usually have about 14 GB of free space. This is
not enough to build a full Fedora kernel with debug symbols.

Disabling the debug symbols is not an option, because they are needed
to generate data for BPF programs, and because it would mean more
patches to the spec file which I would like to avoid.

The runners contain a bunch of software that is entirely useless for
our purposes. For example, a full .NET distribution, the entire Android
SDK, and a bunch of cached container images that arent Fedora.

However, with GitHubs container directive it is not possible to remove
these, because no custom code is ever executed on the runner itself.

To execute code on the runner, the container has to be created and used
manually. To simplify this, all of the actual logic has been moved to a
dedicated script, similar to how the CI on iptsd is set up.
This commit is contained in:
Dorian Stoll 2023-07-22 20:27:44 +02:00
parent 0095650bf0
commit 7c7ba21c90
No known key found for this signature in database
GPG key ID: F1DACD02C619442A
6 changed files with 403 additions and 222 deletions

33
.github/scripts/container/create.sh vendored Normal file
View file

@ -0,0 +1,33 @@
#!/usr/bin/env bash
set -euxo pipefail
if [ -z "${GITHUB_REPOSITORY_ID:-}" ]; then
echo "GITHUB_REPOSITORY_ID is unset!"
exit 1
fi
if [ -z "${GITHUB_WORKSPACE:-}" ]; then
echo "GITHUB_WORKSPACE is unset!"
exit 1
fi
IMAGE="${1:-}"
if [ -z "${IMAGE}" ]; then
echo "Container image is unset!"
exit 1
fi
if command -v docker &> /dev/null; then
DOCKER="docker"
elif command -v podman &> /dev/null; then
DOCKER="podman"
else
echo "Could not find docker / podman!"
exit 1
fi
exec "${DOCKER}" run -d --name "${GITHUB_REPOSITORY_ID}" \
-v "${GITHUB_WORKSPACE}:/working" --workdir "/working" \
--entrypoint "tail" "${IMAGE}" -f /dev/null

46
.github/scripts/container/exec.sh vendored Normal file
View file

@ -0,0 +1,46 @@
#!/usr/bin/env bash
set -euxo pipefail
if [ -z "${GITHUB_REPOSITORY_ID:-}" ]; then
echo "GITHUB_REPOSITORY_ID is unset!"
exit 1
fi
if [ -z "${1:-}" ]; then
echo "Arguments are unset!"
exit 1
fi
ENVVARS=()
COMMAND=()
while (( "${#}" )); do
case "$1" in
-e)
ENVVARS+=("-e")
shift
ENVVARS+=("$1")
shift
;;
--)
shift
while (( "${#}" )); do
COMMAND+=("$1")
shift
done
;;
esac
done
if command -v docker &> /dev/null; then
DOCKER="docker"
elif command -v podman &> /dev/null; then
DOCKER="podman"
else
echo "Could not find docker / podman!"
exit 1
fi
exec "${DOCKER}" exec "${ENVVARS[@]}" "${GITHUB_REPOSITORY_ID}" "${COMMAND[@]}"

76
.github/scripts/package/fedora.sh vendored Normal file
View file

@ -0,0 +1,76 @@
#!/usr/bin/env bash
set -euxo pipefail
if [ -z "$1" ]; then
$0 setup-builddeps
$0 setup-secureboot
$0 build-packages
$0 sign-packages
exit
fi
dnf()
{
command dnf -y "$@"
}
case "$1" in
setup-builddeps)
# Setup build environment
dnf distro-sync
dnf install @rpm-development-tools git rpm-sign
# Install build dependencies
dnf builddep kernel
# Install additional build dependencies
dnf install sbsigntools
;;
setup-secureboot)
if [ -z "${SB_KEY:-}" ]; then
echo "WARNING: No secureboot key configured, skipping signing."
exit
fi
# Install the surface secureboot certificate
echo "${SB_KEY}" | base64 -d > pkg/fedora/kernel-surface/secureboot/MOK.key
cp pkg/keys/surface.crt pkg/fedora/kernel-surface/secureboot/MOK.crt
;;
build-packages)
pushd pkg/fedora/kernel-surface || exit 1
# setup git
git config --global user.name "surfacebot"
git config --global user.email "surfacebot@users.noreply.github.com"
# Build source RPM packages
python3 build-linux-surface.py --mode srpm --ark-dir kernel-ark --outdir srpm
# Remove the kernel-ark tree to get as much free disk space as possible
rm -rf kernel-ark
# Build binary RPM packages
find srpm -name '*.src.rpm' -type f -exec rpmbuild -rb \
--define "_topdir ${PWD}/rpmbuild" --define "_rpmdir ${PWD}/out" {} \;
popd || exit 1
;;
sign-packages)
if [ -z "${GPG_KEY:-}" ] || [ -z "${GPG_KEY_ID:-}" ]; then
echo "WARNING: No GPG key configured, skipping signing."
exit
fi
pushd pkg/fedora/kernel-surface/out/x86_64 || exit 1
# import GPG key
echo "${GPG_KEY}" | base64 -d | gpg --import --no-tty --batch --yes
# sign packages
find . -name '*.rpm' -type f -exec \
rpm --resign {} --define "_gpg_name ${GPG_KEY_ID}" \;
popd || exit 1
;;
esac

72
.github/scripts/repository/fedora.sh vendored Normal file
View file

@ -0,0 +1,72 @@
#!/usr/bin/env bash
set -euxo pipefail
dnf()
{
command dnf -y "$@"
}
if [ -z "${GIT_REF:-}" ]; then
echo "GIT_REF is unset!"
exit 1
fi
if [ -z "${GITHUB_REPOSITORY:-}" ]; then
echo "GITHUB_REPOSITORY is unset!"
exit 1
fi
if [ -z "${SURFACEBOT_TOKEN:-}" ]; then
echo "SURFACEBOT_TOKEN is unset!"
exit 1
fi
if [ -z "${BRANCH_STAGING:-}" ]; then
echo "BRANCH_STAGING is unset!"
exit 1
fi
FEDORA="${1:-}"
if [ -z "${FEDORA}" ]; then
echo "Fedora version is unset!"
exit 1
fi
REPONAME="$(echo "${GITHUB_REPOSITORY}" | cut -d'/' -f2)"
REPO="https://surfacebot:${SURFACEBOT_TOKEN}@github.com/linux-surface/repo.git"
# parse git tag from ref
GIT_TAG="${GIT_REF#refs/tags/}"
# Install dependencies
dnf install git findutils
# clone package repository
git clone -b "${BRANCH_STAGING}" "${REPO}" repo
# copy packages
find "fedora-${FEDORA}-latest" -type f -exec cp {} "repo/fedora/f${FEDORA}" \;
pushd "repo/fedora/f${FEDORA}" || exit 1
# convert packages into references
while read -rd $'\n' FILE; do
echo "${REPONAME}:${GIT_TAG}/$(basename "${FILE}")" > "${FILE}.blob"
rm "${FILE}"
done <<< "$(find . -name '*.rpm' -type f)"
RAND="$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 32 | head -n 1)"
BRANCH="${BRANCH_STAGING}-${RAND}"
# set git identity
git config --global user.name "surfacebot"
git config --global user.email "surfacebot@users.noreply.github.com"
# commit and push
git checkout -b "${BRANCH}"
git add .
git commit -m "Update Fedora ${FEDORA} ${REPONAME} package"
git push --set-upstream origin "${BRANCH}"
popd || exit 1

View file

@ -1,144 +1,121 @@
name: Fedora 37
env:
FEDORA: 37
GPG_KEY_ID: 56C464BAAC421453
on: on:
push: push:
tags: tags:
- 'fedora-37-*' - 'fedora-37-*'
name: Fedora 37
env:
GPG_KEY_ID: 56C464BAAC421453
jobs: jobs:
build: build:
name: Build Kernel name: Build Kernel
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: registry.fedoraproject.org/fedora:37
steps: steps:
- name: Checkout code - name: Maximize disk space
uses: actions/checkout@v3 uses: easimon/maximize-build-space@master
with:
root-reserve-mb: 5120
remove-dotnet: true
remove-android: true
remove-docker-images: true
- name: Install build dependencies - name: Checkout code
run: | uses: actions/checkout@v3
dnf distro-sync -y
dnf install -y git make gcc flex bison bzip2 rpm-build
dnf install -y rpmdevtools rpm-sign 'dnf-command(builddep)'
# Install build dependencies - name: Initialize containers
dnf builddep -y kernel run: |
bash ./.github/scripts/container/create.sh \
registry.fedoraproject.org/fedora:${{ env.FEDORA }}
# Install additional build dependencies - name: Install build dependencies
dnf install -y sbsigntools run: |
bash ./.github/scripts/container/exec.sh \
-- \
bash ./.github/scripts/package/fedora.sh setup-builddeps
- name: Setup secureboot certificate - name: Setup secureboot certificate
env: env:
SB_KEY: ${{ secrets.SURFACE_SB_KEY }} SB_KEY: ${{ secrets.SURFACE_SB_KEY }}
run: | run: |
# Install the surface secureboot certificate bash ./.github/scripts/container/exec.sh \
echo "$SB_KEY" | base64 -d > pkg/fedora/kernel-surface/secureboot/MOK.key -e SB_KEY \
cp pkg/keys/surface.crt pkg/fedora/kernel-surface/secureboot/MOK.crt -- \
bash ./.github/scripts/package/fedora.sh setup-secureboot
- name: Build packages - name: Build packages
run: | run: |
cd pkg/fedora/kernel-surface bash ./.github/scripts/container/exec.sh \
-- \
bash ./.github/scripts/package/fedora.sh build-packages
# setup git - name: Sign packages
git config --global user.email "surfacebot@users.noreply.github.com" env:
git config --global user.name "surfacebot" GPG_KEY: ${{ secrets.LINUX_SURFACE_GPG_KEY }}
run: |
bash ./.github/scripts/container/exec.sh \
-e GPG_KEY \
-e GPG_KEY_ID \
-- \
bash ./.github/scripts/package/fedora.sh sign-packages
# Build source RPM packages - name: Upload artifacts
python3 build-linux-surface.py --mode srpm --ark-dir kernel-ark --outdir srpm uses: actions/upload-artifact@v3
with:
# Remove the kernel-ark tree to get as much free disk space as possible name: fedora-${{ env.FEDORA }}-latest
rm -rf kernel-ark path: pkg/fedora/kernel-surface/out/x86_64
# Build binary RPM packages
rpmbuild -rb --define "_rpmdir $PWD/out" srpm/*.src.rpm
- name: Sign packages
env:
GPG_KEY: ${{ secrets.LINUX_SURFACE_GPG_KEY }}
run: |
cd pkg/fedora/kernel-surface/out/x86_64
# import GPG key
echo "$GPG_KEY" | base64 -d | gpg --import --no-tty --batch --yes
# sign packages
rpm --resign *.rpm --define "_gpg_name $GPG_KEY_ID"
- name: Upload artifacts
uses: actions/upload-artifact@v3
with:
name: fedora-37-latest
path: pkg/fedora/kernel-surface/out/x86_64
release: release:
name: Publish release name: Publish release
needs: [build] needs: [build]
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Download artifacts - name: Download artifacts
uses: actions/download-artifact@v3 uses: actions/download-artifact@v3
with: with:
name: fedora-37-latest name: fedora-${{ env.FEDORA }}-latest
path: fedora-37-latest path: fedora-${{ env.FEDORA }}-latest
- name: Upload assets - name: Upload assets
uses: svenstaro/upload-release-action@v2 uses: svenstaro/upload-release-action@v2
with: with:
repo_token: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }} repo_token: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }}
file: ./*-latest/* file: ./*-latest/*
tag: ${{ github.ref }} tag: ${{ github.ref }}
overwrite: true overwrite: true
file_glob: true file_glob: true
repo: repo:
name: Update package repository name: Update package repository
needs: [release] needs: [release]
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: registry.fedoraproject.org/fedora:37
steps: steps:
- name: Install dependencies - name: Checkout repository
run: | uses: actions/checkout@v3
dnf install -y git findutils
- name: Download artifacts - name: Download artifacts
uses: actions/download-artifact@v3 uses: actions/download-artifact@v3
with: with:
name: fedora-37-latest name: fedora-${{ env.FEDORA }}-latest
path: fedora-37-latest path: fedora-${{ env.FEDORA }}-latest
- name: Update repository - name: Initialize containers
env: run: |
SURFACEBOT_TOKEN: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }} bash ./.github/scripts/container/create.sh \
BRANCH_STAGING: u/staging registry.fedoraproject.org/fedora:${{ env.FEDORA }}
GIT_REF: ${{ github.ref }}
run: |
repo="https://surfacebot:${SURFACEBOT_TOKEN}@github.com/linux-surface/repo.git"
# clone package repository - name: Update repository
git clone -b "${BRANCH_STAGING}" "${repo}" repo env:
SURFACEBOT_TOKEN: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }}
# copy packages GIT_REF: ${{ github.ref }}
cp fedora-37-latest/* repo/fedora/f37 BRANCH_STAGING: u/staging
cd repo/fedora/f37 run: |
bash ./.github/scripts/container/exec.sh \
# parse git tag from ref -e SURFACEBOT_TOKEN \
GIT_TAG=$(echo $GIT_REF | sed 's|^refs/tags/||g') -e GIT_REF \
-e BRANCH_STAGING \
# convert packages into references -e GITHUB_REPOSITORY \
for pkg in $(find . -name '*.rpm'); do -- \
echo "linux-surface:$GIT_TAG/$(basename $pkg)" > $pkg.blob bash ./.github/scripts/repository/fedora.sh ${{ env.FEDORA }}
rm $pkg
done
# set git identity
git config --global user.email "surfacebot@users.noreply.github.com"
git config --global user.name "surfacebot"
# commit and push
update_branch="${BRANCH_STAGING}-$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)"
git checkout -b "${update_branch}"
git add .
git commit -m "Update Fedora 37 kernel"
git push --set-upstream origin "${update_branch}"

View file

@ -1,144 +1,121 @@
name: Fedora 38
env:
FEDORA: 38
GPG_KEY_ID: 56C464BAAC421453
on: on:
push: push:
tags: tags:
- 'fedora-38-*' - 'fedora-38-*'
name: Fedora 38
env:
GPG_KEY_ID: 56C464BAAC421453
jobs: jobs:
build: build:
name: Build Kernel name: Build Kernel
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: registry.fedoraproject.org/fedora:38
steps: steps:
- name: Checkout code - name: Maximize disk space
uses: actions/checkout@v3 uses: easimon/maximize-build-space@master
with:
root-reserve-mb: 5120
remove-dotnet: true
remove-android: true
remove-docker-images: true
- name: Install build dependencies - name: Checkout code
run: | uses: actions/checkout@v3
dnf distro-sync -y
dnf install -y git make gcc flex bison bzip2 rpm-build
dnf install -y rpmdevtools rpm-sign 'dnf-command(builddep)'
# Install build dependencies - name: Initialize containers
dnf builddep -y kernel run: |
bash ./.github/scripts/container/create.sh \
registry.fedoraproject.org/fedora:${{ env.FEDORA }}
# Install additional build dependencies - name: Install build dependencies
dnf install -y sbsigntools run: |
bash ./.github/scripts/container/exec.sh \
-- \
bash ./.github/scripts/package/fedora.sh setup-builddeps
- name: Setup secureboot certificate - name: Setup secureboot certificate
env: env:
SB_KEY: ${{ secrets.SURFACE_SB_KEY }} SB_KEY: ${{ secrets.SURFACE_SB_KEY }}
run: | run: |
# Install the surface secureboot certificate bash ./.github/scripts/container/exec.sh \
echo "$SB_KEY" | base64 -d > pkg/fedora/kernel-surface/secureboot/MOK.key -e SB_KEY \
cp pkg/keys/surface.crt pkg/fedora/kernel-surface/secureboot/MOK.crt -- \
bash ./.github/scripts/package/fedora.sh setup-secureboot
- name: Build packages - name: Build packages
run: | run: |
cd pkg/fedora/kernel-surface bash ./.github/scripts/container/exec.sh \
-- \
bash ./.github/scripts/package/fedora.sh build-packages
# setup git - name: Sign packages
git config --global user.email "surfacebot@users.noreply.github.com" env:
git config --global user.name "surfacebot" GPG_KEY: ${{ secrets.LINUX_SURFACE_GPG_KEY }}
run: |
bash ./.github/scripts/container/exec.sh \
-e GPG_KEY \
-e GPG_KEY_ID \
-- \
bash ./.github/scripts/package/fedora.sh sign-packages
# Build source RPM packages - name: Upload artifacts
python3 build-linux-surface.py --mode srpm --ark-dir kernel-ark --outdir srpm uses: actions/upload-artifact@v3
with:
# Remove the kernel-ark tree to get as much free disk space as possible name: fedora-${{ env.FEDORA }}-latest
rm -rf kernel-ark path: pkg/fedora/kernel-surface/out/x86_64
# Build binary RPM packages
rpmbuild -rb --define "_rpmdir $PWD/out" srpm/*.src.rpm
- name: Sign packages
env:
GPG_KEY: ${{ secrets.LINUX_SURFACE_GPG_KEY }}
run: |
cd pkg/fedora/kernel-surface/out/x86_64
# import GPG key
echo "$GPG_KEY" | base64 -d | gpg --import --no-tty --batch --yes
# sign packages
rpm --resign *.rpm --define "_gpg_name $GPG_KEY_ID"
- name: Upload artifacts
uses: actions/upload-artifact@v3
with:
name: fedora-38-latest
path: pkg/fedora/kernel-surface/out/x86_64
release: release:
name: Publish release name: Publish release
needs: [build] needs: [build]
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Download artifacts - name: Download artifacts
uses: actions/download-artifact@v3 uses: actions/download-artifact@v3
with: with:
name: fedora-38-latest name: fedora-${{ env.FEDORA }}-latest
path: fedora-38-latest path: fedora-${{ env.FEDORA }}-latest
- name: Upload assets - name: Upload assets
uses: svenstaro/upload-release-action@v2 uses: svenstaro/upload-release-action@v2
with: with:
repo_token: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }} repo_token: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }}
file: ./*-latest/* file: ./*-latest/*
tag: ${{ github.ref }} tag: ${{ github.ref }}
overwrite: true overwrite: true
file_glob: true file_glob: true
repo: repo:
name: Update package repository name: Update package repository
needs: [release] needs: [release]
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: registry.fedoraproject.org/fedora:38
steps: steps:
- name: Install dependencies - name: Checkout repository
run: | uses: actions/checkout@v3
dnf install -y git findutils
- name: Download artifacts - name: Download artifacts
uses: actions/download-artifact@v3 uses: actions/download-artifact@v3
with: with:
name: fedora-38-latest name: fedora-${{ env.FEDORA }}-latest
path: fedora-38-latest path: fedora-${{ env.FEDORA }}-latest
- name: Update repository - name: Initialize containers
env: run: |
SURFACEBOT_TOKEN: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }} bash ./.github/scripts/container/create.sh \
BRANCH_STAGING: u/staging registry.fedoraproject.org/fedora:${{ env.FEDORA }}
GIT_REF: ${{ github.ref }}
run: |
repo="https://surfacebot:${SURFACEBOT_TOKEN}@github.com/linux-surface/repo.git"
# clone package repository - name: Update repository
git clone -b "${BRANCH_STAGING}" "${repo}" repo env:
SURFACEBOT_TOKEN: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }}
# copy packages GIT_REF: ${{ github.ref }}
cp fedora-38-latest/* repo/fedora/f38 BRANCH_STAGING: u/staging
cd repo/fedora/f38 run: |
bash ./.github/scripts/container/exec.sh \
# parse git tag from ref -e SURFACEBOT_TOKEN \
GIT_TAG=$(echo $GIT_REF | sed 's|^refs/tags/||g') -e GIT_REF \
-e BRANCH_STAGING \
# convert packages into references -e GITHUB_REPOSITORY \
for pkg in $(find . -name '*.rpm'); do -- \
echo "linux-surface:$GIT_TAG/$(basename $pkg)" > $pkg.blob bash ./.github/scripts/repository/fedora.sh ${{ env.FEDORA }}
rm $pkg
done
# set git identity
git config --global user.email "surfacebot@users.noreply.github.com"
git config --global user.name "surfacebot"
# commit and push
update_branch="${BRANCH_STAGING}-$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)"
git checkout -b "${update_branch}"
git add .
git commit -m "Update Fedora 38 kernel"
git push --set-upstream origin "${update_branch}"