From ffa8b7c606748cb8dcb0575c0223ed0a413b3a1d Mon Sep 17 00:00:00 2001 From: Dorian Stoll Date: Sun, 23 Jul 2023 11:53:50 +0200 Subject: [PATCH] pkg: debian: Move to script based workflow For the future we should consider adapting the Fedora python scripts for Debian and move the logic out of the CI script into a dedicated packaging script. --- .github/scripts/package/debian.sh | 146 ++++++++++++++ .github/scripts/repository/debian.sh | 66 +++++++ .github/workflows/debian.yml | 275 +++++++++------------------ pkg/debian/kernel/version.conf | 3 + 4 files changed, 302 insertions(+), 188 deletions(-) create mode 100644 .github/scripts/package/debian.sh create mode 100644 .github/scripts/repository/debian.sh create mode 100644 pkg/debian/kernel/version.conf diff --git a/.github/scripts/package/debian.sh b/.github/scripts/package/debian.sh new file mode 100644 index 000000000..74cdce3a1 --- /dev/null +++ b/.github/scripts/package/debian.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +if [ -z "${1:-}" ]; then + $0 setup-builddeps + $0 setup-secureboot + $0 build-packages + $0 sign-packages + exit +fi + +apt-get() +{ + command apt-get -y "$@" +} + +MAINLINE_REPO="git://git.launchpad.net/~ubuntu-kernel-test/ubuntu/+source/linux/+git/mainline-crack" +MAINLINE_BRANCH="cod/mainline" + +case "${1:-}" in +setup-builddeps) + SOURCES="$(sed 's/^deb /deb-src /' /etc/apt/sources.list)" + echo "${SOURCES}" >> /etc/apt/sources.list + + ln -snf /usr/share/zoneinfo/UTC /etc/localtime + echo UTC > /etc/timezone + + apt-get update + apt-get upgrade + apt-get install build-essential fakeroot rsync git wget software-properties-common \ + zstd lz4 sbsigntool debhelper dpkg-dev dpkg-sig + apt-get build-dep linux + + # install python 3.11, required for configuring the kernel via Ubuntu's annotation format + add-apt-repository -y ppa:deadsnakes + + apt-get update + apt-get upgrade + apt-get install python3.11 + + rm -f /usr/bin/python + rm -f /usr/bin/python3 + ln -s /usr/bin/python3.11 /usr/bin/python + ln -s /usr/bin/python3.11 /usr/bin/python3 + ;; +setup-secureboot) + if [ -z "${SB_KEY:-}" ]; then + echo "WARNING: No secureboot key configured, skipping signing." + exit + fi + + mkdir -p pkg/debian/kernel/keys + + # Install the surface secureboot certificate + echo "${SB_KEY}" | base64 -d > pkg/debian/kernel/keys/MOK.key + cp pkg/keys/surface.crt pkg/debian/kernel/keys/MOK.crt + ;; +build-packages) + pushd pkg/debian/kernel || exit 1 + + . version.conf + + # setup git + git config --global user.name "surfacebot" + git config --global user.email "surfacebot@users.noreply.github.com" + + # get ubuntu mainline source + # see https://kernel.ubuntu.com/~kernel-ppa/mainline + git clone "${MAINLINE_REPO}" --branch "${MAINLINE_BRANCH}/v${KERNEL_VERSION}" --depth 1 linux + + if [ -d "keys" ]; then + mv keys linux + fi + + pushd linux || exit 1 + + # apply surface build/packaging patches + find .. -name '*.patch' -type f -exec git apply --index --reject {} \; + + git add . + git commit --allow-empty -m "Apply linux-surface packaging patches" + + KERNEL_MAJORVER="${KERNEL_VERSION%.*}" + + # apply surface patches + find "../../../../patches/${KERNEL_MAJORVER}" -name '*.patch' -type f -exec \ + git apply --index --reject {} \; + + git add . + git commit --allow-empty -m "Apply linux-surface patches" + + # generate base config + ./debian/scripts/misc/annotations --arch amd64 --flavour generic --export > ../base.config + + # merge configs + ./scripts/kconfig/merge_config.sh \ + ../base.config \ + ../ubuntu.config \ + "../../../../configs/surface-${KERNEL_MAJORVER}.config" + + # Explicitly set package version, including revision. This is picked up by 'make bindeb-pkg'. + export KDEB_PKGVERSION="${KERNEL_VERSION}${KERNEL_LOCALVERSION}-${KERNEL_REVISION}" + + # The DPKG in Ubuntu 22.04 defaults to using ZSTD, which is not yet supported by the DPKG in Debian 11 + export KDEB_COMPRESS="xz" + + make bindeb-pkg -j "$(nproc)" + + popd || exit 1 + popd || exit 1 + + pushd pkg/debian/meta || exit 1 + + ./mkdebian.sh "$(make -C ../kernel/linux -s kernelrelease)" "${KERNEL_REVISION}" + dpkg-buildpackage -b -Zxz + + popd || exit 1 + + pushd pkg/debian || exit 1 + + mkdir release + + find . -name 'linux-libc-dev*.deb' -type f -exec rm {} \; + find . -name '*.deb' -type f -exec cp {} release \; + + popd || exit 1 + ;; +sign-packages) + if [ -z "${GPG_KEY:-}" ] || [ -z "${GPG_KEY_ID:-}" ]; then + echo "WARNING: No GPG key configured, skipping signing." + exit + fi + + pushd pkg/debian/release || exit 1 + + # import GPG key + echo "${GPG_KEY}" | base64 -d | gpg --import --no-tty --batch --yes + + # sign packages + find . -name '*.deb' -type f -exec \ + dpkg-sig -g "--batch --no-tty" --sign builder -k "${GPG_KEY_ID}" {} \; + + popd || exit 1 + ;; +esac diff --git a/.github/scripts/repository/debian.sh b/.github/scripts/repository/debian.sh new file mode 100644 index 000000000..83a8c9863 --- /dev/null +++ b/.github/scripts/repository/debian.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +apt-get() +{ + command apt-get -y "$@" +} + +if [ -z "${GIT_REF:-}" ]; then + echo "GIT_REF is unset!" + exit 1 +fi + +if [ -z "${GITHUB_REPOSITORY:-}" ]; then + echo "GITHUB_REPOSITORY is unset!" + exit 1 +fi + +if [ -z "${SURFACEBOT_TOKEN:-}" ]; then + echo "SURFACEBOT_TOKEN is unset!" + exit 1 +fi + +if [ -z "${BRANCH_STAGING:-}" ]; then + echo "BRANCH_STAGING is unset!" + exit 1 +fi + +REPONAME="$(echo "${GITHUB_REPOSITORY}" | cut -d'/' -f2)" +REPO="https://surfacebot:${SURFACEBOT_TOKEN}@github.com/linux-surface/repo.git" + +# parse git tag from ref +GIT_TAG="${GIT_REF#refs/tags/}" + +# Install dependencies +apt-get update +apt-get install git + +# clone package repository +git clone -b "${BRANCH_STAGING}" "${REPO}" repo + +# copy packages +find debian-latest -type f -exec cp {} repo/debian \; +pushd repo/debian || exit 1 + +# convert packages into references +while read -rd $'\n' FILE; do + echo "${REPONAME}:${GIT_TAG}/$(basename "${FILE}")" > "${FILE}.blob" + rm "${FILE}" +done <<< "$(find . -name '*.deb')" + +RAND="$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 32 | head -n 1)" +BRANCH="${BRANCH_STAGING}-${RAND}" + +# set git identity +git config --global user.name "surfacebot" +git config --global user.email "surfacebot@users.noreply.github.com" + +# commit and push +git checkout -b "${BRANCH}" +git add . +git commit -m "Update Debian ${REPONAME} package" +git push --set-upstream origin "${BRANCH}" + +popd || exit 1 \ No newline at end of file diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 2def8ea83..e248d1c05 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -1,221 +1,120 @@ +name: Debian + +env: + GPG_KEY_ID: 56C464BAAC421453 + on: push: tags: - 'debian-*' -name: Debian - -env: - GPG_KEY_ID: 56C464BAAC421453 - KERNEL_VERSION: 6.4.4 - KERNEL_REVISION: 3 - LOCALVERSION: -surface - MAINLINE_REPO: git://git.launchpad.net/~ubuntu-kernel-test/ubuntu/+source/linux/+git/mainline-crack - MAINLINE_BRANCH: cod/mainline - TZ: UTC - jobs: build: name: Build Kernel runs-on: ubuntu-latest - container: ubuntu:20.04 steps: - - name: Checkout code - uses: actions/checkout@v3 + - name: Maximize disk space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 5120 + remove-dotnet: true + remove-android: true + remove-docker-images: true - - name: Install build dependencies - run: | - sed 's/^deb /deb-src /' /etc/apt/sources.list >> /etc/apt/sources.list - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime - echo $TZ > /etc/timezone - apt-get -y update - apt-get -y upgrade - apt-get -y install build-essential fakeroot rsync git wget software-properties-common \ - zstd lz4 sbsigntool debhelper dpkg-dev dpkg-sig - apt-get -y build-dep linux + - name: Checkout code + uses: actions/checkout@v3 - - name: Install Python 3.11 - run: | - # install python 3.11, required for configuring the kernel via Ubuntu's annotation format - add-apt-repository -y ppa:deadsnakes + - name: Initialize containers + run: | + bash ./.github/scripts/container/create.sh \ + ubuntu:20.04 - apt-get -y update - apt-get -y upgrade + - name: Install build dependencies + run: | + bash ./.github/scripts/container/exec.sh \ + -- \ + bash ./.github/scripts/package/debian.sh setup-builddeps - apt-get -y install python3.11 + - name: Setup secureboot certificate + env: + SB_KEY: ${{ secrets.SURFACE_SB_KEY }} + run: | + bash ./.github/scripts/container/exec.sh \ + -e SB_KEY \ + -- \ + bash ./.github/scripts/package/debian.sh setup-secureboot - rm -f /usr/bin/python - rm -f /usr/bin/python3 - ln -s /usr/bin/python3.11 /usr/bin/python - ln -s /usr/bin/python3.11 /usr/bin/python3 + - name: Build packages + run: | + bash ./.github/scripts/container/exec.sh \ + -- \ + bash ./.github/scripts/package/debian.sh build-packages - - name: Prepare kernel source - run: | - cd pkg/debian/kernel + - name: Sign packages + env: + GPG_KEY: ${{ secrets.LINUX_SURFACE_GPG_KEY }} + run: | + bash ./.github/scripts/container/exec.sh \ + -e GPG_KEY \ + -e GPG_KEY_ID \ + -- \ + bash ./.github/scripts/package/debian.sh sign-packages - # setup git - git config --global user.email "surfacebot@users.noreply.github.com" - git config --global user.name "surfacebot" - - # get ubuntu mainline source - # see https://kernel.ubuntu.com/~kernel-ppa/mainline - git clone "$MAINLINE_REPO" --branch "$MAINLINE_BRANCH/v$KERNEL_VERSION" --depth 1 linux - - cd linux - - # apply surface build/packaging patches - for PATCH in ../*.patch; do - git apply --index --reject ${PATCH} - done - - git add . - git commit --allow-empty -m "Apply linux-surface packaging patches" - - # apply surface patches - for PATCH in ../../../../patches/${KERNEL_VERSION%.*}/*.patch; do - git apply --index --reject ${PATCH} - done - - git add . - git commit --allow-empty -m "Apply linux-surface patches" - - - name: Configure - run: | - cd pkg/debian/kernel/linux - - # generate base config - ./debian/scripts/misc/annotations --arch amd64 --flavour generic --export > ../base.config - - # merge configs - ./scripts/kconfig/merge_config.sh \ - ../base.config \ - ../ubuntu.config \ - ../../../../configs/surface-${KERNEL_VERSION%.*}.config - - - name: Setup secureboot certificate - env: - SB_KEY: ${{ secrets.SURFACE_SB_KEY }} - run: | - cd pkg - - mkdir -p debian/kernel/linux/keys - - # unlock/copy key and certificate - echo "$SB_KEY" | base64 -d > debian/kernel/linux/keys/MOK.key - cp keys/surface.crt debian/kernel/linux/keys/MOK.crt - - - name: Build kernel - env: - # The DPKG in Ubuntu 22.04 defaults to using ZSTD, - # which is not yet supported by the DPKG in Debian 11 - KDEB_COMPRESS: xz - run: | - cd pkg/debian/kernel/linux - - # Explicitly set package version, including revision. This is picked up - # by 'make bindeb-pkg'. - export KDEB_PKGVERSION="${KERNEL_VERSION}${LOCALVERSION}-${KERNEL_REVISION}" - - make bindeb-pkg -j2 - - - name: Build meta-package - run: | - cd pkg/debian/meta - ./mkdebian.sh $(make -C ../kernel/linux -s kernelrelease) ${KERNEL_REVISION} - dpkg-buildpackage -b -Zxz - - - name: Prepare release - run: | - mkdir release - rm pkg/debian/kernel/linux-libc-dev*.deb - mv pkg/debian/kernel/*.deb release - mv pkg/debian/*.deb release - - - name: Sign packages - env: - GPG_KEY: ${{ secrets.LINUX_SURFACE_GPG_KEY }} - run: | - # import GPG key - echo "$GPG_KEY" | base64 -d | gpg --import --no-tty --batch --yes - export GPG_TTY=$(tty) - - # sign package - dpkg-sig -g "--batch --no-tty" --sign builder -k $GPG_KEY_ID release/*.deb - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: debian-latest - path: release + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: debian-latest + path: pkg/debian/release release: name: Publish release needs: [build] runs-on: ubuntu-latest steps: - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - name: debian-latest - path: debian-latest + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + name: debian-latest + path: debian-latest - - name: Upload assets - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }} - file: ./*-latest/* - tag: ${{ github.ref }} - overwrite: true - file_glob: true + - name: Upload assets + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }} + file: ./*-latest/* + tag: ${{ github.ref }} + overwrite: true + file_glob: true repo: name: Update package repository needs: [release] runs-on: ubuntu-latest - container: debian:sid steps: - - name: Install dependencies - run: | - apt-get update - apt-get install -y git + - name: Checkout repository + uses: actions/checkout@v3 - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - name: debian-latest - path: debian-latest + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + name: debian-latest + path: debian-latest - - name: Update repository - env: - SURFACEBOT_TOKEN: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }} - BRANCH_STAGING: u/staging - GIT_REF: ${{ github.ref }} - run: | - repo="https://surfacebot:${SURFACEBOT_TOKEN}@github.com/linux-surface/repo.git" + - name: Initialize containers + run: | + bash ./.github/scripts/container/create.sh \ + debian:sid - # clone package repository - git clone -b "${BRANCH_STAGING}" "${repo}" repo - - # copy packages - cp debian-latest/* repo/debian/ - cd repo/debian - - # parse git tag from ref - GIT_TAG=$(echo $GIT_REF | sed 's|^refs/tags/||g') - - # convert packages into references - for pkg in $(find . -name '*.deb'); do - echo "linux-surface:$GIT_TAG/$(basename $pkg)" > $pkg.blob - rm $pkg - done - - # set git identity - git config --global user.email "surfacebot@users.noreply.github.com" - git config --global user.name "surfacebot" - - # commit and push - update_branch="${BRANCH_STAGING}-$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)" - git switch -c "${update_branch}" - git add . - git commit -m "Update Debian kernel" - git push --set-upstream origin "${update_branch}" + - name: Update repository + env: + SURFACEBOT_TOKEN: ${{ secrets.LINUX_SURFACE_BOT_TOKEN }} + GIT_REF: ${{ github.ref }} + BRANCH_STAGING: u/staging + run: | + bash ./.github/scripts/container/exec.sh \ + -e SURFACEBOT_TOKEN \ + -e GIT_REF \ + -e BRANCH_STAGING \ + -e GITHUB_REPOSITORY \ + -- \ + bash ./.github/scripts/repository/debian.sh diff --git a/pkg/debian/kernel/version.conf b/pkg/debian/kernel/version.conf new file mode 100644 index 000000000..183bf4229 --- /dev/null +++ b/pkg/debian/kernel/version.conf @@ -0,0 +1,3 @@ +KERNEL_VERSION="6.4.4" +KERNEL_REVISION="2" +KERNEL_LOCALVERSION="-surface" \ No newline at end of file