devops: Update release process for Github

This commit is contained in:
Thomas Buckley-Houston 2022-07-18 12:58:27 -04:00
parent 87bf16dd20
commit 3bc427f416
15 changed files with 239 additions and 123 deletions

View File

@ -1,13 +1,15 @@
name: Test/Release
name: Test-Release
on: [push]
jobs:
test_maybe_release:
name: Test (then release if new version)
tests:
name: "Tests (webextension, interfacer: unit, tty, http-server)"
runs-on: ubuntu-latest
env:
GOPATH: ${{ github.workspace }}
GOBIN: ${{ github.workspace }}/bin
outputs:
is_new_version: ${{ steps.check_versions.outputs.is_new_version }}
steps:
- name: Checkout
uses: actions/checkout@v3
@ -16,15 +18,15 @@ jobs:
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: 1.18.x
go-version-file: 'interfacer/go.mod'
- name: Setup node
uses: actions/setup-node@v3
with:
node-version: 16
node-version-file: '.nvmrc'
- name: Install Firefox
uses: browser-actions/setup-firefox@latest
with:
firefox-version: 102.0.1 # TODO: Use same version in Dockerfile
firefox-version: 102.0.1
- run: firefox --version
# Web extension tests
@ -33,14 +35,13 @@ jobs:
- name: Web extension tests
run: npm test
working-directory: ./webext
- run: npm run build_webextension
working-directory: ./webext
# Interfacer tests
- name: Build
run: go build ./cmd/browsh
working-directory: ./interfacer
- name: Interfacer tests setup
run: |
touch interfacer/src/browsh/browsh.xpi
cd webext
npm run build:dev
- name: Unit tests
run: go test -v $(find src/browsh -name '*.go' | grep -v windows)
working-directory: ./interfacer
@ -51,19 +52,64 @@ jobs:
if: ${{ failure() }}
run: cat ./interfacer/test/tty/debug.log || echo "No log file"
- name: HTTP Server tests
run: go test test/http-server/*.go -v -ginkgo.slowSpecThreshold=30 -ginkgo.flakeAttempts=3
working-directory: ./interfacer
uses: nick-fields/retry@v2
with:
max_attempts: 3
retry_on: error
timeout_minutes: 15
command: |
cd interfacer
go test test/http-server/*.go -v -ginkgo.slowSpecThreshold=30 -ginkgo.flakeAttempts=3
- name: HTTP Server debug log
if: ${{ failure() }}
run: cat ./interfacer/test/http-server/debug.log || echo "No log file"
# Release
- name: Check for new version
id: check_versions
run: ./ctl.sh github_actions_output_version_status
- name: Release
if: contains(steps.check_versions.outputs.is_new_version, 'true')
release:
name: Release
runs-on: ubuntu-latest
needs: tests
if: github.ref == 'refs/heads/master' && contains(needs.tests.outputs.is_new_version, 'true')
env:
GOPATH: ${{ github.workspace }}
GOBIN: ${{ github.workspace }}/bin
MDN_KEY: ${{ secrets.MDN_KEY }}
DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: Setup Deploy Keys
uses: webfactory/ssh-agent@v0.5.4
with:
# Note that these private keys depend on having an ssh-keygen'd comment with the
# Git remote URL. This is because Github Actions use the *first* matching private
# key and fails if it doesn't match. webfactory/ssh-agent
ssh-private-key: |
${{ secrets.HOMEBREW_DEPLOY_KEY }}
${{ secrets.WWW_DEPLOY_KEY }}
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Setup node
uses: actions/setup-node@v3
with:
node-version-file: '.nvmrc'
- name: Setup go
uses: actions/setup-go@v3
with:
go-version-file: 'interfacer/go.mod'
- run: npm ci
working-directory: ./webext
- name: Binary Release
run: ./ctl.sh release
- name: Push new tag
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
tags: true
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
@ -71,5 +117,7 @@ jobs:
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
- name: Docker Release
run: ./ctl.sh docker_release
- name: Update Homebrew Tap
run: ./ctl.sh update_homebrew_tap_with_new_version
- name: Update Browsh Website
run: ./ctl.sh update_browsh_website_with_new_version

11
.gitignore vendored
View File

@ -12,3 +12,14 @@ webext/node_modules
webext/dist/*
dist
*.xpi
# This is because of an odd permissions quirk on Github Actions. I can't seem to find a
# way to delete these files in CI, so let's just ignore them. Otherwise Goreleaser complains
# about a dirty working tree.
/pkg
/bin
# Goreleaser needs to upload the webextension as an extra file in the release. But it doesn't
# like Git to be in a dirty state. Also Goreleaser is run at PWD ./interfacer, so we can't
# reference the webextension with something like ../webext/...
interfacer/browsh-*.xpi

View File

@ -24,21 +24,19 @@ ADD .github .github
ADD scripts scripts
ADD ctl.sh .
# Install Golang
# Install Golang and Browsh
ENV GOROOT=/go
ENV GOPATH=/go-home
ENV PATH=$GOROOT/bin:$GOPATH/bin:$PATH
RUN /build/ctl.sh install_golang
ENV BASE=$GOPATH/src/browsh/interfacer
ADD interfacer $BASE
WORKDIR $BASE
RUN /build/ctl.sh install_golang $BASE
RUN /build/ctl.sh build_browsh_binary $BASE
# Install firefox
RUN /build/ctl.sh install_firefox
# Build Browsh
ENV BASE=$GOPATH/src/browsh/interfacer
WORKDIR $BASE
ADD interfacer $BASE
RUN /build/ctl.sh build_browsh_binary $BASE
###########################
# Actual final Docker image

View File

@ -35,8 +35,8 @@ is running somewhere else on mains electricity.
## Installation
Download a binary from the [releases](https://github.com/browsh-org/browsh/releases) (~7MB).
You will need to have [Firefox 63](https://www.mozilla.org/en-US/firefox/new/) (or higher) already installed.
Download a binary from the [releases](https://github.com/browsh-org/browsh/releases) (~11MB).
You will need to have [Firefox](https://www.mozilla.org/en-US/firefox/new/) already installed.
Or download and run the Docker image (~230MB) with:
`docker run --rm -it browsh/browsh`

10
ctl.sh
View File

@ -1,11 +1,12 @@
#!/bin/env bash
#!/usr/bin/env bash
set -e
function_to_run=$1
export PROJECT_ROOT && PROJECT_ROOT=$(git rev-parse --show-toplevel)
export PROJECT_ROOT
export GORELEASER_VERSION=1.10.2
export GOBINDATA_VERSION=3.23.0
PROJECT_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
function _includes_path {
echo "$PROJECT_ROOT"/scripts
@ -26,4 +27,7 @@ if [[ $(type -t "$function_to_run") != function ]]; then
fi
shift
pushd "$PROJECT_ROOT" || _panic
"$function_to_run" "$@"
popd || _panic

View File

@ -57,15 +57,14 @@ nfpms:
brews:
- name: browsh
tap:
owner: browsh-org
name: homebrew-browsh
commit_author:
name: Goreleaser Bot care of Github Actions
email: ci@github.com
homepage: "https://www.brow.sh"
description: "The modern, text-based browser"
caveats: "You need Firefox 57 or newer to run Browsh"
# We do the upload manually because Goreleaser doesn't support Deploy Keys and Github
# doesn't support repo-specific Access Tokens 🙄
skip_upload: true
release:
extra_files:
- glob: ./webext/dist/web-ext-artifacts/browsh-{{ Env.BROWSH_VERSION }}-an+fx.xpi
- glob: ./browsh-*.xpi

View File

@ -1,26 +0,0 @@
#!/usr/bin/env bash
set -ex
if [ ! -f .travis.yml ]; then
PROJECT_ROOT=$(git rev-parse --show-toplevel)
else
PROJECT_ROOT=.
fi
line=$(grep 'firefox: "' < $PROJECT_ROOT/.travis.yml)
version=$(echo $line | grep -o '".*"' | cut -d " " -f 1 | sed 's/"//g')
# Firefox is needed both for testing in Travis and embedding in the Docker
# image used by the Browsh as a Service platform. So we need to be able to
# give a specific and consistent version pin.
FIREFOX_VERSION=$version
mkdir -p $HOME/bin
pushd $HOME/bin
curl -L -o firefox.tar.bz2 https://ftp.mozilla.org/pub/firefox/releases/$FIREFOX_VERSION/linux-x86_64/en-US/firefox-$FIREFOX_VERSION.tar.bz2
bzip2 -d firefox.tar.bz2
tar xf firefox.tar
popd
firefox --version

View File

@ -230,8 +230,6 @@ func firefoxMarionette() {
sendFirefoxCommand("WebDriver:NewSession", map[string]interface{}{})
}
// Install the Browsh extension that was bundled with `go-bindata` under
// `webextension.go`.
func installWebextension() {
data, err := browshXpi.ReadFile("browsh.xpi")
if err != nil {

View File

@ -1,3 +1,3 @@
package browsh
var browshVersion = "1.6.4"
var browshVersion = "1.8.0"

View File

@ -1,7 +1,23 @@
#!/usr/bin/env bash
function build_webextension() {
local NODE_BIN=$PROJECT_ROOT/webext/node_modules/.bin
export XPI_PATH="$PROJECT_ROOT"/interfacer/src/browsh/browsh.xpi
export XPI_SOURCE_DIR=$PROJECT_ROOT/webext/dist/web-ext-artifacts
export NODE_BIN=$PROJECT_ROOT/webext/node_modules/.bin
MDN_USER="user:13243312:78"
function versioned_xpi_file() {
echo "$XPI_SOURCE_DIR/browsh-$(browsh_version).xpi"
}
# You'll want to use this with `go run ./cmd/browsh --debug --firefox.use-existing`
function build_webextension_watch() {
"$NODE_BIN"/web-ext run \
--firefox contrib/firefoxheadless.sh \
--verbose
}
function build_webextension_production() {
local version && version=$(browsh_version)
cd "$PROJECT_ROOT"/webext && "$NODE_BIN"/webpack
cd "$PROJECT_ROOT"/webext/dist && rm ./*.map
@ -13,36 +29,63 @@ function build_webextension() {
"$NODE_BIN"/web-ext build --overwrite-dest
ls -alh web-ext-artifacts
version=$(browsh_version)
webextension_sign
local source_file && source_file=$(versioned_xpi_file)
local source_file
local source_dir=$PROJECT_ROOT/webext/dist/web-ext-artifacts
local bundle_file=$PROJECT_ROOT/interfacer/src/browsh/browsh.xpi
echo "Bundling $source_file to $XPI_PATH"
cp -f "$source_file" "$XPI_PATH"
if [ "$BROWSH_ENV" == "RELEASE" ]; then
# The signed version. There can only be one canonical XPI for each semantic
# version.
source_file="$source_dir/browsh-$version-an+fx.xpi"
echo "Making extra copy for Goreleaser to put in Github release:"
local goreleaser_pwd="$PROJECT_ROOT"/interfacer/
cp -a "$source_file" "$goreleaser_pwd"
ls -alh "$goreleaser_pwd"
}
# It is possible to use unsigned webextensions in Firefox but it requires that Firefox
# uses problematically insecure config. I know it's a hassle having to jump through all
# these signing hoops, but I think it's better to use a standard Firefox configuration.
# Moving away from the webextension alltogether is another story, but something I'm still
# thinking about.
#
# NB: There can only be one canonical XPI for each semantic version.
#
# shellcheck disable=2120
function webextension_sign() {
local use_existing=$1
if [ "$use_existing" == "" ]; then
"$NODE_BIN"/web-ext sign --api-key "$MDN_USER" --api-secret "$MDN_KEY"
_rename_built_xpi
else
# TODO: This doesn't currently work with the Marionettte `tempAddon`
# installation method. Just use `web-ext run` and Browsh's `use-existing-ff`
# flag - which is better anyway as it auto-reloads the extension when files
# change. NB: If you fix this, don't forget to change the filename loaded
# by `Asset()` in `main.go`.
# In development/testing, we want to be able to bundle the webextension
# frequently without having to resort to version bumps.
source_file="$source_dir/browsh-$version.zip"
echo "Skipping signing, downloading existing webextension"
local base="https://github.com/browsh-org/browsh/releases/download"
curl -L \
-o "$(versioned_xpi_file)" \
"$base/v$LATEST_TAGGED_VERSION/browsh-$LATEST_TAGGED_VERSION.xpi"
fi
}
echo "Bundling $source_file to $bundle_file"
cp -f "$source_file" "$bundle_file"
function _rename_built_xpi() {
pushd "$XPI_SOURCE_DIR" || _panic
local xpi_file
xpi_file="$(
find ./*.xpi \
-printf "%T@ %f\n" |
sort |
cut -d' ' -f2 |
tail -n1
)"
cp -a "$xpi_file" "$(versioned_xpi_file)"
popd || _panic
}
function bundle_production_webextension() {
local version && version=$(browsh_version)
local base='https://github.com/browsh-org/browsh/releases/download'
local release_url="$base/v$version/browsh-$version-an.fx.xpi"
local xpi_file=$PROJECT_ROOT/interfacer/src/browsh/browsh.xpi
curl -L -o "$xpi_file" "$release_url"
echo "Downloading webextension from: $release_url"
local size && size=$(wc -c <"$XPI_PATH")
curl -L -o "$XPI_PATH" "$release_url"
if [ "$size" -lt 500 ]; then
_panic "Problem downloading latest webextension XPI"
fi
}

View File

@ -1,7 +1,23 @@
#!/bin/env bash
#!/usr/bin/env bash
# shellcheck disable=2120
function _panic() {
local message=$1
echo >&2 "$message"
exit 1
}
function _md5() {
local path=$1
md5sum "$path" | cut -d' ' -f1
}
function pushd() {
# shellcheck disable=2119
command pushd "$@" >/dev/null || _panic
}
function popd() {
# shellcheck disable=2119
command popd "$@" >/dev/null || _panic
}

View File

@ -1,6 +1,7 @@
#!/usr/bin/env bash
function docker_image_name() {
_export_versions
echo browsh/browsh:v"$BROWSH_VERSION"
}

View File

@ -36,15 +36,19 @@ function install_firefox() {
popd || _panic
}
function parse_golang_version_from_ci_config() {
local line && line=$(grep 'go-version:' <"$PROJECT_ROOT"/.github/workflows/main.yml)
local version && version=$(echo "$line" | tr -s ' ' | cut -d ' ' -f 3)
[ "$version" = "" ] && _panic "Couldn't parse Golang version"
function parse_golang_version_from_go_mod() {
local path=$1
[ "$path" = "" ] && _panic "Path to Golang interfacer code not passed"
local line && line=$(grep '^go ' <"$path"/go.mod)
local version && version=$(echo "$line" | tr -s ' ' | cut -d ' ' -f 2)
[ "$(echo "$version" | tr -s ' ')" == "" ] && _panic "Couldn't parse Golang version"
echo -n "$version"
}
function install_golang() {
local version && version=$(parse_golang_version_from_ci_config)
local path=$1
[ "$path" = "" ] && _panic "Path to Golang interfacer code not passed"
local version && version=$(parse_golang_version_from_go_mod "$path")
[ "$GOPATH" = "" ] && _panic "GOPATH not set"
[ "$GOROOT" = "" ] && _panic "GOROOT not set"
echo "Installing Golang v$version... to $GOROOT"
@ -56,14 +60,3 @@ function install_golang() {
tar -C "$GOROOT/.." -xzf go.tar.gz
go version
}
function build_browsh_binary() {
local path=$1
pushd "$path" || _panic
local webextension="src/browsh/browsh.xpi"
[ ! -f "$webextension" ] && _panic "browsh.xpi not present"
md5sum "$webextension"
go build ./cmd/browsh
./browsh --version
popd || _panic
}

View File

@ -1,4 +1,4 @@
#!/bin/env bash
#!/usr/bin/env bash
export BROWSH_VERSION
export LATEST_TAGGED_VERSION
@ -8,8 +8,12 @@ function _goreleaser_production() {
echo "Installing \`goreleaser'..."
go install github.com/goreleaser/goreleaser@v"$GORELEASER_VERSION"
fi
pushd "$PROJECT_ROOT"/interfacer/src || _panic
goreleaser release
pushd "$PROJECT_ROOT"/interfacer || _panic
_export_versions
[ "$BROWSH_VERSION" = "" ] && _panic "BROWSH_VERSION unset (goreleaser needs it)"
goreleaser release \
--config "$PROJECT_ROOT"/goreleaser.yml \
--rm-dist
popd || _panic
}
@ -21,9 +25,9 @@ function _export_versions() {
}
function _parse_browsh_version() {
version_file=$PROJECT_ROOT/interfacer/src/browsh/version.go
line=$(grep 'browshVersion' <"$version_file")
version=$(echo "$line" | grep -o '".*"' | sed 's/"//g')
local version_file=$PROJECT_ROOT/interfacer/src/browsh/version.go
local line && line=$(grep 'browshVersion' <"$version_file")
local version && version=$(echo "$line" | grep -o '".*"' | sed 's/"//g')
echo -n "$version"
}
@ -39,7 +43,7 @@ function _tag_on_version_change() {
echo_versions
if ! _is_new_version; then
echo "Not running release as there's no new version."
echo "Not tagging as there's no new version."
exit 0
fi
@ -47,15 +51,13 @@ function _tag_on_version_change() {
git show v"$BROWSH_VERSION" --quiet
git config --global user.email "ci@github.com"
git config --global user.name "Github Actions"
# `/dev/null` needed to prevent Github token appearing in logs
git push --tags --quiet https://"$GITHUB_TOKEN"@github.com/browsh-org/browsh >/dev/null 2>&1
git add --all
git reset --hard v"$BROWSH_VERSION"
}
function echo_versions() {
_export_versions
echo "Browsh Golang version: $BROWSH_VERSION"
echo "Browsh binary version: $BROWSH_VERSION"
echo "Git latest tag: $LATEST_TAGGED_VERSION"
}
@ -72,21 +74,35 @@ function github_actions_output_version_status() {
echo "::set-output name=is_new_version::$status"
}
function npm_build_release() {
function webext_build_release() {
pushd "$PROJECT_ROOT"/webext || _panic
BROWSH_ENV=RELEASE npm run build_webextension
build_webextension_production
popd || _panic
}
function update_browsh_website_with_new_version() {
_export_versions
local remote="git@github.com:browsh-org/www.brow.sh.git"
pushd /tmp || _panic
git clone https://github.com/browsh-org/www.brow.sh.git
cd www.brow.sh || exit 1
git clone "$remote"
cd www.brow.sh || _panic
echo "latest_version: $BROWSH_VERSION" >_data/browsh.yml
git add _data/browsh.yml
git commit -m "Github Actions: updated Browsh version to $BROWSH_VERSION"
# `/dev/null` needed to prevent Github token appearing in logs
git push --quiet https://"$GITHUB_TOKEN"@github.com/browsh-org/www.brow.sh >/dev/null 2>&1
git push "$remote"
popd || _panic
}
function update_homebrew_tap_with_new_version() {
_export_versions
local remote="git@github.com:browsh-org/homebrew-browsh.git"
pushd /tmp || _panic
git clone "$remote"
cd homebrew-browsh || _panic
cp -f "$PROJECT_ROOT"/interfacer/dist/browsh.rb browsh.rb
git add browsh.rb
git commit -m "Github Actions: updated to $BROWSH_VERSION"
git push "$remote"
popd || _panic
}
@ -99,9 +115,23 @@ function goreleaser_local_only() {
popd || _panic
}
function build_browsh_binary() {
# Requires $path argument because it's used in the Dockerfile where the GOROOT is
# outside .git/
local path=$1
pushd "$path" || _panic
local webextension="src/browsh/browsh.xpi"
[ ! -f "$webextension" ] && _panic "browsh.xpi not present"
md5sum "$webextension"
go build ./cmd/browsh
echo "Freshly built \`browsh' version: $(./browsh --version 2>&1)"
popd || _panic
}
function release() {
npm_build_release
[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ] && _panic "Not releasing unless on the master branch"
webext_build_release
build_browsh_binary "$PROJECT_ROOT"/interfacer
_tag_on_version_change
_goreleaser_production
update_browsh_website_with_new_version
}

View File

@ -1,7 +1,8 @@
{
"type": "module",
"scripts": {
"build_webextension": "../ctl.sh build_webextension",
"build:dev": "webpack",
"build:watch": "webpack --watch",
"lint": "prettier --list-different '{src,test}/**/*.js'",
"test": "NODE_PATH=src:test mocha"
},