single workflow for all tests, with bats coverage (#1413)

This commit is contained in:
mmetc 2022-04-05 11:00:11 +02:00 committed by GitHub
parent 9cf2d5ab5c
commit bf4bc0c9fc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 304 additions and 181 deletions

View file

@ -1,12 +1,12 @@
name: Hub tests
on:
push:
branches:
- master
pull_request:
branches:
- master
workflow_call:
secrets:
GIST_BADGES_SECRET:
required: true
GIST_BADGES_ID:
required: true
jobs:
build:
@ -21,13 +21,13 @@ jobs:
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go 1.17"
uses: actions/setup-go@v1
uses: actions/setup-go@v3
with:
go-version: 1.17
id: go
- name: "Clone CrowdSec"
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true

View file

@ -1,12 +1,14 @@
name: Functional tests with MySQL
on:
push:
branches:
- master
pull_request:
branches:
- master
workflow_call:
inputs:
database_image:
required: true
type: string
secrets:
DATABASE_PASSWORD:
required: true
jobs:
build:
@ -15,7 +17,7 @@ jobs:
timeout-minutes: 20
services:
database:
image: mysql:latest
image: ${{ inputs.database_image }}
env:
MYSQL_ROOT_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
ports:
@ -28,13 +30,13 @@ jobs:
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go 1.17"
uses: actions/setup-go@v1
uses: actions/setup-go@v3
with:
go-version: 1.17
id: go
- name: "Clone CrowdSec"
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true

View file

@ -1,12 +1,10 @@
name: Functional tests with PostgreSQL
on:
push:
branches:
- master
pull_request:
branches:
- master
workflow_call:
secrets:
DATABASE_PASSWORD:
required: true
jobs:
build:
@ -33,13 +31,13 @@ jobs:
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go 1.17"
uses: actions/setup-go@v1
uses: actions/setup-go@v3
with:
go-version: 1.17
id: go
- name: "Clone CrowdSec"
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true

View file

@ -0,0 +1,47 @@
name: Functional tests with sqlite
on:
workflow_call:
jobs:
build:
name: "Build + tests"
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: "Force machineid"
run: |
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go 1.17"
uses: actions/setup-go@v3
with:
go-version: 1.17
id: go
- name: "Clone CrowdSec"
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Install bats dependencies"
run: |
sudo apt install -y -qq build-essential daemonize jq netcat-openbsd
GO111MODULE=on go get github.com/mikefarah/yq/v4
go install github.com/wadey/gocovmerge@latest
sudo cp -u ~/go/bin/yq ~/go/bin/gocovmerge /usr/local/bin/
- name: "Build crowdsec and fixture"
run: TEST_COVERAGE=true make bats-clean bats-build bats-fixture
- name: "Run tests (with coverage)"
run: TEST_COVERAGE=true make bats-test
- name: Upload coverage report
uses: actions/upload-artifact@v2
with:
name: coverage-bats.out
path: ./tests/local/var/lib/coverage/coverage-bats.out

View file

@ -1,68 +0,0 @@
name: Functional tests with MariaDB
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build:
name: "Build + tests"
runs-on: ubuntu-latest
timeout-minutes: 20
services:
database:
image: mariadb:latest
env:
MYSQL_ROOT_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
ports:
- 3306:3306
steps:
- name: "Force machineid"
run: |
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go 1.17"
uses: actions/setup-go@v1
with:
go-version: 1.17
id: go
- name: "Clone CrowdSec"
uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: true
- name: "Install bats dependencies"
run: |
sudo apt install -y -qq build-essential daemonize jq netcat-openbsd
GO111MODULE=on go get github.com/mikefarah/yq/v4
- name: "Build crowdsec and fixture"
run: make bats-clean bats-build bats-fixture
env:
DB_BACKEND: mysql
MYSQL_HOST: 127.0.0.1
MYSQL_PORT: 3306
MYSQL_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
MYSQL_USER: root
- name: "Run tests"
run: make bats-test
env:
DB_BACKEND: mysql
MYSQL_HOST: 127.0.0.1
MYSQL_PORT: 3306
MYSQL_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
MYSQL_USER: root
- name: "Show database logs"
run: docker logs "${{ job.services.database.id }}"
if: ${{ always() }}

View file

@ -1,44 +0,0 @@
name: bats / functional tests with sqlite
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build:
name: "Build the application"
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- name: "Force machineid"
run: |
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go 1.17"
uses: actions/setup-go@v1
with:
go-version: 1.17
id: go
- name: "Clone CrowdSec"
uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: true
- name: "Install bats dependencies"
run: |
sudo apt install -qq daemonize netcat-openbsd
GO111MODULE=on go get github.com/mikefarah/yq/v4
- name: "BATS: build crowdsec"
run: make bats-clean bats-build bats-fixture
- name: "BATS: run tests"
run: make bats-test

View file

@ -19,7 +19,7 @@ jobs:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v2
with:

98
.github/workflows/ci_tests.yml vendored Normal file
View file

@ -0,0 +1,98 @@
name: Tests
# Main workflow for tests, it calls all the others through parallel jobs.
#
# A final step collects and merges coverage output, then pushes it to
# coveralls.io
#
# https://docs.github.com/en/actions/using-workflows/reusing-workflows
on:
push:
branches:
- master
- testing*
paths-ignore:
- 'README.md'
pull_request:
branches:
- master
- testing*
paths-ignore:
- 'README.md'
jobs:
go-tests:
uses: ./.github/workflows/go-tests.yml
bats-sqlite:
uses: ./.github/workflows/bats-sqlite-coverage.yml
bats-mariadb:
uses: ./.github/workflows/bats-mysql.yml
with:
database_image: mariadb:latest
secrets:
DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD}}
bats-mysql:
uses: ./.github/workflows/bats-mysql.yml
with:
database_image: mysql:latest
secrets:
DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD}}
bats-postgres:
uses: ./.github/workflows/bats-postgres.yml
secrets:
DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD}}
bats-hub:
uses: ./.github/workflows/bats-hub.yml
secrets:
GIST_BADGES_ID: ${{ secrets.GIST_BADGES_ID }}
GIST_BADGES_SECRET: ${{ secrets.GIST_BADGES_SECRET }}
coverage:
needs: [go-tests, bats-sqlite]
name: Coverage
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.17
uses: actions/setup-go@v3
with:
go-version: 1.17
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v3
- name: Download unit report
uses: actions/download-artifact@v3
with:
name: coverage.out
- name: Download bats report
uses: actions/download-artifact@v3
with:
name: coverage-bats.out
- name: merge coverage reports
run: |
go get -u github.com/wadey/gocovmerge
~/go/bin/gocovmerge coverage.out coverage-bats.out > coverage-all.out
- name: gcov2lcov
uses: jandelgado/gcov2lcov-action@v1.0.8
with:
infile: coverage-all.out
outfile: coverage-all.txt
- name: Coveralls
uses: coverallsapp/github-action@master
continue-on-error: true
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
path-to-lcov: coverage-all.txt

View file

@ -39,7 +39,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL

View file

@ -13,18 +13,7 @@ env:
KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2"
on:
push:
branches: [ master ]
paths-ignore:
- 'docs/**'
- 'mkdocs.yml'
- 'README.md'
pull_request:
branches: [ master ]
paths-ignore:
- 'docs/**'
- 'mkdocs.yml'
- 'README.md'
workflow_call:
jobs:
@ -55,24 +44,20 @@ jobs:
--health-retries=3
steps:
- name: Set up Go 1.17
uses: actions/setup-go@v1
uses: actions/setup-go@v3
with:
go-version: 1.17
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Build
run: make build && go get -u github.com/jandelgado/gcov2lcov && go get -u github.com/ory/go-acc
- name: All tests
run: go run github.com/ory/go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models
- name: gcov2lcov
uses: jandelgado/gcov2lcov-action@v1.0.2
- name: Upload coverage report
uses: actions/upload-artifact@v2
with:
infile: coverage.out
outfile: coverage.txt
- name: Coveralls
uses: coverallsapp/github-action@master
continue-on-error: true
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
path-to-lcov: coverage.txt
name: coverage.out
path: ./coverage.out

View file

@ -11,12 +11,12 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.17
uses: actions/setup-go@v1
uses: actions/setup-go@v3
with:
go-version: 1.17
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Build the binaries
run: make release
- name: Upload to release
@ -35,7 +35,7 @@ jobs:
go-version: 1.17
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Build the binaries
run: make release_static
- name: Upload to release
@ -70,7 +70,7 @@ jobs:
cd crowdsec-${{ steps.fetch_prerelease_version.outputs.release }}
sudo ./wizard.sh --unattended
- name: Check out code to get functional tests scripts
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: "Test post-install base"
run: |
cd scripts/func_tests/

View file

@ -12,7 +12,7 @@ jobs:
steps:
-
name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
-
name: Prepare
id: prep

View file

@ -11,7 +11,7 @@ jobs:
steps:
-
name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
-
name: Prepare
id: prep

View file

@ -13,7 +13,7 @@ jobs:
steps:
-
name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
-
name: Update docker hub README
uses: ms-jpq/sync-dockerhub-readme@v1

View file

@ -18,7 +18,7 @@ build: clean
@$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v
build-bincover: clean
$(GOTEST) . -tags testrunmain -coverpkg=all -covermode=atomic $(LD_OPTS) -c -o $(BINARY_NAME_COVER)
$(GOTEST) . -tags testrunmain -coverpkg=$(go list github.com/crowdsecurity/crowdsec/... | grep -v -e 'pkg/database' -e 'plugins/notifications' -e 'pkg/protobufs' -e 'pkg/cwversions' -e 'pkg/cstest' -e 'pkg/models') -covermode=atomic $(LD_OPTS) -c -o $(BINARY_NAME_COVER)
static: clean
@$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -v -a -tags netgo

View file

@ -23,7 +23,7 @@ build: clean
$(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN) -v
build-bincover: clean
$(GOTEST) . -tags testrunmain -coverpkg=all -covermode=atomic $(LD_OPTS) -c -o $(CROWDSEC_BIN_COVER)
$(GOTEST) . -tags testrunmain -coverpkg=$(go list github.com/crowdsecurity/crowdsec/... | grep -v -e 'pkg/database' -e 'plugins/notifications' -e 'pkg/protobufs' -e 'pkg/cwversions' -e 'pkg/cstest' -e 'pkg/models') -covermode=atomic $(LD_OPTS) -c -o $(CROWDSEC_BIN_COVER)
static: clean
$(GOBUILD) $(LD_OPTS_STATIC) -o $(CROWDSEC_BIN) -v -a -tags netgo

View file

@ -1,12 +1,20 @@
#!/usr/bin/env bash
pgrep crowdsec >/dev/null || exit 0
is_crowdsec_running() {
PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover')
}
# removing this second test causes CI to fail sometimes
is_crowdsec_running || exit 0
# The process can be slow, especially on CI and during test coverage.
# Give it some time, maybe it's quitting soon.
sleep 2
pgrep crowdsec >/dev/null || exit 0
is_crowdsec_running || exit 0
sleep 2
is_crowdsec_running || exit 0
msg="A CrowdSec process is already running. Please terminate it and run the tests again."
PIDS=$(echo "$PIDS" | sed ':a;N;$!ba;s/\n/ /g')
msg="CrowdSec is already running (PID $PIDS). Please terminate it and run the tests again."
# Are we inside a setup() or @test? Is file descriptor 3 open?
if { true >&3; } 2>/dev/null; then

View file

@ -27,11 +27,25 @@ PID_DIR = $(LOCAL_DIR)/var/run
PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins
DB_BACKEND ?= sqlite
ifdef TEST_COVERAGE
CROWDSEC = "$(TEST_DIR)/crowdsec-wrapper"
CSCLI = "$(TEST_DIR)/cscli-wrapper"
else
# the wrappers should work here too - it detects TEST_COVERAGE - but we allow
# overriding the path to the binaries
CROWDSEC ?= "$(BIN_DIR)/crowdsec"
CSCLI ?= "$(BIN_DIR)/cscli"
endif
# If you change the name of the crowdsec executable, make sure the pgrep
# parameters are correct in $(TEST_DIR)/assert-crowdsec-not-running
define ENV :=
export TEST_DIR="$(TEST_DIR)"
export LOCAL_DIR="$(LOCAL_DIR)"
export CROWDSEC="$(BIN_DIR)/crowdsec"
export CSCLI="$(BIN_DIR)/cscli"
export BIN_DIR="$(BIN_DIR)"
export CROWDSEC="$(CROWDSEC)"
export CSCLI="$(CSCLI)"
export CONFIG_YAML="$(CONFIG_DIR)/config.yaml"
export LOCAL_INIT_DIR="$(LOCAL_INIT_DIR)"
export LOG_DIR="$(LOG_DIR)"
@ -41,6 +55,7 @@ export DB_BACKEND="$(DB_BACKEND)"
export INIT_BACKEND="$(INIT_BACKEND)"
export CONFIG_BACKEND="$(CONFIG_BACKEND)"
export PACKAGE_TESTING="$(PACKAGE_TESTING)"
export TEST_COVERAGE="$(TEST_COVERAGE)"
endef
bats-all: bats-clean bats-build bats-fixture bats-test bats-test-hub

View file

@ -62,10 +62,18 @@ check_daemonizer() {
esac
}
check_gocovmerge() {
if ! command -v gocovmerge >/dev/null; then
die "missing required program 'gocovmerge'. You can install it with \"go install github.com/wadey/gocovmerge@latest\""
fi
}
check_bats_core
check_python3
check_nc
check_yq
check_daemonizer
if [ -n "${TEST_COVERAGE}" ]; then
check_gocovmerge
fi

22
tests/crowdsec-wrapper Executable file
View file

@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -eu
#
# Delegate operations to an instrumented binary and collects coverage data.
#
#shellcheck disable=SC1007
THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
# no need to change directory, and doing it here would break hub tests
#shellcheck disable=SC1090
. "${THIS_DIR}/.environment.sh"
# Arguments to crowdsec are passed through a temporary, newline-delimited
# file courtesy of github.com/confluentinc/bincover. Coverage data will be
# merged at the end of the test run.
# The '=' between flags and values is required.
exec "${BIN_DIR}/crowdsec.cover" \
-test.run="^TestBincoverRunMain$" \
-test.coverprofile="${LOCAL_DIR}/var/lib/coverage/$(date +'%s')-$$.out" \
-args-file=<(for i; do echo "$i"; done) # Behold the amazing parameter contraption!

38
tests/cscli-wrapper Executable file
View file

@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -eu
#
# Delegate operations to an instrumented binary and collects coverage data.
#
#shellcheck disable=SC1007
THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
# no need to change directory, and doing it here would break hub tests
#shellcheck disable=SC1090
. "${THIS_DIR}/.environment.sh"
set -o pipefail # don't let sed hide the statuscode
mkdir -p "${LOCAL_DIR}/var/lib/coverage"
# we collect rc and output by hand, because setting -o pipefail would trigger a
# SIGPIPE.
set +e
# Arguments to cscli are passed through a temporary, newline-delimited
# file courtesy of github.com/confluentinc/bincover. Coverage data will be
# merged at the end of the test run.
# The '=' between flags and values is required.
output=$("${BIN_DIR}/cscli.cover" \
-test.run="^TestBincoverRunMain$" \
-test.coverprofile="${LOCAL_DIR}/var/lib/coverage/$(date +'%s')-$$.out" \
-args-file=<(for i; do echo "$i"; done))
rc=$?
# We also cut the metadata stuff that we don't need.
echo -n "$output" | tr '\n' '\f' | sed 's/START_BINCOVER_METADATA.*//' | tr '\f' '\n'
# this does not work because cscli output does not always end with \n
# echo -n "$output" | sed -n '/START_BINCOVER_METADATA/q;p'
exit $rc

View file

@ -15,7 +15,7 @@ about() {
#shellcheck disable=SC1007
THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "${THIS_DIR}"/../../
#shellcheck disable=SC1090
#shellcheck disable=SC1091
. ./.environment.sh
# you have not removed set -u above, have you?

View file

@ -15,7 +15,7 @@ about() {
#shellcheck disable=SC1007
THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "${THIS_DIR}"/../../
#shellcheck disable=SC1090
#shellcheck disable=SC1091
. ./.environment.sh
# you have not removed set -u above, have you?
@ -57,8 +57,9 @@ config_generate() {
"${CONFIG_DIR}/notifications/"
yq '
.common.daemonize=false |
.common.daemonize=true |
del(.common.pid_dir) |
.common.log_level="info" |
.common.log_dir=strenv(LOG_DIR) |
.config_paths.config_dir=strenv(CONFIG_DIR) |
.config_paths.data_dir=strenv(DATA_DIR) |
@ -95,6 +96,7 @@ make_init_data() {
"${CSCLI}" collections install crowdsecurity/linux
"${TEST_DIR}/instance-crowdsec" start
[[ "$DB_BACKEND" =~ ^postgres|pgx$ ]] && sleep 4
"${CSCLI}" lapi status
"${TEST_DIR}/instance-crowdsec" stop

View file

@ -16,6 +16,14 @@ TEST_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
echo "Running tests..."
echo "DB_BACKEND: $DB_BACKEND"
if [ -z "$TEST_COVERAGE" ]; then
echo "Coverage report: no"
else
echo "Coverage report: yes"
rm -f "${LOCAL_DIR}/var/lib/coverage/*"
mkdir -p "${LOCAL_DIR}/var/lib/coverage"
fi
dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")"
if [ "$DB_BACKEND" != "$dump_backend" ]; then
@ -35,3 +43,7 @@ else
--print-output-on-failure \
-T "${TEST_DIR}/bats" "${TEST_DIR}/dyn-bats"
fi
if [ -n "$TEST_COVERAGE" ]; then
gocovmerge "${LOCAL_DIR}"/var/lib/coverage/* > "${LOCAL_DIR}/var/lib/coverage/coverage-bats.out"
fi