From bf4bc0c9fcedff3a1724b26020a37a53bce9966c Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 5 Apr 2022 11:00:11 +0200 Subject: [PATCH] single workflow for all tests, with bats coverage (#1413) --- .../{ci_bats_hub.yaml => bats-hub.yml} | 16 +-- .../{ci_bats_mysql.yaml => bats-mysql.yml} | 20 ++-- ...i_bats_postgres.yaml => bats-postgres.yml} | 14 ++- .github/workflows/bats-sqlite-coverage.yml | 47 +++++++++ .github/workflows/ci_bats_mariadb.yaml | 68 ------------- .github/workflows/ci_bats_sqlite.yaml | 44 --------- .github/workflows/ci_golangci-lint.yml | 2 +- .github/workflows/ci_tests.yml | 98 +++++++++++++++++++ .github/workflows/codeql-analysis.yml | 2 +- .../{ci_go-test.yml => go-tests.yml} | 33 ++----- .github/workflows/release_publish-package.yml | 8 +- .../release_publish_docker-image-debian.yml | 2 +- .../release_publish_docker-image.yml | 2 +- .github/workflows/update_docker_hub_doc.yml | 2 +- cmd/crowdsec-cli/Makefile | 2 +- cmd/crowdsec/Makefile | 2 +- tests/assert-crowdsec-not-running | 16 ++- tests/bats.mk | 19 +++- tests/check-requirements | 8 ++ tests/crowdsec-wrapper | 22 +++++ tests/cscli-wrapper | 38 +++++++ tests/lib/config/config-global | 2 +- tests/lib/config/config-local | 6 +- tests/run-tests | 12 +++ 24 files changed, 304 insertions(+), 181 deletions(-) rename .github/workflows/{ci_bats_hub.yaml => bats-hub.yml} (90%) rename .github/workflows/{ci_bats_mysql.yaml => bats-mysql.yml} (84%) rename .github/workflows/{ci_bats_postgres.yaml => bats-postgres.yml} (93%) create mode 100644 .github/workflows/bats-sqlite-coverage.yml delete mode 100644 .github/workflows/ci_bats_mariadb.yaml delete mode 100644 .github/workflows/ci_bats_sqlite.yaml create mode 100644 .github/workflows/ci_tests.yml rename .github/workflows/{ci_go-test.yml => go-tests.yml} (75%) create mode 100755 tests/crowdsec-wrapper create mode 100755 tests/cscli-wrapper diff --git a/.github/workflows/ci_bats_hub.yaml b/.github/workflows/bats-hub.yml similarity index 90% rename from .github/workflows/ci_bats_hub.yaml rename to .github/workflows/bats-hub.yml index e03e062a5..82cb40ada 100644 --- a/.github/workflows/ci_bats_hub.yaml +++ b/.github/workflows/bats-hub.yml @@ -1,12 +1,12 @@ name: Hub tests on: - push: - branches: - - master - pull_request: - branches: - - master + workflow_call: + secrets: + GIST_BADGES_SECRET: + required: true + GIST_BADGES_ID: + required: true jobs: build: @@ -21,13 +21,13 @@ jobs: echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - name: "Set up Go 1.17" - uses: actions/setup-go@v1 + uses: actions/setup-go@v3 with: go-version: 1.17 id: go - name: "Clone CrowdSec" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true diff --git a/.github/workflows/ci_bats_mysql.yaml b/.github/workflows/bats-mysql.yml similarity index 84% rename from .github/workflows/ci_bats_mysql.yaml rename to .github/workflows/bats-mysql.yml index 53711c804..cd905239a 100644 --- a/.github/workflows/ci_bats_mysql.yaml +++ b/.github/workflows/bats-mysql.yml @@ -1,12 +1,14 @@ name: Functional tests with MySQL on: - push: - branches: - - master - pull_request: - branches: - - master + workflow_call: + inputs: + database_image: + required: true + type: string + secrets: + DATABASE_PASSWORD: + required: true jobs: build: @@ -15,7 +17,7 @@ jobs: timeout-minutes: 20 services: database: - image: mysql:latest + image: ${{ inputs.database_image }} env: MYSQL_ROOT_PASSWORD: ${{ secrets.DATABASE_PASSWORD }} ports: @@ -28,13 +30,13 @@ jobs: echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - name: "Set up Go 1.17" - uses: actions/setup-go@v1 + uses: actions/setup-go@v3 with: go-version: 1.17 id: go - name: "Clone CrowdSec" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true diff --git a/.github/workflows/ci_bats_postgres.yaml b/.github/workflows/bats-postgres.yml similarity index 93% rename from .github/workflows/ci_bats_postgres.yaml rename to .github/workflows/bats-postgres.yml index 2149fd929..f63aaca9b 100644 --- a/.github/workflows/ci_bats_postgres.yaml +++ b/.github/workflows/bats-postgres.yml @@ -1,12 +1,10 @@ name: Functional tests with PostgreSQL on: - push: - branches: - - master - pull_request: - branches: - - master + workflow_call: + secrets: + DATABASE_PASSWORD: + required: true jobs: build: @@ -33,13 +31,13 @@ jobs: echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - name: "Set up Go 1.17" - uses: actions/setup-go@v1 + uses: actions/setup-go@v3 with: go-version: 1.17 id: go - name: "Clone CrowdSec" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true diff --git a/.github/workflows/bats-sqlite-coverage.yml b/.github/workflows/bats-sqlite-coverage.yml new file mode 100644 index 000000000..055c688bb --- /dev/null +++ b/.github/workflows/bats-sqlite-coverage.yml @@ -0,0 +1,47 @@ +name: Functional tests with sqlite + +on: + workflow_call: + +jobs: + build: + name: "Build + tests" + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + + - name: "Force machineid" + run: | + sudo chmod +w /etc/machine-id + echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id + + - name: "Set up Go 1.17" + uses: actions/setup-go@v3 + with: + go-version: 1.17 + id: go + + - name: "Clone CrowdSec" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: "Install bats dependencies" + run: | + sudo apt install -y -qq build-essential daemonize jq netcat-openbsd + GO111MODULE=on go get github.com/mikefarah/yq/v4 + go install github.com/wadey/gocovmerge@latest + sudo cp -u ~/go/bin/yq ~/go/bin/gocovmerge /usr/local/bin/ + + - name: "Build crowdsec and fixture" + run: TEST_COVERAGE=true make bats-clean bats-build bats-fixture + + - name: "Run tests (with coverage)" + run: TEST_COVERAGE=true make bats-test + + - name: Upload coverage report + uses: actions/upload-artifact@v2 + with: + name: coverage-bats.out + path: ./tests/local/var/lib/coverage/coverage-bats.out diff --git a/.github/workflows/ci_bats_mariadb.yaml b/.github/workflows/ci_bats_mariadb.yaml deleted file mode 100644 index 87feed92b..000000000 --- a/.github/workflows/ci_bats_mariadb.yaml +++ /dev/null @@ -1,68 +0,0 @@ -name: Functional tests with MariaDB - -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - build: - name: "Build + tests" - runs-on: ubuntu-latest - timeout-minutes: 20 - services: - database: - image: mariadb:latest - env: - MYSQL_ROOT_PASSWORD: ${{ secrets.DATABASE_PASSWORD }} - ports: - - 3306:3306 - steps: - - - name: "Force machineid" - run: | - sudo chmod +w /etc/machine-id - echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - - name: "Set up Go 1.17" - uses: actions/setup-go@v1 - with: - go-version: 1.17 - id: go - - - name: "Clone CrowdSec" - uses: actions/checkout@v2 - with: - fetch-depth: 0 - submodules: true - - - name: "Install bats dependencies" - run: | - sudo apt install -y -qq build-essential daemonize jq netcat-openbsd - GO111MODULE=on go get github.com/mikefarah/yq/v4 - - - name: "Build crowdsec and fixture" - run: make bats-clean bats-build bats-fixture - env: - DB_BACKEND: mysql - MYSQL_HOST: 127.0.0.1 - MYSQL_PORT: 3306 - MYSQL_PASSWORD: ${{ secrets.DATABASE_PASSWORD }} - MYSQL_USER: root - - - name: "Run tests" - run: make bats-test - env: - DB_BACKEND: mysql - MYSQL_HOST: 127.0.0.1 - MYSQL_PORT: 3306 - MYSQL_PASSWORD: ${{ secrets.DATABASE_PASSWORD }} - MYSQL_USER: root - - - name: "Show database logs" - run: docker logs "${{ job.services.database.id }}" - if: ${{ always() }} - diff --git a/.github/workflows/ci_bats_sqlite.yaml b/.github/workflows/ci_bats_sqlite.yaml deleted file mode 100644 index 2c5de1ba9..000000000 --- a/.github/workflows/ci_bats_sqlite.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: bats / functional tests with sqlite - -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - build: - name: "Build the application" - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - - name: "Force machineid" - run: | - sudo chmod +w /etc/machine-id - echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - - name: "Set up Go 1.17" - uses: actions/setup-go@v1 - with: - go-version: 1.17 - id: go - - - name: "Clone CrowdSec" - uses: actions/checkout@v2 - with: - fetch-depth: 0 - submodules: true - - - name: "Install bats dependencies" - run: | - sudo apt install -qq daemonize netcat-openbsd - GO111MODULE=on go get github.com/mikefarah/yq/v4 - - - name: "BATS: build crowdsec" - run: make bats-clean bats-build bats-fixture - - - name: "BATS: run tests" - run: make bats-test diff --git a/.github/workflows/ci_golangci-lint.yml b/.github/workflows/ci_golangci-lint.yml index 8f2c2ff28..bf0f57f22 100644 --- a/.github/workflows/ci_golangci-lint.yml +++ b/.github/workflows/ci_golangci-lint.yml @@ -19,7 +19,7 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v2 with: diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml new file mode 100644 index 000000000..f20b31397 --- /dev/null +++ b/.github/workflows/ci_tests.yml @@ -0,0 +1,98 @@ +name: Tests + +# Main workflow for tests, it calls all the others through parallel jobs. +# +# A final step collects and merges coverage output, then pushes it to +# coveralls.io +# +# https://docs.github.com/en/actions/using-workflows/reusing-workflows + +on: + push: + branches: + - master + - testing* + paths-ignore: + - 'README.md' + pull_request: + branches: + - master + - testing* + paths-ignore: + - 'README.md' + +jobs: + + go-tests: + uses: ./.github/workflows/go-tests.yml + + bats-sqlite: + uses: ./.github/workflows/bats-sqlite-coverage.yml + + bats-mariadb: + uses: ./.github/workflows/bats-mysql.yml + with: + database_image: mariadb:latest + secrets: + DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD}} + + bats-mysql: + uses: ./.github/workflows/bats-mysql.yml + with: + database_image: mysql:latest + secrets: + DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD}} + + bats-postgres: + uses: ./.github/workflows/bats-postgres.yml + secrets: + DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD}} + + bats-hub: + uses: ./.github/workflows/bats-hub.yml + secrets: + GIST_BADGES_ID: ${{ secrets.GIST_BADGES_ID }} + GIST_BADGES_SECRET: ${{ secrets.GIST_BADGES_SECRET }} + + coverage: + needs: [go-tests, bats-sqlite] + name: Coverage + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.17 + uses: actions/setup-go@v3 + with: + go-version: 1.17 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Download unit report + uses: actions/download-artifact@v3 + with: + name: coverage.out + + - name: Download bats report + uses: actions/download-artifact@v3 + with: + name: coverage-bats.out + + - name: merge coverage reports + run: | + go get -u github.com/wadey/gocovmerge + ~/go/bin/gocovmerge coverage.out coverage-bats.out > coverage-all.out + + - name: gcov2lcov + uses: jandelgado/gcov2lcov-action@v1.0.8 + with: + infile: coverage-all.out + outfile: coverage-all.txt + + - name: Coveralls + uses: coverallsapp/github-action@master + continue-on-error: true + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: coverage-all.txt + diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 292ed326a..6907bfa60 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/ci_go-test.yml b/.github/workflows/go-tests.yml similarity index 75% rename from .github/workflows/ci_go-test.yml rename to .github/workflows/go-tests.yml index 5b5ea5242..a01891ca0 100644 --- a/.github/workflows/ci_go-test.yml +++ b/.github/workflows/go-tests.yml @@ -13,18 +13,7 @@ env: KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2" on: - push: - branches: [ master ] - paths-ignore: - - 'docs/**' - - 'mkdocs.yml' - - 'README.md' - pull_request: - branches: [ master ] - paths-ignore: - - 'docs/**' - - 'mkdocs.yml' - - 'README.md' + workflow_call: jobs: @@ -55,24 +44,20 @@ jobs: --health-retries=3 steps: - name: Set up Go 1.17 - uses: actions/setup-go@v1 + uses: actions/setup-go@v3 with: go-version: 1.17 id: go - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build run: make build && go get -u github.com/jandelgado/gcov2lcov && go get -u github.com/ory/go-acc - name: All tests run: go run github.com/ory/go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models - - name: gcov2lcov - uses: jandelgado/gcov2lcov-action@v1.0.2 + + - name: Upload coverage report + uses: actions/upload-artifact@v2 with: - infile: coverage.out - outfile: coverage.txt - - name: Coveralls - uses: coverallsapp/github-action@master - continue-on-error: true - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - path-to-lcov: coverage.txt + name: coverage.out + path: ./coverage.out + diff --git a/.github/workflows/release_publish-package.yml b/.github/workflows/release_publish-package.yml index 6f0d1e94b..0bd029e48 100644 --- a/.github/workflows/release_publish-package.yml +++ b/.github/workflows/release_publish-package.yml @@ -11,12 +11,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go 1.17 - uses: actions/setup-go@v1 + uses: actions/setup-go@v3 with: go-version: 1.17 id: go - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build the binaries run: make release - name: Upload to release @@ -35,7 +35,7 @@ jobs: go-version: 1.17 id: go - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build the binaries run: make release_static - name: Upload to release @@ -70,7 +70,7 @@ jobs: cd crowdsec-${{ steps.fetch_prerelease_version.outputs.release }} sudo ./wizard.sh --unattended - name: Check out code to get functional tests scripts - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Test post-install base" run: | cd scripts/func_tests/ diff --git a/.github/workflows/release_publish_docker-image-debian.yml b/.github/workflows/release_publish_docker-image-debian.yml index cc8e0e3b4..f6c29d81b 100644 --- a/.github/workflows/release_publish_docker-image-debian.yml +++ b/.github/workflows/release_publish_docker-image-debian.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Prepare id: prep diff --git a/.github/workflows/release_publish_docker-image.yml b/.github/workflows/release_publish_docker-image.yml index 093fe4726..0149c2b68 100644 --- a/.github/workflows/release_publish_docker-image.yml +++ b/.github/workflows/release_publish_docker-image.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Prepare id: prep diff --git a/.github/workflows/update_docker_hub_doc.yml b/.github/workflows/update_docker_hub_doc.yml index 9e6b65073..243144e4e 100644 --- a/.github/workflows/update_docker_hub_doc.yml +++ b/.github/workflows/update_docker_hub_doc.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Update docker hub README uses: ms-jpq/sync-dockerhub-readme@v1 diff --git a/cmd/crowdsec-cli/Makefile b/cmd/crowdsec-cli/Makefile index 4bdf675a3..6f94bbcda 100644 --- a/cmd/crowdsec-cli/Makefile +++ b/cmd/crowdsec-cli/Makefile @@ -18,7 +18,7 @@ build: clean @$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v build-bincover: clean - $(GOTEST) . -tags testrunmain -coverpkg=all -covermode=atomic $(LD_OPTS) -c -o $(BINARY_NAME_COVER) + $(GOTEST) . -tags testrunmain -coverpkg=$(go list github.com/crowdsecurity/crowdsec/... | grep -v -e 'pkg/database' -e 'plugins/notifications' -e 'pkg/protobufs' -e 'pkg/cwversions' -e 'pkg/cstest' -e 'pkg/models') -covermode=atomic $(LD_OPTS) -c -o $(BINARY_NAME_COVER) static: clean @$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -v -a -tags netgo diff --git a/cmd/crowdsec/Makefile b/cmd/crowdsec/Makefile index 894efba0a..4d9a90c61 100644 --- a/cmd/crowdsec/Makefile +++ b/cmd/crowdsec/Makefile @@ -23,7 +23,7 @@ build: clean $(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN) -v build-bincover: clean - $(GOTEST) . -tags testrunmain -coverpkg=all -covermode=atomic $(LD_OPTS) -c -o $(CROWDSEC_BIN_COVER) + $(GOTEST) . -tags testrunmain -coverpkg=$(go list github.com/crowdsecurity/crowdsec/... | grep -v -e 'pkg/database' -e 'plugins/notifications' -e 'pkg/protobufs' -e 'pkg/cwversions' -e 'pkg/cstest' -e 'pkg/models') -covermode=atomic $(LD_OPTS) -c -o $(CROWDSEC_BIN_COVER) static: clean $(GOBUILD) $(LD_OPTS_STATIC) -o $(CROWDSEC_BIN) -v -a -tags netgo diff --git a/tests/assert-crowdsec-not-running b/tests/assert-crowdsec-not-running index 627831c80..45372ad51 100755 --- a/tests/assert-crowdsec-not-running +++ b/tests/assert-crowdsec-not-running @@ -1,12 +1,20 @@ #!/usr/bin/env bash -pgrep crowdsec >/dev/null || exit 0 +is_crowdsec_running() { + PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover') +} -# removing this second test causes CI to fail sometimes +is_crowdsec_running || exit 0 + +# The process can be slow, especially on CI and during test coverage. +# Give it some time, maybe it's quitting soon. sleep 2 -pgrep crowdsec >/dev/null || exit 0 +is_crowdsec_running || exit 0 +sleep 2 +is_crowdsec_running || exit 0 -msg="A CrowdSec process is already running. Please terminate it and run the tests again." +PIDS=$(echo "$PIDS" | sed ':a;N;$!ba;s/\n/ /g') +msg="CrowdSec is already running (PID $PIDS). Please terminate it and run the tests again." # Are we inside a setup() or @test? Is file descriptor 3 open? if { true >&3; } 2>/dev/null; then diff --git a/tests/bats.mk b/tests/bats.mk index 3857cdf1b..a1ea4fc56 100644 --- a/tests/bats.mk +++ b/tests/bats.mk @@ -27,11 +27,25 @@ PID_DIR = $(LOCAL_DIR)/var/run PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins DB_BACKEND ?= sqlite +ifdef TEST_COVERAGE + CROWDSEC = "$(TEST_DIR)/crowdsec-wrapper" + CSCLI = "$(TEST_DIR)/cscli-wrapper" +else + # the wrappers should work here too - it detects TEST_COVERAGE - but we allow + # overriding the path to the binaries + CROWDSEC ?= "$(BIN_DIR)/crowdsec" + CSCLI ?= "$(BIN_DIR)/cscli" +endif + +# If you change the name of the crowdsec executable, make sure the pgrep +# parameters are correct in $(TEST_DIR)/assert-crowdsec-not-running + define ENV := export TEST_DIR="$(TEST_DIR)" export LOCAL_DIR="$(LOCAL_DIR)" -export CROWDSEC="$(BIN_DIR)/crowdsec" -export CSCLI="$(BIN_DIR)/cscli" +export BIN_DIR="$(BIN_DIR)" +export CROWDSEC="$(CROWDSEC)" +export CSCLI="$(CSCLI)" export CONFIG_YAML="$(CONFIG_DIR)/config.yaml" export LOCAL_INIT_DIR="$(LOCAL_INIT_DIR)" export LOG_DIR="$(LOG_DIR)" @@ -41,6 +55,7 @@ export DB_BACKEND="$(DB_BACKEND)" export INIT_BACKEND="$(INIT_BACKEND)" export CONFIG_BACKEND="$(CONFIG_BACKEND)" export PACKAGE_TESTING="$(PACKAGE_TESTING)" +export TEST_COVERAGE="$(TEST_COVERAGE)" endef bats-all: bats-clean bats-build bats-fixture bats-test bats-test-hub diff --git a/tests/check-requirements b/tests/check-requirements index 40e619fcf..f86544506 100755 --- a/tests/check-requirements +++ b/tests/check-requirements @@ -62,10 +62,18 @@ check_daemonizer() { esac } +check_gocovmerge() { + if ! command -v gocovmerge >/dev/null; then + die "missing required program 'gocovmerge'. You can install it with \"go install github.com/wadey/gocovmerge@latest\"" + fi +} check_bats_core check_python3 check_nc check_yq check_daemonizer +if [ -n "${TEST_COVERAGE}" ]; then + check_gocovmerge +fi diff --git a/tests/crowdsec-wrapper b/tests/crowdsec-wrapper new file mode 100755 index 000000000..40628dc37 --- /dev/null +++ b/tests/crowdsec-wrapper @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eu + +# +# Delegate operations to an instrumented binary and collects coverage data. +# + +#shellcheck disable=SC1007 +THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +# no need to change directory, and doing it here would break hub tests +#shellcheck disable=SC1090 +. "${THIS_DIR}/.environment.sh" + +# Arguments to crowdsec are passed through a temporary, newline-delimited +# file courtesy of github.com/confluentinc/bincover. Coverage data will be +# merged at the end of the test run. +# The '=' between flags and values is required. +exec "${BIN_DIR}/crowdsec.cover" \ + -test.run="^TestBincoverRunMain$" \ + -test.coverprofile="${LOCAL_DIR}/var/lib/coverage/$(date +'%s')-$$.out" \ + -args-file=<(for i; do echo "$i"; done) # Behold the amazing parameter contraption! diff --git a/tests/cscli-wrapper b/tests/cscli-wrapper new file mode 100755 index 000000000..54386227d --- /dev/null +++ b/tests/cscli-wrapper @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -eu + +# +# Delegate operations to an instrumented binary and collects coverage data. +# + +#shellcheck disable=SC1007 +THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +# no need to change directory, and doing it here would break hub tests +#shellcheck disable=SC1090 +. "${THIS_DIR}/.environment.sh" + +set -o pipefail # don't let sed hide the statuscode +mkdir -p "${LOCAL_DIR}/var/lib/coverage" + +# we collect rc and output by hand, because setting -o pipefail would trigger a +# SIGPIPE. +set +e + +# Arguments to cscli are passed through a temporary, newline-delimited +# file courtesy of github.com/confluentinc/bincover. Coverage data will be +# merged at the end of the test run. +# The '=' between flags and values is required. +output=$("${BIN_DIR}/cscli.cover" \ + -test.run="^TestBincoverRunMain$" \ + -test.coverprofile="${LOCAL_DIR}/var/lib/coverage/$(date +'%s')-$$.out" \ + -args-file=<(for i; do echo "$i"; done)) +rc=$? + +# We also cut the metadata stuff that we don't need. +echo -n "$output" | tr '\n' '\f' | sed 's/START_BINCOVER_METADATA.*//' | tr '\f' '\n' + +# this does not work because cscli output does not always end with \n +# echo -n "$output" | sed -n '/START_BINCOVER_METADATA/q;p' + +exit $rc diff --git a/tests/lib/config/config-global b/tests/lib/config/config-global index a2206691e..899876841 100755 --- a/tests/lib/config/config-global +++ b/tests/lib/config/config-global @@ -15,7 +15,7 @@ about() { #shellcheck disable=SC1007 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) cd "${THIS_DIR}"/../../ -#shellcheck disable=SC1090 +#shellcheck disable=SC1091 . ./.environment.sh # you have not removed set -u above, have you? diff --git a/tests/lib/config/config-local b/tests/lib/config/config-local index e20858384..d19edcb9a 100755 --- a/tests/lib/config/config-local +++ b/tests/lib/config/config-local @@ -15,7 +15,7 @@ about() { #shellcheck disable=SC1007 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) cd "${THIS_DIR}"/../../ -#shellcheck disable=SC1090 +#shellcheck disable=SC1091 . ./.environment.sh # you have not removed set -u above, have you? @@ -57,8 +57,9 @@ config_generate() { "${CONFIG_DIR}/notifications/" yq ' - .common.daemonize=false | + .common.daemonize=true | del(.common.pid_dir) | + .common.log_level="info" | .common.log_dir=strenv(LOG_DIR) | .config_paths.config_dir=strenv(CONFIG_DIR) | .config_paths.data_dir=strenv(DATA_DIR) | @@ -95,6 +96,7 @@ make_init_data() { "${CSCLI}" collections install crowdsecurity/linux "${TEST_DIR}/instance-crowdsec" start + [[ "$DB_BACKEND" =~ ^postgres|pgx$ ]] && sleep 4 "${CSCLI}" lapi status "${TEST_DIR}/instance-crowdsec" stop diff --git a/tests/run-tests b/tests/run-tests index 90fc57228..5edc9ec69 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -16,6 +16,14 @@ TEST_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) echo "Running tests..." echo "DB_BACKEND: $DB_BACKEND" +if [ -z "$TEST_COVERAGE" ]; then + echo "Coverage report: no" +else + echo "Coverage report: yes" + rm -f "${LOCAL_DIR}/var/lib/coverage/*" + mkdir -p "${LOCAL_DIR}/var/lib/coverage" +fi + dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")" if [ "$DB_BACKEND" != "$dump_backend" ]; then @@ -35,3 +43,7 @@ else --print-output-on-failure \ -T "${TEST_DIR}/bats" "${TEST_DIR}/dyn-bats" fi + +if [ -n "$TEST_COVERAGE" ]; then + gocovmerge "${LOCAL_DIR}"/var/lib/coverage/* > "${LOCAL_DIR}/var/lib/coverage/coverage-bats.out" +fi