fixed coverage reporting for functional tests; added cscli (#1568)

This commit is contained in:
mmetc 2022-05-31 10:01:30 +02:00 committed by GitHub
parent 88a4801d6a
commit df7c51f34e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 95 additions and 38 deletions

View file

@ -51,7 +51,8 @@ MINIMUM_SUPPORTED_GO_MINOR_VERSION = 17
GO_VERSION_VALIDATION_ERR_MSG = Golang version ($(BUILD_GOVERSION)) is not supported, please use at least $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION).$(MINIMUM_SUPPORTED_GO_MINOR_VERSION) GO_VERSION_VALIDATION_ERR_MSG = Golang version ($(BUILD_GOVERSION)) is not supported, please use at least $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION).$(MINIMUM_SUPPORTED_GO_MINOR_VERSION)
LD_OPTS_VARS= \ LD_OPTS_VARS= \
-X github.com/crowdsecurity/crowdsec/cmd/crowdsec/main.bincoverTesting=$(BINCOVER_TESTING) \ -X github.com/crowdsecurity/crowdsec/cmd/crowdsec.bincoverTesting=$(BINCOVER_TESTING) \
-X github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli.bincoverTesting=$(BINCOVER_TESTING) \
-X github.com/crowdsecurity/crowdsec/pkg/cwversion.Version=$(BUILD_VERSION) \ -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Version=$(BUILD_VERSION) \
-X github.com/crowdsecurity/crowdsec/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP) \ -X github.com/crowdsecurity/crowdsec/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP) \
-X github.com/crowdsecurity/crowdsec/pkg/cwversion.Codename=$(BUILD_CODENAME) \ -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Codename=$(BUILD_CODENAME) \

View file

@ -12,6 +12,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/go-openapi/strfmt" "github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@ -30,7 +31,7 @@ func NewCapiCmd() *cobra.Command {
DisableAutoGenTag: true, DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error { PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI { if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
log.Fatal("Local API is disabled, please run this command on the local API machine") return errors.Wrap(err, "Local API is disabled, please run this command on the local API machine")
} }
if csConfig.API.Server.OnlineClient == nil { if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath) log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
@ -113,7 +114,7 @@ func NewCapiCmd() *cobra.Command {
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
var err error var err error
if csConfig.API.Server == nil { if csConfig.API.Server == nil {
log.Fatalln("There is no configuration on 'api_client:'") log.Fatalln("There is no configuration on 'api.server:'")
} }
if csConfig.API.Server.OnlineClient == nil { if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath) log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
@ -134,8 +135,8 @@ func NewCapiCmd() *cobra.Command {
} }
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index") log.Infoln("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to load hub index : %s", err)
} }
scenarios, err := cwhub.GetInstalledScenariosAsString() scenarios, err := cwhub.GetInstalledScenariosAsString()
if err != nil { if err != nil {

View file

@ -7,6 +7,7 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"github.com/confluentinc/bincover"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/cwversion"
@ -17,6 +18,8 @@ import (
"github.com/spf13/cobra/doc" "github.com/spf13/cobra/doc"
) )
var bincoverTesting = ""
var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
var ConfigFilePath string var ConfigFilePath string
@ -184,7 +187,17 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.AddCommand(NewExplainCmd()) rootCmd.AddCommand(NewExplainCmd())
rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewHubTestCmd())
rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewNotificationsCmd())
if err := rootCmd.Execute(); err != nil { if err := rootCmd.Execute(); err != nil {
log.Fatal(err) if bincoverTesting != "" {
log.Debug("coverage report is enabled")
}
exitCode := 1
log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err)
if bincoverTesting == "" {
os.Exit(exitCode)
}
bincover.ExitCode = exitCode
} }
} }

View file

@ -49,7 +49,7 @@ var (
pluginBroker csplugin.PluginBroker pluginBroker csplugin.PluginBroker
) )
const bincoverTesting = false var bincoverTesting = ""
type Flags struct { type Flags struct {
ConfigFile string ConfigFile string

View file

@ -15,7 +15,6 @@ import (
) )
func StartRunSvc() { func StartRunSvc() {
var ( var (
cConfig *csconfig.Config cConfig *csconfig.Config
err error err error
@ -44,6 +43,10 @@ func StartRunSvc() {
log.Infof("Crowdsec %s", cwversion.VersionStr()) log.Infof("Crowdsec %s", cwversion.VersionStr())
if bincoverTesting != "" {
log.Debug("coverage report is enabled")
}
// Enable profiling early // Enable profiling early
if cConfig.Prometheus != nil { if cConfig.Prometheus != nil {
go registerPrometheus(cConfig.Prometheus) go registerPrometheus(cConfig.Prometheus)
@ -56,7 +59,7 @@ func StartRunSvc() {
// is not going to change in logrus to keep backward // is not going to change in logrus to keep backward
// compatibility), and allows us to report coverage. // compatibility), and allows us to report coverage.
log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err) log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err)
if !bincoverTesting { if bincoverTesting == "" {
os.Exit(exitCode) os.Exit(exitCode)
} }
bincover.ExitCode = exitCode bincover.ExitCode = exitCode

View file

@ -84,6 +84,10 @@ func WindowsRun() {
log.Infof("Crowdsec %s", cwversion.VersionStr()) log.Infof("Crowdsec %s", cwversion.VersionStr())
if bincoverTesting != "" {
log.Debug("coverage report is enabled")
}
// Enable profiling early // Enable profiling early
if cConfig.Prometheus != nil { if cConfig.Prometheus != nil {
go registerPrometheus(cConfig.Prometheus) go registerPrometheus(cConfig.Prometheus)
@ -91,8 +95,12 @@ func WindowsRun() {
if exitCode, err := Serve(cConfig); err != nil { if exitCode, err := Serve(cConfig); err != nil {
if err != nil { if err != nil {
log.Errorf(err.Error()) // this method of logging a fatal error does not
if !bincoverTesting { // trigger a program exit (as stated by the authors, it
// is not going to change in logrus to keep backward
// compatibility), and allows us to report coverage.
log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err)
if bincoverTesting != "" {
os.Exit(exitCode) os.Exit(exitCode)
} }
bincover.ExitCode = exitCode bincover.ExitCode = exitCode

View file

@ -28,15 +28,16 @@ PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins
DB_BACKEND ?= sqlite DB_BACKEND ?= sqlite
ifdef TEST_COVERAGE ifdef TEST_COVERAGE
CROWDSEC = "$(TEST_DIR)/crowdsec-wrapper" CROWDSEC = $(TEST_DIR)/crowdsec-wrapper
CSCLI = "$(TEST_DIR)/cscli-wrapper" CSCLI = $(TEST_DIR)/cscli-wrapper
BINCOVER_TESTING = true BINCOVER_TESTING = true
else else
# the wrappers should work here too - it detects TEST_COVERAGE - but we allow # the wrappers should work here too - it detects TEST_COVERAGE - but we allow
# overriding the path to the binaries # overriding the path to the binaries
CROWDSEC ?= "$(BIN_DIR)/crowdsec" CROWDSEC ?= "$(BIN_DIR)/crowdsec"
CSCLI ?= "$(BIN_DIR)/cscli" CSCLI ?= "$(BIN_DIR)/cscli"
BINCOVER_TESTING = false # any value is considered true
BINCOVER_TESTING =
endif endif
# If you change the name of the crowdsec executable, make sure the pgrep # If you change the name of the crowdsec executable, make sure the pgrep

View file

@ -93,6 +93,12 @@ declare stderr
assert_output --partial "Trying to authenticate with username" assert_output --partial "Trying to authenticate with username"
assert_output --partial " on https://api.crowdsec.net/" assert_output --partial " on https://api.crowdsec.net/"
assert_output --partial "You can successfully interact with Central API (CAPI)" assert_output --partial "You can successfully interact with Central API (CAPI)"
ONLINE_API_CREDENTIALS_YAML="$(config_yq '.api.server.online_client.credentials_path')"
rm "${ONLINE_API_CREDENTIALS_YAML}"
run -1 --separate-stderr cscli capi status
run -0 echo "${stderr}"
assert_output --partial "Local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory"
} }
@test "${FILE} cscli config show -o human" { @test "${FILE} cscli config show -o human" {

View file

@ -49,7 +49,8 @@ declare stderr
run -1 --separate-stderr cscli capi status run -1 --separate-stderr cscli capi status
run -0 echo "$stderr" run -0 echo "$stderr"
assert_output --partial "Local API is disabled, please run this command on the local API machine" assert_output --partial "crowdsec local API is disabled"
assert_output --partial "There is no configuration on 'api.server:'"
} }
@test "$FILE cscli config show -o human" { @test "$FILE cscli config show -o human" {

View file

@ -12,11 +12,29 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
#shellcheck disable=SC1090 #shellcheck disable=SC1090
. "${THIS_DIR}/.environment.sh" . "${THIS_DIR}/.environment.sh"
set -o pipefail # don't let sed hide the statuscode
mkdir -p "${LOCAL_DIR}/var/lib/coverage"
# we collect rc and output by hand, because setting -o pipefail would trigger a
# SIGPIPE.
set +e
# Arguments to crowdsec are passed through a temporary, newline-delimited # Arguments to crowdsec are passed through a temporary, newline-delimited
# file courtesy of github.com/confluentinc/bincover. Coverage data will be # file courtesy of github.com/confluentinc/bincover. Coverage data will be
# merged at the end of the test run. # merged at the end of the test run.
# The '=' between flags and values is required. # The '=' between flags and values is required.
exec "${BIN_DIR}/crowdsec.cover" \ output=$("${BIN_DIR}/crowdsec.cover" \
-test.run="^TestBincoverRunMain$" \ -test.run="^TestBincoverRunMain$" \
-test.coverprofile="${LOCAL_DIR}/var/lib/coverage/crowdsec-$(date +'%s')-$$.out" \ -test.coverprofile="${LOCAL_DIR}/var/lib/coverage/crowdsec-$(date +'%s')-$$-${RANDOM}.out" \
-args-file=<(for i; do echo "$i"; done) # Behold the amazing parameter contraption! -args-file=<(for i; do echo "$i"; done))
rc=$?
# If there is bincover metadata, we take the status code from there. Otherwise,
# we keep the status from the above command.
if [[ ${output} =~ (.*)(START_BINCOVER_METADATA[[:space:]]*)(.*)([[:space:]]END_BINCOVER_METADATA) ]]; then
echo -n "${BASH_REMATCH[1]}"
exit "$(jq '.exit_code' <<< "${BASH_REMATCH[3]}")"
fi
echo -n "${output}"
exit "${rc}"

View file

@ -25,14 +25,16 @@ set +e
# The '=' between flags and values is required. # The '=' between flags and values is required.
output=$("${BIN_DIR}/cscli.cover" \ output=$("${BIN_DIR}/cscli.cover" \
-test.run="^TestBincoverRunMain$" \ -test.run="^TestBincoverRunMain$" \
-test.coverprofile="${LOCAL_DIR}/var/lib/coverage/cscli-$(date +'%s')-$$.out" \ -test.coverprofile="${LOCAL_DIR}/var/lib/coverage/cscli-$(date +'%s')-$$-${RANDOM}.out" \
-args-file=<(for i; do echo "$i"; done)) -args-file=<(for i; do echo "${i}"; done))
rc=$? rc=$?
# We also cut the metadata stuff that we don't need. # If there is bincover metadata, we take the status code from there. Otherwise,
echo -n "$output" | tr '\n' '\f' | sed 's/START_BINCOVER_METADATA.*//' | tr '\f' '\n' # we keep the status from the above command.
if [[ ${output} =~ (.*)(START_BINCOVER_METADATA[[:space:]]*)(.*)([[:space:]]END_BINCOVER_METADATA) ]]; then
echo -n "${BASH_REMATCH[1]}"
exit "$(jq '.exit_code' <<< "${BASH_REMATCH[3]}")"
fi
# this does not work because cscli output does not always end with \n echo -n "${output}"
# echo -n "$output" | sed -n '/START_BINCOVER_METADATA/q;p' exit "${rc}"
exit $rc

View file

@ -1,20 +1,20 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -eu set -eu
script_name=$0 script_name=$0
die() { die() {
echo >&2 "$@" echo >&2 "$@"
exit 1 exit 1
} }
about() { about() {
die "usage: $script_name [ start | stop ]" die "usage: $script_name [ start | stop ]"
} }
#shellcheck disable=SC1007 #shellcheck disable=SC1007
THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "${THIS_DIR}"/../../ cd "${THIS_DIR}"/../../
#shellcheck disable=SC1091 #shellcheck disable=SC1091
. ./.environment.sh . ./.environment.sh

View file

@ -45,6 +45,9 @@ else
fi fi
if [ -n "$TEST_COVERAGE" ]; then if [ -n "$TEST_COVERAGE" ]; then
# empty files just to avoid merge errors
touch "${LOCAL_DIR}"/var/lib/coverage/crowdsec-
touch "${LOCAL_DIR}"/var/lib/coverage/cscli-
gocovmerge "${LOCAL_DIR}"/var/lib/coverage/crowdsec-* > "${LOCAL_DIR}/var/lib/coverage/coverage-crowdsec.out" gocovmerge "${LOCAL_DIR}"/var/lib/coverage/crowdsec-* > "${LOCAL_DIR}/var/lib/coverage/coverage-crowdsec.out"
gocovmerge "${LOCAL_DIR}"/var/lib/coverage/cscli-* > "${LOCAL_DIR}/var/lib/coverage/coverage-cscli.out" gocovmerge "${LOCAL_DIR}"/var/lib/coverage/cscli-* > "${LOCAL_DIR}/var/lib/coverage/coverage-cscli.out"
fi fi