fixed coverage reporting for functional tests; added cscli (#1568)
This commit is contained in:
parent
88a4801d6a
commit
df7c51f34e
3
Makefile
3
Makefile
|
@ -51,7 +51,8 @@ MINIMUM_SUPPORTED_GO_MINOR_VERSION = 17
|
||||||
GO_VERSION_VALIDATION_ERR_MSG = Golang version ($(BUILD_GOVERSION)) is not supported, please use at least $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION).$(MINIMUM_SUPPORTED_GO_MINOR_VERSION)
|
GO_VERSION_VALIDATION_ERR_MSG = Golang version ($(BUILD_GOVERSION)) is not supported, please use at least $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION).$(MINIMUM_SUPPORTED_GO_MINOR_VERSION)
|
||||||
|
|
||||||
LD_OPTS_VARS= \
|
LD_OPTS_VARS= \
|
||||||
-X github.com/crowdsecurity/crowdsec/cmd/crowdsec/main.bincoverTesting=$(BINCOVER_TESTING) \
|
-X github.com/crowdsecurity/crowdsec/cmd/crowdsec.bincoverTesting=$(BINCOVER_TESTING) \
|
||||||
|
-X github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli.bincoverTesting=$(BINCOVER_TESTING) \
|
||||||
-X github.com/crowdsecurity/crowdsec/pkg/cwversion.Version=$(BUILD_VERSION) \
|
-X github.com/crowdsecurity/crowdsec/pkg/cwversion.Version=$(BUILD_VERSION) \
|
||||||
-X github.com/crowdsecurity/crowdsec/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP) \
|
-X github.com/crowdsecurity/crowdsec/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP) \
|
||||||
-X github.com/crowdsecurity/crowdsec/pkg/cwversion.Codename=$(BUILD_CODENAME) \
|
-X github.com/crowdsecurity/crowdsec/pkg/cwversion.Codename=$(BUILD_CODENAME) \
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
|
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/models"
|
"github.com/crowdsecurity/crowdsec/pkg/models"
|
||||||
"github.com/go-openapi/strfmt"
|
"github.com/go-openapi/strfmt"
|
||||||
|
"github.com/pkg/errors"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
@ -30,7 +31,7 @@ func NewCapiCmd() *cobra.Command {
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
||||||
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
|
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
|
||||||
log.Fatal("Local API is disabled, please run this command on the local API machine")
|
return errors.Wrap(err, "Local API is disabled, please run this command on the local API machine")
|
||||||
}
|
}
|
||||||
if csConfig.API.Server.OnlineClient == nil {
|
if csConfig.API.Server.OnlineClient == nil {
|
||||||
log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
|
log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
|
||||||
|
@ -113,7 +114,7 @@ func NewCapiCmd() *cobra.Command {
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
var err error
|
var err error
|
||||||
if csConfig.API.Server == nil {
|
if csConfig.API.Server == nil {
|
||||||
log.Fatalln("There is no configuration on 'api_client:'")
|
log.Fatalln("There is no configuration on 'api.server:'")
|
||||||
}
|
}
|
||||||
if csConfig.API.Server.OnlineClient == nil {
|
if csConfig.API.Server.OnlineClient == nil {
|
||||||
log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
|
log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
|
||||||
|
@ -134,8 +135,8 @@ func NewCapiCmd() *cobra.Command {
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
|
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
|
||||||
log.Fatalf("Failed to load hub index : %s", err)
|
|
||||||
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
|
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
|
||||||
|
log.Fatalf("Failed to load hub index : %s", err)
|
||||||
}
|
}
|
||||||
scenarios, err := cwhub.GetInstalledScenariosAsString()
|
scenarios, err := cwhub.GetInstalledScenariosAsString()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/confluentinc/bincover"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
|
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
|
||||||
|
@ -17,6 +18,8 @@ import (
|
||||||
"github.com/spf13/cobra/doc"
|
"github.com/spf13/cobra/doc"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var bincoverTesting = ""
|
||||||
|
|
||||||
var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
|
var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
|
||||||
|
|
||||||
var ConfigFilePath string
|
var ConfigFilePath string
|
||||||
|
@ -184,7 +187,17 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
|
||||||
rootCmd.AddCommand(NewExplainCmd())
|
rootCmd.AddCommand(NewExplainCmd())
|
||||||
rootCmd.AddCommand(NewHubTestCmd())
|
rootCmd.AddCommand(NewHubTestCmd())
|
||||||
rootCmd.AddCommand(NewNotificationsCmd())
|
rootCmd.AddCommand(NewNotificationsCmd())
|
||||||
|
|
||||||
if err := rootCmd.Execute(); err != nil {
|
if err := rootCmd.Execute(); err != nil {
|
||||||
log.Fatal(err)
|
if bincoverTesting != "" {
|
||||||
|
log.Debug("coverage report is enabled")
|
||||||
|
}
|
||||||
|
|
||||||
|
exitCode := 1
|
||||||
|
log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err)
|
||||||
|
if bincoverTesting == "" {
|
||||||
|
os.Exit(exitCode)
|
||||||
|
}
|
||||||
|
bincover.ExitCode = exitCode
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -49,7 +49,7 @@ var (
|
||||||
pluginBroker csplugin.PluginBroker
|
pluginBroker csplugin.PluginBroker
|
||||||
)
|
)
|
||||||
|
|
||||||
const bincoverTesting = false
|
var bincoverTesting = ""
|
||||||
|
|
||||||
type Flags struct {
|
type Flags struct {
|
||||||
ConfigFile string
|
ConfigFile string
|
||||||
|
|
|
@ -15,7 +15,6 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func StartRunSvc() {
|
func StartRunSvc() {
|
||||||
|
|
||||||
var (
|
var (
|
||||||
cConfig *csconfig.Config
|
cConfig *csconfig.Config
|
||||||
err error
|
err error
|
||||||
|
@ -44,6 +43,10 @@ func StartRunSvc() {
|
||||||
|
|
||||||
log.Infof("Crowdsec %s", cwversion.VersionStr())
|
log.Infof("Crowdsec %s", cwversion.VersionStr())
|
||||||
|
|
||||||
|
if bincoverTesting != "" {
|
||||||
|
log.Debug("coverage report is enabled")
|
||||||
|
}
|
||||||
|
|
||||||
// Enable profiling early
|
// Enable profiling early
|
||||||
if cConfig.Prometheus != nil {
|
if cConfig.Prometheus != nil {
|
||||||
go registerPrometheus(cConfig.Prometheus)
|
go registerPrometheus(cConfig.Prometheus)
|
||||||
|
@ -56,7 +59,7 @@ func StartRunSvc() {
|
||||||
// is not going to change in logrus to keep backward
|
// is not going to change in logrus to keep backward
|
||||||
// compatibility), and allows us to report coverage.
|
// compatibility), and allows us to report coverage.
|
||||||
log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err)
|
log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err)
|
||||||
if !bincoverTesting {
|
if bincoverTesting == "" {
|
||||||
os.Exit(exitCode)
|
os.Exit(exitCode)
|
||||||
}
|
}
|
||||||
bincover.ExitCode = exitCode
|
bincover.ExitCode = exitCode
|
||||||
|
|
|
@ -84,6 +84,10 @@ func WindowsRun() {
|
||||||
|
|
||||||
log.Infof("Crowdsec %s", cwversion.VersionStr())
|
log.Infof("Crowdsec %s", cwversion.VersionStr())
|
||||||
|
|
||||||
|
if bincoverTesting != "" {
|
||||||
|
log.Debug("coverage report is enabled")
|
||||||
|
}
|
||||||
|
|
||||||
// Enable profiling early
|
// Enable profiling early
|
||||||
if cConfig.Prometheus != nil {
|
if cConfig.Prometheus != nil {
|
||||||
go registerPrometheus(cConfig.Prometheus)
|
go registerPrometheus(cConfig.Prometheus)
|
||||||
|
@ -91,8 +95,12 @@ func WindowsRun() {
|
||||||
|
|
||||||
if exitCode, err := Serve(cConfig); err != nil {
|
if exitCode, err := Serve(cConfig); err != nil {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf(err.Error())
|
// this method of logging a fatal error does not
|
||||||
if !bincoverTesting {
|
// trigger a program exit (as stated by the authors, it
|
||||||
|
// is not going to change in logrus to keep backward
|
||||||
|
// compatibility), and allows us to report coverage.
|
||||||
|
log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err)
|
||||||
|
if bincoverTesting != "" {
|
||||||
os.Exit(exitCode)
|
os.Exit(exitCode)
|
||||||
}
|
}
|
||||||
bincover.ExitCode = exitCode
|
bincover.ExitCode = exitCode
|
||||||
|
|
|
@ -28,15 +28,16 @@ PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins
|
||||||
DB_BACKEND ?= sqlite
|
DB_BACKEND ?= sqlite
|
||||||
|
|
||||||
ifdef TEST_COVERAGE
|
ifdef TEST_COVERAGE
|
||||||
CROWDSEC = "$(TEST_DIR)/crowdsec-wrapper"
|
CROWDSEC = $(TEST_DIR)/crowdsec-wrapper
|
||||||
CSCLI = "$(TEST_DIR)/cscli-wrapper"
|
CSCLI = $(TEST_DIR)/cscli-wrapper
|
||||||
BINCOVER_TESTING = true
|
BINCOVER_TESTING = true
|
||||||
else
|
else
|
||||||
# the wrappers should work here too - it detects TEST_COVERAGE - but we allow
|
# the wrappers should work here too - it detects TEST_COVERAGE - but we allow
|
||||||
# overriding the path to the binaries
|
# overriding the path to the binaries
|
||||||
CROWDSEC ?= "$(BIN_DIR)/crowdsec"
|
CROWDSEC ?= "$(BIN_DIR)/crowdsec"
|
||||||
CSCLI ?= "$(BIN_DIR)/cscli"
|
CSCLI ?= "$(BIN_DIR)/cscli"
|
||||||
BINCOVER_TESTING = false
|
# any value is considered true
|
||||||
|
BINCOVER_TESTING =
|
||||||
endif
|
endif
|
||||||
|
|
||||||
# If you change the name of the crowdsec executable, make sure the pgrep
|
# If you change the name of the crowdsec executable, make sure the pgrep
|
||||||
|
|
|
@ -93,6 +93,12 @@ declare stderr
|
||||||
assert_output --partial "Trying to authenticate with username"
|
assert_output --partial "Trying to authenticate with username"
|
||||||
assert_output --partial " on https://api.crowdsec.net/"
|
assert_output --partial " on https://api.crowdsec.net/"
|
||||||
assert_output --partial "You can successfully interact with Central API (CAPI)"
|
assert_output --partial "You can successfully interact with Central API (CAPI)"
|
||||||
|
|
||||||
|
ONLINE_API_CREDENTIALS_YAML="$(config_yq '.api.server.online_client.credentials_path')"
|
||||||
|
rm "${ONLINE_API_CREDENTIALS_YAML}"
|
||||||
|
run -1 --separate-stderr cscli capi status
|
||||||
|
run -0 echo "${stderr}"
|
||||||
|
assert_output --partial "Local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "${FILE} cscli config show -o human" {
|
@test "${FILE} cscli config show -o human" {
|
||||||
|
|
|
@ -49,7 +49,8 @@ declare stderr
|
||||||
run -1 --separate-stderr cscli capi status
|
run -1 --separate-stderr cscli capi status
|
||||||
|
|
||||||
run -0 echo "$stderr"
|
run -0 echo "$stderr"
|
||||||
assert_output --partial "Local API is disabled, please run this command on the local API machine"
|
assert_output --partial "crowdsec local API is disabled"
|
||||||
|
assert_output --partial "There is no configuration on 'api.server:'"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "$FILE cscli config show -o human" {
|
@test "$FILE cscli config show -o human" {
|
||||||
|
|
|
@ -12,11 +12,29 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||||
#shellcheck disable=SC1090
|
#shellcheck disable=SC1090
|
||||||
. "${THIS_DIR}/.environment.sh"
|
. "${THIS_DIR}/.environment.sh"
|
||||||
|
|
||||||
|
set -o pipefail # don't let sed hide the statuscode
|
||||||
|
mkdir -p "${LOCAL_DIR}/var/lib/coverage"
|
||||||
|
|
||||||
|
# we collect rc and output by hand, because setting -o pipefail would trigger a
|
||||||
|
# SIGPIPE.
|
||||||
|
set +e
|
||||||
|
|
||||||
# Arguments to crowdsec are passed through a temporary, newline-delimited
|
# Arguments to crowdsec are passed through a temporary, newline-delimited
|
||||||
# file courtesy of github.com/confluentinc/bincover. Coverage data will be
|
# file courtesy of github.com/confluentinc/bincover. Coverage data will be
|
||||||
# merged at the end of the test run.
|
# merged at the end of the test run.
|
||||||
# The '=' between flags and values is required.
|
# The '=' between flags and values is required.
|
||||||
exec "${BIN_DIR}/crowdsec.cover" \
|
output=$("${BIN_DIR}/crowdsec.cover" \
|
||||||
-test.run="^TestBincoverRunMain$" \
|
-test.run="^TestBincoverRunMain$" \
|
||||||
-test.coverprofile="${LOCAL_DIR}/var/lib/coverage/crowdsec-$(date +'%s')-$$.out" \
|
-test.coverprofile="${LOCAL_DIR}/var/lib/coverage/crowdsec-$(date +'%s')-$$-${RANDOM}.out" \
|
||||||
-args-file=<(for i; do echo "$i"; done) # Behold the amazing parameter contraption!
|
-args-file=<(for i; do echo "$i"; done))
|
||||||
|
rc=$?
|
||||||
|
|
||||||
|
# If there is bincover metadata, we take the status code from there. Otherwise,
|
||||||
|
# we keep the status from the above command.
|
||||||
|
if [[ ${output} =~ (.*)(START_BINCOVER_METADATA[[:space:]]*)(.*)([[:space:]]END_BINCOVER_METADATA) ]]; then
|
||||||
|
echo -n "${BASH_REMATCH[1]}"
|
||||||
|
exit "$(jq '.exit_code' <<< "${BASH_REMATCH[3]}")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo -n "${output}"
|
||||||
|
exit "${rc}"
|
||||||
|
|
|
@ -25,14 +25,16 @@ set +e
|
||||||
# The '=' between flags and values is required.
|
# The '=' between flags and values is required.
|
||||||
output=$("${BIN_DIR}/cscli.cover" \
|
output=$("${BIN_DIR}/cscli.cover" \
|
||||||
-test.run="^TestBincoverRunMain$" \
|
-test.run="^TestBincoverRunMain$" \
|
||||||
-test.coverprofile="${LOCAL_DIR}/var/lib/coverage/cscli-$(date +'%s')-$$.out" \
|
-test.coverprofile="${LOCAL_DIR}/var/lib/coverage/cscli-$(date +'%s')-$$-${RANDOM}.out" \
|
||||||
-args-file=<(for i; do echo "$i"; done))
|
-args-file=<(for i; do echo "${i}"; done))
|
||||||
rc=$?
|
rc=$?
|
||||||
|
|
||||||
# We also cut the metadata stuff that we don't need.
|
# If there is bincover metadata, we take the status code from there. Otherwise,
|
||||||
echo -n "$output" | tr '\n' '\f' | sed 's/START_BINCOVER_METADATA.*//' | tr '\f' '\n'
|
# we keep the status from the above command.
|
||||||
|
if [[ ${output} =~ (.*)(START_BINCOVER_METADATA[[:space:]]*)(.*)([[:space:]]END_BINCOVER_METADATA) ]]; then
|
||||||
|
echo -n "${BASH_REMATCH[1]}"
|
||||||
|
exit "$(jq '.exit_code' <<< "${BASH_REMATCH[3]}")"
|
||||||
|
fi
|
||||||
|
|
||||||
# this does not work because cscli output does not always end with \n
|
echo -n "${output}"
|
||||||
# echo -n "$output" | sed -n '/START_BINCOVER_METADATA/q;p'
|
exit "${rc}"
|
||||||
|
|
||||||
exit $rc
|
|
||||||
|
|
|
@ -45,6 +45,9 @@ else
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -n "$TEST_COVERAGE" ]; then
|
if [ -n "$TEST_COVERAGE" ]; then
|
||||||
|
# empty files just to avoid merge errors
|
||||||
|
touch "${LOCAL_DIR}"/var/lib/coverage/crowdsec-
|
||||||
|
touch "${LOCAL_DIR}"/var/lib/coverage/cscli-
|
||||||
gocovmerge "${LOCAL_DIR}"/var/lib/coverage/crowdsec-* > "${LOCAL_DIR}/var/lib/coverage/coverage-crowdsec.out"
|
gocovmerge "${LOCAL_DIR}"/var/lib/coverage/crowdsec-* > "${LOCAL_DIR}/var/lib/coverage/coverage-crowdsec.out"
|
||||||
gocovmerge "${LOCAL_DIR}"/var/lib/coverage/cscli-* > "${LOCAL_DIR}/var/lib/coverage/coverage-cscli.out"
|
gocovmerge "${LOCAL_DIR}"/var/lib/coverage/cscli-* > "${LOCAL_DIR}/var/lib/coverage/coverage-cscli.out"
|
||||||
fi
|
fi
|
||||||
|
|
Loading…
Reference in a new issue