CI: Static builds by default; replace bincover with go -cover from 1.20 (#2150)

* Makefile: build static binaries only
* Replace bincover with go -cover from 1.20
* CI: Fix timing issue between lapi and agent containers
This commit is contained in:
mmetc 2023-03-30 15:05:09 +02:00 committed by GitHub
parent 61bea26486
commit ea6401ce09
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 122 additions and 357 deletions

View file

@ -55,7 +55,6 @@ jobs:
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
go install github.com/wadey/gocovmerge@latest
- name: "Build crowdsec and fixture"
run: |
@ -64,6 +63,20 @@ jobs:
- name: "Run tests"
run: make bats-test
- name: "Collect coverage data"
run: |
go tool covdata textfmt -i test/coverage -o coverage-bats-raw.out
# filter out unwanted packages, should match the argument to "go-acc --ignore"
grep -v \
-e '/pkg/database' \
-e '/plugins/notifications' \
-e '/pkg/protobufs' \
-e '/pkg/cwversions' \
-e '/pkg/cstest' \
-e '/pkg/models' \
< coverage-bats-raw.out \
> coverage-bats.out
#
# In case you need to inspect the database status after the failure of a given test
#
@ -87,11 +100,5 @@ jobs:
- name: Upload crowdsec coverage to codecov
uses: codecov/codecov-action@v3
with:
files: ./test/local/var/lib/coverage/coverage-crowdsec.out
flags: func-crowdsec
- name: Upload cscli coverage to codecov
uses: codecov/codecov-action@v3
with:
files: ./test/local/var/lib/coverage/coverage-cscli.out
flags: func-cscli
files: ./coverage-bats.out
flags: bats

View file

@ -141,12 +141,6 @@ jobs:
make build
make go-acc | richgo testfilter
- name: Build and run tests (static)
run: |
make clean build BUILD_STATIC=yes
make test \
| richgo testfilter
- name: Upload unit coverage to Codecov
uses: codecov/codecov-action@v3
with:

View file

@ -41,53 +41,10 @@ jobs:
- name: Build the binaries
run: make release
- name: Upload to release
uses: JasonEtco/upload-to-release@master
with:
args: crowdsec-release.tgz application/x-gzip
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
build_static:
strategy:
matrix:
go-version: ["1.20.1"]
name: Build and upload binary package
runs-on: ubuntu-latest
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: Cache Go modules
uses: actions/cache@v3
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build the binaries
run: |
make release BUILD_STATIC=yes
mv crowdsec-release.tgz crowdsec-release-static.tgz
- name: Upload to release
uses: JasonEtco/upload-to-release@master
with:
args: crowdsec-release-static.tgz application/x-gzip
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

3
.gitignore vendored
View file

@ -15,8 +15,9 @@
# Test binaries, built from *_test.go
pkg/csplugin/tests/cs_plugin_test*
# Output of go-acc
# Output of go-acc, go -cover
*.out
test/coverage/*
# Development artifacts, backups, etc
*.swp

View file

@ -27,8 +27,6 @@ BUILD_CMD = build
GO_MODULE_NAME = github.com/crowdsecurity/crowdsec
LD_OPTS_VARS= \
-X '$(GO_MODULE_NAME)/cmd/crowdsec.bincoverTesting=$(BINCOVER_TESTING)' \
-X '$(GO_MODULE_NAME)/cmd/crowdsec-cli.bincoverTesting=$(BINCOVER_TESTING)' \
-X '$(GO_MODULE_NAME)/pkg/cwversion.Version=$(BUILD_VERSION)' \
-X '$(GO_MODULE_NAME)/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP)' \
-X '$(GO_MODULE_NAME)/pkg/cwversion.Codename=$(BUILD_CODENAME)' \
@ -41,9 +39,14 @@ LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.System=docker'
endif
ifdef BUILD_STATIC
export LD_OPTS=-ldflags "-s -w $(LD_OPTS_VARS) -extldflags '-static'" -tags netgo,osusergo,sqlite_omit_load_extension
else
export LD_OPTS=-ldflags "-s -w $(LD_OPTS_VARS)"
$(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.)
endif
export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \
-trimpath -tags netgo,osusergo,sqlite_omit_load_extension
ifneq (,$(TEST_COVERAGE))
LD_OPTS += -cover
endif
GOCMD = go
@ -68,7 +71,6 @@ clean: testclean
@$(RM) $(CSCLI_BIN) $(WIN_IGNORE_ERR)
@$(RM) *.log $(WIN_IGNORE_ERR)
@$(RM) crowdsec-release.tgz $(WIN_IGNORE_ERR)
@$(RM) crowdsec-release-static.tgz $(WIN_IGNORE_ERR)
@$(RM) $(HTTP_PLUGIN_FOLDER)/$(HTTP_PLUGIN_BIN) $(WIN_IGNORE_ERR)
@$(RM) $(SLACK_PLUGIN_FOLDER)/$(SLACK_PLUGIN_BIN) $(WIN_IGNORE_ERR)
@$(RM) $(SPLUNK_PLUGIN_FOLDER)/$(SPLUNK_PLUGIN_BIN) $(WIN_IGNORE_ERR)
@ -79,15 +81,9 @@ clean: testclean
cscli: goversion
@$(MAKE) -C $(CSCLI_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
cscli-bincover: goversion
@GOARCH=$(GOARCH) GOOS=$(GOOS) $(MAKE) -C $(CSCLI_FOLDER) build-bincover --no-print-directory
crowdsec: goversion
@$(MAKE) -C $(CROWDSEC_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
crowdsec-bincover: goversion
@GOARCH=$(GOARCH) GOOS=$(GOOS) $(MAKE) -C $(CROWDSEC_FOLDER) build-bincover --no-print-directory
http-plugin: goversion
@$(MAKE) -C $(HTTP_PLUGIN_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"

View file

@ -12,8 +12,6 @@ GOTEST = $(GOCMD) test
GOGET = $(GOCMD) get
BINARY_NAME = cscli$(EXT)
# names longer than 15 chars break 'pgrep'
BINARY_NAME_COVER = $(BINARY_NAME).cover
PREFIX ?= "/"
BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
@ -23,9 +21,6 @@ all: clean build
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
build-bincover: clean
$(GOTEST) . -tags testrunmain -coverpkg=$(go list github.com/crowdsecurity/crowdsec/... | grep -v -e 'pkg/database' -e 'plugins/notifications' -e 'pkg/protobufs' -e 'pkg/cwversions' -e 'pkg/cstest' -e 'pkg/models') -covermode=atomic $(LD_OPTS) -c -o $(BINARY_NAME_COVER)
.PHONY: install
install: install-conf install-bin
@ -39,4 +34,4 @@ uninstall:
@$(RM) $(BIN_PREFIX)$(BINARY_NAME) $(WIN_IGNORE_ERR)
clean:
@$(RM) $(BINARY_NAME) $(BINARY_NAME_COVER) $(WIN_IGNORE_ERR)
@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)

View file

@ -7,7 +7,6 @@ import (
"path/filepath"
"strings"
"github.com/confluentinc/bincover"
"github.com/fatih/color"
cc "github.com/ivanpirog/coloredcobra"
log "github.com/sirupsen/logrus"
@ -21,8 +20,6 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/fflag"
)
var bincoverTesting = ""
var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
var ConfigFilePath string
@ -252,15 +249,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
}
if err := rootCmd.Execute(); err != nil {
if bincoverTesting != "" {
log.Debug("coverage report is enabled")
}
exitCode := 1
log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err)
if bincoverTesting == "" {
os.Exit(exitCode)
}
bincover.ExitCode = exitCode
os.Exit(1)
}
}

View file

@ -1,13 +0,0 @@
//go:build testrunmain
package main
import (
"testing"
"github.com/confluentinc/bincover"
)
func TestBincoverRunMain(t *testing.T) {
bincover.RunTest(main)
}

View file

@ -13,7 +13,6 @@ GOGET = $(GOCMD) get
CROWDSEC_BIN = crowdsec$(EXT)
# names longer than 15 chars break 'pgrep'
CROWDSEC_BIN_COVER = $(CROWDSEC_BIN).cover
PREFIX ?= "/"
CFG_PREFIX = $(PREFIX)"/etc/crowdsec/config/"
BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
@ -28,14 +27,11 @@ all: clean test build
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(CROWDSEC_BIN)
build-bincover: clean
$(GOTEST) . -tags testrunmain -coverpkg=$(go list github.com/crowdsecurity/crowdsec/... | grep -v -e 'pkg/database' -e 'plugins/notifications' -e 'pkg/protobufs' -e 'pkg/cwversions' -e 'pkg/cstest' -e 'pkg/models') -covermode=atomic $(LD_OPTS) -c -o $(CROWDSEC_BIN_COVER)
test:
$(GOTEST) $(LD_OPTS) -v ./...
clean:
@$(RM) $(CROWDSEC_BIN) $(CROWDSEC_BIN).test $(CROWDSEC_BIN_COVER) $(WIN_IGNORE_ERR)
@$(RM) $(CROWDSEC_BIN) $(CROWDSEC_BIN).test $(WIN_IGNORE_ERR)
.PHONY: install
install: install-conf install-bin

View file

@ -9,7 +9,6 @@ import (
"strings"
"time"
"github.com/confluentinc/bincover"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
@ -51,8 +50,6 @@ var (
pluginBroker csplugin.PluginBroker
)
var bincoverTesting = ""
type Flags struct {
ConfigFile string
TraceLevel bool
@ -296,25 +293,6 @@ func LoadConfig(cConfig *csconfig.Config) error {
return nil
}
// exitWithCode must be called right before the program termination,
// to allow measuring functional test coverage in case of abnormal exit.
//
// without bincover: log error and exit with code
// with bincover: log error and tell bincover the exit code, then return
func exitWithCode(exitCode int, err error) {
if err != nil {
// this method of logging a fatal error does not
// trigger a program exit (as stated by the authors, it
// is not going to change in logrus to keep backward
// compatibility), and allows us to report coverage.
log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err)
}
if bincoverTesting == "" {
os.Exit(exitCode)
}
bincover.ExitCode = exitCode
}
// crowdsecT0 can be used to measure start time of services,
// or uptime of the application
var crowdsecT0 time.Time
@ -332,8 +310,6 @@ func main() {
crowdsecT0 = time.Now()
defer types.CatchPanic("crowdsec/main")
log.Debugf("os.Args: %v", os.Args)
// Handle command line arguments
@ -344,20 +320,17 @@ func main() {
fmt.Fprintf(os.Stderr, "argument provided but not defined: %s\n", flag.Args()[0])
flag.Usage()
// the flag package exits with 2 in case of unknown flag
exitWithCode(2, nil)
return
os.Exit(2)
}
if flags.PrintVersion {
cwversion.Show()
exitWithCode(0, nil)
return
os.Exit(0)
}
exitCode := 0
err := StartRunSvc()
if err != nil {
exitCode = 1
log.Fatal(err)
}
exitWithCode(exitCode, err)
os.Exit(0)
}

View file

@ -1,13 +0,0 @@
//go:build testrunmain
package main
import (
"github.com/confluentinc/bincover"
"testing"
)
func TestBincoverRunMain(t *testing.T) {
bincover.RunTest(main)
}

View file

@ -4,6 +4,7 @@
package main
import (
"fmt"
"os"
log "github.com/sirupsen/logrus"
@ -12,6 +13,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func StartRunSvc() error {
@ -20,6 +22,8 @@ func StartRunSvc() error {
err error
)
defer types.CatchPanic("crowdsec/StartRunSvc")
// Set a default logger with level=fatal on stderr,
// in addition to the one we configure afterwards
log.AddHook(&writer.Hook{
@ -40,10 +44,6 @@ func StartRunSvc() error {
log.Infof("Crowdsec %s", cwversion.VersionStr())
if bincoverTesting != "" {
log.Debug("coverage report is enabled")
}
apiReady := make(chan bool, 1)
agentReady := make(chan bool, 1)
@ -56,7 +56,7 @@ func StartRunSvc() error {
dbClient, err = database.NewClient(cConfig.DbConfig)
if err != nil {
log.Fatalf("unable to create database client: %s", err)
return fmt.Errorf("unable to create database client: %s", err)
}
}
registerPrometheus(cConfig.Prometheus)

View file

@ -10,12 +10,15 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func StartRunSvc() error {
const svcName = "CrowdSec"
const svcDescription = "Crowdsec IPS/IDS"
defer types.CatchPanic("crowdsec/StartRunSvc")
isRunninginService, err := svc.IsWindowsService()
if err != nil {
return errors.Wrap(err, "failed to determine if we are running in windows service mode")
@ -68,10 +71,6 @@ func WindowsRun() error {
// Configure logging
log.Infof("Crowdsec %s", cwversion.VersionStr())
if bincoverTesting != "" {
log.Debug("coverage report is enabled")
}
apiReady := make(chan bool, 1)
agentReady := make(chan bool, 1)
@ -84,7 +83,7 @@ func WindowsRun() error {
dbClient, err = database.NewClient(cConfig.DbConfig)
if err != nil {
log.Fatalf("unable to create database client: %s", err)
return fmt.Errorf("unable to create database client: %s", err)
}
}
registerPrometheus(cConfig.Prometheus)

View file

@ -28,11 +28,12 @@ def test_split_lapi_agent(crowdsec, flavor):
cs_lapi = crowdsec(name=lapiname, environment=lapi_env, flavor=flavor)
cs_agent = crowdsec(name=agentname, environment=agent_env, flavor=flavor)
with cs_lapi as lapi, cs_agent as agent:
with cs_lapi as lapi:
lapi.wait_for_log("*CrowdSec Local API listening on 0.0.0.0:8080*")
agent.wait_for_log("*Starting processing data*")
lapi.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
res = agent.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
with cs_agent as agent:
agent.wait_for_log("*Starting processing data*")
res = agent.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout

View file

@ -168,22 +168,26 @@ def test_tls_split_lapi_agent(crowdsec, flavor, certs_dir):
certs_dir(lapi_hostname=lapiname): {'bind': '/etc/ssl/crowdsec', 'mode': 'ro'},
}
with crowdsec(flavor=flavor, name=lapiname, environment=lapi_env, volumes=volumes) as lapi, crowdsec(flavor=flavor, name=agentname, environment=agent_env, volumes=volumes) as agent:
cs_lapi = crowdsec(flavor=flavor, name=lapiname, environment=lapi_env, volumes=volumes)
cs_agent = crowdsec(flavor=flavor, name=agentname, environment=agent_env, volumes=volumes)
with cs_lapi as lapi:
lapi.wait_for_log([
"*(tls) Client Auth Type set to VerifyClientCertIfGiven*",
"*CrowdSec Local API listening on 0.0.0.0:8080*"
])
# TODO: wait_for_https
lapi.wait_for_http(8080, '/health', want_status=None)
agent.wait_for_log("*Starting processing data*")
res = agent.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
res = lapi.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
with cs_agent as agent:
agent.wait_for_log("*Starting processing data*")
res = agent.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
res = lapi.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
def test_tls_mutual_split_lapi_agent(crowdsec, flavor, certs_dir):
@ -215,22 +219,26 @@ def test_tls_mutual_split_lapi_agent(crowdsec, flavor, certs_dir):
certs_dir(lapi_hostname=lapiname): {'bind': '/etc/ssl/crowdsec', 'mode': 'ro'},
}
with crowdsec(flavor=flavor, name=lapiname, environment=lapi_env, volumes=volumes) as lapi, crowdsec(flavor=flavor, name=agentname, environment=agent_env, volumes=volumes) as agent:
cs_lapi = crowdsec(flavor=flavor, name=lapiname, environment=lapi_env, volumes=volumes)
cs_agent = crowdsec(flavor=flavor, name=agentname, environment=agent_env, volumes=volumes)
with cs_lapi as lapi:
lapi.wait_for_log([
"*(tls) Client Auth Type set to VerifyClientCertIfGiven*",
"*CrowdSec Local API listening on 0.0.0.0:8080*"
])
# TODO: wait_for_https
lapi.wait_for_http(8080, '/health', want_status=None)
agent.wait_for_log("*Starting processing data*")
res = agent.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
res = lapi.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
with cs_agent as agent:
agent.wait_for_log("*Starting processing data*")
res = agent.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
res = lapi.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
def test_tls_client_ou(crowdsec, certs_dir):
@ -262,26 +270,40 @@ def test_tls_client_ou(crowdsec, certs_dir):
certs_dir(lapi_hostname=lapiname, agent_ou='custom-client-ou'): {'bind': '/etc/ssl/crowdsec', 'mode': 'ro'},
}
with crowdsec(name=lapiname, environment=lapi_env, volumes=volumes) as lapi, crowdsec(name=agentname, environment=agent_env, volumes=volumes) as agent:
lapi.wait_for_log([
"*client certificate OU (?custom-client-ou?) doesn't match expected OU (?agent-ou?)*",
])
cs_lapi = crowdsec(name=lapiname, environment=lapi_env, volumes=volumes)
cs_agent = crowdsec(name=agentname, environment=agent_env, volumes=volumes)
lapi_env['AGENTS_ALLOWED_OU'] = 'custom-client-ou'
with crowdsec(name=lapiname, environment=lapi_env, volumes=volumes) as lapi, crowdsec(name=agentname, environment=agent_env, volumes=volumes) as agent:
with cs_lapi as lapi:
lapi.wait_for_log([
"*(tls) Client Auth Type set to VerifyClientCertIfGiven*",
"*CrowdSec Local API listening on 0.0.0.0:8080*"
])
# TODO: wait_for_https
lapi.wait_for_http(8080, '/health', want_status=None)
agent.wait_for_log("*Starting processing data*")
res = agent.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
res = lapi.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
with cs_agent as agent:
lapi.wait_for_log([
"*client certificate OU (?custom-client-ou?) doesn't match expected OU (?agent-ou?)*",
])
lapi_env['AGENTS_ALLOWED_OU'] = 'custom-client-ou'
cs_lapi = crowdsec(name=lapiname, environment=lapi_env, volumes=volumes)
cs_agent = crowdsec(name=agentname, environment=agent_env, volumes=volumes)
with cs_lapi as lapi:
lapi.wait_for_log([
"*(tls) Client Auth Type set to VerifyClientCertIfGiven*",
"*CrowdSec Local API listening on 0.0.0.0:8080*"
])
# TODO: wait_for_https
lapi.wait_for_http(8080, '/health', want_status=None)
with cs_agent as agent:
agent.wait_for_log("*Starting processing data*")
res = agent.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout
res = lapi.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
stdout = res.output.decode()
assert "You can successfully interact with Local API (LAPI)" in stdout

3
go.mod
View file

@ -12,7 +12,6 @@ require (
github.com/aws/aws-sdk-go v1.42.25
github.com/buger/jsonparser v1.1.1
github.com/c-robinson/iplib v1.0.3
github.com/confluentinc/bincover v0.2.0
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
github.com/crowdsecurity/grokky v0.2.1
@ -84,6 +83,7 @@ require (
github.com/shirou/gopsutil/v3 v3.22.12
github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c
github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
github.com/wasilibs/go-re2 v0.2.1
golang.org/x/sys v0.5.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/apiserver v0.22.5
@ -177,7 +177,6 @@ require (
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/ugorji/go/codec v1.2.6 // indirect
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
github.com/wasilibs/go-re2 v0.2.1 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
github.com/zclconf/go-cty v1.8.0 // indirect
go.mongodb.org/mongo-driver v1.9.0 // indirect

12
go.sum
View file

@ -88,10 +88,6 @@ github.com/alexliesenfeld/health v0.5.1 h1:cohQdtQbJdA6bj0aMD4gdXA9xQyvh9NxWO9XL
github.com/alexliesenfeld/health v0.5.1/go.mod h1:N4NDIeQtlWumG+6z1ne1v62eQxktz5ylEgGgH9emdMw=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/antonmedv/expr v1.12.2 h1:nlRcu4uHI6oSKCf6GHJTcT7hIf7dFAjgfvG0MWb7Cu0=
github.com/antonmedv/expr v1.12.2/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
github.com/antonmedv/expr v1.12.4 h1:YRkeF7r0cejMS47bDYe3Jyes7L9t1AhpunC+Duq+R9k=
github.com/antonmedv/expr v1.12.4/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
github.com/antonmedv/expr v1.12.5 h1:Fq4okale9swwL3OeLLs9WD9H6GbgBLJyN/NUHRv+n0E=
github.com/antonmedv/expr v1.12.5/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
@ -156,8 +152,6 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe
github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/confluentinc/bincover v0.2.0 h1:WSS3MqzwJbosCLMOuF3tJ0pMpALzBfrm80Tb+/3gbQs=
github.com/confluentinc/bincover v0.2.0/go.mod h1:qeI1wx0RxdGTZtrJY0HVlgJ4NqC/X2Z+fHbvy87tgHE=
github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns=
github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@ -181,8 +175,6 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
github.com/crowdsecurity/grokky v0.2.0 h1:8u3dCU1kqSlOu2phP6S+HF2OTohZ/JBGcZCWu1dhnF0=
github.com/crowdsecurity/grokky v0.2.0/go.mod h1:qdyzFHtnY6nmdYZJFHVbdms2HMFLiEXzmqq5tCUmiGE=
github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
@ -927,10 +919,9 @@ github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaU
github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
github.com/wasilibs/go-re2 v0.1.0 h1:+pcjlvge6E9WWU4eaMlULRikEUkhPDXSFfCF5p7htMI=
github.com/wasilibs/go-re2 v0.1.0/go.mod h1:F91Yac+zPNDFrrd8fl4mSd7+TTu2tYiX56BEIEPQl2M=
github.com/wasilibs/go-re2 v0.2.1 h1:bZVXxlqGHXnHlQt5ys6CsHyypsCsP952m1oKBB5O9Dc=
github.com/wasilibs/go-re2 v0.2.1/go.mod h1:jrB0h+KSTmTwsL0lI2sEyu8TcRi1htLiR/TbSURMwMM=
github.com/wasilibs/nottinygc v0.2.0 h1:cXz2Ac9bVMLkpuOlUlPQMWowjw0K2cOErXZOFdAj7yE=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
@ -1426,7 +1417,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

View file

@ -27,18 +27,8 @@ PID_DIR = $(LOCAL_DIR)/var/run
PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins
DB_BACKEND ?= sqlite
ifdef TEST_COVERAGE
CROWDSEC = $(TEST_DIR)/bin/crowdsec-wrapper
CSCLI = $(TEST_DIR)/bin/cscli-wrapper
BINCOVER_TESTING = true
else
# the wrappers should work here too - it detects TEST_COVERAGE - but we allow
# overriding the path to the binaries
CROWDSEC ?= $(BIN_DIR)/crowdsec
CSCLI ?= $(BIN_DIR)/cscli
# any value is considered true
BINCOVER_TESTING =
endif
CROWDSEC ?= $(BIN_DIR)/crowdsec
CSCLI ?= $(BIN_DIR)/cscli
# If you change the name of the crowdsec executable, make sure the pgrep
# parameters are correct in $(TEST_DIR)/assert-crowdsec-not-running
@ -59,6 +49,7 @@ export INIT_BACKEND="$(INIT_BACKEND)"
export CONFIG_BACKEND="$(CONFIG_BACKEND)"
export PACKAGE_TESTING="$(PACKAGE_TESTING)"
export TEST_COVERAGE="$(TEST_COVERAGE)"
export GOCOVERDIR="$(TEST_DIR)/coverage"
endef
bats-all: bats-clean bats-build bats-fixture bats-test bats-test-hub
@ -76,11 +67,9 @@ bats-check-requirements:
# Build and installs crowdsec in a local directory. Rebuilds if already exists.
bats-build: bats-environment bats-check-requirements
@mkdir -p $(BIN_DIR) $(LOG_DIR) $(PID_DIR) $(PLUGIN_DIR)
@BINCOVER_TESTING=$(BINCOVER_TESTING) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec cscli plugins
@TEST_COVERAGE=$(TEST_COVERAGE) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec cscli plugins
@install -m 0755 cmd/crowdsec/crowdsec cmd/crowdsec-cli/cscli $(BIN_DIR)/
@install -m 0755 plugins/notifications/*/notification-* $(PLUGIN_DIR)/
@BINCOVER_TESTING=$(BINCOVER_TESTING) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec-bincover cscli-bincover
@install -m 0755 cmd/crowdsec/crowdsec.cover cmd/crowdsec-cli/cscli.cover $(BIN_DIR)/
# Create a reusable package with initial configuration + data
bats-fixture:

View file

@ -77,8 +77,8 @@ teardown() {
log_old="${logdir1}/crowdsec.log"
config_set ".common.log_dir=\"${logdir1}\""
rune -0 ./instance-crowdsec start
# PID="$output"
rune -0 ./instance-crowdsec start-pid
PID="$output"
assert_file_exist "$log_old"
assert_file_contains "$log_old" "Starting processing data"
@ -90,15 +90,7 @@ teardown() {
sleep 5
# this won't work as crowdsec-wrapper does not relay the signal
# rune -0 kill -HUP "$PID"
# During functional tests, crowdsec is often run from a wrapper script,
# which captures its output (for coverage reports) and cannot relay signals
# at the same time. So instead of sending a SIGHUP to the wrapper, we send
# it to the crowdsec process by name - with or without coverage.
rune pkill -HUP -f "$BIN_DIR/crowdsec.cover"
rune pkill -HUP -f "$BIN_DIR/crowdsec"
rune -0 kill -HUP "$PID"
for ((i=0; i<10; i++)); do
sleep 1

View file

@ -2,7 +2,7 @@
is_crowdsec_running() {
# ignore processes in containers
PIDS=$(pgrep --ns $$ -x 'crowdsec|crowdsec.test|crowdsec.cover')
PIDS=$(pgrep --ns $$ -x 'crowdsec|crowdsec.test')
}
# The process can be slow, especially on CI and during test coverage.

View file

@ -87,12 +87,6 @@ check_cfssljson() {
fi
}
check_gocovmerge() {
if ! command -v gocovmerge >/dev/null; then
die "missing required program 'gocovmerge'. You can install it with \"go install github.com/wadey/gocovmerge@latest\""
fi
}
check_bats_core
check_curl
check_daemonizer
@ -104,7 +98,3 @@ check_base64
check_python3
check_yq
check_pkill
if [[ -n "${TEST_COVERAGE}" ]]; then
check_gocovmerge
fi

View file

@ -1,49 +0,0 @@
#!/usr/bin/env bash
set -eu
#
# Delegate operations to an instrumented binary and collects coverage data.
#
#shellcheck disable=SC1007
THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
# no need to change directory, and doing it here would break hub tests
# shellcheck disable=SC1091
. "${THIS_DIR}/../.environment.sh"
set -o pipefail # don't let sed hide the statuscode
mkdir -p "${LOCAL_DIR}/var/lib/coverage"
# this would be nice but doesn't work, since the binary is not running in background
#_hup() {
# echo "pkill -1 crowdsec.cover"
# pkill -HUP crowdsec.cover
#}
#
## relay the "configuration reload" signal
#trap _hup SIGHUP
# we collect rc and output by hand, because setting -o pipefail would trigger a
# SIGPIPE.
set +e
# Arguments to crowdsec are passed through a temporary, newline-delimited
# file courtesy of github.com/confluentinc/bincover. Coverage data will be
# merged at the end of the test run.
# The '=' between flags and values is required.
output=$("${BIN_DIR}/crowdsec.cover" \
-test.run="^TestBincoverRunMain$" \
-test.coverprofile="${LOCAL_DIR}/var/lib/coverage/crowdsec-$(date +'%s')-$$-${RANDOM}.out" \
-args-file=<(for i; do echo "${i}"; done))
rc=$?
# If there is bincover metadata, we take the status code from there. Otherwise,
# we keep the status from the above command.
if [[ ${output} =~ (.*)(START_BINCOVER_METADATA[[:space:]]*)(.*)([[:space:]]END_BINCOVER_METADATA) ]]; then
echo -n "${BASH_REMATCH[1]}"
exit "$(jq '.exit_code' <<< "${BASH_REMATCH[3]}")"
fi
echo -n "${output}"
exit "${rc}"

View file

@ -1,40 +0,0 @@
#!/usr/bin/env bash
set -eu
#
# Delegate operations to an instrumented binary and collects coverage data.
#
#shellcheck disable=SC1007
THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
# no need to change directory, and doing it here would break hub tests
# shellcheck disable=SC1091
. "${THIS_DIR}/../.environment.sh"
set -o pipefail # don't let sed hide the statuscode
mkdir -p "${LOCAL_DIR}/var/lib/coverage"
# we collect rc and output by hand, because setting -o pipefail would trigger a
# SIGPIPE.
set +e
# Arguments to cscli are passed through a temporary, newline-delimited
# file courtesy of github.com/confluentinc/bincover. Coverage data will be
# merged at the end of the test run.
# The '=' between flags and values is required.
output=$("${BIN_DIR}/cscli.cover" \
-test.run="^TestBincoverRunMain$" \
-test.coverprofile="${LOCAL_DIR}/var/lib/coverage/cscli-$(date +'%s')-$$-${RANDOM}.out" \
-args-file=<(for i; do echo "${i}"; done))
rc=$?
# If there is bincover metadata, we take the status code from there. Otherwise,
# we keep the status from the above command.
if [[ ${output} =~ (.*)(START_BINCOVER_METADATA[[:space:]]*)(.*)([[:space:]]END_BINCOVER_METADATA) ]]; then
echo -n "${BASH_REMATCH[1]}"
exit "$(jq '.exit_code' <<< "${BASH_REMATCH[3]}")"
fi
echo -n "${output}"
exit "${rc}"

View file

View file

@ -120,11 +120,11 @@ make_init_data() {
./instance-db config-yaml
./instance-db setup
"${CSCLI}" machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
"${CSCLI}" hub update
"${CSCLI}" collections install crowdsecurity/linux
"$CSCLI" machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
"$CSCLI" hub update
"$CSCLI" collections install crowdsecurity/linux
mkdir -p "${LOCAL_INIT_DIR}"
mkdir -p "$LOCAL_INIT_DIR"
./instance-db dump "${LOCAL_INIT_DIR}/database"

View file

@ -20,8 +20,6 @@ if [[ -z "${TEST_COVERAGE}" ]]; then
echo "Coverage report: no"
else
echo "Coverage report: yes"
rm -f "${LOCAL_DIR}/var/lib/coverage/*"
mkdir -p "${LOCAL_DIR}/var/lib/coverage"
fi
dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")"
@ -44,11 +42,3 @@ else
--print-output-on-failure \
"${TEST_DIR}/bats" "${TEST_DIR}/dyn-bats"
fi
if [[ -n "${TEST_COVERAGE}" ]]; then
# empty files just to avoid merge errors
touch "${LOCAL_DIR}"/var/lib/coverage/crowdsec-
touch "${LOCAL_DIR}"/var/lib/coverage/cscli-
gocovmerge "${LOCAL_DIR}"/var/lib/coverage/crowdsec-* > "${LOCAL_DIR}/var/lib/coverage/coverage-crowdsec.out"
gocovmerge "${LOCAL_DIR}"/var/lib/coverage/cscli-* > "${LOCAL_DIR}/var/lib/coverage/coverage-cscli.out"
fi