merge master branch

.github/workflows/bats-hub.yml

@@ -15,7 +15,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.3"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -37,7 +37,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:

.github/workflows/bats-mysql.yml

@@ -14,7 +14,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.3"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -44,7 +44,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:

.github/workflows/bats-postgres.yml

@@ -10,14 +10,14 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.3"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
     timeout-minutes: 30
     services:
       database:
-        image: postgres:15
+        image: postgres:16
         env:
           POSTGRES_PASSWORD: "secret"
         ports:
@@ -30,13 +30,13 @@ jobs:
 
     steps:
 
-    - name: "Install pg_dump v15"
+    - name: "Install pg_dump v16"
       # we can remove this when it's released on ubuntu-latest
       run: |
           sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
           wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null
           sudo apt update
-          sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-15
+          sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-16
 
     - name: "Force machineid"
       run: |
@@ -53,7 +53,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:

.github/workflows/bats-sqlite-coverage.yml

@@ -11,7 +11,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.3"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -34,7 +34,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:

.github/workflows/ci-windows-build-msi.yml

@@ -23,7 +23,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.3"]
 
     name: Build
     runs-on: windows-2019
@@ -40,7 +40,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: Build
       run: make windows_installer BUILD_RE2_WASM=1

.github/workflows/codeql-analysis.yml

@@ -45,6 +45,9 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
+      with:
+        # required to pick up tags for BUILD_VERSION
+        fetch-depth: 0
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
@@ -58,8 +61,8 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
+    # - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+    #   uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,9 +71,14 @@ jobs:
     #    and modify them (or add more) to build your code if your project
     #    uses a compiled language
 
-    #- run: |
+    - name: "Set up Go"
-    #   make bootstrap
+      uses: actions/setup-go@v4
-    #   make release
+      with:
+        go-version: "1.21.0"
+        cache-dependency-path: "**/go.sum"
+
+    - run: |
+       make clean build BUILD_RE2_WASM=1
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2

.github/workflows/go-tests-windows.yml

@@ -22,7 +22,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.3"]
 
     name: "Build + tests"
     runs-on: windows-2022
@@ -39,7 +39,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: Build
       run: |
@@ -61,7 +60,7 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v3
       with:
-        version: v1.51
+        version: v1.54
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false
         # the cache is already managed above, enabling it here

.github/workflows/go-tests.yml

@@ -34,7 +34,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.3"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -120,7 +120,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: Build and run tests, static
       run: |
@@ -145,7 +144,7 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v3
       with:
-        version: v1.51
+        version: v1.54
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false
         # the cache is already managed above, enabling it here

.github/workflows/release_publish-package.yml

@@ -14,7 +14,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.3"]
 
     name: Build and upload binary package
     runs-on: ubuntu-latest
@@ -30,7 +30,6 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go-version }}
-          cache-dependency-path: "**/go.sum"
 
       - name: Build the binaries
         run: |
@@ -42,4 +41,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           tag_name="${GITHUB_REF##*/}"
-          hub release edit -a crowdsec-release.tgz -a vendor.tgz -m "" "$tag_name"
+          hub release edit -a crowdsec-release.tgz -a vendor.tgz -a *-vendor.tar.xz -m "" "$tag_name"

.gitignore

@@ -41,12 +41,7 @@ vendor.tgz
 # crowdsec binaries
 cmd/crowdsec-cli/cscli
 cmd/crowdsec/crowdsec
-plugins/notifications/http/notification-http
+cmd/notification-*/notification-*
-plugins/notifications/slack/notification-slack
-plugins/notifications/splunk/notification-splunk
-plugins/notifications/email/notification-email
-plugins/notifications/dummy/notification-dummy
-plugins/notifications/sentinel/notification-sentinel
 
 # Test cache (downloaded files)
 .cache

.golangci.yml

@@ -105,6 +105,7 @@ linters:
     # Recommended? (easy)
     #
 
+    - depguard              # Go linter that checks if package imports are in a list of acceptable packages
     - dogsled               # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f())
     - errchkjson            # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted.
     - errorlint             # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
@@ -121,10 +122,10 @@ linters:
     - nosprintfhostport     # Checks for misuse of Sprintf to construct a host with port in a URL.
     - promlinter            # Check Prometheus metrics naming via promlint
     - revive                # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint.
+    - tagalign              # check that struct tags are well aligned [fast: true, auto-fix: true]
     - thelper               # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers
     - wastedassign          # wastedassign finds wasted assignment statements.
     - wrapcheck             # Checks that errors returned from external packages are wrapped
-    - depguard              # Go linter that checks if package imports are in a list of acceptable packages
 
     #
     # Recommended? (requires some work)
@@ -198,6 +199,18 @@ issues:
         - govet
       text: "shadow: declaration of \"err\" shadows declaration"
 
+    #
+    # typecheck
+    #
+
+    - linters:
+        - typecheck
+      text: "undefined: min"
+
+    - linters:
+        - typecheck
+      text: "undefined: max"
+
     #
     # errcheck
     #

Dockerfile

@@ -1,5 +1,5 @@
 # vim: set ft=dockerfile:
-ARG GOVERSION=1.20.6
+ARG GOVERSION=1.21.3
 
 FROM golang:${GOVERSION}-alpine AS build
 
@@ -52,11 +52,11 @@ FROM slim as plugins
 
 # Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
 # The files are here for reference, as users will need to mount a new version to be actually able to use notifications
-COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
 COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
 
 FROM slim as geoip

Dockerfile.debian

@@ -1,5 +1,5 @@
 # vim: set ft=dockerfile:
-ARG GOVERSION=1.20.6
+ARG GOVERSION=1.21.3
 
 FROM golang:${GOVERSION}-bookworm AS build
 
@@ -68,11 +68,11 @@ FROM slim as plugins
 
 # Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
 # The files are here for reference, as users will need to mount a new version to be actually able to use notifications
-COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
 COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
 
 FROM slim as geoip

Makefile

@@ -23,11 +23,11 @@ BUILD_RE2_WASM ?= 0
 BUILD_STATIC ?= 0
 
 # List of plugins to build
-PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*))
+PLUGINS ?= $(patsubst ./cmd/notification-%,%,$(wildcard ./cmd/notification-*))
 
 # Can be overriden, if you can deal with the consequences
 BUILD_REQUIRE_GO_MAJOR ?= 1
-BUILD_REQUIRE_GO_MINOR ?= 20
+BUILD_REQUIRE_GO_MINOR ?= 21
 
 #--------------------------------------
 
@@ -38,7 +38,7 @@ BUILD_CODENAME ?= alphaga
 
 CROWDSEC_FOLDER = ./cmd/crowdsec
 CSCLI_FOLDER = ./cmd/crowdsec-cli/
-PLUGINS_DIR = ./plugins/notifications
+PLUGINS_DIR_PREFIX = ./cmd/notification-
 
 CROWDSEC_BIN = crowdsec$(EXT)
 CSCLI_BIN = cscli$(EXT)
@@ -64,7 +64,7 @@ bool = $(if $(filter $(call lc, $1),1 yes true),1,0)
 
 #--------------------------------------
 #
-# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/ and plugins/
+# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/
 #
 
 MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
@@ -92,7 +92,6 @@ ifeq ($(PKG_CONFIG),)
 endif
 
 ifeq ($(RE2_CHECK),)
-# we could detect the platform and suggest the command to install
 RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version"
 else
 # += adds a space that we don't want
@@ -101,6 +100,7 @@ LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++'
 endif
 endif
 
+# Build static to avoid the runtime dependency on libre2.so
 ifeq ($(call bool,$(BUILD_STATIC)),1)
 BUILD_TYPE = static
 EXTLDFLAGS := -extldflags '-static'
@@ -109,10 +109,19 @@ BUILD_TYPE = dynamic
 EXTLDFLAGS :=
 endif
 
-export LD_OPTS=-ldflags "-s -w $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
+# Build with debug symbols, and disable optimizations + inlining, to use Delve
-	-trimpath -tags $(GO_TAGS)
+ifeq ($(call bool,$(DEBUG)),1)
+STRIP_SYMBOLS :=
+DISABLE_OPTIMIZATION := -gcflags "-N -l"
+else
+STRIP_SYMBOLS := -s -w
+DISABLE_OPTIMIZATION :=
+endif
 
-ifneq (,$(TEST_COVERAGE))
+export LD_OPTS=-ldflags "$(STRIP_SYMBOLS) $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
+	-trimpath -tags $(GO_TAGS) $(DISABLE_OPTIMIZATION)
+
+ifeq ($(call bool,$(TEST_COVERAGE)),1)
 LD_OPTS += -cover
 endif
 
@@ -135,19 +144,47 @@ ifneq (,$(RE2_CHECK))
 else
 	$(info Fallback to WebAssembly regexp library. To use the C++ version, make sure you have installed libre2-dev and pkg-config.)
 endif
+
+ifeq ($(call bool,$(DEBUG)),1)
+	$(info Building with debug symbols and disabled optimizations)
+endif
+
+ifeq ($(call bool,$(TEST_COVERAGE)),1)
+	$(info Test coverage collection enabled)
+endif
+
 	$(info )
 
+
 .PHONY: all
 all: clean test build
 
 .PHONY: plugins
 plugins:
 	@$(foreach plugin,$(PLUGINS), \
-		$(MAKE) -C $(PLUGINS_DIR)/$(plugin) build $(MAKE_FLAGS); \
+		$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) build $(MAKE_FLAGS); \
 	)
 
+# same as "$(MAKE) -f debian/rules clean" but without the dependency on debhelper
+.PHONY: clean-debian
+clean-debian:
+	@$(RM) -r debian/crowdsec
+	@$(RM) -r debian/crowdsec
+	@$(RM) -r debian/files
+	@$(RM) -r debian/.debhelper
+	@$(RM) -r debian/*.substvars
+	@$(RM) -r debian/*-stamp
+
+.PHONY: clean-rpm
+clean-rpm:
+	@$(RM) -r rpm/BUILD
+	@$(RM) -r rpm/BUILDROOT
+	@$(RM) -r rpm/RPMS
+	@$(RM) -r rpm/SOURCES/*.tar.gz
+	@$(RM) -r rpm/SRPMS
+
 .PHONY: clean
-clean: testclean
+clean: clean-debian clean-rpm testclean
 	@$(MAKE) -C $(CROWDSEC_FOLDER) clean $(MAKE_FLAGS)
 	@$(MAKE) -C $(CSCLI_FOLDER) clean $(MAKE_FLAGS)
 	@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)
@@ -155,7 +192,7 @@ clean: testclean
 	@$(RM) *.log $(WIN_IGNORE_ERR)
 	@$(RM) crowdsec-release.tgz $(WIN_IGNORE_ERR)
 	@$(foreach plugin,$(PLUGINS), \
-		$(MAKE) -C $(PLUGINS_DIR)/$(plugin) clean $(MAKE_FLAGS); \
+		$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) clean $(MAKE_FLAGS); \
 	)
 
 .PHONY: cscli
@@ -166,6 +203,12 @@ cscli: goversion
 crowdsec: goversion
 	@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
 
+.PHONY: notification-email
+notification-email: goversion
+	@$(MAKE) -C cmd/notification-email build $(MAKE_FLAGS)
+
+
+
 .PHONY: testclean
 testclean: bats-clean
 	@$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR)
@@ -201,37 +244,17 @@ localstack:
 localstack-stop:
 	docker-compose -f test/localstack/docker-compose.yml down
 
-# list of plugins that contain go.mod
-PLUGIN_VENDOR = $(foreach plugin,$(PLUGINS),$(shell if [ -f $(PLUGINS_DIR)/$(plugin)/go.mod ]; then echo $(PLUGINS_DIR)/$(plugin); fi))
-
 # build vendor.tgz to be distributed with the release
 .PHONY: vendor
-vendor:
+vendor: vendor-remove
-	$(foreach plugin_dir,$(PLUGIN_VENDOR), \
-		cd $(plugin_dir) >/dev/null && \
-		$(GO) mod vendor && \
-		cd - >/dev/null; \
-	)
 	$(GO) mod vendor
-	tar -czf vendor.tgz vendor $(foreach plugin_dir,$(PLUGIN_VENDOR),$(plugin_dir)/vendor)
+	tar czf vendor.tgz vendor
-
+	tar --create --auto-compress --file=$(RELDIR)-vendor.tar.xz vendor
-.PHONY: tidy
-tidy:
-	$(GO) mod tidy
-	$(foreach plugin_dir,$(PLUGIN_VENDOR), \
-		cd $(plugin_dir) >/dev/null && \
-		$(GO) mod tidy && \
-		cd - >/dev/null; \
-	)
 
 # remove vendor directories and vendor.tgz
 .PHONY: vendor-remove
 vendor-remove:
-	$(foreach plugin_dir,$(PLUGIN_VENDOR), \
+	$(RM) vendor vendor.tgz *-vendor.tar.xz
-		$(RM) $(plugin_dir)/vendor; \
-	)
-	$(RM) vendor vendor.tgz
-
 
 .PHONY: package
 package:
@@ -242,9 +265,9 @@ package:
 	@$(CP) $(CSCLI_FOLDER)/$(CSCLI_BIN) $(RELDIR)/cmd/crowdsec-cli
 
 	@$(foreach plugin,$(PLUGINS), \
-		$(MKDIR) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
+		$(MKDIR) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
-		$(CP) $(PLUGINS_DIR)/$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
+		$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
-		$(CP) $(PLUGINS_DIR)/$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR)/$(plugin)/; \
+		$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin)/; \
 	)
 
 	@$(CPR) ./config $(RELDIR)

azure-pipelines.yml

@@ -27,7 +27,7 @@ stages:
           - task: GoTool@0
             displayName: "Install Go 1.20"
             inputs:
-                version: '1.20.6'
+                version: '1.21.3'
 
           - pwsh: |
               choco install -y make

cmd/crowdsec-cli/Makefile

@@ -4,10 +4,8 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-# Go parameters
 GO = go
 GOBUILD = $(GO) build
-GOTEST = $(GO) test
 
 BINARY_NAME = cscli$(EXT)
 PREFIX ?= "/"
@@ -17,7 +15,7 @@ BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
 all: clean build
 
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
 
 .PHONY: install
 install: install-conf install-bin

cmd/crowdsec-cli/alerts.go

@@ -126,6 +126,12 @@ func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
 		}
 		csvwriter.Flush()
 	} else if csConfig.Cscli.Output == "json" {
+		if *alerts == nil {
+			// avoid returning "null" in json
+			// could be cleaner if we used slice of alerts directly
+			fmt.Println("[]")
+			return nil
+		}
 		x, _ := json.MarshalIndent(alerts, "", " ")
 		fmt.Printf("%s", string(x))
 	} else if csConfig.Cscli.Output == "human" {
@@ -208,6 +214,7 @@ func NewAlertsCmd() *cobra.Command {
 		Short:             "Manage alerts",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
+		Aliases:           []string{"alert"},
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			var err error
 			if err := csConfig.LoadAPIClient(); err != nil {

cmd/crowdsec-cli/bouncers.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"slices"
 	"strings"
 	"time"
 
@@ -12,7 +13,6 @@ import (
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 
 	middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
@@ -160,7 +160,7 @@ func runBouncersDelete(cmd *cobra.Command, args []string) error {
 func NewBouncersDeleteCmd() *cobra.Command {
 	cmdBouncersDelete := &cobra.Command{
 		Use:               "delete MyBouncerName",
-		Short:             "delete a single bouncer from the database",
+		Short:             "delete bouncer(s) from the database",
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"remove"},
 		DisableAutoGenTag: true,

cmd/crowdsec-cli/capi.go

@@ -60,16 +60,16 @@ func NewCapiRegisterCmd() *cobra.Command {
 		Short:             "Register to Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			var err error
 			capiUser, err := generateID(capiUserPrefix)
 			if err != nil {
-				log.Fatalf("unable to generate machine id: %s", err)
+				return fmt.Errorf("unable to generate machine id: %s", err)
 			}
 			password := strfmt.Password(generatePassword(passwordLength))
 			apiurl, err := url.Parse(types.CAPIBaseURL)
 			if err != nil {
-				log.Fatalf("unable to parse api url %s : %s", types.CAPIBaseURL, err)
+				return fmt.Errorf("unable to parse api url %s: %w", types.CAPIBaseURL, err)
 			}
 			_, err = apiclient.RegisterClient(&apiclient.Config{
 				MachineID:     capiUser,
@@ -80,7 +80,7 @@ func NewCapiRegisterCmd() *cobra.Command {
 			}, nil)
 
 			if err != nil {
-				log.Fatalf("api client register ('%s'): %s", types.CAPIBaseURL, err)
+				return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err)
 			}
 			log.Printf("Successfully registered to Central API (CAPI)")
 
@@ -103,12 +103,12 @@ func NewCapiRegisterCmd() *cobra.Command {
 			}
 			apiConfigDump, err := yaml.Marshal(apiCfg)
 			if err != nil {
-				log.Fatalf("unable to marshal api credentials: %s", err)
+				return fmt.Errorf("unable to marshal api credentials: %w", err)
 			}
 			if dumpFile != "" {
 				err = os.WriteFile(dumpFile, apiConfigDump, 0600)
 				if err != nil {
-					log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
+					return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
 				}
 				log.Printf("Central API credentials dumped to '%s'", dumpFile)
 			} else {
@@ -116,6 +116,8 @@ func NewCapiRegisterCmd() *cobra.Command {
 			}
 
 			log.Warning(ReloadMessage())
+
+			return nil
 		},
 	}
 	cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
@@ -133,53 +135,56 @@ func NewCapiStatusCmd() *cobra.Command {
 		Short:             "Check status with the Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if csConfig.API.Server.OnlineClient == nil {
-				log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+				return fmt.Errorf("please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 			}
 
 			if csConfig.API.Server.OnlineClient.Credentials == nil {
-				log.Fatalf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+				return fmt.Errorf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 			}
 
 			password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
+
 			apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
 			if err != nil {
-				log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
+				return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
 			}
 
-			if err := csConfig.LoadHub(); err != nil {
+			if err := require.Hub(csConfig); err != nil {
-				log.Fatal(err)
+				return err
 			}
 
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+			scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to load hub index : %s", err)
-			}
-			scenarios, err := cwhub.GetInstalledScenariosAsString()
 			if err != nil {
-				log.Fatalf("failed to get scenarios : %s", err)
+				return fmt.Errorf("failed to get scenarios: %w", err)
 			}
+
 			if len(scenarios) == 0 {
-				log.Fatalf("no scenarios installed, abort")
+				return fmt.Errorf("no scenarios installed, abort")
 			}
 
 			Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
 			if err != nil {
-				log.Fatalf("init default client: %s", err)
+				return fmt.Errorf("init default client: %w", err)
 			}
+
 			t := models.WatcherAuthRequest{
 				MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login,
 				Password:  &password,
 				Scenarios: scenarios,
 			}
+
 			log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 			log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl)
+
 			_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
 			if err != nil {
-				log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err)
+				return fmt.Errorf("failed to authenticate to Central API (CAPI): %w", err)
 			}
 			log.Infof("You can successfully interact with Central API (CAPI)")
+
+			return nil
 		},
 	}
 

cmd/crowdsec-cli/collections.go

@@ -7,6 +7,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -20,20 +21,8 @@ func NewCollectionsCmd() *cobra.Command {
 		Aliases:           []string{"collection"},
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
+			if err := require.Hub(csConfig); err != nil {
-				log.Fatal(err)
+				return err
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
-			}
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
 			}
 
 			return nil
@@ -47,6 +36,7 @@ func NewCollectionsCmd() *cobra.Command {
 	}
 
 	var ignoreError bool
+
 	var cmdCollectionsInstall = &cobra.Command{
 		Use:     "install collection",
 		Short:   "Install given collection(s)",
@@ -57,7 +47,7 @@ func NewCollectionsCmd() *cobra.Command {
 			return compAllItems(cwhub.COLLECTIONS, args, toComplete)
 		},
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, name := range args {
 				t := cwhub.GetItem(cwhub.COLLECTIONS, name)
 				if t == nil {
@@ -67,11 +57,12 @@ func NewCollectionsCmd() *cobra.Command {
 				}
 				if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
 					if !ignoreError {
-						log.Fatalf("Error while installing '%s': %s", name, err)
+						return fmt.Errorf("error while installing '%s': %w", name, err)
 					}
 					log.Errorf("Error while installing '%s': %s", name, err)
 				}
 			}
+			return nil
 		},
 	}
 	cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
@@ -89,21 +80,21 @@ func NewCollectionsCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
-				return
+				return nil
 			}
 
 			if len(args) == 0 {
-				log.Fatal("Specify at least one collection to remove or '--all' flag.")
+				return fmt.Errorf("specify at least one collection to remove or '--all'")
 			}
 
 			for _, name := range args {
 				if !forceAction {
 					item := cwhub.GetItem(cwhub.COLLECTIONS, name)
 					if item == nil {
-						log.Fatalf("unable to retrieve: %s\n", name)
+						return fmt.Errorf("unable to retrieve: %s", name)
 					}
 					if len(item.BelongsToCollections) > 0 {
 						log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
@@ -113,6 +104,7 @@ func NewCollectionsCmd() *cobra.Command {
 				}
 				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
 			}
+			return nil
 		},
 	}
 	cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
@@ -129,17 +121,18 @@ func NewCollectionsCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
 			} else {
 				if len(args) == 0 {
-					log.Fatalf("no target collection to upgrade")
+					return fmt.Errorf("specify at least one collection to upgrade or '--all'")
 				}
 				for _, name := range args {
 					cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
 				}
 			}
+			return nil
 		},
 	}
 	cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")

cmd/crowdsec-cli/config_backup.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -9,9 +10,76 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
-/* Backup crowdsec configurations to directory <dirPath> :
+func backupHub(dirPath string) error {
+	var err error
+	var itemDirectory string
+	var upstreamParsers []string
+
+	for _, itemType := range cwhub.ItemTypes {
+		clog := log.WithFields(log.Fields{
+			"type": itemType,
+		})
+		itemMap := cwhub.GetItemMap(itemType)
+		if itemMap == nil {
+			clog.Infof("No %s to backup.", itemType)
+			continue
+		}
+		itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
+		if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
+			return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
+		}
+		upstreamParsers = []string{}
+		for k, v := range itemMap {
+			clog = clog.WithFields(log.Fields{
+				"file": v.Name,
+			})
+			if !v.Installed { //only backup installed ones
+				clog.Debugf("[%s] : not installed", k)
+				continue
+			}
+
+			//for the local/tainted ones, we backup the full file
+			if v.Tainted || v.Local || !v.UpToDate {
+				//we need to backup stages for parsers
+				if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
+					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
+					if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
+						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
+					}
+				}
+				clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
+				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
+				if err = CopyFile(v.LocalPath, tfile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
+				}
+				clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
+				continue
+			}
+			clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
+			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
+			upstreamParsers = append(upstreamParsers, v.Name)
+		}
+		//write the upstream items
+		upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
+		upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
+		if err != nil {
+			return fmt.Errorf("failed marshaling upstream parsers : %s", err)
+		}
+		err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
+		if err != nil {
+			return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
+		}
+		clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
+	}
+
+	return nil
+}
+
+/*
+	Backup crowdsec configurations to directory <dirPath>:
 
 - Main config (config.yaml)
 - Profiles config (profiles.yaml)
@@ -19,6 +87,7 @@ import (
 - Backup of API credentials (local API and online API)
 - List of scenarios, parsers, postoverflows and collections that are up-to-date
 - Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+- Acquisition files (acquis.yaml, acquis.d/*.yaml)
 */
 func backupConfigToDirectory(dirPath string) error {
 	var err error
@@ -120,7 +189,7 @@ func backupConfigToDirectory(dirPath string) error {
 		log.Infof("Saved profiles to %s", backupProfiles)
 	}
 
-	if err = BackupHub(dirPath); err != nil {
+	if err = backupHub(dirPath); err != nil {
 		return fmt.Errorf("failed to backup hub config: %s", err)
 	}
 
@@ -128,15 +197,10 @@ func backupConfigToDirectory(dirPath string) error {
 }
 
 func runConfigBackup(cmd *cobra.Command, args []string) error {
-	if err := csConfig.LoadHub(); err != nil {
+	if err := require.Hub(csConfig); err != nil {
 		return err
 	}
 
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		log.Info("Run 'sudo cscli hub update' to get the hub index")
-		return fmt.Errorf("failed to get Hub index: %w", err)
-	}
-
 	if err := backupConfigToDirectory(args[0]); err != nil {
 		return fmt.Errorf("failed to backup config: %w", err)
 	}

cmd/crowdsec-cli/config_feature_flags.go

@@ -2,10 +2,12 @@ package main
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"github.com/fatih/color"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 )
 
@@ -87,7 +89,14 @@ func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
 
 	fmt.Println("To enable a feature you can: ")
 	fmt.Println("  - set the environment variable CROWDSEC_FEATURE_<uppercase_feature_name> to true")
-	fmt.Printf("  - add the line '- <feature_name>' to the file %s/feature.yaml\n", csConfig.ConfigPaths.ConfigDir)
+
+	featurePath, err := filepath.Abs(csconfig.GetFeatureFilePath(ConfigFilePath))
+	if err != nil {
+		// we already read the file, shouldn't happen
+		return err
+	}
+
+	fmt.Printf("  - add the line '- <feature_name>' to the file %s\n", featurePath)
 	fmt.Println()
 
 	if len(enabled) == 0 && len(disabled) == 0 {

cmd/crowdsec-cli/config_restore.go

@@ -11,6 +11,7 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
@@ -20,7 +21,126 @@ type OldAPICfg struct {
 	Password  string `json:"password"`
 }
 
-/* Restore crowdsec configurations to directory <dirPath> :
+// it's a rip of the cli version, but in silent-mode
+func silentInstallItem(name string, obtype string) (string, error) {
+	var item = cwhub.GetItem(obtype, name)
+	if item == nil {
+		return "", fmt.Errorf("error retrieving item")
+	}
+	if downloadOnly && item.Downloaded && item.UpToDate {
+		return fmt.Sprintf("%s is already downloaded and up-to-date", item.Name), nil
+	}
+	err := cwhub.DownloadLatest(csConfig.Hub, item, forceAction, false)
+	if err != nil {
+		return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
+	}
+	if err := cwhub.AddItem(obtype, *item); err != nil {
+		return "", err
+	}
+
+	if downloadOnly {
+		return fmt.Sprintf("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath), nil
+	}
+	err = cwhub.EnableItem(csConfig.Hub, item)
+	if err != nil {
+		return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
+	}
+	if err := cwhub.AddItem(obtype, *item); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("Enabled %s", item.Name), nil
+}
+
+func restoreHub(dirPath string) error {
+	var err error
+
+	if err := csConfig.LoadHub(); err != nil {
+		return err
+	}
+
+	cwhub.SetHubBranch()
+
+	for _, itype := range cwhub.ItemTypes {
+		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
+		if _, err = os.Stat(itemDirectory); err != nil {
+			log.Infof("no %s in backup", itype)
+			continue
+		}
+		/*restore the upstream items*/
+		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
+		file, err := os.ReadFile(upstreamListFN)
+		if err != nil {
+			return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
+		}
+		var upstreamList []string
+		err = json.Unmarshal(file, &upstreamList)
+		if err != nil {
+			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
+		}
+		for _, toinstall := range upstreamList {
+			label, err := silentInstallItem(toinstall, itype)
+			if err != nil {
+				log.Errorf("Error while installing %s : %s", toinstall, err)
+			} else if label != "" {
+				log.Infof("Installed %s : %s", toinstall, label)
+			} else {
+				log.Printf("Installed %s : ok", toinstall)
+			}
+		}
+
+		/*restore the local and tainted items*/
+		files, err := os.ReadDir(itemDirectory)
+		if err != nil {
+			return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
+		}
+		for _, file := range files {
+			//this was the upstream data
+			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
+				continue
+			}
+			if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
+				//we expect a stage here
+				if !file.IsDir() {
+					continue
+				}
+				stage := file.Name()
+				stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
+				log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
+				if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
+					return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
+				}
+				/*find items*/
+				ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
+				if err != nil {
+					return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
+				}
+				//finally copy item
+				for _, tfile := range ifiles {
+					log.Infof("Going to restore local/tainted [%s]", tfile.Name())
+					sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
+					destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
+					if err = CopyFile(sourceFile, destinationFile); err != nil {
+						return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
+					}
+					log.Infof("restored %s to %s", sourceFile, destinationFile)
+				}
+			} else {
+				log.Infof("Going to restore local/tainted [%s]", file.Name())
+				sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
+				destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
+				if err = CopyFile(sourceFile, destinationFile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
+				}
+				log.Infof("restored %s to %s", sourceFile, destinationFile)
+			}
+
+		}
+	}
+	return nil
+}
+
+/*
+	Restore crowdsec configurations to directory <dirPath>:
 
 - Main config (config.yaml)
 - Profiles config (profiles.yaml)
@@ -28,6 +148,7 @@ type OldAPICfg struct {
 - Backup of API credentials (local API and online API)
 - List of scenarios, parsers, postoverflows and collections that are up-to-date
 - Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+- Acquisition files (acquis.yaml, acquis.d/*.yaml)
 */
 func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 	var err error
@@ -111,7 +232,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 
 	/*if there is a acquisition dir, restore its content*/
 	if csConfig.Crowdsec.AcquisitionDirPath != "" {
-		if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
+		if err = os.MkdirAll(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
 			return fmt.Errorf("error while creating %s : %s", csConfig.Crowdsec.AcquisitionDirPath, err)
 		}
 	}
@@ -166,7 +287,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 		}
 	}
 
-	if err = RestoreHub(dirPath); err != nil {
+	if err = restoreHub(dirPath); err != nil {
 		return fmt.Errorf("failed to restore hub config : %s", err)
 	}
 
@@ -181,15 +302,10 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if err := csConfig.LoadHub(); err != nil {
+	if err := require.Hub(csConfig); err != nil {
 		return err
 	}
 
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		log.Info("Run 'sudo cscli hub update' to get the hub index")
-		return fmt.Errorf("failed to get Hub index: %w", err)
-	}
-
 	if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
 		return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
 	}

cmd/crowdsec-cli/config_show.go

@@ -57,7 +57,6 @@ func showConfigKey(key string) error {
 var configShowTemplate = `Global:
 
 {{- if .ConfigPaths }}
-   - Configuration Folder   : {{.ConfigPaths.ConfigDir}}
    - Configuration Folder   : {{.ConfigPaths.ConfigDir}}
    - Data Folder            : {{.ConfigPaths.DataDir}}
    - Hub Folder             : {{.ConfigPaths.HubDir}}
@@ -164,6 +163,12 @@ Central API:
       - User                : {{.DbConfig.User}}
       - DB Name             : {{.DbConfig.DbName}}
 {{- end }}
+{{- if .DbConfig.MaxOpenConns }}
+      - Max Open Conns      : {{.DbConfig.MaxOpenConns}}
+{{- end }}
+{{- if ne .DbConfig.DecisionBulkSize 0 }}
+      - Decision Bulk Size  : {{.DbConfig.DecisionBulkSize}}
+{{- end }}
 {{- if .DbConfig.Flush }}
 {{- if .DbConfig.Flush.MaxAge }}
       - Flush age           : {{.DbConfig.Flush.MaxAge}}

cmd/crowdsec-cli/console.go

@@ -39,7 +39,7 @@ func NewConsoleCmd() *cobra.Command {
 			if err := require.CAPI(csConfig); err != nil {
 				return err
 			}
-			if err := require.Enrolled(csConfig); err != nil {
+			if err := require.CAPIRegistered(csConfig); err != nil {
 				return err
 			}
 			return nil
@@ -64,25 +64,20 @@ After running this command your will need to validate the enrollment in the weba
 `,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
 			apiURL, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
 			if err != nil {
-				log.Fatalf("Could not parse CAPI URL : %s", err)
+				return fmt.Errorf("could not parse CAPI URL: %s", err)
 			}
 
-			if err := csConfig.LoadHub(); err != nil {
+			if err := require.Hub(csConfig); err != nil {
-				log.Fatal(err)
+				return err
 			}
 
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+			scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
-				log.Fatalf("Failed to load hub index : %s", err)
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-			}
-
-			scenarios, err := cwhub.GetInstalledScenariosAsString()
 			if err != nil {
-				log.Fatalf("failed to get scenarios : %s", err)
+				return fmt.Errorf("failed to get installed scenarios: %s", err)
 			}
 
 			if len(scenarios) == 0 {
@@ -99,20 +94,21 @@ After running this command your will need to validate the enrollment in the weba
 			})
 			resp, err := c.Auth.EnrollWatcher(context.Background(), args[0], name, tags, overwrite)
 			if err != nil {
-				log.Fatalf("Could not enroll instance: %s", err)
+				return fmt.Errorf("could not enroll instance: %s", err)
 			}
 			if resp.Response.StatusCode == 200 && !overwrite {
 				log.Warning("Instance already enrolled. You can use '--overwrite' to force enroll")
-				return
+				return nil
 			}
 
-			SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
+			if err := SetConsoleOpts([]string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}, true); err != nil {
-			if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
+				return err
-				log.Fatalf("failed writing console config : %s", err)
 			}
-			log.Infof("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
+
-			log.Infof("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
+			log.Info("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
-			log.Infof("Please restart crowdsec after accepting the enrollment.")
+			log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
+			log.Info("Please restart crowdsec after accepting the enrollment.")
+			return nil
 		},
 	}
 	cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console")
@@ -130,21 +126,23 @@ After running this command your will need to validate the enrollment in the weba
 Enable given information push to the central API. Allows to empower the console`,
 		ValidArgs:         csconfig.CONSOLE_CONFIGS,
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if enableAll {
-				SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
+				if err := SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true); err != nil {
+					return err
+				}
 				log.Infof("All features have been enabled successfully")
 			} else {
 				if len(args) == 0 {
-					log.Fatalf("You must specify at least one feature to enable")
+					return fmt.Errorf("you must specify at least one feature to enable")
+				}
+				if err := SetConsoleOpts(args, true); err != nil {
+					return err
 				}
-				SetConsoleOpts(args, true)
 				log.Infof("%v have been enabled", args)
 			}
-			if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
-				log.Fatalf("failed writing console config : %s", err)
-			}
 			log.Infof(ReloadMessage())
+			return nil
 		},
 	}
 	cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
@@ -157,49 +155,55 @@ Enable given information push to the central API. Allows to empower the console`
 		Long: `
 Disable given information push to the central API.`,
 		ValidArgs:         csconfig.CONSOLE_CONFIGS,
-		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if disableAll {
-				SetConsoleOpts(csconfig.CONSOLE_CONFIGS, false)
+				if err := SetConsoleOpts(csconfig.CONSOLE_CONFIGS, false); err != nil {
-			} else {
+					return err
-				SetConsoleOpts(args, false)
 				}
-
-			if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
-				log.Fatalf("failed writing console config : %s", err)
-			}
-			if disableAll {
 				log.Infof("All features have been disabled")
 			} else {
+				if err := SetConsoleOpts(args, false); err != nil {
+					return err
+				}
 				log.Infof("%v have been disabled", args)
 			}
+
 			log.Infof(ReloadMessage())
+			return nil
 		},
 	}
 	cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
 	cmdConsole.AddCommand(cmdDisable)
 
 	cmdConsoleStatus := &cobra.Command{
-		Use:               "status [option]",
+		Use:               "status",
-		Short:             "Shows status of one or all console options",
+		Short:             "Shows status of the console options",
 		Example:           `sudo cscli console status`,
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			switch csConfig.Cscli.Output {
 			case "human":
 				cmdConsoleStatusTable(color.Output, *csConfig)
 			case "json":
-				data, err := json.MarshalIndent(csConfig.API.Server.ConsoleConfig, "", "  ")
+				c := csConfig.API.Server.ConsoleConfig
-				if err != nil {
+				out := map[string](*bool){
-					log.Fatalf("failed to marshal configuration: %s", err)
+					csconfig.SEND_MANUAL_SCENARIOS: c.ShareManualDecisions,
+					csconfig.SEND_CUSTOM_SCENARIOS: c.ShareCustomScenarios,
+					csconfig.SEND_TAINTED_SCENARIOS: c.ShareTaintedScenarios,
+					csconfig.SEND_CONTEXT: c.ShareContext,
+					csconfig.CONSOLE_MANAGEMENT: c.ConsoleManagement,
 				}
-				fmt.Printf("%s\n", string(data))
+				data, err := json.MarshalIndent(out, "", "  ")
+				if err != nil {
+					return fmt.Errorf("failed to marshal configuration: %s", err)
+				}
+				fmt.Println(string(data))
 			case "raw":
 				csvwriter := csv.NewWriter(os.Stdout)
 				err := csvwriter.Write([]string{"option", "enabled"})
 				if err != nil {
-					log.Fatal(err)
+					return err
 				}
 
 				rows := [][]string{
@@ -212,11 +216,12 @@ Disable given information push to the central API.`,
 				for _, row := range rows {
 					err = csvwriter.Write(row)
 					if err != nil {
-						log.Fatal(err)
+						return err
 					}
 				}
 				csvwriter.Flush()
 			}
+			return nil
 		},
 	}
 	cmdConsole.AddCommand(cmdConsoleStatus)
@@ -224,7 +229,25 @@ Disable given information push to the central API.`,
 	return cmdConsole
 }
 
-func SetConsoleOpts(args []string, wanted bool) {
+func dumpConsoleConfig(c *csconfig.LocalApiServerCfg) error {
+	out, err := yaml.Marshal(c.ConsoleConfig)
+	if err != nil {
+		return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleConfigPath, err)
+	}
+
+	if c.ConsoleConfigPath == "" {
+		c.ConsoleConfigPath = csconfig.DefaultConsoleConfigFilePath
+		log.Debugf("Empty console_path, defaulting to %s", c.ConsoleConfigPath)
+	}
+
+	if err := os.WriteFile(c.ConsoleConfigPath, out, 0600); err != nil {
+		return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleConfigPath, err)
+	}
+
+	return nil
+}
+
+func SetConsoleOpts(args []string, wanted bool) error {
 	for _, arg := range args {
 		switch arg {
 		case csconfig.CONSOLE_MANAGEMENT:
@@ -255,12 +278,12 @@ func SetConsoleOpts(args []string, wanted bool) {
 				if changed {
 					fileContent, err := yaml.Marshal(csConfig.API.Server.OnlineClient.Credentials)
 					if err != nil {
-						log.Fatalf("Cannot marshal credentials: %s", err)
+						return fmt.Errorf("cannot marshal credentials: %s", err)
 					}
 					log.Infof("Updating credentials file: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 					err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0600)
 					if err != nil {
-						log.Fatalf("Cannot write credentials file: %s", err)
+						return fmt.Errorf("cannot write credentials file: %s", err)
 					}
 				}
 			}
@@ -317,8 +340,13 @@ func SetConsoleOpts(args []string, wanted bool) {
 				csConfig.API.Server.ConsoleConfig.ShareContext = ptr.Of(wanted)
 			}
 		default:
-			log.Fatalf("unknown flag %s", arg)
+			return fmt.Errorf("unknown flag %s", arg)
 		}
 	}
 
+	if err := dumpConsoleConfig(csConfig.API.Server); err != nil {
+		return fmt.Errorf("failed writing console config: %s", err)
+	}
+
+	return nil
 }

cmd/crowdsec-cli/decisions.go

@@ -81,6 +81,12 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
 		}
 		csvwriter.Flush()
 	} else if csConfig.Cscli.Output == "json" {
+		if *alerts == nil {
+			// avoid returning "null" in `json"
+			// could be cleaner if we used slice of alerts directly
+			fmt.Println("[]")
+			return nil
+		}
 		x, _ := json.MarshalIndent(alerts, "", " ")
 		fmt.Printf("%s", string(x))
 	} else if csConfig.Cscli.Output == "human" {

cmd/crowdsec-cli/decisions_import.go

@@ -188,7 +188,9 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error  {
 		}
 	}
 
-	alerts := models.AddAlertsRequest{}
+	if len(decisions) > 1000 {
+		log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
+	}
 
 	for _, chunk := range slicetools.Chunks(decisions, batchSize) {
 		log.Debugf("Processing chunk of %d decisions", len(chunk))
@@ -212,17 +214,12 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error  {
 			ScenarioVersion: ptr.Of(""),
 			Decisions:       chunk,
 		}
-		alerts = append(alerts, &importAlert)
-	}
 
-	if len(decisions) > 1000 {
+		_, _, err = Client.Alerts.Add(context.Background(), models.AddAlertsRequest{&importAlert})
-		log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
-	}
-
-	_, _, err = Client.Alerts.Add(context.Background(), alerts)
 		if err != nil {
 			return err
 		}
+	}
 
 	log.Infof("Imported %d decisions", len(decisions))
 	return nil

cmd/crowdsec-cli/explain.go

@@ -12,9 +12,22 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
+func GetLineCountForFile(filepath string) (int, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	lc := 0
+	fs := bufio.NewScanner(f)
+	for fs.Scan() {
+		lc++
+	}
+	return lc, nil
+}
+
 func runExplain(cmd *cobra.Command, args []string) error {
 	flags := cmd.Flags()
 
@@ -61,6 +74,11 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	labels, err := flags.GetString("labels")
+	if err != nil {
+		return err
+	}
+
 	fileInfo, _ := os.Stdin.Stat()
 
 	if logType == "" || (logLine == "" && logFile == "" && dsn == "") {
@@ -123,9 +141,12 @@ func runExplain(cmd *cobra.Command, args []string) error {
 			return fmt.Errorf("unable to get absolute path of '%s', exiting", logFile)
 		}
 		dsn = fmt.Sprintf("file://%s", absolutePath)
-		lineCount := types.GetLineCountForFile(absolutePath)
+		lineCount, err := GetLineCountForFile(absolutePath)
+		if err != nil {
+			return err
+		}
 		if lineCount > 100 {
-			log.Warnf("log file contains %d lines. This may take lot of resources.", lineCount)
+			log.Warnf("The log file contains %d lines. This may take a lot of resources.", lineCount)
 		}
 	}
 
@@ -134,6 +155,10 @@ func runExplain(cmd *cobra.Command, args []string) error {
 	}
 
 	cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", dir, "-no-api"}
+	if labels != "" {
+		log.Debugf("adding labels %s", labels)
+		cmdArgs = append(cmdArgs, "-label", labels)
+	}
 	crowdsecCmd := exec.Command(crowdsec, cmdArgs...)
 	output, err := crowdsecCmd.CombinedOutput()
 	if err != nil {
@@ -193,6 +218,7 @@ tail -n 5 myfile.log | cscli explain --type nginx -f -
 	flags.StringP("dsn", "d", "", "DSN to test")
 	flags.StringP("log", "l", "", "Log line to test")
 	flags.StringP("type", "t", "", "Type of the acquisition to test")
+	flags.String("labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
 	flags.BoolP("verbose", "v", false, "Display individual changes")
 	flags.Bool("failures", false, "Only show failed lines")
 	flags.Bool("only-successful-parsers", false, "Only show successful parsers")

cmd/crowdsec-cli/hub.go

@@ -8,6 +8,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -50,17 +51,13 @@ func NewHubListCmd() *cobra.Command {
 		Short:             "List installed configs",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := require.Hub(csConfig); err != nil {
+				return err
+			}
 
-			if err := csConfig.LoadHub(); err != nil {
+			// use LocalSync to get warnings about tainted / outdated items
-				log.Fatal(err)
+			warn, _ := cwhub.LocalSync(csConfig.Hub)
-			}
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			//use LocalSync to get warnings about tainted / outdated items
-			_, warn := cwhub.LocalSync(csConfig.Hub)
 			for _, v := range warn {
 				log.Info(v)
 			}
@@ -68,6 +65,8 @@ func NewHubListCmd() *cobra.Command {
 			ListItems(color.Output, []string{
 				cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
 			}, args, true, false, all)
+
+			return nil
 		},
 	}
 	cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
@@ -89,31 +88,31 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := cwhub.SetHubBranch(); err != nil {
+			cwhub.SetHubBranch()
-				return fmt.Errorf("error while setting hub branch: %s", err)
+
-			}
 			return nil
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
+				return err
 			}
 			if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-				if errors.Is(err, cwhub.ErrIndexNotFound) {
+				if !errors.Is(err, cwhub.ErrIndexNotFound) {
+					return fmt.Errorf("failed to get Hub index : %w", err)
+				}
 				log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
 				cwhub.HubBranch = "master"
 				if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-						log.Fatalf("Failed to get Hub index after retry : %v", err)
+					return fmt.Errorf("failed to get Hub index after retry: %w", err)
-					}
-				} else {
-					log.Fatalf("Failed to get Hub index : %v", err)
 				}
 			}
-			//use LocalSync to get warnings about tainted / outdated items
+			// use LocalSync to get warnings about tainted / outdated items
-			_, warn := cwhub.LocalSync(csConfig.Hub)
+			warn, _ := cwhub.LocalSync(csConfig.Hub)
 			for _, v := range warn {
 				log.Info(v)
 			}
+
+			return nil
 		},
 	}
 
@@ -134,18 +133,13 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := cwhub.SetHubBranch(); err != nil {
+			cwhub.SetHubBranch()
-				return fmt.Errorf("error while setting hub branch: %s", err)
+
-			}
 			return nil
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
+			if err := require.Hub(csConfig); err != nil {
-				log.Fatal(err)
+				return err
-			}
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
 			}
 
 			log.Infof("Upgrading collections")
@@ -156,6 +150,8 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
 			cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
 			log.Infof("Upgrading postoverflows")
 			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
+
+			return nil
 		},
 	}
 	cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")

cmd/crowdsec-cli/lapi.go

@@ -5,17 +5,18 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"slices"
 	"sort"
 	"strings"
 
 	"github.com/go-openapi/strfmt"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/go-cs-lib/version"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@@ -36,15 +37,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
 	}
-	if err := csConfig.LoadHub(); err != nil {
+
+	if err := require.Hub(csConfig); err != nil {
 		log.Fatal(err)
 	}
 
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
-		log.Info("Run 'sudo cscli hub update' to get the hub index")
-		log.Fatalf("Failed to load hub index : %s", err)
-	}
-	scenarios, err := cwhub.GetInstalledScenariosAsString()
 	if err != nil {
 		log.Fatalf("failed to get scenarios : %s", err)
 	}
@@ -216,6 +214,29 @@ func NewLapiCmd() *cobra.Command {
 	return cmdLapi
 }
 
+func AddContext(key string, values []string) error {
+	if err := alertcontext.ValidateContextExpr(key, values); err != nil {
+		return fmt.Errorf("invalid context configuration :%s", err)
+	}
+	if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok {
+		csConfig.Crowdsec.ContextToSend[key] = make([]string, 0)
+		log.Infof("key '%s' added", key)
+	}
+	data := csConfig.Crowdsec.ContextToSend[key]
+	for _, val := range values {
+		if !slices.Contains(data, val) {
+			log.Infof("value '%s' added to key '%s'", val, key)
+			data = append(data, val)
+		}
+		csConfig.Crowdsec.ContextToSend[key] = data
+	}
+	if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func NewLapiContextCmd() *cobra.Command {
 	cmdContext := &cobra.Command{
 		Use:               "context [command]",
@@ -246,32 +267,29 @@ func NewLapiContextCmd() *cobra.Command {
 		Short: "Add context to send with alerts. You must specify the output key with the expr value you want",
 		Example: `cscli lapi context add --key source_ip --value evt.Meta.source_ip
 cscli lapi context add --key file_source --value evt.Line.Src
+cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user 
 		`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := alertcontext.ValidateContextExpr(keyToAdd, valuesToAdd); err != nil {
+			if keyToAdd != "" {
-				log.Fatalf("invalid context configuration :%s", err)
+				if err := AddContext(keyToAdd, valuesToAdd); err != nil {
-			}
-			if _, ok := csConfig.Crowdsec.ContextToSend[keyToAdd]; !ok {
-				csConfig.Crowdsec.ContextToSend[keyToAdd] = make([]string, 0)
-				log.Infof("key '%s' added", keyToAdd)
-			}
-			data := csConfig.Crowdsec.ContextToSend[keyToAdd]
-			for _, val := range valuesToAdd {
-				if !slices.Contains(data, val) {
-					log.Infof("value '%s' added to key '%s'", val, keyToAdd)
-					data = append(data, val)
-				}
-				csConfig.Crowdsec.ContextToSend[keyToAdd] = data
-			}
-			if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
 					log.Fatalf(err.Error())
 				}
+				return
+			}
+
+			for _, v := range valuesToAdd {
+				keySlice := strings.Split(v, ".")
+				key := keySlice[len(keySlice)-1]
+				value := []string{v}
+				if err := AddContext(key, value); err != nil {
+					log.Fatalf(err.Error())
+				}
+			}
 		},
 	}
 	cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
 	cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
-	cmdContextAdd.MarkFlagRequired("key")
 	cmdContextAdd.MarkFlagRequired("value")
 	cmdContext.AddCommand(cmdContextAdd)
 

cmd/crowdsec-cli/machines.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"math/big"
 	"os"
+	"slices"
 	"strings"
 	"time"
 
@@ -17,7 +18,6 @@ import (
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/machineid"

cmd/crowdsec-cli/main.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"github.com/fatih/color"
@@ -11,7 +12,6 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
-	"golang.org/x/exp/slices"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@@ -53,11 +53,11 @@ func initConfig() {
 	}
 
 	if !slices.Contains(NoNeedConfig, os.Args[1]) {
+		log.Debugf("Using %s as configuration file", ConfigFilePath)
 		csConfig, mergedConfig, err = csconfig.NewConfig(ConfigFilePath, false, false, true)
 		if err != nil {
 			log.Fatal(err)
 		}
-		log.Debugf("Using %s as configuration file", ConfigFilePath)
 		if err := csConfig.LoadCSCLI(); err != nil {
 			log.Fatal(err)
 		}
@@ -188,7 +188,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	/*usage*/
 	var cmdVersion = &cobra.Command{
 		Use:               "version",
-		Short:             "Display version and exit.",
+		Short:             "Display version",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {

cmd/crowdsec-cli/parsers.go

@@ -7,10 +7,10 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-
 func NewParsersCmd() *cobra.Command {
 	var cmdParsers = &cobra.Command{
 		Use:   "parsers [action] [config]",
@@ -25,21 +25,10 @@ cscli parsers remove crowdsecurity/sshd-logs
 		Aliases:           []string{"parser"},
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
+			if err := require.Hub(csConfig); err != nil {
-				log.Fatal(err)
+				return err
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
-			}
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
 			return nil
 		},
 		PersistentPostRun: func(cmd *cobra.Command, args []string) {
@@ -59,7 +48,6 @@ cscli parsers remove crowdsecurity/sshd-logs
 	return cmdParsers
 }
 
-
 func NewParsersInstallCmd() *cobra.Command {
 	var ignoreError bool
 
@@ -73,7 +61,7 @@ func NewParsersInstallCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compAllItems(cwhub.PARSERS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, name := range args {
 				t := cwhub.GetItem(cwhub.PARSERS, name)
 				if t == nil {
@@ -82,15 +70,16 @@ func NewParsersInstallCmd() *cobra.Command {
 					continue
 				}
 				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
-					if ignoreError {
+					if !ignoreError {
+						return fmt.Errorf("error while installing '%s': %w", name, err)
+					}
 					log.Errorf("Error while installing '%s': %s", name, err)
-					} else {
-						log.Fatalf("Error while installing '%s': %s", name, err)
-					}
 				}
 			}
+			return nil
 		},
 	}
+
 	cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
 	cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
 	cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
@@ -98,33 +87,35 @@ func NewParsersInstallCmd() *cobra.Command {
 	return cmdParsersInstall
 }
 
-
 func NewParsersRemoveCmd() *cobra.Command {
-	var cmdParsersRemove = &cobra.Command{
+	cmdParsersRemove := &cobra.Command{
 		Use:               "remove [config]",
 		Short:             "Remove given parser(s)",
 		Long:              `Remove given parse(s) from hub`,
-		Aliases:           []string{"delete"},
 		Example:           `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
+		Aliases:           []string{"delete"},
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
-				return
+				return nil
 			}
 
 			if len(args) == 0 {
-				log.Fatalf("Specify at least one parser to remove or '--all' flag.")
+				return fmt.Errorf("specify at least one parser to remove or '--all'")
 			}
 
 			for _, name := range args {
 				cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
 			}
+
+			return nil
 		},
 	}
+
 	cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
 	cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
 	cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
@@ -132,9 +123,8 @@ func NewParsersRemoveCmd() *cobra.Command {
 	return cmdParsersRemove
 }
 
-
 func NewParsersUpgradeCmd() *cobra.Command {
-	var cmdParsersUpgrade = &cobra.Command{
+	cmdParsersUpgrade := &cobra.Command{
 		Use:               "upgrade [config]",
 		Short:             "Upgrade given parser(s)",
 		Long:              `Fetch and upgrade given parser(s) from hub`,
@@ -143,26 +133,27 @@ func NewParsersUpgradeCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
 			} else {
 				if len(args) == 0 {
-					log.Fatalf("no target parser to upgrade")
+					return fmt.Errorf("specify at least one parser to upgrade or '--all'")
 				}
 				for _, name := range args {
 					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
 				}
 			}
+			return nil
 		},
 	}
+
 	cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
 	cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
 
 	return cmdParsersUpgrade
 }
 
-
 func NewParsersInspectCmd() *cobra.Command {
 	var cmdParsersInspect = &cobra.Command{
 		Use:               "inspect [name]",
@@ -178,12 +169,12 @@ func NewParsersInspectCmd() *cobra.Command {
 			InspectItem(args[0], cwhub.PARSERS)
 		},
 	}
+
 	cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
 
 	return cmdParsersInspect
 }
 
-
 func NewParsersListCmd() *cobra.Command {
 	var cmdParsersList = &cobra.Command{
 		Use:   "list [name]",
@@ -196,6 +187,7 @@ cscli parser list crowdsecurity/xxx`,
 			ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
 		},
 	}
+
 	cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
 
 	return cmdParsersList

cmd/crowdsec-cli/postoverflows.go

@@ -7,9 +7,46 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
+func NewPostOverflowsCmd() *cobra.Command {
+	cmdPostOverflows := &cobra.Command{
+		Use:   "postoverflows [action] [config]",
+		Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
+		Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
+		cscli postoverflows inspect crowdsecurity/cdn-whitelist
+		cscli postoverflows upgrade crowdsecurity/cdn-whitelist
+		cscli postoverflows list
+		cscli postoverflows remove crowdsecurity/cdn-whitelist`,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{"postoverflow"},
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := require.Hub(csConfig); err != nil {
+				return err
+			}
+
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof(ReloadMessage())
+		},
+	}
+
+	cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
+
+	return cmdPostOverflows
+}
+
 func NewPostOverflowsInstallCmd() *cobra.Command {
 	var ignoreError bool
 
@@ -23,7 +60,7 @@ func NewPostOverflowsInstallCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, name := range args {
 				t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
 				if t == nil {
@@ -32,13 +69,13 @@ func NewPostOverflowsInstallCmd() *cobra.Command {
 					continue
 				}
 				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
-					if ignoreError {
+					if !ignoreError {
+						return fmt.Errorf("error while installing '%s': %w", name, err)
+					}
 					log.Errorf("Error while installing '%s': %s", name, err)
-					} else {
-						log.Fatalf("Error while installing '%s': %s", name, err)
-					}
 				}
 			}
+			return nil
 		},
 	}
 
@@ -55,24 +92,26 @@ func NewPostOverflowsRemoveCmd() *cobra.Command {
 		Short:             "Remove given postoverflow(s)",
 		Long:              `remove given postoverflow(s)`,
 		Example:           `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
 		Aliases:           []string{"delete"},
+		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
-				return
+				return nil
 			}
 
 			if len(args) == 0 {
-				log.Fatalf("Specify at least one postoverflow to remove or '--all' flag.")
+				return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
 			}
 
 			for _, name := range args {
 				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
 			}
+
+			return nil
 		},
 	}
 
@@ -93,17 +132,18 @@ func NewPostOverflowsUpgradeCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
 			} else {
 				if len(args) == 0 {
-					log.Fatalf("no target postoverflow to upgrade")
+					return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
 				}
 				for _, name := range args {
 					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
 				}
 			}
+			return nil
 		},
 	}
 
@@ -120,10 +160,10 @@ func NewPostOverflowsInspectCmd() *cobra.Command {
 		Long:              `Inspect given postoverflow`,
 		Example:           `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
 		DisableAutoGenTag: true,
+		Args:              cobra.MinimumNArgs(1),
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
 		},
-		Args: cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			InspectItem(args[0], cwhub.PARSERS_OVFLW)
 		},
@@ -149,52 +189,3 @@ cscli postoverflows list crowdsecurity/xxx`,
 
 	return cmdPostOverflowsList
 }
-
-
-
-func NewPostOverflowsCmd() *cobra.Command {
-	cmdPostOverflows := &cobra.Command{
-		Use:   "postoverflows [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
-		Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
-		cscli postoverflows inspect crowdsecurity/cdn-whitelist
-		cscli postoverflows upgrade crowdsecurity/cdn-whitelist
-		cscli postoverflows list
-		cscli postoverflows remove crowdsecurity/cdn-whitelist`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"postoverflow"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
-			}
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
-
-	return cmdPostOverflows
-}

cmd/crowdsec-cli/require/require.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
 func LAPI(c *csconfig.Config) error {
@@ -32,7 +33,7 @@ func PAPI(c *csconfig.Config) error {
 	return nil
 }
 
-func Enrolled(c *csconfig.Config) error {
+func CAPIRegistered(c *csconfig.Config) error {
 	if c.API.Server.OnlineClient.Credentials == nil {
 		return fmt.Errorf("the Central API (CAPI) must be configured with 'cscli capi register'")
 	}
@@ -63,3 +64,20 @@ func Notifications(c *csconfig.Config) error {
 	return nil
 }
 
+func Hub (c *csconfig.Config) error {
+	if err := c.LoadHub(); err != nil {
+		return err
+	}
+
+	if c.Hub == nil {
+		return fmt.Errorf("you must configure cli before interacting with hub")
+	}
+
+	cwhub.SetHubBranch()
+
+	if err := cwhub.GetHubIdx(c.Hub); err != nil {
+		return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
+	}
+
+	return nil
+}

cmd/crowdsec-cli/scenarios.go

@@ -7,6 +7,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -24,20 +25,8 @@ cscli scenarios remove crowdsecurity/ssh-bf
 		Aliases:           []string{"scenario"},
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
+			if err := require.Hub(csConfig); err != nil {
-				log.Fatal(err)
+				return err
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("while setting hub branch: %w", err)
-			}
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
 			}
 
 			return nil
@@ -72,7 +61,7 @@ func NewCmdScenariosInstall() *cobra.Command {
 			return compAllItems(cwhub.SCENARIOS, args, toComplete)
 		},
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, name := range args {
 				t := cwhub.GetItem(cwhub.SCENARIOS, name)
 				if t == nil {
@@ -81,13 +70,13 @@ func NewCmdScenariosInstall() *cobra.Command {
 					continue
 				}
 				if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
-					if ignoreError {
+					if !ignoreError {
+						return fmt.Errorf("error while installing '%s': %w", name, err)
+					}
 					log.Errorf("Error while installing '%s': %s", name, err)
-					} else {
-						log.Fatalf("Error while installing '%s': %s", name, err)
-					}
 				}
 			}
+			return nil
 		},
 	}
 	cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
@@ -108,19 +97,20 @@ func NewCmdScenariosRemove() *cobra.Command {
 			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
 		},
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
-				return
+				return nil
 			}
 
 			if len(args) == 0 {
-				log.Fatalf("Specify at least one scenario to remove or '--all' flag.")
+				return fmt.Errorf("specify at least one scenario to remove or '--all'")
 			}
 
 			for _, name := range args {
 				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
 			}
+			return nil
 		},
 	}
 	cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
@@ -140,17 +130,18 @@ func NewCmdScenariosUpgrade() *cobra.Command {
 			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
 		},
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
 			} else {
 				if len(args) == 0 {
-					log.Fatalf("no target scenario to upgrade")
+					return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
 				}
 				for _, name := range args {
 					cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
 				}
 			}
+			return nil
 		},
 	}
 	cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")

cmd/crowdsec-cli/setup.go

@@ -112,6 +112,20 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	var detectReader *os.File
+
+	switch detectConfigFile {
+	case "-":
+		log.Tracef("Reading detection rules from stdin")
+		detectReader = os.Stdin
+	default:
+		log.Tracef("Reading detection rules: %s", detectConfigFile)
+		detectReader, err = os.Open(detectConfigFile)
+		if err != nil {
+			return err
+		}
+	}
+
 	listSupportedServices, err := flags.GetBool("list-supported-services")
 	if err != nil {
 		return err
@@ -171,7 +185,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 	}
 
 	if listSupportedServices {
-		supported, err := setup.ListSupported(detectConfigFile)
+		supported, err := setup.ListSupported(detectReader)
 		if err != nil {
 			return err
 		}
@@ -195,7 +209,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 		SnubSystemd:  snubSystemd,
 	}
 
-	hubSetup, err := setup.Detect(detectConfigFile, opts)
+	hubSetup, err := setup.Detect(detectReader, opts)
 	if err != nil {
 		return fmt.Errorf("detecting services: %w", err)
 	}

cmd/crowdsec-cli/simulation.go

@@ -3,12 +3,13 @@ package main
 import (
 	"fmt"
 	"os"
+	"slices"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -18,7 +19,7 @@ func addToExclusion(name string) error {
 }
 
 func removeFromExclusion(name string) error {
-	index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
+	index := slices.Index(csConfig.Cscli.SimulationConfig.Exclusions, name)
 
 	// Remove element from the slice
 	csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
@@ -144,13 +145,9 @@ func NewSimulationEnableCmd() *cobra.Command {
 		Example:           `cscli simulation enable`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := csConfig.LoadHub(); err != nil {
+			if err := require.Hub(csConfig); err != nil {
 				log.Fatal(err)
 			}
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
 
 			if len(args) > 0 {
 				for _, scenario := range args {

cmd/crowdsec-cli/support.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/crowdsecurity/go-cs-lib/version"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
@@ -131,24 +132,6 @@ func collectOSInfo() ([]byte, error) {
 	return w.Bytes(), nil
 }
 
-func initHub() error {
-	if err := csConfig.LoadHub(); err != nil {
-		return fmt.Errorf("cannot load hub: %s", err)
-	}
-	if csConfig.Hub == nil {
-		return fmt.Errorf("hub not configured")
-	}
-
-	if err := cwhub.SetHubBranch(); err != nil {
-		return fmt.Errorf("cannot set hub branch: %s", err)
-	}
-
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		return fmt.Errorf("no hub index found: %s", err)
-	}
-	return nil
-}
-
 func collectHubItems(itemType string) []byte {
 	out := bytes.NewBuffer(nil)
 	log.Infof("Collecting %s list", itemType)
@@ -184,7 +167,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
 	if err != nil {
 		return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
 	}
-	scenarios, err := cwhub.GetInstalledScenariosAsString()
+	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 	if err != nil {
 		return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
 	}
@@ -312,8 +295,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				skipAgent = true
 			}
 
-			err = initHub()
+			if err := require.Hub(csConfig); err != nil {
-			if err != nil {
 				log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
 				skipHub = true
 				infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())

cmd/crowdsec-cli/utils.go

@@ -8,7 +8,7 @@ import (
 	"math"
 	"net"
 	"net/http"
-	"os"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -19,11 +19,11 @@ import (
 	"github.com/prometheus/prom2json"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -38,34 +38,6 @@ func printHelp(cmd *cobra.Command) {
 	}
 }
 
-func indexOf(s string, slice []string) int {
-	for i, elem := range slice {
-		if s == elem {
-			return i
-		}
-	}
-	return -1
-}
-
-func LoadHub() error {
-	if err := csConfig.LoadHub(); err != nil {
-		log.Fatal(err)
-	}
-	if csConfig.Hub == nil {
-		return fmt.Errorf("unable to load hub")
-	}
-
-	if err := cwhub.SetHubBranch(); err != nil {
-		log.Warningf("unable to set hub branch (%s), default to master", err)
-	}
-
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
-	}
-
-	return nil
-}
-
 func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
 	errMsg := ""
 	if score < MaxDistance {
@@ -100,7 +72,7 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
 }
 
 func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := LoadHub(); err != nil {
+	if err := require.Hub(csConfig); err != nil {
 		return nil, cobra.ShellCompDirectiveDefault
 	}
 
@@ -116,31 +88,16 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string,
 }
 
 func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := LoadHub(); err != nil {
+	if err := require.Hub(csConfig); err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	var items []string
-	var err error
-	switch itemType {
-	case cwhub.PARSERS:
-		items, err = cwhub.GetInstalledParsersAsString()
-	case cwhub.SCENARIOS:
-		items, err = cwhub.GetInstalledScenariosAsString()
-	case cwhub.PARSERS_OVFLW:
-		items, err = cwhub.GetInstalledPostOverflowsAsString()
-	case cwhub.COLLECTIONS:
-		items, err = cwhub.GetInstalledCollectionsAsString()
-	case cwhub.WAAP_RULES:
-		items, err = cwhub.GetInstalledWafRulesAsString()
-	default:
 		return nil, cobra.ShellCompDirectiveDefault
 	}
 
+	items, err := cwhub.GetInstalledItemsAsString(itemType)
 	if err != nil {
 		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
 		return nil, cobra.ShellCompDirectiveDefault
 	}
+
 	comp := make([]string, 0)
 
 	if toComplete != "" {
@@ -470,37 +427,6 @@ func GetScenarioMetric(url string, itemName string) map[string]int {
 	return stats
 }
 
-// it's a rip of the cli version, but in silent-mode
-func silenceInstallItem(name string, obtype string) (string, error) {
-	var item = cwhub.GetItem(obtype, name)
-	if item == nil {
-		return "", fmt.Errorf("error retrieving item")
-	}
-	it := *item
-	if downloadOnly && it.Downloaded && it.UpToDate {
-		return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
-	}
-	it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
-	if err != nil {
-		return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
-	}
-	if err := cwhub.AddItem(obtype, it); err != nil {
-		return "", err
-	}
-
-	if downloadOnly {
-		return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
-	}
-	it, err = cwhub.EnableItem(csConfig.Hub, it)
-	if err != nil {
-		return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
-	}
-	if err := cwhub.AddItem(obtype, it); err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("Enabled %s", it.Name), nil
-}
-
 func GetPrometheusMetric(url string) []*prom2json.Family {
 	mfChan := make(chan *dto.MetricFamily, 1024)
 
@@ -529,160 +455,6 @@ func GetPrometheusMetric(url string) []*prom2json.Family {
 	return result
 }
 
-func RestoreHub(dirPath string) error {
-	var err error
-
-	if err := csConfig.LoadHub(); err != nil {
-		return err
-	}
-	if err := cwhub.SetHubBranch(); err != nil {
-		return fmt.Errorf("error while setting hub branch: %s", err)
-	}
-
-	for _, itype := range cwhub.ItemTypes {
-		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
-		if _, err = os.Stat(itemDirectory); err != nil {
-			log.Infof("no %s in backup", itype)
-			continue
-		}
-		/*restore the upstream items*/
-		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
-		file, err := os.ReadFile(upstreamListFN)
-		if err != nil {
-			return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
-		}
-		var upstreamList []string
-		err = json.Unmarshal(file, &upstreamList)
-		if err != nil {
-			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
-		}
-		for _, toinstall := range upstreamList {
-			label, err := silenceInstallItem(toinstall, itype)
-			if err != nil {
-				log.Errorf("Error while installing %s : %s", toinstall, err)
-			} else if label != "" {
-				log.Infof("Installed %s : %s", toinstall, label)
-			} else {
-				log.Printf("Installed %s : ok", toinstall)
-			}
-		}
-
-		/*restore the local and tainted items*/
-		files, err := os.ReadDir(itemDirectory)
-		if err != nil {
-			return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
-		}
-		for _, file := range files {
-			//this was the upstream data
-			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
-				continue
-			}
-			if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
-				//we expect a stage here
-				if !file.IsDir() {
-					continue
-				}
-				stage := file.Name()
-				stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
-				log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
-				if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
-					return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
-				}
-				/*find items*/
-				ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
-				if err != nil {
-					return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
-				}
-				//finally copy item
-				for _, tfile := range ifiles {
-					log.Infof("Going to restore local/tainted [%s]", tfile.Name())
-					sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
-					destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
-					if err = CopyFile(sourceFile, destinationFile); err != nil {
-						return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
-					}
-					log.Infof("restored %s to %s", sourceFile, destinationFile)
-				}
-			} else {
-				log.Infof("Going to restore local/tainted [%s]", file.Name())
-				sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
-				destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
-				if err = CopyFile(sourceFile, destinationFile); err != nil {
-					return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
-				}
-				log.Infof("restored %s to %s", sourceFile, destinationFile)
-			}
-
-		}
-	}
-	return nil
-}
-
-func BackupHub(dirPath string) error {
-	var err error
-	var itemDirectory string
-	var upstreamParsers []string
-
-	for _, itemType := range cwhub.ItemTypes {
-		clog := log.WithFields(log.Fields{
-			"type": itemType,
-		})
-		itemMap := cwhub.GetItemMap(itemType)
-		if itemMap == nil {
-			clog.Infof("No %s to backup.", itemType)
-			continue
-		}
-		itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
-		if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
-			return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
-		}
-		upstreamParsers = []string{}
-		for k, v := range itemMap {
-			clog = clog.WithFields(log.Fields{
-				"file": v.Name,
-			})
-			if !v.Installed { //only backup installed ones
-				clog.Debugf("[%s] : not installed", k)
-				continue
-			}
-
-			//for the local/tainted ones, we backup the full file
-			if v.Tainted || v.Local || !v.UpToDate {
-				//we need to backup stages for parsers
-				if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
-					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
-					if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
-						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
-					}
-				}
-				clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
-				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
-				if err = CopyFile(v.LocalPath, tfile); err != nil {
-					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
-				}
-				clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
-				continue
-			}
-			clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
-			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
-			upstreamParsers = append(upstreamParsers, v.Name)
-		}
-		//write the upstream items
-		upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
-		upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
-		if err != nil {
-			return fmt.Errorf("failed marshaling upstream parsers : %s", err)
-		}
-		err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
-		if err != nil {
-			return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
-		}
-		clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
-	}
-
-	return nil
-}
-
 type unit struct {
 	value  int64
 	symbol string

cmd/crowdsec-cli/utils_table.go

@@ -17,7 +17,7 @@ func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatu
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	for _, status := range statuses {
-		t.AddRow(status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath)
+		t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
 	}
 	renderTableTitle(out, title)
 	t.Render()

cmd/crowdsec-cli/waap_rules.go

@@ -31,9 +31,7 @@ cscli waap-rules remove crowdsecurity/core-rule-set
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := cwhub.SetHubBranch(); err != nil {
+			cwhub.SetHubBranch()
-				return fmt.Errorf("error while setting hub branch: %s", err)
-			}
 
 			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
 				log.Info("Run 'sudo cscli hub update' to get the hub index")

cmd/crowdsec/Makefile

@@ -4,7 +4,6 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-# Go parameters
 GO = go
 GOBUILD = $(GO) build
 GOTEST = $(GO) test
@@ -23,7 +22,7 @@ SYSTEMD_PATH_FILE = "/etc/systemd/system/crowdsec.service"
 all: clean test build
 
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(CROWDSEC_BIN)
+	$(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN)
 
 test:
 	$(GOTEST) $(LD_OPTS) -v ./...

cmd/crowdsec/main.go

@@ -138,11 +138,13 @@ func (l *labelsMap) String() string {
 }
 
 func (l labelsMap) Set(label string) error {
-	split := strings.Split(label, ":")
+	for _, pair := range strings.Split(label, ",") {
+		split := strings.Split(pair, ":")
 		if len(split) != 2 {
-		return errors.Wrapf(errors.New("Bad Format"), "for Label '%s'", label)
+			return fmt.Errorf("invalid format for label '%s', must be key:value", pair)
 		}
 		l[split[0]] = split[1]
+	}
 	return nil
 }
 

cmd/crowdsec/output.go

@@ -70,7 +70,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 	var cache []types.RuntimeAlert
 	var cacheMutex sync.Mutex
 
-	scenarios, err := cwhub.GetInstalledScenariosAsString()
+	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 	if err != nil {
 		return fmt.Errorf("loading list of installed hub scenarios: %w", err)
 	}
@@ -93,7 +93,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 		URL:            apiURL,
 		PapiURL:        papiURL,
 		VersionPrefix:  "v1",
-		UpdateScenario: cwhub.GetInstalledScenariosAsString,
+		UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
 	})
 	if err != nil {
 		return fmt.Errorf("new client api: %w", err)

cmd/crowdsec/serve.go

@@ -141,12 +141,24 @@ func ShutdownCrowdsecRoutines() error {
 	time.Sleep(1 * time.Second) // ugly workaround for now
 	outputsTomb.Kill(nil)
 
-	if err := outputsTomb.Wait(); err != nil {
+	done := make(chan error, 1)
-		log.Warningf("Ouputs returned error : %s", err)
+	go func() {
+		done <- outputsTomb.Wait()
+	}()
+
+	// wait for outputs to finish, max 3 seconds
+	select {
+	case err := <-done:
+		if err != nil {
+			log.Warningf("Outputs returned error : %s", err)
 			reterr = err
 		}
-
 		log.Debugf("outputs are done")
+	case <-time.After(3 * time.Second):
+		// this can happen if outputs are stuck in a http retry loop
+		log.Warningf("Outputs didn't finish in time, some events may have not been flushed")
+	}
+
 	// He's dead, Jim.
 	crowdsecTomb.Kill(nil)
 

cmd/notification-dummy/Makefile

@@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-PLUGIN=http
-BINARY_NAME = notification-$(PLUGIN)$(EXT)
-
 GO = go
 GOBUILD = $(GO) build
 
+BINARY_NAME = notification-dummy$(EXT)
+
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
 
 .PHONY: clean
 clean:

cmd/notification-email/Makefile

@@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-PLUGIN=slack
-BINARY_NAME = notification-$(PLUGIN)$(EXT)
-
 GO = go
 GOBUILD = $(GO) build
 
+BINARY_NAME = notification-email$(EXT)
+
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
 
 .PHONY: clean
 clean:

cmd/notification-email/email.yaml

@@ -15,12 +15,14 @@ timeout: 20s          # Time to wait for response from the plugin before conside
 # The following template receives a list of models.Alert objects
 # The output goes in the email message body
 format: |
+  <html><body>
   {{range . -}}
     {{$alert := . -}}
     {{range .Decisions -}}
-      <html><body><p><a href=https://www.whois.com/whois/{{.Value}}>{{.Value}}</a> will get <b>{{.Type}}</b> for next <b>{{.Duration}}</b> for triggering <b>{{.Scenario}}</b> on machine <b>{{$alert.MachineID}}</b>.</p> <p><a href=https://app.crowdsec.net/cti/{{.Value}}>CrowdSec CTI</a></p></body></html>
+      <p><a href="https://www.whois.com/whois/{{.Value}}">{{.Value}}</a> will get <b>{{.Type}}</b> for next <b>{{.Duration}}</b> for triggering <b>{{.Scenario}}</b> on machine <b>{{$alert.MachineID}}</b>.</p> <p><a href="https://app.crowdsec.net/cti/{{.Value}}">CrowdSec CTI</a></p>
     {{end -}}
   {{end -}}
+  </body></html>
 
 smtp_host:            # example: smtp.gmail.com
 smtp_username:        # Replace with your actual username
@@ -35,7 +37,15 @@ receiver_emails:
 # - email2@gmail.com
 
 # One of "ssltls", "starttls", "none"
-encryption_type: ssltls
+encryption_type: "ssltls"
+
+# If you need to set the HELO hostname:
+# helo_host: "localhost"
+
+# If the email server is hitting the default timeouts (10 seconds), you can increase them here
+#
+# connect_timeout: 10s
+# send_timeout: 10s
 
 ---
 

cmd/notification-email/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/crowdsecurity/crowdsec/pkg/protobufs"
 	"github.com/hashicorp/go-hclog"
@@ -47,6 +48,8 @@ type PluginConfig struct {
 	EncryptionType string   `yaml:"encryption_type"`
 	AuthType       string   `yaml:"auth_type"`
 	HeloHost       string   `yaml:"helo_host"`
+	ConnectTimeout string   `yaml:"connect_timeout"`
+	SendTimeout    string   `yaml:"send_timeout"`
 }
 
 type EmailPlugin struct {
@@ -77,7 +80,7 @@ func (n *EmailPlugin) Configure(ctx context.Context, config *protobufs.Config) (
 	}
 
 	if d.ReceiverEmails == nil || len(d.ReceiverEmails) == 0 {
-		return nil, fmt.Errorf("Receiver emails are not set")
+		return nil, fmt.Errorf("receiver emails are not set")
 	}
 
 	n.ConfigByName[d.Name] = d
@@ -108,6 +111,24 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
 	server.Authentication = AuthStringToType[cfg.AuthType]
 	server.Helo = cfg.HeloHost
 
+	var err error
+
+	if cfg.ConnectTimeout != "" {
+		server.ConnectTimeout, err = time.ParseDuration(cfg.ConnectTimeout)
+		if err != nil {
+			logger.Warn(fmt.Sprintf("invalid connect timeout '%s', using default '10s'", cfg.ConnectTimeout))
+			server.ConnectTimeout = 10 * time.Second
+		}
+	}
+
+	if cfg.SendTimeout != "" {
+		server.SendTimeout, err = time.ParseDuration(cfg.SendTimeout)
+		if err != nil {
+			logger.Warn(fmt.Sprintf("invalid send timeout '%s', using default '10s'", cfg.SendTimeout))
+			server.SendTimeout = 10 * time.Second
+		}
+	}
+
 	logger.Debug("making smtp connection")
 	smtpClient, err := server.Connect()
 	if err != nil {

cmd/notification-http/Makefile

@@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-PLUGIN=splunk
-BINARY_NAME = notification-$(PLUGIN)$(EXT)
-
 GO = go
 GOBUILD = $(GO) build
 
+BINARY_NAME = notification-http$(EXT)
+
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
 
 .PHONY: clean
 clean:

cmd/notification-http/main.go

@@ -63,7 +63,7 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
 		logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue))
 		request.Header.Add(headerName, headerValue)
 	}
-	logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, string(notification.Text)))
+	logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
 	resp, err := client.Do(request)
 	if err != nil {
 		logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))

cmd/notification-sentinel/Makefile

@@ -4,14 +4,14 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-# Go parameters
 GO = go
 GOBUILD = $(GO) build
 
 BINARY_NAME = notification-sentinel$(EXT)
 
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
 
+.PHONY: clean
 clean:
 	@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)

cmd/notification-sentinel/main.go

@@ -78,7 +78,7 @@ func (s *SentinelPlugin) Notify(ctx context.Context, notification *protobufs.Not
 		return &protobufs.Empty{}, err
 	}
 
-	req, err := http.NewRequest("POST", url, body)
+	req, err := http.NewRequest(http.MethodPost, url, body)
 	if err != nil {
 		logger.Error("failed to create request", "error", err)
 		return &protobufs.Empty{}, err
@@ -98,7 +98,7 @@ func (s *SentinelPlugin) Notify(ctx context.Context, notification *protobufs.Not
 	defer resp.Body.Close()
 	logger.Debug("sent notification to sentinel", "status", resp.Status)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		return &protobufs.Empty{}, fmt.Errorf("failed to send notification to sentinel: %s", resp.Status)
 	}
 

cmd/notification-slack/Makefile

@@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-PLUGIN = dummy
-BINARY_NAME = notification-$(PLUGIN)$(EXT)
-
 GO = go
 GOBUILD = $(GO) build
 
+BINARY_NAME = notification-slack$(EXT)
+
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
 
 .PHONY: clean
 clean:

cmd/notification-splunk/Makefile

@@ -0,0 +1,17 @@
+ifeq ($(OS), Windows_NT)
+	SHELL := pwsh.exe
+	.SHELLFLAGS := -NoProfile -Command
+	EXT = .exe
+endif
+
+GO = go
+GOBUILD = $(GO) build
+
+BINARY_NAME = notification-splunk$(EXT)
+
+build: clean
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
+
+.PHONY: clean
+clean:
+	@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)

cmd/notification-splunk/main.go

@@ -58,7 +58,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
 		return &protobufs.Empty{}, err
 	}
 
-	req, err := http.NewRequest("POST", cfg.URL, strings.NewReader(string(data)))
+	req, err := http.NewRequest(http.MethodPost, cfg.URL, strings.NewReader(string(data)))
 	if err != nil {
 		return &protobufs.Empty{}, err
 	}
@@ -70,7 +70,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
 		return &protobufs.Empty{}, err
 	}
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		content, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return &protobufs.Empty{}, fmt.Errorf("got non 200 response and failed to read error %s", err)

debian/install

@@ -6,8 +6,8 @@ config/patterns/*       etc/crowdsec/patterns
 config/crowdsec.service lib/systemd/system
 
 # Referenced configs:
-plugins/notifications/slack/slack.yaml      etc/crowdsec/notifications/
+cmd/notification-slack/slack.yaml        etc/crowdsec/notifications/
-plugins/notifications/http/http.yaml        etc/crowdsec/notifications/
+cmd/notification-http/http.yaml          etc/crowdsec/notifications/
-plugins/notifications/splunk/splunk.yaml    etc/crowdsec/notifications/
+cmd/notification-splunk/splunk.yaml      etc/crowdsec/notifications/
-plugins/notifications/email/email.yaml      etc/crowdsec/notifications/
+cmd/notification-email/email.yaml        etc/crowdsec/notifications/
-plugins/notifications/sentinel/sentinel.yaml      etc/crowdsec/notifications/
+cmd/notification-sentinel/sentinel.yaml  etc/crowdsec/notifications/

debian/rules

@@ -25,11 +25,11 @@ override_dh_auto_install:
 	mkdir -p debian/crowdsec/usr/lib/crowdsec/plugins/
 	mkdir -p debian/crowdsec/etc/crowdsec/notifications/
 
-	install -m 551 plugins/notifications/slack/notification-slack debian/crowdsec/usr/lib/crowdsec/plugins/
+	install -m 551 cmd/notification-slack/notification-slack debian/crowdsec/usr/lib/crowdsec/plugins/
-	install -m 551 plugins/notifications/http/notification-http debian/crowdsec/usr/lib/crowdsec/plugins/
+	install -m 551 cmd/notification-http/notification-http debian/crowdsec/usr/lib/crowdsec/plugins/
-	install -m 551 plugins/notifications/splunk/notification-splunk debian/crowdsec/usr/lib/crowdsec/plugins/
+	install -m 551 cmd/notification-splunk/notification-splunk debian/crowdsec/usr/lib/crowdsec/plugins/
-	install -m 551 plugins/notifications/email/notification-email debian/crowdsec/usr/lib/crowdsec/plugins/
+	install -m 551 cmd/notification-email/notification-email debian/crowdsec/usr/lib/crowdsec/plugins/
-	install -m 551 plugins/notifications/sentinel/notification-sentinel debian/crowdsec/usr/lib/crowdsec/plugins/
+	install -m 551 cmd/notification-sentinel/notification-sentinel debian/crowdsec/usr/lib/crowdsec/plugins/
 
 	cp cmd/crowdsec/crowdsec debian/crowdsec/usr/bin
 	cp cmd/crowdsec-cli/cscli debian/crowdsec/usr/bin

docker/README.md

@@ -19,11 +19,7 @@ All the following images are available on Docker Hub for the architectures
 
  - `crowdsecurity/crowdsec:{version}`
 
-Recommended for production usage. Also available on GitHub (ghcr.io).
+Latest stable release recommended for production usage. Also available on GitHub (ghcr.io).
-
- - `crowdsecurity/crowdsec:dev`
-
-The latest stable release.
 
  - `crowdsecurity/crowdsec:dev`
 
@@ -190,6 +186,14 @@ It is not recommended anymore to bind-mount the full config.yaml file and you sh
 
 If you want to use the [notification system](https://docs.crowdsec.net/docs/notification_plugins/intro), you have to use the full image (not slim) and mount at least a custom `profiles.yaml` and a notification configuration to `/etc/crowdsec/notifications`
 
+```shell
+docker run -d \
+    -v ./profiles.yaml:/etc/crowdsec/profiles.yaml \
+    -v ./http_notification.yaml:/etc/crowdsec/notifications/http_notification.yaml \
+    -p 8080:8080 -p 6060:6060 \
+    --name crowdsec crowdsecurity/crowdsec
+```
+
 # Deployment use cases
 
 Crowdsec is composed of an `agent` that parses logs and creates `alerts`, and a

docker/test/Pipfile

@@ -1,7 +1,7 @@
 [packages]
 pytest-dotenv = "0.5.2"
 pytest-xdist = "3.3.1"
-pytest-cs = {ref = "0.7.16", git = "https://github.com/crowdsecurity/pytest-cs.git"}
+pytest-cs = {ref = "0.7.18", git = "https://github.com/crowdsecurity/pytest-cs.git"}
 
 [dev-packages]
 gnureadline = "8.1.2"

docker/test/Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "78f693678e411b7bdb5dd0280b7d6f8d9880069b331d44d96d32ba697275e30d"
+            "sha256": "64085783c9fec3a9eda976b7700b5bad7abd2b7a0f0670fa2209c52f3647be7f"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -176,32 +176,32 @@
         },
         "cryptography": {
             "hashes": [
-                "sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711",
+                "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67",
-                "sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7",
+                "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311",
-                "sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd",
+                "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8",
-                "sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e",
+                "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13",
-                "sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58",
+                "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143",
-                "sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0",
+                "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f",
-                "sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d",
+                "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829",
-                "sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83",
+                "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd",
-                "sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831",
+                "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397",
-                "sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766",
+                "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac",
-                "sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b",
+                "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d",
-                "sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c",
+                "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a",
-                "sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182",
+                "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839",
-                "sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f",
+                "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e",
-                "sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa",
+                "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6",
-                "sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4",
+                "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9",
-                "sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a",
+                "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860",
-                "sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2",
+                "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca",
-                "sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76",
+                "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91",
-                "sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5",
+                "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d",
-                "sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee",
+                "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714",
-                "sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f",
+                "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb",
-                "sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14"
+                "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==41.0.2"
+            "version": "==41.0.4"
         },
         "docker": {
             "hashes": [
@@ -245,11 +245,11 @@
         },
         "pluggy": {
             "hashes": [
-                "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849",
+                "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12",
-                "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"
+                "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version >= '3.8'",
-            "version": "==1.2.0"
+            "version": "==1.3.0"
         },
         "psutil": {
             "hashes": [
@@ -280,15 +280,15 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32",
+                "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002",
-                "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"
+                "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.4.0"
+            "version": "==7.4.2"
         },
         "pytest-cs": {
             "git": "https://github.com/crowdsecurity/pytest-cs.git",
-            "ref": "4a3451084215053af8a48ff37507b4f86bf75c10"
+            "ref": "df835beabc539be7f7f627b21caa0d6ad333daae"
         },
         "pytest-datadir": {
             "hashes": [
@@ -324,7 +324,9 @@
         },
         "pyyaml": {
             "hashes": [
+                "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
                 "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
+                "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
                 "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
                 "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
                 "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
@@ -332,7 +334,10 @@
                 "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
                 "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
                 "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
+                "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
+                "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
                 "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
+                "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
                 "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
                 "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
                 "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
@@ -340,9 +345,12 @@
                 "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
                 "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
                 "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
+                "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
                 "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
                 "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
                 "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
+                "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
+                "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
                 "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
                 "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
                 "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
@@ -357,7 +365,9 @@
                 "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
                 "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
                 "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
+                "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
                 "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
+                "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
                 "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
                 "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
                 "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
@@ -386,28 +396,28 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11",
+                "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594",
-                "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"
+                "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.0.4"
+            "version": "==2.0.5"
         },
         "websocket-client": {
             "hashes": [
-                "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd",
+                "sha256:3aad25d31284266bcfcfd1fd8a743f63282305a364b8d0948a43bd606acc652f",
-                "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"
+                "sha256:6cfc30d051ebabb73a5fa246efdcc14c8fbebbd0330f8984ac3bb6d9edd2ad03"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version >= '3.8'",
-            "version": "==1.6.1"
+            "version": "==1.6.3"
         }
     },
     "develop": {
         "asttokens": {
             "hashes": [
-                "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3",
+                "sha256:2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e",
-                "sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c"
+                "sha256:cf8fc9e61a86461aa9fb161a14a0841a03c405fa829ac6b202670b3495d2ce69"
             ],
-            "version": "==2.2.1"
+            "version": "==2.4.0"
         },
         "backcall": {
             "hashes": [
@@ -474,19 +484,19 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:1d197b907b6ba441b692c48cf2a3a2de280dc0ac91a3405b39349a50272ca0a1",
+                "sha256:2baeb5be6949eeebf532150f81746f8333e2ccce02de1c7eedde3f23ed5e9f1e",
-                "sha256:248aca623f5c99a6635bc3857677b7320b9b8039f99f070ee0d20a5ca5a8e6bf"
+                "sha256:45a2c3a529296870a97b7de34eda4a31bee16bc7bf954e07d39abe49caf8f887"
             ],
             "markers": "python_version >= '3.11'",
-            "version": "==8.14.0"
+            "version": "==8.15.0"
         },
         "jedi": {
             "hashes": [
-                "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e",
+                "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4",
-                "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"
+                "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.18.2"
+            "version": "==0.19.0"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -543,11 +553,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c",
+                "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692",
-                "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"
+                "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.15.1"
+            "version": "==2.16.1"
         },
         "six": {
             "hashes": [
@@ -566,11 +576,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8",
+                "sha256:417745a96681fbb358e723d5346a547521f36e9bd0d50ba7ab368fff5d67aa54",
-                "sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9"
+                "sha256:f584ea209240466e66e91f3c81aa7d004ba4cf794990b0c775938a1544217cd1"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version >= '3.8'",
-            "version": "==5.9.0"
+            "version": "==5.10.0"
         },
         "wcwidth": {
             "hashes": [

docker/test/tests/test_tls.py

@@ -4,7 +4,7 @@
 Test agent-lapi and cscli-lapi communication via TLS, on the same container.
 """
 
-import random
+import uuid
 
 from pytest_cs import Status
 
@@ -140,7 +140,7 @@ def test_tls_lapi_var(crowdsec, flavor, certs_dir):
 def test_tls_split_lapi_agent(crowdsec, flavor, certs_dir):
     """Server-only certificate, split containers"""
 
-    rand = random.randint(0, 10000)
+    rand = uuid.uuid1()
     lapiname = 'lapi-' + str(rand)
     agentname = 'agent-' + str(rand)
 
@@ -193,7 +193,7 @@ def test_tls_split_lapi_agent(crowdsec, flavor, certs_dir):
 def test_tls_mutual_split_lapi_agent(crowdsec, flavor, certs_dir):
     """Server and client certificates, split containers"""
 
-    rand = random.randint(0, 10000)
+    rand = uuid.uuid1()
     lapiname = 'lapi-' + str(rand)
     agentname = 'agent-' + str(rand)
 
@@ -244,7 +244,7 @@ def test_tls_mutual_split_lapi_agent(crowdsec, flavor, certs_dir):
 def test_tls_client_ou(crowdsec, certs_dir):
     """Check behavior of client certificate vs AGENTS_ALLOWED_OU"""
 
-    rand = random.randint(0, 10000)
+    rand = uuid.uuid1()
     lapiname = 'lapi-' + str(rand)
     agentname = 'agent-' + str(rand)
 
@@ -287,6 +287,19 @@ def test_tls_client_ou(crowdsec, certs_dir):
 
     lapi_env['AGENTS_ALLOWED_OU'] = 'custom-client-ou'
 
+    # change container names to avoid conflict
+    # recreate certificates because they need the new hostname
+
+    rand = uuid.uuid1()
+    lapiname = 'lapi-' + str(rand)
+    agentname = 'agent-' + str(rand)
+
+    agent_env['LOCAL_API_URL'] = f'https://{lapiname}:8080'
+
+    volumes = {
+        certs_dir(lapi_hostname=lapiname, agent_ou='custom-client-ou'): {'bind': '/etc/ssl/crowdsec', 'mode': 'ro'},
+    }
+
     cs_lapi = crowdsec(name=lapiname, environment=lapi_env, volumes=volumes)
     cs_agent = crowdsec(name=agentname, environment=agent_env, volumes=volumes)
 

go.mod

@@ -1,9 +1,13 @@
 module github.com/crowdsecurity/crowdsec
 
-go 1.20
+go 1.21
+
+// Don't use the toolchain directive to avoid uncontrolled downloads during
+// a build, especially in sandboxed environments (freebsd, gentoo...).
+// toolchain go1.21.3
 
 require (
-	entgo.io/ent v0.11.3
+	entgo.io/ent v0.12.4
 	github.com/AlecAivazis/survey/v2 v2.2.7
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/Masterminds/sprig/v3 v3.2.2
@@ -21,7 +25,7 @@ require (
 	github.com/c-robinson/iplib v1.0.3
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
-	github.com/crowdsecurity/go-cs-lib v0.0.3
+	github.com/crowdsecurity/go-cs-lib v0.0.4
 	github.com/crowdsecurity/grokky v0.2.1
 	github.com/crowdsecurity/machineid v1.0.2
 	github.com/davecgh/go-spew v1.1.1
@@ -68,12 +72,14 @@ require (
 	github.com/segmentio/kafka-go v0.4.34
 	github.com/shirou/gopsutil/v3 v3.23.5
 	github.com/sirupsen/logrus v1.9.3
+	github.com/slack-go/slack v0.12.2
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
 	github.com/wasilibs/go-re2 v1.3.0
 	golang.org/x/crypto v0.10.0
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
+	github.com/xhit/go-simple-mail/v2 v2.16.0
 	golang.org/x/mod v0.11.0
 	golang.org/x/sys v0.10.0
 	google.golang.org/grpc v1.56.1
@@ -90,7 +96,7 @@ require (
 )
 
 require (
-	ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a // indirect
+	ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 // indirect
@@ -126,6 +132,7 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/hcl/v2 v2.13.0 // indirect
 	github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
@@ -184,6 +191,7 @@ require (
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
@@ -196,7 +204,7 @@ require (
 	golang.org/x/term v0.10.0 // indirect
 	golang.org/x/text v0.11.0 // indirect
 	golang.org/x/time v0.2.0 // indirect
-	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect

go.sum

@@ -1,6 +1,7 @@
-ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a h1:6/nt4DODfgxzHTTg3tYy7YkVzruGQGZ/kRvXpA45KUo=
+ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935 h1:JnYs/y8RJ3+MiIUp+3RgyyeO48VHLAZimqiaZYnMKk8=
-ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a/go.mod h1:ft47uSh5hWGDCmQC9DsztZg6Xk+KagM5Ts/mZYKb9JE=
+ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935/go.mod h1:isZrlzJ5cpoCoKFoY9knZug7Lq4pP1cm8g3XciLZ0Pw=
 bitbucket.org/creachadair/stringset v0.0.9 h1:L4vld9nzPt90UZNrXjNelTshD74ps4P5NGs3Iq6yN3o=
+bitbucket.org/creachadair/stringset v0.0.9/go.mod h1:t+4WcQ4+PXTa8aQdNKe40ZP6iwesoMFWAxPGd3UGjyY=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -34,14 +35,16 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-entgo.io/ent v0.11.3 h1:F5FBGAWiDCGder7YT+lqMnyzXl6d0xU3xMBM/SO3CMc=
+entgo.io/ent v0.12.4 h1:LddPnAyxls/O7DTXZvUGDj0NZIdGSu317+aoNLJWbD8=
-entgo.io/ent v0.11.3/go.mod h1:mvDhvynOzAsOe7anH7ynPPtMjA/eeXP96kAfweevyxc=
+entgo.io/ent v0.12.4/go.mod h1:Y3JVAjtlIk8xVZYSn3t3mf8xlZIn5SAOXZQxD6kKI+Q=
 github.com/AlecAivazis/survey/v2 v2.2.7 h1:5NbxkF4RSKmpywYdcRgUmos1o+roJY8duCLZXbVjoig=
 github.com/AlecAivazis/survey/v2 v2.2.7/go.mod h1:9DYvHgXtiXm6nCn+jXnOXLKbH+Yo9u8fAS/SduGdoPk=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
+github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
@@ -138,8 +141,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f h1:7MgSs0r
 github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f/go.mod h1:YwM+m6iBdUn6P1eQKu+F+83bzkP0AzSEBCcVL//zh9c=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
-github.com/crowdsecurity/go-cs-lib v0.0.3 h1:NPSHTLS83A3wFyzV5R9Py8fBbTrVHu/1PQeaD7id4+I=
+github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims=
-github.com/crowdsecurity/go-cs-lib v0.0.3/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
+github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
 github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
@@ -288,6 +291,7 @@ github.com/go-openapi/validate v0.20.0 h1:pzutNCCBZGZlE+u8HD3JZyWdc/TVbtVwlWUp8/
 github.com/go-openapi/validate v0.20.0/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
@@ -302,7 +306,8 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfC
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
+github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
+github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
 github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
 github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
@@ -385,6 +390,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
@@ -412,6 +418,9 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e h1:XmA6L9IPRdUr28a+SK/oMchGgQy159wvzXA5tJ7l+40=
 github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e/go.mod h1:AFIo+02s+12CEg8Gzz9kzhCbmbq6JcKNrhHffCGA9z4=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
 github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk=
@@ -488,6 +497,7 @@ github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
 github.com/jarcoal/httpmock v1.1.0 h1:F47ChZj1Y2zFsCXxNkBPwNNKnAyOATcdQibk0qEdVCE=
 github.com/jarcoal/httpmock v1.1.0/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
+github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -529,6 +539,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
@@ -538,14 +549,15 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
 github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
 github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
@@ -693,6 +705,7 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
@@ -719,6 +732,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ=
+github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
@@ -764,6 +779,8 @@ github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+Kd
 github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
 github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
 github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
+github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 h1:PM5hJF7HVfNWmCjMdEfbuOBNXSVF2cMFGgQTPdKCbwM=
+github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208/go.mod h1:BzWtXXrXzZUvMacR0oF/fbDDgUPO8L36tDMmRAf14ns=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
@@ -774,6 +791,7 @@ github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 h1:UFHFmFfixpmf
 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26/go.mod h1:IGhd0qMDsUa9acVjsbsT7bu3ktadtGOHI79+idTew/M=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vjeantet/grok v1.0.1 h1:2rhIR7J4gThTgcZ1m2JY4TrJZNgjn985U28kT2wQrJ4=
+github.com/vjeantet/grok v1.0.1/go.mod h1:ax1aAchzC6/QMXMcyzHQGZWaW1l195+uMYIkCWPCNIo=
 github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
 github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
@@ -781,6 +799,7 @@ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgq
 github.com/wasilibs/go-re2 v1.3.0 h1:LFhBNzoStM3wMie6rN2slD1cuYH2CGiHpvNL3UtcsMw=
 github.com/wasilibs/go-re2 v1.3.0/go.mod h1:AafrCXVvGRJJOImMajgJ2M7rVmWyisVK7sFshbxnVrg=
 github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ=
+github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
@@ -790,6 +809,8 @@ github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49
 github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4=
 github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xhit/go-simple-mail/v2 v2.16.0 h1:ouGy/Ww4kuaqu2E2UrDw7SvLaziWTB60ICLkIkNVccA=
+github.com/xhit/go-simple-mail/v2 v2.16.0/go.mod h1:b7P5ygho6SYE+VIqpxA6QkYfv4teeyG4MKqB3utRu98=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -858,8 +879,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
-golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1090,8 +1109,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
+golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 h1:0wxTF6pSjIIhNt7mo9GvjDfzyCOiWhmICgtO/Ah948s=
-golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
+golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1241,3 +1260,4 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h6
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

pkg/acquisition/modules/cloudwatch/cloudwatch.go

@@ -370,7 +370,7 @@ func (cw *CloudwatchSource) LogStreamManager(in chan LogStreamTailConfig, outCha
 			}
 
 			if cw.Config.StreamRegexp != nil {
-				match, err := regexp.Match(*cw.Config.StreamRegexp, []byte(newStream.StreamName))
+				match, err := regexp.MatchString(*cw.Config.StreamRegexp, newStream.StreamName)
 				if err != nil {
 					cw.logger.Warningf("invalid regexp : %s", err)
 				} else if !match {

pkg/acquisition/modules/docker/docker.go

@@ -392,14 +392,14 @@ func (d *DockerSource) EvalContainer(container dockerTypes.Container) (*Containe
 	}
 
 	for _, cont := range d.compiledContainerID {
-		if matched := cont.Match([]byte(container.ID)); matched {
+		if matched := cont.MatchString(container.ID); matched {
 			return &ContainerConfig{ID: container.ID, Name: container.Names[0], Labels: d.Config.Labels, Tty: d.getContainerTTY(container.ID)}, true
 		}
 	}
 
 	for _, cont := range d.compiledContainerName {
 		for _, name := range container.Names {
-			if matched := cont.Match([]byte(name)); matched {
+			if matched := cont.MatchString(name); matched {
 				return &ContainerConfig{ID: container.ID, Name: name, Labels: d.Config.Labels, Tty: d.getContainerTTY(container.ID)}, true
 			}
 		}

pkg/acquisition/modules/docker/docker_test.go

@@ -193,7 +193,7 @@ container_name_regexp:
 					actualLines++
 					ticker.Reset(1 * time.Second)
 				case <-ticker.C:
-					log.Infof("no more line to read")
+					log.Infof("no more lines to read")
 					dockerSource.t.Kill(nil)
 					return nil
 				}

pkg/acquisition/modules/file/file_test.go

@@ -410,9 +410,7 @@ force_inotify: true`, testPattern),
 
 			if tc.expectedLines != 0 {
 				fd, err := os.Create("test_files/stream.log")
-				if err != nil {
+				require.NoError(t, err, "could not create test file")
-					t.Fatalf("could not create test file : %s", err)
-				}
 
 				for i := 0; i < 5; i++ {
 					_, err = fmt.Fprintf(fd, "%d\n", i)
@@ -424,7 +422,7 @@ force_inotify: true`, testPattern),
 
 				fd.Close()
 				// we sleep to make sure we detect the new file
-				time.Sleep(1 * time.Second)
+				time.Sleep(3 * time.Second)
 				os.Remove("test_files/stream.log")
 				assert.Equal(t, tc.expectedLines, actualLines)
 			}

pkg/acquisition/modules/kafka/kafka.go

@@ -149,7 +149,9 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error {
 				return nil
 			}
 			k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err))
+			continue
 		}
+		k.logger.Tracef("got message: %s", string(m.Value))
 		l := types.Line{
 			Raw:     string(m.Value),
 			Labels:  k.Config.Labels,
@@ -223,7 +225,6 @@ func (kc *KafkaConfiguration) NewTLSConfig() (*tls.Config, error) {
 	caCertPool.AppendCertsFromPEM(caCert)
 	tlsConfig.RootCAs = caCertPool
 
-	tlsConfig.BuildNameToCertificate()
 	return &tlsConfig, err
 }
 

pkg/alertcontext/alertcontext.go

@@ -3,12 +3,12 @@ package alertcontext
 import (
 	"encoding/json"
 	"fmt"
+	"slices"
 	"strconv"
 
 	"github.com/antonmedv/expr"
 	"github.com/antonmedv/expr/vm"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/exp/slices"
 
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/models"

pkg/apiclient/auth.go

@@ -96,10 +96,16 @@ func (r retryRoundTripper) ShouldRetry(statusCode int) bool {
 func (r retryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
 	var resp *http.Response
 	var err error
+
 	backoff := 0
-	for i := 0; i < r.maxAttempts; i++ {
+	maxAttempts := r.maxAttempts
+	if fflag.DisableHttpRetryBackoff.IsEnabled() {
+		maxAttempts = 1
+	}
+
+	for i := 0; i < maxAttempts; i++ {
 		if i > 0 {
-			if r.withBackOff && !fflag.DisableHttpRetryBackoff.IsEnabled() {
+			if r.withBackOff {
 				backoff += 10 + rand.Intn(20)
 			}
 			log.Infof("retrying in %d seconds (attempt %d of %d)", backoff, i+1, r.maxAttempts)
@@ -115,7 +121,10 @@ func (r retryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error)
 		clonedReq := cloneRequest(req)
 		resp, err = r.next.RoundTrip(clonedReq)
 		if err != nil {
-			log.Errorf("error while performing request: %s; %d retries left", err, r.maxAttempts-i-1)
+			left := maxAttempts - i - 1
+			if left > 0 {
+				log.Errorf("error while performing request: %s; %d retries left", err, left)
+			}
 			continue
 		}
 		if !r.ShouldRetry(resp.StatusCode) {
@@ -264,7 +273,9 @@ func (t *JWTTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 		return resp, fmt.Errorf("performing jwt auth: %w", err)
 	}
 
+	if resp != nil {
 		log.Debugf("resp-jwt: %d", resp.StatusCode)
+	}
 
 	return resp, nil
 }

pkg/apiserver/apic.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -15,7 +16,6 @@ import (
 	"github.com/go-openapi/strfmt"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/exp/slices"
 	"gopkg.in/tomb.v2"
 
 	"github.com/crowdsecurity/go-cs-lib/ptr"
@@ -43,8 +43,6 @@ const (
 	metricsIntervalDelta   = time.Minute * 15
 )
 
-var SCOPE_CAPI_ALIAS_ALIAS string = "crowdsecurity/community-blocklist" //we don't use "CAPI" directly, to make it less confusing for the user
-
 type apic struct {
 	// when changing the intervals in tests, always set *First too
 	// or they can be negative
@@ -620,59 +618,57 @@ func (a *apic) PullTop(forcePull bool) error {
 	return nil
 }
 
+// if decisions is whitelisted: return representation of the whitelist ip or cidr
+// if not whitelisted: empty string
+func (a *apic) whitelistedBy(decision *models.Decision) string {
+	if decision.Value == nil {
+		return ""
+	}
+	ipval := net.ParseIP(*decision.Value)
+	for _, cidr := range a.whitelists.Cidrs {
+		if cidr.Contains(ipval) {
+			return cidr.String()
+		}
+	}
+	for _, ip := range a.whitelists.Ips {
+		if ip != nil && ip.Equal(ipval) {
+			return ip.String()
+		}
+	}
+	return ""
+}
+
 func (a *apic) ApplyApicWhitelists(decisions []*models.Decision) []*models.Decision {
-	if a.whitelists == nil {
+	if a.whitelists == nil || len(a.whitelists.Cidrs) == 0 && len(a.whitelists.Ips) == 0 {
 		return decisions
 	}
 	//deal with CAPI whitelists for fire. We want to avoid having a second list, so we shrink in place
 	outIdx := 0
 	for _, decision := range decisions {
-		if decision.Value == nil {
+		whitelister := a.whitelistedBy(decision)
+		if whitelister != "" {
+			log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, whitelister)
 			continue
 		}
-		skip := false
-		ipval := net.ParseIP(*decision.Value)
-		for _, cidr := range a.whitelists.Cidrs {
-			if skip {
-				break
-			}
-			if cidr.Contains(ipval) {
-				log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, cidr.String())
-				skip = true
-			}
-		}
-		for _, ip := range a.whitelists.Ips {
-			if skip {
-				break
-			}
-			if ip != nil && ip.Equal(ipval) {
-				log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, ip.String())
-				skip = true
-			}
-		}
-		if !skip {
 		decisions[outIdx] = decision
 		outIdx++
 	}
-
-	}
 	//shrink the list, those are deleted items
-	decisions = decisions[:outIdx]
+	return decisions[:outIdx]
-	return decisions
 }
 
 func (a *apic) SaveAlerts(alertsFromCapi []*models.Alert, add_counters map[string]map[string]int, delete_counters map[string]map[string]int) error {
-	for idx, alert := range alertsFromCapi {
+	for _, alert := range alertsFromCapi {
-		alertsFromCapi[idx] = setAlertScenario(add_counters, delete_counters, alert)
+		setAlertScenario(alert, add_counters, delete_counters)
-		log.Debugf("%s has %d decisions", *alertsFromCapi[idx].Source.Scope, len(alertsFromCapi[idx].Decisions))
+		log.Debugf("%s has %d decisions", *alert.Source.Scope, len(alert.Decisions))
 		if a.dbClient.Type == "sqlite" && (a.dbClient.WalMode == nil || !*a.dbClient.WalMode) {
 			log.Warningf("sqlite is not using WAL mode, LAPI might become unresponsive when inserting the community blocklist")
 		}
-		alertID, inserted, deleted, err := a.dbClient.UpdateCommunityBlocklist(alertsFromCapi[idx])
+		alertID, inserted, deleted, err := a.dbClient.UpdateCommunityBlocklist(alert)
 		if err != nil {
-			return fmt.Errorf("while saving alert from %s: %w", *alertsFromCapi[idx].Source.Scope, err)
+			return fmt.Errorf("while saving alert from %s: %w", *alert.Source.Scope, err)
 		}
-		log.Printf("%s : added %d entries, deleted %d entries (alert:%d)", *alertsFromCapi[idx].Source.Scope, inserted, deleted, alertID)
+		log.Printf("%s : added %d entries, deleted %d entries (alert:%d)", *alert.Source.Scope, inserted, deleted, alertID)
 	}
 
 	return nil
@@ -708,6 +704,60 @@ func (a *apic) ShouldForcePullBlocklist(blocklist *modelscapi.BlocklistLink) (bo
 	return false, nil
 }
 
+func (a *apic) updateBlocklist(client *apiclient.ApiClient, blocklist *modelscapi.BlocklistLink, add_counters map[string]map[string]int) error {
+	if blocklist.Scope == nil {
+		log.Warningf("blocklist has no scope")
+		return nil
+	}
+	if blocklist.Duration == nil {
+		log.Warningf("blocklist has no duration")
+		return nil
+	}
+	forcePull, err := a.ShouldForcePullBlocklist(blocklist)
+	if err != nil {
+		return fmt.Errorf("while checking if we should force pull blocklist %s: %w", *blocklist.Name, err)
+	}
+	blocklistConfigItemName := fmt.Sprintf("blocklist:%s:last_pull", *blocklist.Name)
+	var lastPullTimestamp *string
+	if !forcePull {
+		lastPullTimestamp, err = a.dbClient.GetConfigItem(blocklistConfigItemName)
+		if err != nil {
+			return fmt.Errorf("while getting last pull timestamp for blocklist %s: %w", *blocklist.Name, err)
+		}
+	}
+	decisions, hasChanged, err := client.Decisions.GetDecisionsFromBlocklist(context.Background(), blocklist, lastPullTimestamp)
+	if err != nil {
+		return fmt.Errorf("while getting decisions from blocklist %s: %w", *blocklist.Name, err)
+	}
+	if !hasChanged {
+		if lastPullTimestamp == nil {
+			log.Infof("blocklist %s hasn't been modified or there was an error reading it, skipping", *blocklist.Name)
+		} else {
+			log.Infof("blocklist %s hasn't been modified since %s, skipping", *blocklist.Name, *lastPullTimestamp)
+		}
+		return nil
+	}
+	err = a.dbClient.SetConfigItem(blocklistConfigItemName, time.Now().UTC().Format(http.TimeFormat))
+	if err != nil {
+		return fmt.Errorf("while setting last pull timestamp for blocklist %s: %w", *blocklist.Name, err)
+	}
+	if len(decisions) == 0 {
+		log.Infof("blocklist %s has no decisions", *blocklist.Name)
+		return nil
+	}
+	//apply APIC specific whitelists
+	decisions = a.ApplyApicWhitelists(decisions)
+	alert := createAlertForDecision(decisions[0])
+	alertsFromCapi := []*models.Alert{alert}
+	alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, add_counters)
+
+	err = a.SaveAlerts(alertsFromCapi, add_counters, nil)
+	if err != nil {
+		return fmt.Errorf("while saving alert from blocklist %s: %w", *blocklist.Name, err)
+	}
+	return nil
+}
+
 func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLinks, add_counters map[string]map[string]int) error {
 	if links == nil {
 		return nil
@@ -722,69 +772,21 @@ func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLink
 		return fmt.Errorf("while creating default client: %w", err)
 	}
 	for _, blocklist := range links.Blocklists {
-		if blocklist.Scope == nil {
+		if err := a.updateBlocklist(defaultClient, blocklist, add_counters); err != nil {
-			log.Warningf("blocklist has no scope")
+			return err
-			continue
-		}
-		if blocklist.Duration == nil {
-			log.Warningf("blocklist has no duration")
-			continue
-		}
-		forcePull, err := a.ShouldForcePullBlocklist(blocklist)
-		if err != nil {
-			return fmt.Errorf("while checking if we should force pull blocklist %s: %w", *blocklist.Name, err)
-		}
-		blocklistConfigItemName := fmt.Sprintf("blocklist:%s:last_pull", *blocklist.Name)
-		var lastPullTimestamp *string
-		if !forcePull {
-			lastPullTimestamp, err = a.dbClient.GetConfigItem(blocklistConfigItemName)
-			if err != nil {
-				return fmt.Errorf("while getting last pull timestamp for blocklist %s: %w", *blocklist.Name, err)
-			}
-		}
-		decisions, has_changed, err := defaultClient.Decisions.GetDecisionsFromBlocklist(context.Background(), blocklist, lastPullTimestamp)
-		if err != nil {
-			return fmt.Errorf("while getting decisions from blocklist %s: %w", *blocklist.Name, err)
-		}
-		if !has_changed {
-			if lastPullTimestamp == nil {
-				log.Infof("blocklist %s hasn't been modified or there was an error reading it, skipping", *blocklist.Name)
-			} else {
-				log.Infof("blocklist %s hasn't been modified since %s, skipping", *blocklist.Name, *lastPullTimestamp)
-			}
-			continue
-		}
-		err = a.dbClient.SetConfigItem(blocklistConfigItemName, time.Now().UTC().Format(http.TimeFormat))
-		if err != nil {
-			return fmt.Errorf("while setting last pull timestamp for blocklist %s: %w", *blocklist.Name, err)
-		}
-		if len(decisions) == 0 {
-			log.Infof("blocklist %s has no decisions", *blocklist.Name)
-			continue
-		}
-		//apply APIC specific whitelists
-		decisions = a.ApplyApicWhitelists(decisions)
-		alert := createAlertForDecision(decisions[0])
-		alertsFromCapi := []*models.Alert{alert}
-		alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, add_counters)
-
-		err = a.SaveAlerts(alertsFromCapi, add_counters, nil)
-		if err != nil {
-			return fmt.Errorf("while saving alert from blocklist %s: %w", *blocklist.Name, err)
 		}
 	}
 	return nil
 }
 
-func setAlertScenario(add_counters map[string]map[string]int, delete_counters map[string]map[string]int, alert *models.Alert) *models.Alert {
+func setAlertScenario(alert *models.Alert, add_counters map[string]map[string]int, delete_counters map[string]map[string]int) {
 	if *alert.Source.Scope == types.CAPIOrigin {
-		*alert.Source.Scope = SCOPE_CAPI_ALIAS_ALIAS
+		*alert.Source.Scope = types.CommunityBlocklistPullSourceScope
 		alert.Scenario = ptr.Of(fmt.Sprintf("update : +%d/-%d IPs", add_counters[types.CAPIOrigin]["all"], delete_counters[types.CAPIOrigin]["all"]))
 	} else if *alert.Source.Scope == types.ListOrigin {
 		*alert.Source.Scope = fmt.Sprintf("%s:%s", types.ListOrigin, *alert.Scenario)
 		alert.Scenario = ptr.Of(fmt.Sprintf("update : +%d/-%d IPs", add_counters[types.ListOrigin][*alert.Scenario], delete_counters[types.ListOrigin][*alert.Scenario]))
 	}
-	return alert
 }
 
 func (a *apic) Pull() error {

pkg/apiserver/apic_metrics.go

@@ -2,10 +2,10 @@ package apiserver
 
 import (
 	"context"
+	"slices"
 	"time"
 
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/exp/slices"
 
 	"github.com/crowdsecurity/go-cs-lib/ptr"
 	"github.com/crowdsecurity/go-cs-lib/trace"
@@ -83,7 +83,7 @@ func (a *apic) SendMetrics(stop chan (bool)) {
 	// intervals must always be > 0
 	metInts := []time.Duration{1*time.Millisecond, a.metricsIntervalFirst, a.metricsInterval}
 
-	log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)",
+	log.Infof("Start sending metrics to CrowdSec Central API (interval: %s once, then %s)",
 		metInts[1].Round(time.Second), metInts[2])
 
 	count := -1
@@ -129,13 +129,16 @@ func (a *apic) SendMetrics(stop chan (bool)) {
 			metTicker.Stop()
 			metrics, err := a.GetMetrics()
 			if err != nil {
-				log.Errorf("unable to get metrics (%s), will retry", err)
+				log.Errorf("unable to get metrics (%s)", err)
 			}
+			// metrics are nil if they could not be retrieved
+			if metrics != nil {
 				log.Info("capi metrics: sending")
 				_, _, err = a.apiClient.Metrics.Add(context.Background(), metrics)
 				if err != nil {
 					log.Errorf("capi metrics: failed: %s", err)
 				}
+			}
 			metTicker.Reset(nextMetInt())
 		case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others?
 			checkTicker.Stop()

pkg/apiserver/apic_test.go

@@ -689,7 +689,7 @@ func TestAPICWhitelists(t *testing.T) {
 		alertScenario[alert.SourceScope]++
 	}
 	assert.Equal(t, 3, len(alertScenario))
-	assert.Equal(t, 1, alertScenario[SCOPE_CAPI_ALIAS_ALIAS])
+	assert.Equal(t, 1, alertScenario[types.CommunityBlocklistPullSourceScope])
 	assert.Equal(t, 1, alertScenario["lists:blocklist1"])
 	assert.Equal(t, 1, alertScenario["lists:blocklist2"])
 
@@ -818,7 +818,7 @@ func TestAPICPullTop(t *testing.T) {
 		alertScenario[alert.SourceScope]++
 	}
 	assert.Equal(t, 3, len(alertScenario))
-	assert.Equal(t, 1, alertScenario[SCOPE_CAPI_ALIAS_ALIAS])
+	assert.Equal(t, 1, alertScenario[types.CommunityBlocklistPullSourceScope])
 	assert.Equal(t, 1, alertScenario["lists:blocklist1"])
 	assert.Equal(t, 1, alertScenario["lists:blocklist2"])
 

pkg/apiserver/jwt_test.go

@@ -55,7 +55,7 @@ func TestLogin(t *testing.T) {
 	router.ServeHTTP(w, req)
 
 	assert.Equal(t, 401, w.Code)
-	assert.Equal(t, "{\"code\":401,\"message\":\"input format error\"}", w.Body.String())
+	assert.Equal(t, "{\"code\":401,\"message\":\"validation failure list:\\npassword in body is required\"}", w.Body.String())
 
 	//Validate machine
 	err = ValidateMachine("test", config.API.Server.DbConfig)

pkg/apiserver/middlewares/v1/api_key.go

@@ -33,7 +33,9 @@ func GenerateAPIKey(n int) (string, error) {
 	if _, err := rand.Read(bytes); err != nil {
 		return "", err
 	}
-	return base64.StdEncoding.EncodeToString(bytes), nil
+	encoded := base64.StdEncoding.EncodeToString(bytes)
+	// the '=' can cause issues on some bouncers
+	return strings.TrimRight(encoded, "="), nil
 }
 
 func NewAPIKey(dbClient *database.Client) *APIKey {

pkg/apiserver/middlewares/v1/jwt.go

@@ -2,6 +2,7 @@ package v1
 
 import (
 	"crypto/rand"
+	"errors"
 	"fmt"
 	"net/http"
 	"os"
@@ -16,7 +17,6 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/gin-gonic/gin"
 	"github.com/go-openapi/strfmt"
-	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/crypto/bcrypt"
 )
@@ -46,40 +46,46 @@ func IdentityHandler(c *gin.Context) interface{} {
 	}
 }
 
-func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
-	var loginInput models.WatcherAuthRequest
-	var scenarios string
-	var err error
-	var scenariosInput []string
-	var clientMachine *ent.Machine
-	var machineID string
 
-	if c.Request.TLS != nil && len(c.Request.TLS.PeerCertificates) > 0 {
+
+type authInput struct {
+	machineID string
+	clientMachine *ent.Machine
+	scenariosInput []string
+}
+
+
+
+func (j *JWT) authTLS(c *gin.Context) (*authInput, error) {
+	ret := authInput{}
+
 	if j.TlsAuth == nil {
 		c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
 		c.Abort()
 		return nil, errors.New("TLS auth is not configured")
 	}
+
 	validCert, extractedCN, err := j.TlsAuth.ValidateCert(c)
 	if err != nil {
 		log.Error(err)
 		c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
 		c.Abort()
-			return nil, errors.Wrap(err, "while trying to validate client cert")
+		return nil, fmt.Errorf("while trying to validate client cert: %w", err)
 	}
+
 	if !validCert {
 		c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
 		c.Abort()
 		return nil, fmt.Errorf("failed cert authentication")
 	}
 
-		machineID = fmt.Sprintf("%s@%s", extractedCN, c.ClientIP())
+	ret.machineID = fmt.Sprintf("%s@%s", extractedCN, c.ClientIP())
-		clientMachine, err = j.DbClient.Ent.Machine.Query().
+	ret.clientMachine, err = j.DbClient.Ent.Machine.Query().
-			Where(machine.MachineId(machineID)).
+		Where(machine.MachineId(ret.machineID)).
 		First(j.DbClient.CTX)
 	if ent.IsNotFound(err) {
 		//Machine was not found, let's create it
-			log.Printf("machine %s not found, create it", machineID)
+		log.Infof("machine %s not found, create it", ret.machineID)
 		//let's use an apikey as the password, doesn't matter in this case (generatePassword is only available in cscli)
 		pwd, err := GenerateAPIKey(dummyAPIKeySize)
 		if err != nil {
@@ -90,98 +96,126 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
 			return nil, fmt.Errorf("error generating password")
 		}
 		password := strfmt.Password(pwd)
-			clientMachine, err = j.DbClient.CreateMachine(&machineID, &password, "", true, true, types.TlsAuthType)
+		ret.clientMachine, err = j.DbClient.CreateMachine(&ret.machineID, &password, "", true, true, types.TlsAuthType)
 		if err != nil {
-				return "", errors.Wrapf(err, "while creating machine entry for %s", machineID)
+			return nil, fmt.Errorf("while creating machine entry for %s: %w", ret.machineID, err)
 		}
 	} else if err != nil {
-			return "", errors.Wrapf(err, "while selecting machine entry for %s", machineID)
+		return nil, fmt.Errorf("while selecting machine entry for %s: %w", ret.machineID, err)
 	} else {
-			if clientMachine.AuthType != types.TlsAuthType {
+		if ret.clientMachine.AuthType != types.TlsAuthType {
-				return "", errors.Errorf("machine %s attempted to auth with TLS cert but it is configured to use %s", machineID, clientMachine.AuthType)
+			return nil, fmt.Errorf("machine %s attempted to auth with TLS cert but it is configured to use %s", ret.machineID, ret.clientMachine.AuthType)
 		}
-			machineID = clientMachine.MachineId
+		ret.machineID = ret.clientMachine.MachineId
+	}
+
 	loginInput := struct {
 		Scenarios []string `json:"scenarios"`
 	}{
 		Scenarios: []string{},
 	}
-			err := c.ShouldBindJSON(&loginInput)
+	err = c.ShouldBindJSON(&loginInput)
 	if err != nil {
-				return "", errors.Wrap(err, "missing scenarios list in login request for TLS auth")
+		return nil, fmt.Errorf("missing scenarios list in login request for TLS auth: %w", err)
-			}
-			scenariosInput = loginInput.Scenarios
 	}
+	ret.scenariosInput = loginInput.Scenarios
 
-	} else {
+	return &ret, nil
-		//normal auth
+}
 
-		if err := c.ShouldBindJSON(&loginInput); err != nil {
+
-			return "", errors.Wrap(err, "missing")
+
+func (j *JWT) authPlain(c *gin.Context) (*authInput, error) {
+	var loginInput models.WatcherAuthRequest
+	var err error
+
+	ret := authInput{}
+
+	if err = c.ShouldBindJSON(&loginInput); err != nil {
+		return nil, fmt.Errorf("missing: %w", err)
 	}
-		if err := loginInput.Validate(strfmt.Default); err != nil {
+	if err = loginInput.Validate(strfmt.Default); err != nil {
-			return "", errors.New("input format error")
+		return nil, err
 	}
-		machineID = *loginInput.MachineID
+	ret.machineID = *loginInput.MachineID
 	password := *loginInput.Password
-		scenariosInput = loginInput.Scenarios
+	ret.scenariosInput = loginInput.Scenarios
 
-		clientMachine, err = j.DbClient.Ent.Machine.Query().
+	ret.clientMachine, err = j.DbClient.Ent.Machine.Query().
-			Where(machine.MachineId(machineID)).
+		Where(machine.MachineId(ret.machineID)).
 		First(j.DbClient.CTX)
 	if err != nil {
-			log.Printf("Error machine login for %s : %+v ", machineID, err)
+		log.Infof("Error machine login for %s : %+v ", ret.machineID, err)
 		return nil, err
 	}
 
-		if clientMachine == nil {
+	if ret.clientMachine == nil {
-			log.Errorf("Nothing for '%s'", machineID)
+		log.Errorf("Nothing for '%s'", ret.machineID)
 		return nil, jwt.ErrFailedAuthentication
 	}
 
-		if clientMachine.AuthType != types.PasswordAuthType {
+	if ret.clientMachine.AuthType != types.PasswordAuthType {
-			return nil, errors.Errorf("machine %s attempted to auth with password but it is configured to use %s", machineID, clientMachine.AuthType)
+		return nil, fmt.Errorf("machine %s attempted to auth with password but it is configured to use %s", ret.machineID, ret.clientMachine.AuthType)
 	}
 
-		if !clientMachine.IsValidated {
+	if !ret.clientMachine.IsValidated {
-			return nil, fmt.Errorf("machine %s not validated", machineID)
+		return nil, fmt.Errorf("machine %s not validated", ret.machineID)
 	}
 
-		if err = bcrypt.CompareHashAndPassword([]byte(clientMachine.Password), []byte(password)); err != nil {
+	if err := bcrypt.CompareHashAndPassword([]byte(ret.clientMachine.Password), []byte(password)); err != nil {
 		return nil, jwt.ErrFailedAuthentication
 	}
 
-		//end of normal auth
+	return &ret, nil
+}
+
+
+func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
+	var err error
+	var auth *authInput
+
+	if c.Request.TLS != nil && len(c.Request.TLS.PeerCertificates) > 0 {
+		auth, err = j.authTLS(c)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		auth, err = j.authPlain(c)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	if len(scenariosInput) > 0 {
+	var scenarios string
-		for _, scenario := range scenariosInput {
+
+	if len(auth.scenariosInput) > 0 {
+		for _, scenario := range auth.scenariosInput {
 			if scenarios == "" {
 				scenarios = scenario
 			} else {
 				scenarios += "," + scenario
 			}
 		}
-		err = j.DbClient.UpdateMachineScenarios(scenarios, clientMachine.ID)
+		err = j.DbClient.UpdateMachineScenarios(scenarios, auth.clientMachine.ID)
 		if err != nil {
-			log.Errorf("Failed to update scenarios list for '%s': %s\n", machineID, err)
+			log.Errorf("Failed to update scenarios list for '%s': %s\n", auth.machineID, err)
 			return nil, jwt.ErrFailedAuthentication
 		}
 	}
 
-	if clientMachine.IpAddress == "" {
+	if auth.clientMachine.IpAddress == "" {
-		err = j.DbClient.UpdateMachineIP(c.ClientIP(), clientMachine.ID)
+		err = j.DbClient.UpdateMachineIP(c.ClientIP(), auth.clientMachine.ID)
 		if err != nil {
-			log.Errorf("Failed to update ip address for '%s': %s\n", machineID, err)
+			log.Errorf("Failed to update ip address for '%s': %s\n", auth.machineID, err)
 			return nil, jwt.ErrFailedAuthentication
 		}
 	}
 
-	if clientMachine.IpAddress != c.ClientIP() && clientMachine.IpAddress != "" {
+	if auth.clientMachine.IpAddress != c.ClientIP() && auth.clientMachine.IpAddress != "" {
-		log.Warningf("new IP address detected for machine '%s': %s (old: %s)", clientMachine.MachineId, c.ClientIP(), clientMachine.IpAddress)
+		log.Warningf("new IP address detected for machine '%s': %s (old: %s)", auth.clientMachine.MachineId, c.ClientIP(), auth.clientMachine.IpAddress)
-		err = j.DbClient.UpdateMachineIP(c.ClientIP(), clientMachine.ID)
+		err = j.DbClient.UpdateMachineIP(c.ClientIP(), auth.clientMachine.ID)
 		if err != nil {
-			log.Errorf("Failed to update ip address for '%s': %s\n", clientMachine.MachineId, err)
+			log.Errorf("Failed to update ip address for '%s': %s\n", auth.clientMachine.MachineId, err)
 			return nil, jwt.ErrFailedAuthentication
 		}
 	}
@@ -192,13 +226,13 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
 		return nil, jwt.ErrFailedAuthentication
 	}
 
-	if err := j.DbClient.UpdateMachineVersion(useragent[1], clientMachine.ID); err != nil {
+	if err := j.DbClient.UpdateMachineVersion(useragent[1], auth.clientMachine.ID); err != nil {
-		log.Errorf("unable to update machine '%s' version '%s': %s", clientMachine.MachineId, useragent[1], err)
+		log.Errorf("unable to update machine '%s' version '%s': %s", auth.clientMachine.MachineId, useragent[1], err)
 		log.Errorf("bad user agent from : %s", c.ClientIP())
 		return nil, jwt.ErrFailedAuthentication
 	}
 	return &models.WatcherAuthRequest{
-		MachineID: &machineID,
+		MachineID: &auth.machineID,
 	}, nil
 
 }

pkg/csconfig/api.go

@@ -3,7 +3,9 @@ package csconfig
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
+	"io"
 	"net"
 	"os"
 	"strings"
@@ -56,7 +58,6 @@ type CTICfg struct {
 }
 
 func (a *CTICfg) Load() error {
-
 	if a.Key == nil {
 		*a.Enabled = false
 	}
@@ -331,40 +332,59 @@ type capiWhitelists struct {
 	Cidrs []string `yaml:"cidrs"`
 }
 
+func parseCapiWhitelists(fd io.Reader) (*CapiWhitelist, error) {
+	fromCfg := capiWhitelists{}
+
+	decoder := yaml.NewDecoder(fd)
+	if err := decoder.Decode(&fromCfg); err != nil {
+		if errors.Is(err, io.EOF) {
+			return nil, fmt.Errorf("empty file")
+		}
+		return nil, err
+	}
+	ret := &CapiWhitelist{
+		Ips:   make([]net.IP, len(fromCfg.Ips)),
+		Cidrs: make([]*net.IPNet, len(fromCfg.Cidrs)),
+	}
+	for idx, v := range fromCfg.Ips {
+		ip := net.ParseIP(v)
+		if ip == nil {
+			return nil, fmt.Errorf("invalid IP address: %s", v)
+		}
+		ret.Ips[idx] = ip
+	}
+	for idx, v := range fromCfg.Cidrs {
+		_, tnet, err := net.ParseCIDR(v)
+		if err != nil {
+			return nil, err
+		}
+		ret.Cidrs[idx] = tnet
+	}
+
+	return ret, nil
+}
+
 func (s *LocalApiServerCfg) LoadCapiWhitelists() error {
 	if s.CapiWhitelistsPath == "" {
 		return nil
 	}
+
 	if _, err := os.Stat(s.CapiWhitelistsPath); os.IsNotExist(err) {
 		return fmt.Errorf("capi whitelist file '%s' does not exist", s.CapiWhitelistsPath)
 	}
+
 	fd, err := os.Open(s.CapiWhitelistsPath)
 	if err != nil {
-		return fmt.Errorf("unable to open capi whitelist file '%s': %s", s.CapiWhitelistsPath, err)
+		return fmt.Errorf("while opening capi whitelist file: %s", err)
 	}
 
-	var fromCfg capiWhitelists
-	s.CapiWhitelists = &CapiWhitelist{}
-
 	defer fd.Close()
-	decoder := yaml.NewDecoder(fd)
+
-	if err := decoder.Decode(&fromCfg); err != nil {
+	s.CapiWhitelists, err = parseCapiWhitelists(fd)
-		return fmt.Errorf("while parsing capi whitelist file '%s': %s", s.CapiWhitelistsPath, err)
-	}
-	for _, v := range fromCfg.Ips {
-		ip := net.ParseIP(v)
-		if ip == nil {
-			return fmt.Errorf("unable to parse ip whitelist '%s'", v)
-		}
-		s.CapiWhitelists.Ips = append(s.CapiWhitelists.Ips, ip)
-	}
-	for _, v := range fromCfg.Cidrs {
-		_, tnet, err := net.ParseCIDR(v)
 	if err != nil {
-			return fmt.Errorf("unable to parse cidr whitelist '%s' : %v", v, err)
+		return fmt.Errorf("while parsing capi whitelist file '%s': %w", s.CapiWhitelistsPath, err)
-		}
-		s.CapiWhitelists.Cidrs = append(s.CapiWhitelists.Cidrs, tnet)
 	}
+
 	return nil
 }
 

pkg/csconfig/api_test.go

@@ -1,7 +1,7 @@
 package csconfig
 
 import (
-	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"strings"
@@ -9,6 +9,7 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/go-cs-lib/cstest"
@@ -25,7 +26,7 @@ func TestLoadLocalApiClientCfg(t *testing.T) {
 		{
 			name: "basic valid configuration",
 			input: &LocalApiClientCfg{
-				CredentialsFilePath: "./tests/lapi-secrets.yaml",
+				CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 			},
 			expected: &ApiCredentialsCfg{
 				URL:      "http://localhost:8080/",
@@ -36,7 +37,7 @@ func TestLoadLocalApiClientCfg(t *testing.T) {
 		{
 			name: "invalid configuration",
 			input: &LocalApiClientCfg{
-				CredentialsFilePath: "./tests/bad_lapi-secrets.yaml",
+				CredentialsFilePath: "./testdata/bad_lapi-secrets.yaml",
 			},
 			expected:    &ApiCredentialsCfg{},
 			expectedErr: "field unknown_key not found in type csconfig.ApiCredentialsCfg",
@@ -44,15 +45,15 @@ func TestLoadLocalApiClientCfg(t *testing.T) {
 		{
 			name: "invalid configuration filepath",
 			input: &LocalApiClientCfg{
-				CredentialsFilePath: "./tests/nonexist_lapi-secrets.yaml",
+				CredentialsFilePath: "./testdata/nonexist_lapi-secrets.yaml",
 			},
 			expected:    nil,
-			expectedErr: "open ./tests/nonexist_lapi-secrets.yaml: " + cstest.FileNotFoundMessage,
+			expectedErr: "open ./testdata/nonexist_lapi-secrets.yaml: " + cstest.FileNotFoundMessage,
 		},
 		{
 			name: "valid configuration with insecure skip verify",
 			input: &LocalApiClientCfg{
-				CredentialsFilePath: "./tests/lapi-secrets.yaml",
+				CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 				InsecureSkipVerify:  ptr.Of(false),
 			},
 			expected: &ApiCredentialsCfg{
@@ -87,7 +88,7 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
 		{
 			name: "basic valid configuration",
 			input: &OnlineApiClientCfg{
-				CredentialsFilePath: "./tests/online-api-secrets.yaml",
+				CredentialsFilePath: "./testdata/online-api-secrets.yaml",
 			},
 			expected: &ApiCredentialsCfg{
 				URL:      "http://crowdsec.api",
@@ -98,7 +99,7 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
 		{
 			name: "invalid configuration",
 			input: &OnlineApiClientCfg{
-				CredentialsFilePath: "./tests/bad_lapi-secrets.yaml",
+				CredentialsFilePath: "./testdata/bad_lapi-secrets.yaml",
 			},
 			expected:    &ApiCredentialsCfg{},
 			expectedErr: "failed unmarshaling api server credentials",
@@ -106,14 +107,14 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
 		{
 			name: "missing field configuration",
 			input: &OnlineApiClientCfg{
-				CredentialsFilePath: "./tests/bad_online-api-secrets.yaml",
+				CredentialsFilePath: "./testdata/bad_online-api-secrets.yaml",
 			},
 			expected: nil,
 		},
 		{
 			name: "invalid configuration filepath",
 			input: &OnlineApiClientCfg{
-				CredentialsFilePath: "./tests/nonexist_online-api-secrets.yaml",
+				CredentialsFilePath: "./testdata/nonexist_online-api-secrets.yaml",
 			},
 			expected:    &ApiCredentialsCfg{},
 			expectedErr: "failed to read api server credentials",
@@ -136,27 +137,23 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
 
 func TestLoadAPIServer(t *testing.T) {
 	tmpLAPI := &LocalApiServerCfg{
-		ProfilesPath: "./tests/profiles.yaml",
+		ProfilesPath: "./testdata/profiles.yaml",
-	}
-	if err := tmpLAPI.LoadProfiles(); err != nil {
-		t.Fatalf("loading tmp profiles: %+v", err)
 	}
+	err := tmpLAPI.LoadProfiles()
+	require.NoError(t, err)
+
+	LogDirFullPath, err := filepath.Abs("./testdata")
+	require.NoError(t, err)
 
-	LogDirFullPath, err := filepath.Abs("./tests")
-	if err != nil {
-		t.Fatal(err)
-	}
 	logLevel := log.InfoLevel
 	config := &Config{}
-	fcontent, err := os.ReadFile("./tests/config.yaml")
+	fcontent, err := os.ReadFile("./testdata/config.yaml")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
+
-	}
 	configData := os.ExpandEnv(string(fcontent))
 	err = yaml.UnmarshalStrict([]byte(configData), &config)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
+
-	}
 	tests := []struct {
 		name        string
 		input       *Config
@@ -171,18 +168,18 @@ func TestLoadAPIServer(t *testing.T) {
 					Server: &LocalApiServerCfg{
 						ListenURI: "http://crowdsec.api",
 						OnlineClient: &OnlineApiClientCfg{
-							CredentialsFilePath: "./tests/online-api-secrets.yaml",
+							CredentialsFilePath: "./testdata/online-api-secrets.yaml",
 						},
-						ProfilesPath: "./tests/profiles.yaml",
+						ProfilesPath: "./testdata/profiles.yaml",
 						PapiLogLevel: &logLevel,
 					},
 				},
 				DbConfig: &DatabaseCfg{
 					Type:   "sqlite",
-					DbPath: "./tests/test.db",
+					DbPath: "./testdata/test.db",
 				},
 				Common: &CommonCfg{
-					LogDir:   "./tests/",
+					LogDir:   "./testdata/",
 					LogMedia: "stdout",
 				},
 				DisableAPI: false,
@@ -192,9 +189,10 @@ func TestLoadAPIServer(t *testing.T) {
 				ListenURI: "http://crowdsec.api",
 				TLS:       nil,
 				DbConfig: &DatabaseCfg{
-					DbPath:       "./tests/test.db",
+					DbPath:           "./testdata/test.db",
 					Type:             "sqlite",
 					MaxOpenConns:     ptr.Of(DEFAULT_MAX_OPEN_CONNS),
+					DecisionBulkSize: defaultDecisionBulkSize,
 				},
 				ConsoleConfigPath: DefaultConfigPath("console.yaml"),
 				ConsoleConfig: &ConsoleConfig{
@@ -207,7 +205,7 @@ func TestLoadAPIServer(t *testing.T) {
 				LogDir:   LogDirFullPath,
 				LogMedia: "stdout",
 				OnlineClient: &OnlineApiClientCfg{
-					CredentialsFilePath: "./tests/online-api-secrets.yaml",
+					CredentialsFilePath: "./testdata/online-api-secrets.yaml",
 					Credentials: &ApiCredentialsCfg{
 						URL:      "http://crowdsec.api",
 						Login:    "test",
@@ -215,7 +213,7 @@ func TestLoadAPIServer(t *testing.T) {
 					},
 				},
 				Profiles:               tmpLAPI.Profiles,
-				ProfilesPath:           "./tests/profiles.yaml",
+				ProfilesPath:           "./testdata/profiles.yaml",
 				UseForwardedForHeaders: false,
 				PapiLogLevel:           &logLevel,
 			},
@@ -228,7 +226,7 @@ func TestLoadAPIServer(t *testing.T) {
 					Server: &LocalApiServerCfg{},
 				},
 				Common: &CommonCfg{
-					LogDir:   "./tests/",
+					LogDir:   "./testdata/",
 					LogMedia: "stdout",
 				},
 				DisableAPI: false,
@@ -241,21 +239,78 @@ func TestLoadAPIServer(t *testing.T) {
 		},
 	}
 
-	for idx, test := range tests {
+	for _, tc := range tests {
-		err := test.input.LoadAPIServer()
+		tc := tc
-		if err == nil && test.expectedErr != "" {
+		t.Run(tc.name, func(t *testing.T) {
-			fmt.Printf("TEST '%s': NOK\n", test.name)
+			err := tc.input.LoadAPIServer()
-			t.Fatalf("Test number %d/%d expected error, didn't get it", idx+1, len(tests))
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
-		} else if test.expectedErr != "" {
+			if tc.expectedErr != "" {
-			fmt.Printf("ERR: %+v\n", err)
+				return
-			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.expectedErr) {
-				fmt.Printf("TEST '%s': NOK\n", test.name)
-				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
-					test.expectedErr,
-					fmt.Sprintf("%s", err))
 			}
 
-			assert.Equal(t, test.expected, test.input.API.Server)
+			assert.Equal(t, tc.expected, tc.input.API.Server)
-		}
+		})
+	}
+}
+
+func mustParseCIDRNet(t *testing.T, s string) *net.IPNet {
+	_, ipNet, err := net.ParseCIDR(s)
+	require.NoError(t, err)
+	return ipNet
+}
+
+func TestParseCapiWhitelists(t *testing.T) {
+	tests := []struct {
+		name        string
+		input       string
+		expected    *CapiWhitelist
+		expectedErr string
+	}{
+		{
+			name:  "empty file",
+			input: "",
+			expected: &CapiWhitelist{
+				Ips:   []net.IP{},
+				Cidrs: []*net.IPNet{},
+			},
+			expectedErr: "empty file",
+		},
+		{
+			name:  "empty ip and cidr",
+			input: `{"ips": [], "cidrs": []}`,
+			expected: &CapiWhitelist{
+				Ips:   []net.IP{},
+				Cidrs: []*net.IPNet{},
+			},
+		},
+		{
+			name:  "some ip",
+			input: `{"ips": ["1.2.3.4"]}`,
+			expected: &CapiWhitelist{
+				Ips:   []net.IP{net.IPv4(1, 2, 3, 4)},
+				Cidrs: []*net.IPNet{},
+			},
+		},
+		{
+			name:  "some cidr",
+			input: `{"cidrs": ["1.2.3.0/24"]}`,
+			expected: &CapiWhitelist{
+				Ips:   []net.IP{},
+				Cidrs: []*net.IPNet{mustParseCIDRNet(t, "1.2.3.0/24")},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			wl, err := parseCapiWhitelists(strings.NewReader(tc.input))
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
+			if tc.expectedErr != "" {
+				return
+			}
+
+			assert.Equal(t, tc.expected, wl)
+		})
 	}
 }

pkg/csconfig/common_test.go

@@ -1,44 +1,41 @@
 package csconfig
 
 import (
-	"fmt"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadCommon(t *testing.T) {
-	pidDirPath := "./tests"
+	pidDirPath := "./testdata"
-	LogDirFullPath, err := filepath.Abs("./tests/log/")
+	LogDirFullPath, err := filepath.Abs("./testdata/log/")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
-	WorkingDirFullPath, err := filepath.Abs("./tests")
+	WorkingDirFullPath, err := filepath.Abs("./testdata")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
 	tests := []struct {
 		name        string
-		Input          *Config
+		input       *Config
-		expectedResult *CommonCfg
+		expected    *CommonCfg
-		err            string
+		expectedErr string
 	}{
 		{
 			name: "basic valid configuration",
-			Input: &Config{
+			input: &Config{
 				Common: &CommonCfg{
 					Daemonize:  true,
-					PidDir:     "./tests",
+					PidDir:     "./testdata",
 					LogMedia:   "file",
-					LogDir:     "./tests/log/",
+					LogDir:     "./testdata/log/",
-					WorkingDir: "./tests/",
+					WorkingDir: "./testdata/",
 				},
 			},
-			expectedResult: &CommonCfg{
+			expected: &CommonCfg{
 				Daemonize:  true,
 				PidDir:     pidDirPath,
 				LogMedia:   "file",
@@ -48,15 +45,15 @@ func TestLoadCommon(t *testing.T) {
 		},
 		{
 			name: "empty working dir",
-			Input: &Config{
+			input: &Config{
 				Common: &CommonCfg{
 					Daemonize: true,
-					PidDir:    "./tests",
+					PidDir:    "./testdata",
 					LogMedia:  "file",
-					LogDir:    "./tests/log/",
+					LogDir:    "./testdata/log/",
 				},
 			},
-			expectedResult: &CommonCfg{
+			expected: &CommonCfg{
 				Daemonize: true,
 				PidDir:    pidDirPath,
 				LogMedia:  "file",
@@ -65,30 +62,22 @@ func TestLoadCommon(t *testing.T) {
 		},
 		{
 			name:        "no common",
-			Input:          &Config{},
+			input:       &Config{},
-			expectedResult: nil,
+			expected:    nil,
+			expectedErr: "no common block provided in configuration file",
 		},
 	}
 
-	for idx, test := range tests {
+	for _, tc := range tests {
-		err := test.Input.LoadCommon()
+		tc := tc
-		if err == nil && test.err != "" {
+		t.Run(tc.name, func(t *testing.T) {
-			fmt.Printf("TEST '%s': NOK\n", test.name)
+			err := tc.input.LoadCommon()
-			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
-		} else if test.err != "" {
+			if tc.expectedErr != "" {
-			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				return
-				fmt.Printf("TEST '%s': NOK\n", test.name)
-				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
-					test.err,
-					fmt.Sprintf("%s", err))
-			}
 			}
 
-		isOk := assert.Equal(t, test.expectedResult, test.Input.Common)
+			assert.Equal(t, tc.expected, tc.input.Common)
-		if !isOk {
+		})
-			t.Fatalf("TEST '%s': NOK", test.name)
-		} else {
-			fmt.Printf("TEST '%s': OK\n", test.name)
-		}
 	}
 }

pkg/csconfig/config.go

@@ -1,3 +1,5 @@
+// Package csconfig contains the configuration structures for crowdsec and cscli.
+
 package csconfig
 
 import (
@@ -37,15 +39,6 @@ type Config struct {
 	Hub          *Hub                `yaml:"-"`
 }
 
-func (c *Config) Dump() error {
-	out, err := yaml.Marshal(c)
-	if err != nil {
-		return fmt.Errorf("failed marshaling config: %w", err)
-	}
-	fmt.Printf("%s", string(out))
-	return nil
-}
-
 func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {
 	patcher := yamlpatch.NewPatcher(configFile, ".local")
 	patcher.SetQuiet(quiet)

pkg/csconfig/config_test.go

@@ -5,42 +5,43 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestNormalLoad(t *testing.T) {
-	_, _, err := NewConfig("./tests/config.yaml", false, false, false)
+	_, _, err := NewConfig("./testdata/config.yaml", false, false, false)
 	require.NoError(t, err)
 
-	_, _, err = NewConfig("./tests/xxx.yaml", false, false, false)
+	_, _, err = NewConfig("./testdata/xxx.yaml", false, false, false)
-	assert.EqualError(t, err, "while reading yaml file: open ./tests/xxx.yaml: "+cstest.FileNotFoundMessage)
+	assert.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
 
-	_, _, err = NewConfig("./tests/simulation.yaml", false, false, false)
+	_, _, err = NewConfig("./testdata/simulation.yaml", false, false, false)
-	assert.EqualError(t, err, "./tests/simulation.yaml: yaml: unmarshal errors:\n  line 1: field simulation not found in type csconfig.Config")
+	assert.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n  line 1: field simulation not found in type csconfig.Config")
 }
 
 func TestNewCrowdSecConfig(t *testing.T) {
 	tests := []struct {
 		name     string
-		expectedResult *Config
+		expected *Config
 	}{
 		{
 			name:     "new configuration: basic",
-			expectedResult: &Config{},
+			expected: &Config{},
 		},
 	}
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			result := &Config{}
-			assert.Equal(t, tc.expectedResult, result)
+			assert.Equal(t, tc.expected, result)
 		})
 	}
 }
 
 func TestDefaultConfig(t *testing.T) {
 	x := NewDefaultConfig()
-	err := x.Dump()
+	_, err := yaml.Marshal(x)
-	require.NoError(t, err)
+	require.NoError(t, err, "failed marshaling config: %s", err)
 }

pkg/csconfig/console.go

@@ -82,23 +82,3 @@ func (c *LocalApiServerCfg) LoadConsoleConfig() error {
 
 	return nil
 }
-
-func (c *LocalApiServerCfg) DumpConsoleConfig() error {
-	var out []byte
-	var err error
-
-	if out, err = yaml.Marshal(c.ConsoleConfig); err != nil {
-		return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleConfigPath, err)
-	}
-	if c.ConsoleConfigPath == "" {
-		c.ConsoleConfigPath = DefaultConsoleConfigFilePath
-		log.Debugf("Empty console_path, defaulting to %s", c.ConsoleConfigPath)
-
-	}
-
-	if err := os.WriteFile(c.ConsoleConfigPath, out, 0600); err != nil {
-		return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleConfigPath, err)
-	}
-
-	return nil
-}

pkg/csconfig/crowdsec_service_test.go

@@ -1,24 +1,23 @@
 package csconfig
 
 import (
-	"fmt"
 	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/crowdsecurity/go-cs-lib/cstest"
 	"github.com/crowdsecurity/go-cs-lib/ptr"
-
-	"github.com/stretchr/testify/require"
 )
 
 func TestLoadCrowdsec(t *testing.T) {
-	acquisFullPath, err := filepath.Abs("./tests/acquis.yaml")
+	acquisFullPath, err := filepath.Abs("./testdata/acquis.yaml")
 	require.NoError(t, err)
 
-	acquisInDirFullPath, err := filepath.Abs("./tests/acquis/acquis.yaml")
+	acquisInDirFullPath, err := filepath.Abs("./testdata/acquis/acquis.yaml")
 	require.NoError(t, err)
 
-	acquisDirFullPath, err := filepath.Abs("./tests/acquis")
+	acquisDirFullPath, err := filepath.Abs("./testdata/acquis")
 	require.NoError(t, err)
 
 	hubFullPath, err := filepath.Abs("./hub")
@@ -27,42 +26,42 @@ func TestLoadCrowdsec(t *testing.T) {
 	dataFullPath, err := filepath.Abs("./data")
 	require.NoError(t, err)
 
-	configDirFullPath, err := filepath.Abs("./tests")
+	configDirFullPath, err := filepath.Abs("./testdata")
 	require.NoError(t, err)
 
 	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
 	require.NoError(t, err)
 
-	contextFileFullPath, err := filepath.Abs("./tests/context.yaml")
+	contextFileFullPath, err := filepath.Abs("./testdata/context.yaml")
 	require.NoError(t, err)
 
 	tests := []struct {
 		name        string
 		input       *Config
-		expectedResult *CrowdsecServiceCfg
+		expected    *CrowdsecServiceCfg
 		expectedErr string
 	}{
 		{
 			name: "basic valid configuration",
 			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir: "./tests",
+					ConfigDir: "./testdata",
 					DataDir:   "./data",
 					HubDir:    "./hub",
 				},
 				API: &APICfg{
 					Client: &LocalApiClientCfg{
-						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+						CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
-					AcquisitionFilePath:       "./tests/acquis.yaml",
+					AcquisitionFilePath:       "./testdata/acquis.yaml",
-					SimulationFilePath:        "./tests/simulation.yaml",
+					SimulationFilePath:        "./testdata/simulation.yaml",
-					ConsoleContextPath:        "./tests/context.yaml",
+					ConsoleContextPath:        "./testdata/context.yaml",
 					ConsoleContextValueLength: 2500,
 				},
 			},
-			expectedResult: &CrowdsecServiceCfg{
+			expected: &CrowdsecServiceCfg{
 				Enable:                    ptr.Of(true),
 				AcquisitionDirPath:        "",
 				ConsoleContextPath:        contextFileFullPath,
@@ -76,7 +75,7 @@ func TestLoadCrowdsec(t *testing.T) {
 				OutputRoutinesCount:       1,
 				ConsoleContextValueLength: 2500,
 				AcquisitionFiles:          []string{acquisFullPath},
-				SimulationFilePath:        "./tests/simulation.yaml",
+				SimulationFilePath:        "./testdata/simulation.yaml",
 				ContextToSend: map[string][]string{
 					"source_ip": {"evt.Parsed.source_ip"},
 				},
@@ -89,23 +88,23 @@ func TestLoadCrowdsec(t *testing.T) {
 			name: "basic valid configuration with acquisition dir",
 			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir: "./tests",
+					ConfigDir: "./testdata",
 					DataDir:   "./data",
 					HubDir:    "./hub",
 				},
 				API: &APICfg{
 					Client: &LocalApiClientCfg{
-						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+						CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
-					AcquisitionFilePath: "./tests/acquis.yaml",
+					AcquisitionFilePath: "./testdata/acquis.yaml",
-					AcquisitionDirPath:  "./tests/acquis/",
+					AcquisitionDirPath:  "./testdata/acquis/",
-					SimulationFilePath:  "./tests/simulation.yaml",
+					SimulationFilePath:  "./testdata/simulation.yaml",
-					ConsoleContextPath:  "./tests/context.yaml",
+					ConsoleContextPath:  "./testdata/context.yaml",
 				},
 			},
-			expectedResult: &CrowdsecServiceCfg{
+			expected: &CrowdsecServiceCfg{
 				Enable:                    ptr.Of(true),
 				AcquisitionDirPath:        acquisDirFullPath,
 				AcquisitionFilePath:       acquisFullPath,
@@ -122,7 +121,7 @@ func TestLoadCrowdsec(t *testing.T) {
 				ContextToSend: map[string][]string{
 					"source_ip": {"evt.Parsed.source_ip"},
 				},
-				SimulationFilePath: "./tests/simulation.yaml",
+				SimulationFilePath: "./testdata/simulation.yaml",
 				SimulationConfig: &SimulationConfig{
 					Simulation: ptr.Of(false),
 				},
@@ -132,13 +131,13 @@ func TestLoadCrowdsec(t *testing.T) {
 			name: "no acquisition file and dir",
 			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir: "./tests",
+					ConfigDir: "./testdata",
 					DataDir:   "./data",
 					HubDir:    "./hub",
 				},
 				API: &APICfg{
 					Client: &LocalApiClientCfg{
-						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+						CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
@@ -146,7 +145,7 @@ func TestLoadCrowdsec(t *testing.T) {
 					ConsoleContextValueLength: 10,
 				},
 			},
-			expectedResult: &CrowdsecServiceCfg{
+			expected: &CrowdsecServiceCfg{
 				Enable:                    ptr.Of(true),
 				AcquisitionDirPath:        "",
 				AcquisitionFilePath:       "",
@@ -173,18 +172,18 @@ func TestLoadCrowdsec(t *testing.T) {
 			name: "non existing acquisition file",
 			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir: "./tests",
+					ConfigDir: "./testdata",
 					DataDir:   "./data",
 					HubDir:    "./hub",
 				},
 				API: &APICfg{
 					Client: &LocalApiClientCfg{
-						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+						CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
 					ConsoleContextPath:  "",
-					AcquisitionFilePath: "./tests/acquis_not_exist.yaml",
+					AcquisitionFilePath: "./testdata/acquis_not_exist.yaml",
 				},
 			},
 			expectedErr: cstest.FileNotFoundMessage,
@@ -193,26 +192,25 @@ func TestLoadCrowdsec(t *testing.T) {
 			name: "agent disabled",
 			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir: "./tests",
+					ConfigDir: "./testdata",
 					DataDir:   "./data",
 					HubDir:    "./hub",
 				},
 			},
-			expectedResult: nil,
+			expected: nil,
 		},
 	}
 
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			fmt.Printf("TEST '%s'\n", tc.name)
 			err := tc.input.LoadCrowdsec()
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			if tc.expectedErr != "" {
 				return
 			}
 
-			require.Equal(t, tc.expectedResult, tc.input.Crowdsec)
+			require.Equal(t, tc.expected, tc.input.Crowdsec)
 		})
 	}
 }

pkg/csconfig/cscli_test.go

@@ -1,52 +1,45 @@
 package csconfig
 
 import (
-	"fmt"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadCSCLI(t *testing.T) {
 	hubFullPath, err := filepath.Abs("./hub")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
 	dataFullPath, err := filepath.Abs("./data")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
-	configDirFullPath, err := filepath.Abs("./tests")
+	configDirFullPath, err := filepath.Abs("./testdata")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
 	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
 	tests := []struct {
 		name        string
-		Input          *Config
+		input       *Config
-		expectedResult *CscliCfg
+		expected    *CscliCfg
-		err            string
+		expectedErr string
 	}{
 		{
 			name: "basic valid configuration",
-			Input: &Config{
+			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir:    "./tests",
+					ConfigDir:    "./testdata",
 					DataDir:      "./data",
 					HubDir:       "./hub",
 					HubIndexFile: "./hub/.index.json",
 				},
 			},
-			expectedResult: &CscliCfg{
+			expected: &CscliCfg{
 				ConfigDir:    configDirFullPath,
 				DataDir:      dataFullPath,
 				HubDir:       hubFullPath,
@@ -55,30 +48,22 @@ func TestLoadCSCLI(t *testing.T) {
 		},
 		{
 			name:        "no configuration path",
-			Input:          &Config{},
+			input:       &Config{},
-			expectedResult: &CscliCfg{},
+			expected:    &CscliCfg{},
+			expectedErr: "no configuration paths provided",
 		},
 	}
 
-	for idx, test := range tests {
+	for _, tc := range tests {
-		err := test.Input.LoadCSCLI()
+		tc := tc
-		if err == nil && test.err != "" {
+		t.Run(tc.name, func(t *testing.T) {
-			fmt.Printf("TEST '%s': NOK\n", test.name)
+			err := tc.input.LoadCSCLI()
-			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
-		} else if test.err != "" {
+			if tc.expectedErr != "" {
-			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				return
-				fmt.Printf("TEST '%s': NOK\n", test.name)
-				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
-					test.err,
-					fmt.Sprintf("%s", err))
-			}
 			}
 
-		isOk := assert.Equal(t, test.expectedResult, test.Input.Cscli)
+			assert.Equal(t, tc.expected, tc.input.Cscli)
-		if !isOk {
+		})
-			t.Fatalf("TEST '%s': NOK", test.name)
-		} else {
-			fmt.Printf("TEST '%s': OK\n", test.name)
-		}
 	}
 }

pkg/csconfig/database.go

@@ -10,7 +10,13 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/ptr"
 )
 
-var DEFAULT_MAX_OPEN_CONNS = 100
+const (
+	DEFAULT_MAX_OPEN_CONNS  = 100
+	defaultDecisionBulkSize = 1000
+	// we need an upper bound due to the sqlite limit of 32k variables in a query
+	// we have 15 variables per decision, so 32768/15 = 2184.5333
+	maxDecisionBulkSize = 2000
+)
 
 type DatabaseCfg struct {
 	User             string      `yaml:"user"`
@@ -25,6 +31,7 @@ type DatabaseCfg struct {
 	LogLevel         *log.Level  `yaml:"log_level"`
 	MaxOpenConns     *int        `yaml:"max_open_conns,omitempty"`
 	UseWal           *bool       `yaml:"use_wal,omitempty"`
+	DecisionBulkSize int         `yaml:"decision_bulk_size,omitempty"`
 }
 
 type AuthGCCfg struct {
@@ -60,11 +67,20 @@ func (c *Config) LoadDBConfig() error {
 		c.DbConfig.MaxOpenConns = ptr.Of(DEFAULT_MAX_OPEN_CONNS)
 	}
 
+	if c.DbConfig.DecisionBulkSize == 0 {
+		log.Tracef("No decision_bulk_size value provided, using default value of %d", defaultDecisionBulkSize)
+		c.DbConfig.DecisionBulkSize = defaultDecisionBulkSize
+	}
+
+	if c.DbConfig.DecisionBulkSize > maxDecisionBulkSize {
+		log.Warningf("decision_bulk_size too high (%d), setting to the maximum value of %d", c.DbConfig.DecisionBulkSize, maxDecisionBulkSize)
+		c.DbConfig.DecisionBulkSize = maxDecisionBulkSize
+	}
+
 	if c.DbConfig.Type == "sqlite" {
 		if c.DbConfig.UseWal == nil {
 			log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.")
 		}
-
 	}
 
 	return nil

pkg/csconfig/database_test.go

@@ -1,28 +1,27 @@
 package csconfig
 
 import (
-	"fmt"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 	"github.com/crowdsecurity/go-cs-lib/ptr"
 )
 
 func TestLoadDBConfig(t *testing.T) {
 	tests := []struct {
 		name        string
-		Input          *Config
+		input       *Config
-		expectedResult *DatabaseCfg
+		expected    *DatabaseCfg
-		err            string
+		expectedErr string
 	}{
 		{
 			name: "basic valid configuration",
-			Input: &Config{
+			input: &Config{
 				DbConfig: &DatabaseCfg{
 					Type:         "sqlite",
-					DbPath:       "./tests/test.db",
+					DbPath:       "./testdata/test.db",
 					MaxOpenConns: ptr.Of(10),
 				},
 				Cscli: &CscliCfg{},
@@ -30,37 +29,31 @@ func TestLoadDBConfig(t *testing.T) {
 					Server: &LocalApiServerCfg{},
 				},
 			},
-			expectedResult: &DatabaseCfg{
+			expected: &DatabaseCfg{
 				Type:             "sqlite",
-				DbPath:       "./tests/test.db",
+				DbPath:           "./testdata/test.db",
 				MaxOpenConns:     ptr.Of(10),
+				DecisionBulkSize: defaultDecisionBulkSize,
 			},
 		},
 		{
 			name:        "no configuration path",
-			Input:          &Config{},
+			input:       &Config{},
-			expectedResult: nil,
+			expected:    nil,
+			expectedErr: "no database configuration provided",
 		},
 	}
 
-	for idx, test := range tests {
+	for _, tc := range tests {
-		err := test.Input.LoadDBConfig()
+		tc := tc
-		if err == nil && test.err != "" {
+		t.Run(tc.name, func(t *testing.T) {
-			fmt.Printf("TEST '%s': NOK\n", test.name)
+			err := tc.input.LoadDBConfig()
-			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
-		} else if test.err != "" {
+			if tc.expectedErr != "" {
-			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				return
-				fmt.Printf("TEST '%s': NOK\n", test.name)
-				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
-					test.err,
-					fmt.Sprintf("%s", err))
-			}
-		}
-		isOk := assert.Equal(t, test.expectedResult, test.Input.DbConfig)
-		if !isOk {
-			t.Fatalf("TEST '%s': NOK", test.name)
-		} else {
-			fmt.Printf("TEST '%s': OK\n", test.name)
 			}
+
+			assert.Equal(t, tc.expected, tc.input.DbConfig)
+		})
 	}
 }

pkg/csconfig/fflag.go

@@ -10,7 +10,6 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 )
 
-
 // LoadFeatureFlagsEnv parses the environment variables to enable feature flags.
 func LoadFeatureFlagsEnv(logger *log.Logger) error {
 	if err := fflag.Crowdsec.SetFromEnv(logger); err != nil {
@@ -19,13 +18,18 @@ func LoadFeatureFlagsEnv(logger *log.Logger) error {
 	return nil
 }
 
-
+// FeatureFlagsFileLocation returns the path to the feature.yaml file.
-// LoadFeatureFlags parses feature.yaml to enable feature flags.
 // The file is in the same directory as config.yaml, which is provided
 // as the fist parameter. This can be different than ConfigPaths.ConfigDir
-func LoadFeatureFlagsFile(configPath string, logger *log.Logger) error {
+// because we have not read config.yaml yet so we don't know the value of ConfigDir.
+func GetFeatureFilePath(configPath string) string {
 	dir := filepath.Dir(configPath)
-	featurePath := filepath.Join(dir, "feature.yaml")
+	return filepath.Join(dir, "feature.yaml")
+}
+
+// LoadFeatureFlags parses feature.yaml to enable feature flags.
+func LoadFeatureFlagsFile(configPath string, logger *log.Logger) error {
+	featurePath := GetFeatureFilePath(configPath)
 
 	if err := fflag.Crowdsec.SetFromYamlFile(featurePath, logger); err != nil {
 		return fmt.Errorf("file %s: %s", featurePath, err)
@@ -33,7 +37,6 @@ func LoadFeatureFlagsFile(configPath string, logger *log.Logger) error {
 	return nil
 }
 
-
 // ListFeatureFlags returns a list of the enabled feature flags.
 func ListFeatureFlags() string {
 	enabledFeatures := fflag.Crowdsec.GetEnabledFeatures()

pkg/csconfig/hub.go

@@ -2,10 +2,10 @@ package csconfig
 
 /*cscli specific config, such as hub directory*/
 type Hub struct {
-	HubDir       string `yaml:"-"`
+	HubIndexFile   string
-	ConfigDir    string `yaml:"-"`
+	HubDir         string
-	HubIndexFile string `yaml:"-"`
+	InstallDir     string
-	DataDir      string `yaml:"-"`
+	InstallDataDir string
 }
 
 func (c *Config) LoadHub() error {
@@ -15,9 +15,9 @@ func (c *Config) LoadHub() error {
 
 	c.Hub = &Hub{
 		HubIndexFile:   c.ConfigPaths.HubIndexFile,
-		ConfigDir:    c.ConfigPaths.ConfigDir,
 		HubDir:         c.ConfigPaths.HubDir,
-		DataDir:      c.ConfigPaths.DataDir,
+		InstallDir:     c.ConfigPaths.ConfigDir,
+		InstallDataDir: c.ConfigPaths.DataDir,
 	}
 
 	return nil

pkg/csconfig/hub_test.go

@@ -1,94 +1,79 @@
 package csconfig
 
 import (
-	"fmt"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadHub(t *testing.T) {
 	hubFullPath, err := filepath.Abs("./hub")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
 	dataFullPath, err := filepath.Abs("./data")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
-	configDirFullPath, err := filepath.Abs("./tests")
+	configDirFullPath, err := filepath.Abs("./testdata")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
 	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
 	tests := []struct {
 		name        string
-		Input          *Config
+		input       *Config
-		expectedResult *Hub
+		expected    *Hub
-		err            string
+		expectedErr string
 	}{
 		{
 			name: "basic valid configuration",
-			Input: &Config{
+			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir:    "./tests",
+					ConfigDir:    "./testdata",
 					DataDir:      "./data",
 					HubDir:       "./hub",
 					HubIndexFile: "./hub/.index.json",
 				},
 			},
-			expectedResult: &Hub{
+			expected: &Hub{
-				ConfigDir:    configDirFullPath,
-				DataDir:      dataFullPath,
 				HubDir:         hubFullPath,
 				HubIndexFile:   hubIndexFileFullPath,
+				InstallDir:     configDirFullPath,
+				InstallDataDir: dataFullPath,
 			},
 		},
 		{
 			name: "no data dir",
-			Input: &Config{
+			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir:    "./tests",
+					ConfigDir:    "./testdata",
 					HubDir:       "./hub",
 					HubIndexFile: "./hub/.index.json",
 				},
 			},
-			expectedResult: nil,
+			expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section",
 		},
 		{
 			name:        "no configuration path",
-			Input:          &Config{},
+			input:       &Config{},
-			expectedResult: nil,
+			expectedErr: "no configuration paths provided",
 		},
 	}
 
-	for idx, test := range tests {
+	for _, tc := range tests {
-		err := test.Input.LoadHub()
+		tc := tc
-		if err == nil && test.err != "" {
+		t.Run(tc.name, func(t *testing.T) {
-			fmt.Printf("TEST '%s': NOK\n", test.name)
+			err := tc.input.LoadHub()
-			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
-		} else if test.err != "" {
+			if tc.expectedErr != "" {
-			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				return
-				fmt.Printf("TEST '%s': NOK\n", test.name)
-				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
-					test.err,
-					fmt.Sprintf("%s", err))
-			}
-		}
-		isOk := assert.Equal(t, test.expectedResult, test.Input.Hub)
-		if !isOk {
-			t.Fatalf("TEST '%s': NOK", test.name)
-		} else {
-			fmt.Printf("TEST '%s': OK\n", test.name)
 			}
+
+			assert.Equal(t, tc.expected, tc.input.Hub)
+		})
 	}
 }

pkg/csconfig/profiles.go

@@ -6,10 +6,11 @@ import (
 	"fmt"
 	"io"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/crowdsecurity/go-cs-lib/yamlpatch"
 
 	"github.com/crowdsecurity/crowdsec/pkg/models"
-	"gopkg.in/yaml.v2"
 )
 
 // var OnErrorDefault = OnErrorIgnore
@@ -43,7 +44,6 @@ func (c *LocalApiServerCfg) LoadProfiles() error {
 	}
 	reader := bytes.NewReader(fcontent)
 
-	//process the yaml
 	dec := yaml.NewDecoder(reader)
 	dec.SetStrict(true)
 	for {

pkg/csconfig/prometheus_test.go

@@ -3,21 +3,21 @@ package csconfig
 import (
 	"testing"
 
-	"github.com/crowdsecurity/go-cs-lib/cstest"
-
 	"github.com/stretchr/testify/require"
+
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadPrometheus(t *testing.T) {
 	tests := []struct {
 		name        string
-		Input       *Config
+		input       *Config
 		expectedURL string
 		expectedErr string
 	}{
 		{
 			name: "basic valid configuration",
-			Input: &Config{
+			input: &Config{
 				Prometheus: &PrometheusCfg{
 					Enabled:    true,
 					Level:      "full",
@@ -33,10 +33,10 @@ func TestLoadPrometheus(t *testing.T) {
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			err := tc.Input.LoadPrometheus()
+			err := tc.input.LoadPrometheus()
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 
-			require.Equal(t, tc.expectedURL, tc.Input.Cscli.PrometheusUrl)
+			require.Equal(t, tc.expectedURL, tc.input.Cscli.PrometheusUrl)
 		})
 	}
 }