From ed3d501081e9b8ff8bb217a7426f23d0ecc4c9e1 Mon Sep 17 00:00:00 2001
From: Laurence Jones <laurence.jones@live.co.uk>
Date: Mon, 4 Dec 2023 10:06:41 +0000
Subject: [PATCH 01/11] [Metabase] QOL Changes and chown wal files (#2627)

* Add detection sqlie wal for dashboard chown

* Lean it down a little

* Change to for loop with extensions

* Keep existing uid on files incase user is running as a unpriviledge user

* I have no idea :shrug:

* Exclude dash.go and update windows

* Update

* Renam

* Remove the os check since we no longer get to this stage for those os's

---------

Co-authored-by: Manuel Sabban <github@sabban.eu>
---
 cmd/crowdsec-cli/dashboard.go             | 79 ++++++++++++++---------
 cmd/crowdsec-cli/dashboard_unsupported.go | 22 +++++++
 pkg/metabase/container.go                 |  9 ---
 3 files changed, 70 insertions(+), 40 deletions(-)
 create mode 100644 cmd/crowdsec-cli/dashboard_unsupported.go

diff --git a/cmd/crowdsec-cli/dashboard.go b/cmd/crowdsec-cli/dashboard.go
index 8eab614e9..a2d6c229a 100644
--- a/cmd/crowdsec-cli/dashboard.go
+++ b/cmd/crowdsec-cli/dashboard.go
@@ -1,3 +1,5 @@
+//go:build linux
+
 package main
 
 import (
@@ -9,6 +11,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"unicode"
 
 	"github.com/AlecAivazis/survey/v2"
@@ -136,6 +139,9 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
 			if err != nil {
 				return err
 			}
+			if err = chownDatabase(dockerGroup.Gid); err != nil {
+				return err
+			}
 			mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
 			if err != nil {
 				return err
@@ -366,45 +372,56 @@ func disclaimer(forceYes *bool) error {
 }
 
 func checkGroups(forceYes *bool) (*user.Group, error) {
-	groupExist := false
 	dockerGroup, err := user.LookupGroup(crowdsecGroup)
 	if err == nil {
-		groupExist = true
+		return dockerGroup, nil
 	}
-	if !groupExist {
-		if !*forceYes {
-			var answer bool
-			prompt := &survey.Confirm{
-				Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
-				Default: true,
-			}
-			if err := survey.AskOne(prompt, &answer); err != nil {
-				return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
-			}
-			if !answer {
-				return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
-			}
+	if !*forceYes {
+		var answer bool
+		prompt := &survey.Confirm{
+			Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
+			Default: true,
 		}
-		groupAddCmd, err := exec.LookPath("groupadd")
-		if err != nil {
-			return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
+		if err := survey.AskOne(prompt, &answer); err != nil {
+			return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
 		}
-
-		groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
-		if err := groupAdd.Run(); err != nil {
-			return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
-		}
-		dockerGroup, err = user.LookupGroup(crowdsecGroup)
-		if err != nil {
-			return dockerGroup, fmt.Errorf("unable to lookup '%s' group: %+v", dockerGroup, err)
+		if !answer {
+			return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
 		}
 	}
-	intID, err := strconv.Atoi(dockerGroup.Gid)
+	groupAddCmd, err := exec.LookPath("groupadd")
 	if err != nil {
-		return dockerGroup, fmt.Errorf("unable to convert group ID to int: %s", err)
+		return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
 	}
-	if err := os.Chown(csConfig.DbConfig.DbPath, 0, intID); err != nil {
-		return dockerGroup, fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
+
+	groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
+	if err := groupAdd.Run(); err != nil {
+		return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
 	}
-	return dockerGroup, nil
+	return user.LookupGroup(crowdsecGroup)
+}
+
+func chownDatabase(gid string) error {
+	intID, err := strconv.Atoi(gid)
+	if err != nil {
+		return fmt.Errorf("unable to convert group ID to int: %s", err)
+	}
+	if stat, err := os.Stat(csConfig.DbConfig.DbPath); !os.IsNotExist(err) {
+		info := stat.Sys()
+		if err := os.Chown(csConfig.DbConfig.DbPath, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
+			return fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
+		}
+	}
+	if csConfig.DbConfig.Type == "sqlite" && csConfig.DbConfig.UseWal != nil && *csConfig.DbConfig.UseWal {
+		for _, ext := range []string{"-wal", "-shm"} {
+			file := csConfig.DbConfig.DbPath + ext
+			if stat, err := os.Stat(file); !os.IsNotExist(err) {
+				info := stat.Sys()
+				if err := os.Chown(file, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
+					return fmt.Errorf("unable to chown sqlite db file '%s': %s", file, err)
+				}
+			}
+		}
+	}
+	return nil
 }
diff --git a/cmd/crowdsec-cli/dashboard_unsupported.go b/cmd/crowdsec-cli/dashboard_unsupported.go
new file mode 100644
index 000000000..f2325ea07
--- /dev/null
+++ b/cmd/crowdsec-cli/dashboard_unsupported.go
@@ -0,0 +1,22 @@
+//go:build !linux
+
+package main
+
+import (
+	"runtime"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func NewDashboardCmd() *cobra.Command {
+	var cmdDashboard = &cobra.Command{
+		Use:               "dashboard",
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			log.Infof("Dashboard command is disabled on %s", runtime.GOOS)
+		},
+	}
+
+	return cmdDashboard
+}
diff --git a/pkg/metabase/container.go b/pkg/metabase/container.go
index b3df7aca7..8b3dd4084 100644
--- a/pkg/metabase/container.go
+++ b/pkg/metabase/container.go
@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"runtime"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -93,14 +92,6 @@ func (c *Container) Create() error {
 		Tty:   true,
 		Env:   env,
 	}
-	os := runtime.GOOS
-	switch os {
-	case "linux":
-	case "windows", "darwin":
-		return fmt.Errorf("mac and windows are not supported yet")
-	default:
-		return fmt.Errorf("OS '%s' is not supported", os)
-	}
 
 	log.Infof("creating container '%s'", c.Name)
 	resp, err := c.CLI.ContainerCreate(ctx, dockerConfig, hostConfig, nil, nil, c.Name)

From bfc92ca1c54238b9799d52a6e65bfd7825c32473 Mon Sep 17 00:00:00 2001
From: Laurence Jones <laurence.jones@live.co.uk>
Date: Mon, 4 Dec 2023 11:48:12 +0000
Subject: [PATCH 02/11] [Explain] Ignore blank lines as crowdsec will anyways
 (#2630)

* Ignore blank lines within file and stdin

* change cleanup to be persistent postrun so if we exit early it always cleans

* When using log flag we should add a newline so we know where EOF is

* Inverse the check for log line since we dont want to modify the line itself

* Wrap run explain with a function that returns the error after cleaning up

* Wrap run explain with a function that returns the error after cleanup

* Use a defer iif instead of global var

* Add invalid len input to err count so it more obvious what is happening

---------

Co-authored-by: Manuel Sabban <github@sabban.eu>
---
 cmd/crowdsec-cli/explain.go | 95 +++++++++++++++++++++++++------------
 1 file changed, 65 insertions(+), 30 deletions(-)

diff --git a/cmd/crowdsec-cli/explain.go b/cmd/crowdsec-cli/explain.go
index 0c33a845e..1a9d24b85 100644
--- a/cmd/crowdsec-cli/explain.go
+++ b/cmd/crowdsec-cli/explain.go
@@ -21,9 +21,15 @@ func GetLineCountForFile(filepath string) (int, error) {
 	}
 	defer f.Close()
 	lc := 0
-	fs := bufio.NewScanner(f)
-	for fs.Scan() {
-		lc++
+	fs := bufio.NewReader(f)
+	for {
+		input, err := fs.ReadBytes('\n')
+		if len(input) > 1 {
+			lc++
+		}
+		if err != nil && err == io.EOF {
+			break
+		}
 	}
 	return lc, nil
 }
@@ -79,19 +85,6 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	fileInfo, _ := os.Stdin.Stat()
-
-	if logType == "" || (logLine == "" && logFile == "" && dsn == "") {
-		printHelp(cmd)
-		fmt.Println()
-		fmt.Printf("Please provide --type flag\n")
-		os.Exit(1)
-	}
-
-	if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
-		return fmt.Errorf("the option -f - is intended to work with pipes")
-	}
-
 	var f *os.File
 
 	// using empty string fallback to /tmp
@@ -99,6 +92,13 @@ func runExplain(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err)
 	}
+	defer func() {
+		if _, err := os.Stat(dir); !os.IsNotExist(err) {
+			if err := os.RemoveAll(dir); err != nil {
+				log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
+			}
+		}
+	}()
 	tmpFile := ""
 	// we create a  temporary log file if a log line/stdin has been provided
 	if logLine != "" || logFile == "-" {
@@ -121,13 +121,15 @@ func runExplain(cmd *cobra.Command, args []string) error {
 				if err != nil && err == io.EOF {
 					break
 				}
-				_, err = f.Write(input)
-				if err != nil {
+				if len(input) > 1 {
+					_, err = f.Write(input)
+				}
+				if err != nil || len(input) <= 1 {
 					errCount++
 				}
 			}
 			if errCount > 0 {
-				log.Warnf("Failed to write %d lines to tmp file", errCount)
+				log.Warnf("Failed to write %d lines to %s", errCount, tmpFile)
 			}
 		}
 		f.Close()
@@ -145,8 +147,12 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		if err != nil {
 			return err
 		}
+		log.Debugf("file %s has %d lines", absolutePath, lineCount)
+		if lineCount == 0 {
+			return fmt.Errorf("the log file is empty: %s", absolutePath)
+		}
 		if lineCount > 100 {
-			log.Warnf("The log file contains %d lines. This may take a lot of resources.", lineCount)
+			log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount)
 		}
 	}
 
@@ -166,12 +172,6 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("fail to run crowdsec for test: %v", err)
 	}
 
-	// rm the temporary log file if only a log line/stdin was provided
-	if tmpFile != "" {
-		if err := os.Remove(tmpFile); err != nil {
-			return fmt.Errorf("unable to remove tmp log file '%s': %+v", tmpFile, err)
-		}
-	}
 	parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
 	bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)
 
@@ -187,10 +187,6 @@ func runExplain(cmd *cobra.Command, args []string) error {
 
 	hubtest.DumpTree(*parserDump, *bucketStateDump, opts)
 
-	if err := os.RemoveAll(dir); err != nil {
-		return fmt.Errorf("unable to delete temporary directory '%s': %s", dir, err)
-	}
-
 	return nil
 }
 
@@ -210,6 +206,45 @@ tail -n 5 myfile.log | cscli explain --type nginx -f -
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
 		RunE:              runExplain,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			flags := cmd.Flags()
+
+			logFile, err := flags.GetString("file")
+			if err != nil {
+				return err
+			}
+
+			dsn, err := flags.GetString("dsn")
+			if err != nil {
+				return err
+			}
+
+			logLine, err := flags.GetString("log")
+			if err != nil {
+				return err
+			}
+
+			logType, err := flags.GetString("type")
+			if err != nil {
+				return err
+			}
+
+			if logLine == "" && logFile == "" && dsn == "" {
+				printHelp(cmd)
+				fmt.Println()
+				return fmt.Errorf("please provide --log, --file or --dsn flag")
+			}
+			if logType == "" {
+				printHelp(cmd)
+				fmt.Println()
+				return fmt.Errorf("please provide --type flag")
+			}
+			fileInfo, _ := os.Stdin.Stat()
+			if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
+				return fmt.Errorf("the option -f - is intended to work with pipes")
+			}
+			return nil
+		},
 	}
 
 	flags := cmdExplain.Flags()

From d1bfaddb6985cfb3a1cec36084edbe473934c543 Mon Sep 17 00:00:00 2001
From: Laurence Jones <laurence.jones@live.co.uk>
Date: Mon, 4 Dec 2023 12:05:26 +0000
Subject: [PATCH 03/11] [Plugin] Pass down ctx and use it (#2626)

* Pass down cancellable context and update http plugin

* Use context where we can
---
 cmd/notification-http/main.go     | 3 +--
 cmd/notification-sentinel/main.go | 2 +-
 cmd/notification-slack/main.go    | 3 +--
 cmd/notification-splunk/main.go   | 2 +-
 pkg/csplugin/notifier.go          | 2 +-
 5 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/cmd/notification-http/main.go b/cmd/notification-http/main.go
index f7908ddda..6d1da788e 100644
--- a/cmd/notification-http/main.go
+++ b/cmd/notification-http/main.go
@@ -58,13 +58,12 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
 	if err != nil {
 		return nil, err
 	}
-
 	for headerName, headerValue := range cfg.Headers {
 		logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue))
 		request.Header.Add(headerName, headerValue)
 	}
 	logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
-	resp, err := client.Do(request)
+	resp, err := client.Do(request.WithContext(ctx))
 	if err != nil {
 		logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))
 		return nil, err
diff --git a/cmd/notification-sentinel/main.go b/cmd/notification-sentinel/main.go
index 18eff1b05..c627f9271 100644
--- a/cmd/notification-sentinel/main.go
+++ b/cmd/notification-sentinel/main.go
@@ -90,7 +90,7 @@ func (s *SentinelPlugin) Notify(ctx context.Context, notification *protobufs.Not
 	req.Header.Set("x-ms-date", now)
 
 	client := &http.Client{}
-	resp, err := client.Do(req)
+	resp, err := client.Do(req.WithContext(ctx))
 	if err != nil {
 		logger.Error("failed to send request", "error", err)
 		return &protobufs.Empty{}, err
diff --git a/cmd/notification-slack/main.go b/cmd/notification-slack/main.go
index 901832381..373cd9527 100644
--- a/cmd/notification-slack/main.go
+++ b/cmd/notification-slack/main.go
@@ -38,10 +38,9 @@ func (n *Notify) Notify(ctx context.Context, notification *protobufs.Notificatio
 	if cfg.LogLevel != nil && *cfg.LogLevel != "" {
 		logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel))
 	}
-
 	logger.Info(fmt.Sprintf("found notify signal for %s config", notification.Name))
 	logger.Debug(fmt.Sprintf("posting to %s webhook, message %s", cfg.Webhook, notification.Text))
-	err := slack.PostWebhook(n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
+	err := slack.PostWebhookContext(ctx, n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
 		Text: notification.Text,
 	})
 	if err != nil {
diff --git a/cmd/notification-splunk/main.go b/cmd/notification-splunk/main.go
index 826986877..b24aa538f 100644
--- a/cmd/notification-splunk/main.go
+++ b/cmd/notification-splunk/main.go
@@ -65,7 +65,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
 
 	req.Header.Add("Authorization", fmt.Sprintf("Splunk %s", cfg.Token))
 	logger.Debug(fmt.Sprintf("posting event %s to %s", string(data), req.URL))
-	resp, err := s.Client.Do(req)
+	resp, err := s.Client.Do(req.WithContext(ctx))
 	if err != nil {
 		return &protobufs.Empty{}, err
 	}
diff --git a/pkg/csplugin/notifier.go b/pkg/csplugin/notifier.go
index 8ab1aa923..a4f5bbc0e 100644
--- a/pkg/csplugin/notifier.go
+++ b/pkg/csplugin/notifier.go
@@ -26,7 +26,7 @@ func (m *GRPCClient) Notify(ctx context.Context, notification *protobufs.Notific
 	done := make(chan error)
 	go func() {
 		_, err := m.client.Notify(
-			context.Background(), &protobufs.Notification{Text: notification.Text, Name: notification.Name},
+			ctx, &protobufs.Notification{Text: notification.Text, Name: notification.Name},
 		)
 		done <- err
 	}()

From f8755be9cd4a027a22459b7af3af2807d39f4f06 Mon Sep 17 00:00:00 2001
From: Laurence Jones <laurence.jones@live.co.uk>
Date: Mon, 4 Dec 2023 15:52:14 +0000
Subject: [PATCH 04/11] Fix formt on documentation (#2577)

When generating decisions import docusarus v3 now does not allow `{` without escaping this adds escaping
---
 cmd/crowdsec-cli/decisions_import.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/crowdsec-cli/decisions_import.go b/cmd/crowdsec-cli/decisions_import.go
index 56fc37c87..31dbc0ead 100644
--- a/cmd/crowdsec-cli/decisions_import.go
+++ b/cmd/crowdsec-cli/decisions_import.go
@@ -232,7 +232,7 @@ func NewDecisionsImportCmd() *cobra.Command {
 		Short: "Import decisions from a file or pipe",
 		Long: "expected format:\n" +
 			"csv  : any of duration,reason,scope,type,value, with a header line\n" +
-			`json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`,
+			"json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`",
 		DisableAutoGenTag: true,
 		Example: `decisions.csv:
 duration,scope,value

From a5ab73d4583b0036afae74ac328a52bd7848bd39 Mon Sep 17 00:00:00 2001
From: mmetc <92726601+mmetc@users.noreply.github.com>
Date: Mon, 4 Dec 2023 22:59:52 +0100
Subject: [PATCH 05/11] cscli machines add: don't overwrite existing credential
 file (#2625)

* cscli machines add: don't overwrite existing credential file
* keep old behavior with --force
Now --force is used both to override the replacement of and existing machine,
and an existing credentials file. To retain the old behavior, the
existence of the file is only checked for the default configuration, not
if explicitly specified.
---
 cmd/crowdsec-cli/capi.go           |  2 +-
 cmd/crowdsec-cli/config_restore.go |  2 +-
 cmd/crowdsec-cli/lapi.go           |  4 ++--
 cmd/crowdsec-cli/machines.go       | 33 ++++++++++++++++++++----------
 test/bats/30_machines.bats         | 16 +++++++++------
 test/lib/config/config-global      |  6 +++---
 test/lib/config/config-local       |  3 ++-
 7 files changed, 41 insertions(+), 25 deletions(-)

diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go
index 0261eab9c..e316abbc6 100644
--- a/cmd/crowdsec-cli/capi.go
+++ b/cmd/crowdsec-cli/capi.go
@@ -110,7 +110,7 @@ func NewCapiRegisterCmd() *cobra.Command {
 				if err != nil {
 					return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
 				}
-				log.Printf("Central API credentials dumped to '%s'", dumpFile)
+				log.Printf("Central API credentials written to '%s'", dumpFile)
 			} else {
 				fmt.Printf("%s\n", string(apiConfigDump))
 			}
diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go
index 56f628281..ab49836a9 100644
--- a/cmd/crowdsec-cli/config_restore.go
+++ b/cmd/crowdsec-cli/config_restore.go
@@ -183,7 +183,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 			if csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
 				apiConfigDumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath
 			}
-			err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o644)
+			err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o600)
 			if err != nil {
 				return fmt.Errorf("write api credentials in '%s' failed: %s", apiConfigDumpFile, err)
 			}
diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go
index b2870cb20..755c2cb6d 100644
--- a/cmd/crowdsec-cli/lapi.go
+++ b/cmd/crowdsec-cli/lapi.go
@@ -150,11 +150,11 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 		log.Fatalf("unable to marshal api credentials: %s", err)
 	}
 	if dumpFile != "" {
-		err = os.WriteFile(dumpFile, apiConfigDump, 0644)
+		err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
 		if err != nil {
 			log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
 		}
-		log.Printf("Local API credentials dumped to '%s'", dumpFile)
+		log.Printf("Local API credentials written to '%s'", dumpFile)
 	} else {
 		fmt.Printf("%s\n", string(apiConfigDump))
 	}
diff --git a/cmd/crowdsec-cli/machines.go b/cmd/crowdsec-cli/machines.go
index 6c97c0109..b7ceed254 100644
--- a/cmd/crowdsec-cli/machines.go
+++ b/cmd/crowdsec-cli/machines.go
@@ -43,6 +43,7 @@ func generatePassword(length int) string {
 	charsetLength := len(charset)
 
 	buf := make([]byte, length)
+
 	for i := 0; i < length; i++ {
 		rInt, err := saferand.Int(saferand.Reader, big.NewInt(int64(charsetLength)))
 		if err != nil {
@@ -190,7 +191,6 @@ cscli machines add MyTestMachine --password MyPassword
 }
 
 func runMachinesAdd(cmd *cobra.Command, args []string) error {
-	var dumpFile string
 	var err error
 
 	flags := cmd.Flags()
@@ -200,7 +200,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	outputFile, err := flags.GetString("file")
+	dumpFile, err := flags.GetString("file")
 	if err != nil {
 		return err
 	}
@@ -220,7 +220,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	forceAdd, err := flags.GetBool("force")
+	force, err := flags.GetBool("force")
 	if err != nil {
 		return err
 	}
@@ -242,17 +242,28 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 	}
 
 	/*check if file already exists*/
-	if outputFile != "" {
-		dumpFile = outputFile
-	} else if csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
-		dumpFile = csConfig.API.Client.CredentialsFilePath
+	if dumpFile == "" && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
+		credFile := csConfig.API.Client.CredentialsFilePath
+		// use the default only if the file does not exist
+		_, err := os.Stat(credFile)
+		switch {
+		case os.IsNotExist(err) || force:
+			dumpFile = csConfig.API.Client.CredentialsFilePath
+		case err != nil:
+			return fmt.Errorf("unable to stat '%s': %s", credFile, err)
+		default:
+			return fmt.Errorf(`credentials file '%s' already exists: please remove it, use "--force" or specify a different file with "-f" ("-f -" for standard output)`, credFile)
+		}
+	}
+
+	if dumpFile == "" {
+		return fmt.Errorf(`please specify a file to dump credentials to, with -f ("-f -" for standard output)`)
 	}
 
 	// create a password if it's not specified by user
 	if machinePassword == "" && !interactive {
 		if !autoAdd {
-			printHelp(cmd)
-			return nil
+			return fmt.Errorf("please specify a password with --password or use --auto")
 		}
 		machinePassword = generatePassword(passwordLength)
 	} else if machinePassword == "" && interactive {
@@ -262,7 +273,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		survey.AskOne(qs, &machinePassword)
 	}
 	password := strfmt.Password(machinePassword)
-	_, err = dbClient.CreateMachine(&machineID, &password, "", true, forceAdd, types.PasswordAuthType)
+	_, err = dbClient.CreateMachine(&machineID, &password, "", true, force, types.PasswordAuthType)
 	if err != nil {
 		return fmt.Errorf("unable to create machine: %s", err)
 	}
@@ -291,7 +302,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		if err != nil {
 			return fmt.Errorf("write api credentials in '%s' failed: %s", dumpFile, err)
 		}
-		log.Printf("API credentials dumped to '%s'", dumpFile)
+		log.Printf("API credentials written to '%s'", dumpFile)
 	} else {
 		fmt.Printf("%s\n", string(apiConfigDump))
 	}
diff --git a/test/bats/30_machines.bats b/test/bats/30_machines.bats
index d5ddf840f..f321b8f3f 100644
--- a/test/bats/30_machines.bats
+++ b/test/bats/30_machines.bats
@@ -23,20 +23,24 @@ teardown() {
 
 #----------
 
-@test "can list machines as regular user" {
-    rune -0 cscli machines list
-}
-
 @test "we have exactly one machine" {
     rune -0 cscli machines list -o json
     rune -0 jq -c '[. | length, .[0].machineId[0:32], .[0].isValidated]' <(output)
     assert_output '[1,"githubciXXXXXXXXXXXXXXXXXXXXXXXX",true]'
 }
 
+@test "don't overwrite local credentials by default" {
+    rune -1 cscli machines add local -a -o json
+    rune -0 jq -r '.msg' <(stderr)
+    assert_output --partial 'already exists: please remove it, use "--force" or specify a different file with "-f"'
+    rune -0 cscli machines add local -a --force
+    assert_stderr --partial "Machine 'local' successfully added to the local API"
+}
+
 @test "add a new machine and delete it" {
     rune -0 cscli machines add -a -f /dev/null CiTestMachine -o human
     assert_stderr --partial "Machine 'CiTestMachine' successfully added to the local API"
-    assert_stderr --partial "API credentials dumped to '/dev/null'"
+    assert_stderr --partial "API credentials written to '/dev/null'"
 
     # we now have two machines
     rune -0 cscli machines list -o json
@@ -56,7 +60,7 @@ teardown() {
 @test "register, validate and then remove a machine" {
     rune -0 cscli lapi register --machine CiTestMachineRegister -f /dev/null -o human
     assert_stderr --partial "Successfully registered to Local API (LAPI)"
-    assert_stderr --partial "Local API credentials dumped to '/dev/null'"
+    assert_stderr --partial "Local API credentials written to '/dev/null'"
 
     # the machine is not validated yet
     rune -0 cscli machines list -o json
diff --git a/test/lib/config/config-global b/test/lib/config/config-global
index 7814cefbb..22ac76df6 100755
--- a/test/lib/config/config-global
+++ b/test/lib/config/config-global
@@ -61,12 +61,12 @@ make_init_data() {
     ./instance-db config-yaml
     ./instance-db setup
 
+    ./bin/preload-hub-items
+
     # when installed packages are always using sqlite, so no need to regenerate
     # local credz for sqlite
 
-    ./bin/preload-hub-items
-
-    [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto
+    [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto --force
 
     mkdir -p "$LOCAL_INIT_DIR"
 
diff --git a/test/lib/config/config-local b/test/lib/config/config-local
index 0941484ff..e3b7bc685 100755
--- a/test/lib/config/config-local
+++ b/test/lib/config/config-local
@@ -115,11 +115,12 @@ make_init_data() {
     ./instance-db config-yaml
     ./instance-db setup
 
-    "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
     "$CSCLI" --warning hub update
 
     ./bin/preload-hub-items
 
+    "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto --force
+
     mkdir -p "$LOCAL_INIT_DIR"
 
     ./instance-db dump "${LOCAL_INIT_DIR}/database"

From 23968e472dd8c32d4a5b949ef4f4428ca6050d30 Mon Sep 17 00:00:00 2001
From: mmetc <92726601+mmetc@users.noreply.github.com>
Date: Mon, 4 Dec 2023 23:06:01 +0100
Subject: [PATCH 06/11] Refact bouncer auth (#2456)

Co-authored-by: blotus <sebastien@crowdsec.net>
---
 cmd/crowdsec-cli/machines.go                |   4 +-
 pkg/apiserver/apiserver.go                  |  12 +-
 pkg/apiserver/apiserver_test.go             |   9 +-
 pkg/apiserver/middlewares/v1/api_key.go     | 213 +++++++++-----------
 pkg/apiserver/middlewares/v1/middlewares.go |   2 +-
 pkg/database/bouncers.go                    |   6 +-
 pkg/database/machines.go                    |  11 +-
 7 files changed, 112 insertions(+), 145 deletions(-)

diff --git a/cmd/crowdsec-cli/machines.go b/cmd/crowdsec-cli/machines.go
index b7ceed254..72e6579cc 100644
--- a/cmd/crowdsec-cli/machines.go
+++ b/cmd/crowdsec-cli/machines.go
@@ -30,9 +30,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
-var (
-	passwordLength = 64
-)
+const passwordLength = 64
 
 func generatePassword(length int) string {
 	upper := "ABCDEFGHIJKLMNOPQRSTUVWXY"
diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go
index 070013298..cfeb13d27 100644
--- a/pkg/apiserver/apiserver.go
+++ b/pkg/apiserver/apiserver.go
@@ -106,13 +106,13 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
 	var flushScheduler *gocron.Scheduler
 	dbClient, err := database.NewClient(config.DbConfig)
 	if err != nil {
-		return &APIServer{}, fmt.Errorf("unable to init database client: %w", err)
+		return nil, fmt.Errorf("unable to init database client: %w", err)
 	}
 
 	if config.DbConfig.Flush != nil {
 		flushScheduler, err = dbClient.StartFlushScheduler(config.DbConfig.Flush)
 		if err != nil {
-			return &APIServer{}, err
+			return nil, err
 		}
 	}
 
@@ -129,7 +129,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
 
 	if config.TrustedProxies != nil && config.UseForwardedForHeaders {
 		if err := router.SetTrustedProxies(*config.TrustedProxies); err != nil {
-			return &APIServer{}, fmt.Errorf("while setting trusted_proxies: %w", err)
+			return nil, fmt.Errorf("while setting trusted_proxies: %w", err)
 		}
 		router.ForwardedByClientIP = true
 	} else {
@@ -215,7 +215,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
 		log.Printf("Loading CAPI manager")
 		apiClient, err = NewAPIC(config.OnlineClient, dbClient, config.ConsoleConfig, config.CapiWhitelists)
 		if err != nil {
-			return &APIServer{}, err
+			return nil, err
 		}
 		log.Infof("CAPI manager configured successfully")
 		isMachineEnrolled = isEnrolled(apiClient.apiClient)
@@ -225,7 +225,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
 				log.Infof("Machine is enrolled in the console, Loading PAPI Client")
 				papiClient, err = NewPAPI(apiClient, dbClient, config.ConsoleConfig, *config.PapiLogLevel)
 				if err != nil {
-					return &APIServer{}, err
+					return nil, err
 				}
 				controller.DecisionDeleteChan = papiClient.Channels.DeleteDecisionChannel
 			} else {
@@ -241,7 +241,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
 	if trustedIPs, err := config.GetTrustedIPs(); err == nil {
 		controller.TrustedIPs = trustedIPs
 	} else {
-		return &APIServer{}, err
+		return nil, err
 	}
 
 	return &APIServer{
diff --git a/pkg/apiserver/apiserver_test.go b/pkg/apiserver/apiserver_test.go
index 435c9601a..6150c351b 100644
--- a/pkg/apiserver/apiserver_test.go
+++ b/pkg/apiserver/apiserver_test.go
@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 	"github.com/crowdsecurity/go-cs-lib/version"
 
 	middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
@@ -295,8 +296,8 @@ func TestWithWrongDBConfig(t *testing.T) {
 	config.API.Server.DbConfig.Type = "test"
 	apiServer, err := NewServer(config.API.Server)
 
-	assert.Equal(t, apiServer, &APIServer{})
-	assert.Equal(t, "unable to init database client: unknown database type 'test'", err.Error())
+	cstest.RequireErrorContains(t, err, "unable to init database client: unknown database type 'test'")
+	assert.Nil(t, apiServer)
 }
 
 func TestWithWrongFlushConfig(t *testing.T) {
@@ -305,8 +306,8 @@ func TestWithWrongFlushConfig(t *testing.T) {
 	config.API.Server.DbConfig.Flush.MaxItems = &maxItems
 	apiServer, err := NewServer(config.API.Server)
 
-	assert.Equal(t, apiServer, &APIServer{})
-	assert.Equal(t, "max_items can't be zero or negative number", err.Error())
+	cstest.RequireErrorContains(t, err, "max_items can't be zero or negative number")
+	assert.Nil(t, apiServer)
 }
 
 func TestUnknownPath(t *testing.T) {
diff --git a/pkg/apiserver/middlewares/v1/api_key.go b/pkg/apiserver/middlewares/v1/api_key.go
index 207f35fc4..1481a0145 100644
--- a/pkg/apiserver/middlewares/v1/api_key.go
+++ b/pkg/apiserver/middlewares/v1/api_key.go
@@ -55,132 +55,110 @@ func HashSHA512(str string) string {
 	return hashStr
 }
 
+func (a *APIKey) authTLS(c *gin.Context, logger *log.Entry) *ent.Bouncer {
+	if a.TlsAuth == nil {
+		logger.Error("TLS Auth is not configured but client presented a certificate")
+		return nil
+	}
+
+	validCert, extractedCN, err := a.TlsAuth.ValidateCert(c)
+	if !validCert {
+		logger.Errorf("invalid client certificate: %s", err)
+		return nil
+	}
+	if err != nil {
+		logger.Error(err)
+		return nil
+	}
+
+	logger = logger.WithFields(log.Fields{
+		"cn": extractedCN,
+	})
+
+	bouncerName := fmt.Sprintf("%s@%s", extractedCN, c.ClientIP())
+	bouncer, err := a.DbClient.SelectBouncerByName(bouncerName)
+
+	//This is likely not the proper way, but isNotFound does not seem to work
+	if err != nil && strings.Contains(err.Error(), "bouncer not found") {
+		//Because we have a valid cert, automatically create the bouncer in the database if it does not exist
+		//Set a random API key, but it will never be used
+		apiKey, err := GenerateAPIKey(dummyAPIKeySize)
+		if err != nil {
+			logger.Errorf("error generating mock api key: %s", err)
+			return nil
+		}
+		logger.Infof("Creating bouncer %s", bouncerName)
+		bouncer, err = a.DbClient.CreateBouncer(bouncerName, c.ClientIP(), HashSHA512(apiKey), types.TlsAuthType)
+		if err != nil {
+			logger.Errorf("while creating bouncer db entry: %s", err)
+			return nil
+		}
+	} else if err != nil {
+		//error while selecting bouncer
+		logger.Errorf("while selecting bouncers: %s", err)
+		return nil
+	} else if bouncer.AuthType != types.TlsAuthType {
+		//bouncer was found in DB
+		logger.Errorf("bouncer isn't allowed to auth by TLS")
+		return nil
+	}
+	return bouncer
+}
+
+func (a *APIKey) authPlain(c *gin.Context, logger *log.Entry) *ent.Bouncer {
+	val, ok := c.Request.Header[APIKeyHeader]
+	if !ok {
+		logger.Errorf("API key not found")
+		return nil
+	}
+	hashStr := HashSHA512(val[0])
+
+	bouncer, err := a.DbClient.SelectBouncer(hashStr)
+	if err != nil {
+		logger.Errorf("while fetching bouncer info: %s", err)
+		return nil
+	}
+
+	if bouncer.AuthType != types.ApiKeyAuthType {
+		logger.Errorf("bouncer %s attempted to login using an API key but it is configured to auth with %s", bouncer.Name, bouncer.AuthType)
+		return nil
+	}
+
+	return bouncer
+}
+
 func (a *APIKey) MiddlewareFunc() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		var bouncer *ent.Bouncer
-		var err error
+
+		logger := log.WithFields(log.Fields{
+			"ip": c.ClientIP(),
+		})
 
 		if c.Request.TLS != nil && len(c.Request.TLS.PeerCertificates) > 0 {
-			if a.TlsAuth == nil {
-				log.WithField("ip", c.ClientIP()).Error("TLS Auth is not configured but client presented a certificate")
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
-			validCert, extractedCN, err := a.TlsAuth.ValidateCert(c)
-			if !validCert {
-				log.WithField("ip", c.ClientIP()).Errorf("invalid client certificate: %s", err)
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
-			if err != nil {
-				log.WithField("ip", c.ClientIP()).Error(err)
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
-			bouncerName := fmt.Sprintf("%s@%s", extractedCN, c.ClientIP())
-			bouncer, err = a.DbClient.SelectBouncerByName(bouncerName)
-			//This is likely not the proper way, but isNotFound does not seem to work
-			if err != nil && strings.Contains(err.Error(), "bouncer not found") {
-				//Because we have a valid cert, automatically create the bouncer in the database if it does not exist
-				//Set a random API key, but it will never be used
-				apiKey, err := GenerateAPIKey(dummyAPIKeySize)
-				if err != nil {
-					log.WithFields(log.Fields{
-						"ip": c.ClientIP(),
-						"cn": extractedCN,
-					}).Errorf("error generating mock api key: %s", err)
-					c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-					c.Abort()
-					return
-				}
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-					"cn": extractedCN,
-				}).Infof("Creating bouncer %s", bouncerName)
-				bouncer, err = a.DbClient.CreateBouncer(bouncerName, c.ClientIP(), HashSHA512(apiKey), types.TlsAuthType)
-				if err != nil {
-					log.WithFields(log.Fields{
-						"ip": c.ClientIP(),
-						"cn": extractedCN,
-					}).Errorf("creating bouncer db entry : %s", err)
-					c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-					c.Abort()
-					return
-				}
-			} else if err != nil {
-				//error while selecting bouncer
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-					"cn": extractedCN,
-				}).Errorf("while selecting bouncers: %s", err)
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			} else if bouncer.AuthType != types.TlsAuthType {
-				//bouncer was found in DB
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-					"cn": extractedCN,
-				}).Errorf("bouncer isn't allowed to auth by TLS")
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
+			bouncer = a.authTLS(c, logger)
 		} else {
-			//API Key Authentication
-			val, ok := c.Request.Header[APIKeyHeader]
-			if !ok {
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-				}).Errorf("API key not found")
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
-			hashStr := HashSHA512(val[0])
-			bouncer, err = a.DbClient.SelectBouncer(hashStr)
-			if err != nil {
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-				}).Errorf("while fetching bouncer info: %s", err)
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
-			if bouncer.AuthType != types.ApiKeyAuthType {
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-				}).Errorf("bouncer %s attempted to login using an API key but it is configured to auth with %s", bouncer.Name, bouncer.AuthType)
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
+			bouncer = a.authPlain(c, logger)
 		}
 
 		if bouncer == nil {
-			log.WithFields(log.Fields{
-				"ip": c.ClientIP(),
-			}).Errorf("bouncer not found")
 			c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
 			c.Abort()
 			return
 		}
 
-		//maybe we want to store the whole bouncer object in the context instead, this would avoid another db query
-		//in StreamDecision
+		logger = logger.WithFields(log.Fields{
+			"name": bouncer.Name,
+		})
+
+		// maybe we want to store the whole bouncer object in the context instead, this would avoid another db query
+		// in StreamDecision
 		c.Set("BOUNCER_NAME", bouncer.Name)
 		c.Set("BOUNCER_HASHED_KEY", bouncer.APIKey)
 
 		if bouncer.IPAddress == "" {
-			err = a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID)
-			if err != nil {
-				log.WithFields(log.Fields{
-					"ip":   c.ClientIP(),
-					"name": bouncer.Name,
-				}).Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err)
+			if err := a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID); err != nil {
+				logger.Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err)
 				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
 				c.Abort()
 				return
@@ -189,12 +167,8 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc {
 
 		if bouncer.IPAddress != c.ClientIP() && bouncer.IPAddress != "" {
 			log.Warningf("new IP address detected for bouncer '%s': %s (old: %s)", bouncer.Name, c.ClientIP(), bouncer.IPAddress)
-			err = a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID)
-			if err != nil {
-				log.WithFields(log.Fields{
-					"ip":   c.ClientIP(),
-					"name": bouncer.Name,
-				}).Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err)
+			if err := a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID); err != nil {
+				logger.Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err)
 				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
 				c.Abort()
 				return
@@ -202,21 +176,14 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc {
 		}
 
 		useragent := strings.Split(c.Request.UserAgent(), "/")
-
 		if len(useragent) != 2 {
-			log.WithFields(log.Fields{
-				"ip":   c.ClientIP(),
-				"name": bouncer.Name,
-			}).Warningf("bad user agent '%s'", c.Request.UserAgent())
+			logger.Warningf("bad user agent '%s'", c.Request.UserAgent())
 			useragent = []string{c.Request.UserAgent(), "N/A"}
 		}
 
 		if bouncer.Version != useragent[1] || bouncer.Type != useragent[0] {
 			if err := a.DbClient.UpdateBouncerTypeAndVersion(useragent[0], useragent[1], bouncer.ID); err != nil {
-				log.WithFields(log.Fields{
-					"ip":   c.ClientIP(),
-					"name": bouncer.Name,
-				}).Errorf("failed to update bouncer version and type: %s", err)
+				logger.Errorf("failed to update bouncer version and type: %s", err)
 				c.JSON(http.StatusForbidden, gin.H{"message": "bad user agent"})
 				c.Abort()
 				return
diff --git a/pkg/apiserver/middlewares/v1/middlewares.go b/pkg/apiserver/middlewares/v1/middlewares.go
index 26879bd8e..ef2d93b92 100644
--- a/pkg/apiserver/middlewares/v1/middlewares.go
+++ b/pkg/apiserver/middlewares/v1/middlewares.go
@@ -14,7 +14,7 @@ func NewMiddlewares(dbClient *database.Client) (*Middlewares, error) {
 
 	ret.JWT, err = NewJWT(dbClient)
 	if err != nil {
-		return &Middlewares{}, err
+		return nil, err
 	}
 
 	ret.APIKey = NewAPIKey(dbClient)
diff --git a/pkg/database/bouncers.go b/pkg/database/bouncers.go
index 804337ecc..496b9b6cc 100644
--- a/pkg/database/bouncers.go
+++ b/pkg/database/bouncers.go
@@ -13,7 +13,7 @@ import (
 func (c *Client) SelectBouncer(apiKeyHash string) (*ent.Bouncer, error) {
 	result, err := c.Ent.Bouncer.Query().Where(bouncer.APIKeyEQ(apiKeyHash)).First(c.CTX)
 	if err != nil {
-		return &ent.Bouncer{}, errors.Wrapf(QueryFail, "select bouncer: %s", err)
+		return nil, err
 	}
 
 	return result, nil
@@ -22,7 +22,7 @@ func (c *Client) SelectBouncer(apiKeyHash string) (*ent.Bouncer, error) {
 func (c *Client) SelectBouncerByName(bouncerName string) (*ent.Bouncer, error) {
 	result, err := c.Ent.Bouncer.Query().Where(bouncer.NameEQ(bouncerName)).First(c.CTX)
 	if err != nil {
-		return &ent.Bouncer{}, errors.Wrapf(QueryFail, "select bouncer: %s", err)
+		return nil, err
 	}
 
 	return result, nil
@@ -31,7 +31,7 @@ func (c *Client) SelectBouncerByName(bouncerName string) (*ent.Bouncer, error) {
 func (c *Client) ListBouncers() ([]*ent.Bouncer, error) {
 	result, err := c.Ent.Bouncer.Query().All(c.CTX)
 	if err != nil {
-		return []*ent.Bouncer{}, errors.Wrapf(QueryFail, "listing bouncer: %s", err)
+		return nil, errors.Wrapf(QueryFail, "listing bouncers: %s", err)
 	}
 	return result, nil
 }
diff --git a/pkg/database/machines.go b/pkg/database/machines.go
index 98992d478..b9834e57e 100644
--- a/pkg/database/machines.go
+++ b/pkg/database/machines.go
@@ -19,8 +19,8 @@ const CapiListsMachineID = types.ListOrigin
 func (c *Client) CreateMachine(machineID *string, password *strfmt.Password, ipAddress string, isValidated bool, force bool, authType string) (*ent.Machine, error) {
 	hashPassword, err := bcrypt.GenerateFromPassword([]byte(*password), bcrypt.DefaultCost)
 	if err != nil {
-		c.Log.Warningf("CreateMachine : %s", err)
-		return nil, errors.Wrap(HashError, "")
+		c.Log.Warningf("CreateMachine: %s", err)
+		return nil, HashError
 	}
 
 	machineExist, err := c.Ent.Machine.
@@ -78,7 +78,7 @@ func (c *Client) QueryMachineByID(machineID string) (*ent.Machine, error) {
 func (c *Client) ListMachines() ([]*ent.Machine, error) {
 	machines, err := c.Ent.Machine.Query().All(c.CTX)
 	if err != nil {
-		return []*ent.Machine{}, errors.Wrapf(QueryFail, "listing machines: %s", err)
+		return nil, errors.Wrapf(QueryFail, "listing machines: %s", err)
 	}
 	return machines, nil
 }
@@ -101,7 +101,7 @@ func (c *Client) QueryPendingMachine() ([]*ent.Machine, error) {
 	machines, err = c.Ent.Machine.Query().Where(machine.IsValidatedEQ(false)).All(c.CTX)
 	if err != nil {
 		c.Log.Warningf("QueryPendingMachine : %s", err)
-		return []*ent.Machine{}, errors.Wrapf(QueryFail, "querying pending machines: %s", err)
+		return nil, errors.Wrapf(QueryFail, "querying pending machines: %s", err)
 	}
 	return machines, nil
 }
@@ -190,12 +190,13 @@ func (c *Client) IsMachineRegistered(machineID string) (bool, error) {
 		return true, nil
 	}
 	if len(exist) > 1 {
-		return false, fmt.Errorf("More than one item with the same machineID in database")
+		return false, fmt.Errorf("more than one item with the same machineID in database")
 	}
 
 	return false, nil
 
 }
+
 func (c *Client) QueryLastValidatedHeartbeatLT(t time.Time) ([]*ent.Machine, error) {
 	return c.Ent.Machine.Query().Where(machine.LastHeartbeatLT(t), machine.IsValidatedEQ(true)).All(c.CTX)
 }

From 0c4093dccac35229ec19c13c07f7d6c52354e92b Mon Sep 17 00:00:00 2001
From: mmetc <92726601+mmetc@users.noreply.github.com>
Date: Mon, 4 Dec 2023 23:09:42 +0100
Subject: [PATCH 07/11] Test for acquisition errors in crowdsec -t (#2629)

---
 test/bats/01_crowdsec.bats | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats
index 7bcc35b1d..fb99d1ec3 100644
--- a/test/bats/01_crowdsec.bats
+++ b/test/bats/01_crowdsec.bats
@@ -227,3 +227,16 @@ teardown() {
     assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
+@test "crowdsec -t (error in acquisition file)" {
+    # we can verify the acquisition configuration without running crowdsec
+    ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
+    config_set "$ACQUIS_YAML" 'del(.filenames)'
+
+    rune -1 wait-for "${CROWDSEC}"
+    assert_stderr --partial "failed to configure datasource file: no filename or filenames configuration provided"
+
+    config_set "$ACQUIS_YAML" '.filenames=["file.log"]'
+    config_set "$ACQUIS_YAML" '.meh=3'
+    rune -1 wait-for "${CROWDSEC}"
+    assert_stderr --partial "field meh not found in type fileacquisition.FileConfiguration"
+}

From 0f3ae64062675884f53f8848a287e7579f0d1935 Mon Sep 17 00:00:00 2001
From: mmetc <92726601+mmetc@users.noreply.github.com>
Date: Tue, 5 Dec 2023 10:38:21 +0100
Subject: [PATCH 08/11] cscli config show: pretty print with package "litter"
 (#2633)

---
 cmd/crowdsec-cli/config_show.go     |  9 +++++----
 go.mod                              |  1 +
 go.sum                              |  5 +++++
 test/bats/01_cscli.bats             | 14 ++++++++++++++
 test/bats/05_config_yaml_local.bats |  8 ++++----
 5 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/cmd/crowdsec-cli/config_show.go b/cmd/crowdsec-cli/config_show.go
index ca7051195..1fd795c87 100644
--- a/cmd/crowdsec-cli/config_show.go
+++ b/cmd/crowdsec-cli/config_show.go
@@ -7,6 +7,7 @@ import (
 	"text/template"
 
 	"github.com/antonmedv/expr"
+	"github.com/sanity-io/litter"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
@@ -35,13 +36,13 @@ func showConfigKey(key string) error {
 
 	switch csConfig.Cscli.Output {
 	case "human", "raw":
+		// Don't use litter for strings, it adds quotes
+		// that we didn't have before
 		switch output.(type) {
 		case string:
-			fmt.Printf("%s\n", output)
-		case int:
-			fmt.Printf("%d\n", output)
+			fmt.Println(output)
 		default:
-			fmt.Printf("%v\n", output)
+			litter.Dump(output)
 		}
 	case "json":
 		data, err := json.MarshalIndent(output, "", "  ")
diff --git a/go.mod b/go.mod
index c6b148377..450926ca2 100644
--- a/go.mod
+++ b/go.mod
@@ -70,6 +70,7 @@ require (
 	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/prom2json v1.3.0
 	github.com/r3labs/diff/v2 v2.14.1
+	github.com/sanity-io/litter v1.5.5
 	github.com/segmentio/kafka-go v0.4.45
 	github.com/shirou/gopsutil/v3 v3.23.5
 	github.com/sirupsen/logrus v1.9.3
diff --git a/go.sum b/go.sum
index 023423876..2703ac09f 100644
--- a/go.sum
+++ b/go.sum
@@ -104,6 +104,7 @@ github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0
 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
 github.com/crowdsecurity/machineid v1.0.2/go.mod h1:XWUSlnS0R0+u/JK5ulidwlbceNT3ZOCKteoVQEn6Luo=
+github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -554,6 +555,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
@@ -593,6 +595,8 @@ github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OK
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo=
+github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/segmentio/kafka-go v0.4.45 h1:prqrZp1mMId4kI6pyPolkLsH6sWOUmDxmmucbL4WS6E=
 github.com/segmentio/kafka-go v0.4.45/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
@@ -630,6 +634,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats
index a49df68bc..518590826 100644
--- a/test/bats/01_cscli.bats
+++ b/test/bats/01_cscli.bats
@@ -108,6 +108,20 @@ teardown() {
     rune -0 cscli config show -o json
     rune -0 jq -c '.API.Client.Credentials | [.url,.login[0:32]]' <(output)
     assert_json '["http://127.0.0.1:8080/","githubciXXXXXXXXXXXXXXXXXXXXXXXX"]'
+
+    # pointer to boolean
+
+    rune -0 cscli config show --key Config.API.Client.InsecureSkipVerify
+    assert_output "&false"
+
+    # complex type
+    rune -0 cscli config show --key Config.PluginConfig
+    assert_output - <<-EOT
+	&csconfig.PluginCfg{
+	  User: "nobody",
+	  Group: "nogroup",
+	}
+	EOT
 }
 
 @test "cscli - required configuration paths" {
diff --git a/test/bats/05_config_yaml_local.bats b/test/bats/05_config_yaml_local.bats
index 6f8b5c28b..b8b6da117 100644
--- a/test/bats/05_config_yaml_local.bats
+++ b/test/bats/05_config_yaml_local.bats
@@ -34,22 +34,22 @@ teardown() {
 @test "config.yaml.local - cscli (log_level)" {
     config_set '.common.log_level="warning"'
     rune -0 cscli config show --key Config.Common.LogLevel
-    assert_output "warning"
+    assert_output "&3"
 
     echo "{'common':{'log_level':'debug'}}" >"${CONFIG_YAML}.local"
     rune -0 cscli config show --key Config.Common.LogLevel
-    assert_output "debug"
+    assert_output "&5"
 }
 
 @test "config.yaml.local - cscli (log_level - with envvar)" {
     config_set '.common.log_level="warning"'
     rune -0 cscli config show --key Config.Common.LogLevel
-    assert_output "warning"
+    assert_output "&3"
 
     export CROWDSEC_LOG_LEVEL=debug
     echo "{'common':{'log_level':'${CROWDSEC_LOG_LEVEL}'}}" >"${CONFIG_YAML}.local"
     rune -0 cscli config show --key Config.Common.LogLevel
-    assert_output "debug"
+    assert_output "&5"
 }
 
 @test "config.yaml.local - crowdsec (listen_url)" {

From 8bb7da399401698f59aa707d27218706cc11b8df Mon Sep 17 00:00:00 2001
From: mmetc <92726601+mmetc@users.noreply.github.com>
Date: Tue, 5 Dec 2023 11:52:04 +0100
Subject: [PATCH 09/11] docker tests: force local machine creation (#2636)

This is required from 1.5.6 to overwrite the local credentials file
---
 docker/docker_start.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/docker_start.sh b/docker/docker_start.sh
index 8a3e55529..61d695f2c 100755
--- a/docker/docker_start.sh
+++ b/docker/docker_start.sh
@@ -202,7 +202,7 @@ if isfalse "$DISABLE_LOCAL_API"; then
             # if the db is persistent but the credentials are not, we need to
             # delete the old machine to generate new credentials
             cscli machines delete "$CUSTOM_HOSTNAME" >/dev/null 2>&1 || true
-            cscli machines add "$CUSTOM_HOSTNAME" --auto
+            cscli machines add "$CUSTOM_HOSTNAME" --auto --force
         fi
     fi
 

From 486f96e7accaf2a8bdf40f30e2a5fc00135f7589 Mon Sep 17 00:00:00 2001
From: mmetc <92726601+mmetc@users.noreply.github.com>
Date: Tue, 5 Dec 2023 12:08:35 +0100
Subject: [PATCH 10/11] cscli context detect: fix nil dereference (#2635)

* cscli context detect: fix nil dereference
* Remove log.warning for missing pattern
---
 cmd/crowdsec-cli/lapi.go | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go
index 755c2cb6d..9b6900a8f 100644
--- a/cmd/crowdsec-cli/lapi.go
+++ b/cmd/crowdsec-cli/lapi.go
@@ -332,7 +332,7 @@ cscli lapi context detect crowdsecurity/sshd-logs
 			}
 
 			// to avoid all the log.Info from the loaders functions
-			log.SetLevel(log.ErrorLevel)
+			log.SetLevel(log.WarnLevel)
 
 			err = exprhelpers.Init(nil)
 			if err != nil {
@@ -499,13 +499,13 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
 
 	if node.Grok.RegexpName != "" {
 		grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName)
-		if err != nil {
-			log.Warningf("Can't get subgrok: %s", err)
-		}
-		for _, capturedField := range grokCompiled.Names() {
-			fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
-			if !slices.Contains(ret, fieldName) {
-				ret = append(ret, fieldName)
+		// ignore error (parser does not exist?)
+		if err == nil {
+			for _, capturedField := range grokCompiled.Names() {
+				fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+				if !slices.Contains(ret, fieldName) {
+					ret = append(ret, fieldName)
+				}
 			}
 		}
 	}
@@ -545,13 +545,13 @@ func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
 		}
 		if subnode.Grok.RegexpName != "" {
 			grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName)
-			if err != nil {
-				log.Warningf("Can't get subgrok: %s", err)
-			}
-			for _, capturedField := range grokCompiled.Names() {
-				fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
-				if !slices.Contains(ret, fieldName) {
-					ret = append(ret, fieldName)
+			if err == nil {
+				// ignore error (parser does not exist?)
+				for _, capturedField := range grokCompiled.Names() {
+					fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+					if !slices.Contains(ret, fieldName) {
+						ret = append(ret, fieldName)
+					}
 				}
 			}
 		}

From 1ab4487b652ed8569336a2e5f5ce9b0b886b888e Mon Sep 17 00:00:00 2001
From: mmetc <92726601+mmetc@users.noreply.github.com>
Date: Tue, 5 Dec 2023 13:38:52 +0100
Subject: [PATCH 11/11] cscli hub list: show only non-empty tables with -o
 human

* agent config: remove unused LintOnly bool
* Item.IsLocal() -> Item.State.IsLocal(); split method InstallStatus()
* cscli hub list: show only non-empty tables with -o human
---
 cmd/crowdsec-cli/config_backup.go |  4 +-
 cmd/crowdsec-cli/hub.go           |  2 +-
 cmd/crowdsec-cli/itemcommands.go  |  2 +-
 cmd/crowdsec-cli/items.go         | 18 ++++--
 cmd/crowdsec-cli/support.go       |  2 +-
 cmd/crowdsec-cli/utils_table.go   |  4 +-
 cmd/crowdsec/main.go              |  4 --
 pkg/csconfig/crowdsec_service.go  |  1 -
 pkg/cwhub/hub.go                  |  2 +-
 pkg/cwhub/item.go                 | 93 +++++++++++++++----------------
 pkg/cwhub/item_test.go            |  4 +-
 pkg/cwhub/iteminstall.go          |  2 +-
 pkg/cwhub/itemremove.go           |  2 +-
 pkg/cwhub/itemupgrade.go          |  4 +-
 pkg/cwhub/sync.go                 |  2 +-
 test/bats/20_hub.bats             |  8 ++-
 16 files changed, 77 insertions(+), 77 deletions(-)

diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go
index 93772d611..c4d09e687 100644
--- a/cmd/crowdsec-cli/config_backup.go
+++ b/cmd/crowdsec-cli/config_backup.go
@@ -46,7 +46,7 @@ func backupHub(dirPath string) error {
 			}
 
 			//for the local/tainted ones, we back up the full file
-			if v.State.Tainted || v.IsLocal() || !v.State.UpToDate {
+			if v.State.Tainted || v.State.IsLocal() || !v.State.UpToDate {
 				//we need to backup stages for parsers
 				if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
 					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
@@ -54,7 +54,7 @@ func backupHub(dirPath string) error {
 						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
 					}
 				}
-				clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.IsLocal(), v.State.UpToDate)
+				clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.State.IsLocal(), v.State.UpToDate)
 				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
 				if err = CopyFile(v.State.LocalPath, tfile); err != nil {
 					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err)
diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go
index 2d139cd90..0a48e0231 100644
--- a/cmd/crowdsec-cli/hub.go
+++ b/cmd/crowdsec-cli/hub.go
@@ -66,7 +66,7 @@ func runHubList(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	err = listItems(color.Output, cwhub.ItemTypes, items)
+	err = listItems(color.Output, cwhub.ItemTypes, items, true)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go
index a95803def..5c699fde9 100644
--- a/cmd/crowdsec-cli/itemcommands.go
+++ b/cmd/crowdsec-cli/itemcommands.go
@@ -580,7 +580,7 @@ func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) err
 			return err
 		}
 
-		if err = listItems(color.Output, []string{it.name}, items); err != nil {
+		if err = listItems(color.Output, []string{it.name}, items, false); err != nil {
 			return err
 		}
 
diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go
index c77ff3f88..6a91ab1f5 100644
--- a/cmd/crowdsec-cli/items.go
+++ b/cmd/crowdsec-cli/items.go
@@ -53,11 +53,19 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b
 	return items, nil
 }
 
-func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item) error {
+func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item, omitIfEmpty bool) error {
 	switch csConfig.Cscli.Output {
 	case "human":
+		nothingToDisplay := true
 		for _, itemType := range itemTypes {
+			if omitIfEmpty && len(items[itemType]) == 0 {
+				continue
+			}
 			listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
+			nothingToDisplay = false
+		}
+		if nothingToDisplay {
+			fmt.Println("No items to display")
 		}
 	case "json":
 		type itemHubStatus struct {
@@ -75,14 +83,15 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item
 			hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
 
 			for i, item := range items[itemType] {
-				status, emo := item.InstallStatus()
+				status := item.State.Text()
+				status_emo := item.State.Emoji()
 				hubStatus[itemType][i] = itemHubStatus{
 					Name:         item.Name,
 					LocalVersion: item.State.LocalVersion,
 					LocalPath:    item.State.LocalPath,
 					Description:  item.Description,
 					Status:       status,
-					UTF8Status:   fmt.Sprintf("%v  %s", emo, status),
+					UTF8Status:   fmt.Sprintf("%v  %s", status_emo, status),
 				}
 			}
 		}
@@ -107,10 +116,9 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item
 
 		for _, itemType := range itemTypes {
 			for _, item := range items[itemType] {
-				status, _ := item.InstallStatus()
 				row := []string{
 					item.Name,
-					status,
+					item.State.Text(),
 					item.State.LocalVersion,
 					item.Description,
 				}
diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go
index 1470d37aa..6395ad6d7 100644
--- a/cmd/crowdsec-cli/support.go
+++ b/cmd/crowdsec-cli/support.go
@@ -140,7 +140,7 @@ func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
 		log.Warnf("could not collect %s list: %s", itemType, err)
 	}
 
-	if err := listItems(out, []string{itemType}, items); err != nil {
+	if err := listItems(out, []string{itemType}, items, false); err != nil {
 		log.Warnf("could not collect %s list: %s", itemType, err)
 	}
 	return out.Bytes()
diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go
index 28b185a01..f3179e40f 100644
--- a/cmd/crowdsec-cli/utils_table.go
+++ b/cmd/crowdsec-cli/utils_table.go
@@ -18,8 +18,8 @@ func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	for _, item := range items {
-		status, emo := item.InstallStatus()
-		t.AddRow(item.Name, fmt.Sprintf("%v  %s", emo, status), item.State.LocalVersion, item.State.LocalPath)
+		status := fmt.Sprintf("%v  %s", item.State.Emoji(), item.State.Text())
+		t.AddRow(item.Name, status, item.State.LocalVersion, item.State.LocalPath)
 	}
 	renderTableTitle(out, title)
 	t.Render()
diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go
index 8c7fb2991..362ed1869 100644
--- a/cmd/crowdsec/main.go
+++ b/cmd/crowdsec/main.go
@@ -262,10 +262,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		return nil, errors.New("You must run at least the API Server or crowdsec")
 	}
 
-	if flags.TestMode && !cConfig.DisableAgent {
-		cConfig.Crowdsec.LintOnly = true
-	}
-
 	if flags.OneShotDSN != "" && flags.SingleFileType == "" {
 		return nil, errors.New("-dsn requires a -type argument")
 	}
diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go
index dc226cfd6..c0b8b5a38 100644
--- a/pkg/csconfig/crowdsec_service.go
+++ b/pkg/csconfig/crowdsec_service.go
@@ -23,7 +23,6 @@ type CrowdsecServiceCfg struct {
 	BucketsRoutinesCount      int               `yaml:"buckets_routines"`
 	OutputRoutinesCount       int               `yaml:"output_routines"`
 	SimulationConfig          *SimulationConfig `yaml:"-"`
-	LintOnly                  bool              `yaml:"-"`                          // if set to true, exit after loading configs
 	BucketStateFile           string            `yaml:"state_input_file,omitempty"` // if we need to unserialize buckets at start
 	BucketStateDumpDir        string            `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
 	BucketsGCEnabled          bool              `yaml:"-"`                          // we need to garbage collect buckets when in forensic mode
diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go
index 18218bd87..bd623e1f9 100644
--- a/pkg/cwhub/hub.go
+++ b/pkg/cwhub/hub.go
@@ -109,7 +109,7 @@ func (h *Hub) ItemStats() []string {
 		loaded += fmt.Sprintf("%d %s, ", len(h.GetItemMap(itemType)), itemType)
 
 		for _, item := range h.GetItemMap(itemType) {
-			if item.IsLocal() {
+			if item.State.IsLocal() {
 				local++
 			}
 
diff --git a/pkg/cwhub/item.go b/pkg/cwhub/item.go
index 9c3ec8cd2..9cf044396 100644
--- a/pkg/cwhub/item.go
+++ b/pkg/cwhub/item.go
@@ -53,6 +53,48 @@ type ItemState struct {
 	BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"`
 }
 
+// IsLocal returns true if the item has been create by a user (not downloaded from the hub).
+func (s *ItemState) IsLocal() bool {
+	return s.Installed && !s.Downloaded
+}
+
+// Text returns the status of the item as a string (eg. "enabled,update-available").
+func (s *ItemState) Text() string {
+	ret := "disabled"
+
+	if s.Installed {
+		ret = "enabled"
+	}
+
+	if s.IsLocal() {
+		ret += ",local"
+	}
+
+	if s.Tainted {
+		ret += ",tainted"
+	} else if !s.UpToDate && !s.IsLocal() {
+		ret += ",update-available"
+	}
+
+	return ret
+}
+
+// Emoji returns the status of the item as an emoji (eg. emoji.Warning).
+func (s *ItemState) Emoji() emoji.Emoji {
+	switch {
+	case s.IsLocal():
+		return emoji.House
+	case !s.Installed:
+		return emoji.Prohibited
+	case s.Tainted || (!s.UpToDate && !s.IsLocal()):
+		return emoji.Warning
+	case s.Installed:
+		return emoji.CheckMark
+	default:
+		return emoji.QuestionMark
+	}
+}
+
 // Item is created from an index file and enriched with local info.
 type Item struct {
 	hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods
@@ -107,11 +149,6 @@ func (i *Item) HasSubItems() bool {
 	return i.Type == COLLECTIONS
 }
 
-// IsLocal returns true if the item has been create by a user (not downloaded from the hub).
-func (i *Item) IsLocal() bool {
-	return i.State.Installed && !i.State.Downloaded
-}
-
 // MarshalJSON is used to prepare the output for "cscli ... inspect -o json".
 // It must not use a pointer receiver.
 func (i Item) MarshalJSON() ([]byte, error) {
@@ -139,7 +176,7 @@ func (i Item) MarshalJSON() ([]byte, error) {
 		UpToDate:             i.State.UpToDate,
 		Tainted:              i.State.Tainted,
 		BelongsToCollections: i.State.BelongsToCollections,
-		Local:                i.IsLocal(),
+		Local:                i.State.IsLocal(),
 	})
 }
 
@@ -155,7 +192,7 @@ func (i Item) MarshalYAML() (interface{}, error) {
 	}{
 		Alias: Alias(i),
 		State: i.State,
-		Local: i.IsLocal(),
+		Local: i.State.IsLocal(),
 	}, nil
 }
 
@@ -290,48 +327,6 @@ func (i *Item) descendants() ([]*Item, error) {
 	return ret, nil
 }
 
-// InstallStatus returns the status of the item as a string and an emoji
-// (eg. "enabled,update-available" and emoji.Warning).
-func (i *Item) InstallStatus() (string, emoji.Emoji) {
-	status := "disabled"
-	ok := false
-
-	if i.State.Installed {
-		ok = true
-		status = "enabled"
-	}
-
-	managed := true
-	if i.IsLocal() {
-		managed = false
-		status += ",local"
-	}
-
-	warning := false
-	if i.State.Tainted {
-		warning = true
-		status += ",tainted"
-	} else if !i.State.UpToDate && !i.IsLocal() {
-		warning = true
-		status += ",update-available"
-	}
-
-	emo := emoji.QuestionMark
-
-	switch {
-	case !managed:
-		emo = emoji.House
-	case !i.State.Installed:
-		emo = emoji.Prohibited
-	case warning:
-		emo = emoji.Warning
-	case ok:
-		emo = emoji.CheckMark
-	}
-
-	return status, emo
-}
-
 // versionStatus returns the status of the item version compared to the hub version.
 // semver requires the 'v' prefix.
 func (i *Item) versionStatus() int {
diff --git a/pkg/cwhub/item_test.go b/pkg/cwhub/item_test.go
index 33a52e8e0..703bbb5cb 100644
--- a/pkg/cwhub/item_test.go
+++ b/pkg/cwhub/item_test.go
@@ -23,7 +23,7 @@ func TestItemStatus(t *testing.T) {
 		item.State.Tainted = false
 		item.State.Downloaded = true
 
-		txt, _ := item.InstallStatus()
+		txt := item.State.Text()
 		require.Equal(t, "enabled,update-available", txt)
 
 		item.State.Installed = true
@@ -31,7 +31,7 @@ func TestItemStatus(t *testing.T) {
 		item.State.Tainted = false
 		item.State.Downloaded = false
 
-		txt, _ = item.InstallStatus()
+		txt = item.State.Text()
 		require.Equal(t, "enabled,local", txt)
 	}
 
diff --git a/pkg/cwhub/iteminstall.go b/pkg/cwhub/iteminstall.go
index 038e5471d..270d4c114 100644
--- a/pkg/cwhub/iteminstall.go
+++ b/pkg/cwhub/iteminstall.go
@@ -13,7 +13,7 @@ func (i *Item) enable() error {
 			return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
 		}
 
-		if i.IsLocal() {
+		if i.State.IsLocal() {
 			return fmt.Errorf("%s is local, won't enable", i.Name)
 		}
 
diff --git a/pkg/cwhub/itemremove.go b/pkg/cwhub/itemremove.go
index cba5a5904..a58bd3fa8 100644
--- a/pkg/cwhub/itemremove.go
+++ b/pkg/cwhub/itemremove.go
@@ -66,7 +66,7 @@ func (i *Item) disable(purge bool, force bool) (bool, error) {
 
 // Remove disables the item, optionally removing the downloaded content.
 func (i *Item) Remove(purge bool, force bool) (bool, error) {
-	if i.IsLocal() {
+	if i.State.IsLocal() {
 		log.Warningf("%s is a local item, please delete manually", i.Name)
 		return false, nil
 	}
diff --git a/pkg/cwhub/itemupgrade.go b/pkg/cwhub/itemupgrade.go
index 5e7c6f71a..07c83b3c4 100644
--- a/pkg/cwhub/itemupgrade.go
+++ b/pkg/cwhub/itemupgrade.go
@@ -20,7 +20,7 @@ import (
 func (i *Item) Upgrade(force bool) (bool, error) {
 	updated := false
 
-	if i.IsLocal() {
+	if i.State.IsLocal() {
 		log.Infof("not upgrading %s: local item", i.Name)
 		return false, nil
 	}
@@ -155,7 +155,7 @@ func (i *Item) fetch() ([]byte, error) {
 
 // download downloads the item from the hub and writes it to the hub directory.
 func (i *Item) download(overwrite bool) (string, error) {
-	if i.IsLocal() {
+	if i.State.IsLocal() {
 		return "", fmt.Errorf("%s is local, can't download", i.Name)
 	}
 	// if user didn't --force, don't overwrite local, tainted, up-to-date files
diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go
index bbd8b2724..ae62b9342 100644
--- a/pkg/cwhub/sync.go
+++ b/pkg/cwhub/sync.go
@@ -420,7 +420,7 @@ func (h *Hub) localSync() error {
 		case versionFuture:
 			warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
 		case versionUnknown:
-			if !item.IsLocal() {
+			if !item.State.IsLocal() {
 				warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
 			}
 		}
diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats
index 5189044b9..f64a64e4b 100644
--- a/test/bats/20_hub.bats
+++ b/test/bats/20_hub.bats
@@ -34,17 +34,19 @@ teardown() {
 
     # no items
     rune -0 cscli hub list
-    assert_output --regexp ".*PARSERS.*POSTOVERFLOWS.*SCENARIOS.*COLLECTIONS.*"
+    assert_output "No items to display"
     rune -0 cscli hub list -o json
     assert_json '{parsers:[],scenarios:[],collections:[],postoverflows:[]}'
     rune -0 cscli hub list -o raw
     assert_output 'name,status,version,description,type'
 
-    # some items
+    # some items: with output=human, show only non-empty tables
     rune -0 cscli parsers install crowdsecurity/whitelists
     rune -0 cscli scenarios install crowdsecurity/telnet-bf
     rune -0 cscli hub list
-    assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*"
+    assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*"
+    refute_output --partial 'POSTOVERFLOWS'
+    refute_output --partial 'COLLECTIONS'
     rune -0 cscli hub list -o json
     rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output)
     rune -0 cscli hub list -o raw