merge master

cmd/crowdsec-cli/capi.go

@@ -110,7 +110,7 @@ func NewCapiRegisterCmd() *cobra.Command {
 				if err != nil {
 					return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
 				}
-				log.Printf("Central API credentials dumped to '%s'", dumpFile)
+				log.Printf("Central API credentials written to '%s'", dumpFile)
 			} else {
 				fmt.Printf("%s\n", string(apiConfigDump))
 			}

cmd/crowdsec-cli/config_backup.go

@@ -46,7 +46,7 @@ func backupHub(dirPath string) error {
 			}
 
 			//for the local/tainted ones, we back up the full file
-			if v.State.Tainted || v.IsLocal() || !v.State.UpToDate {
+			if v.State.Tainted || v.State.IsLocal() || !v.State.UpToDate {
 				//we need to backup stages for parsers
 				if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
 					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
@@ -54,7 +54,7 @@ func backupHub(dirPath string) error {
 						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
 					}
 				}
-				clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.IsLocal(), v.State.UpToDate)
+				clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.State.IsLocal(), v.State.UpToDate)
 				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
 				if err = CopyFile(v.State.LocalPath, tfile); err != nil {
 					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err)

cmd/crowdsec-cli/config_restore.go

@@ -183,7 +183,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 			if csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
 				apiConfigDumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath
 			}
-			err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o644)
+			err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o600)
 			if err != nil {
 				return fmt.Errorf("write api credentials in '%s' failed: %s", apiConfigDumpFile, err)
 			}

cmd/crowdsec-cli/config_show.go

@@ -7,6 +7,7 @@ import (
 	"text/template"
 
 	"github.com/antonmedv/expr"
+	"github.com/sanity-io/litter"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
@@ -35,13 +36,13 @@ func showConfigKey(key string) error {
 
 	switch csConfig.Cscli.Output {
 	case "human", "raw":
+		// Don't use litter for strings, it adds quotes
+		// that we didn't have before
 		switch output.(type) {
 		case string:
-			fmt.Printf("%s\n", output)
+			fmt.Println(output)
-		case int:
-			fmt.Printf("%d\n", output)
 		default:
-			fmt.Printf("%v\n", output)
+			litter.Dump(output)
 		}
 	case "json":
 		data, err := json.MarshalIndent(output, "", "  ")

cmd/crowdsec-cli/dashboard.go

@@ -1,3 +1,5 @@
+//go:build linux
+
 package main
 
 import (
@@ -9,6 +11,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"unicode"
 
 	"github.com/AlecAivazis/survey/v2"
@@ -136,6 +139,9 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
 			if err != nil {
 				return err
 			}
+			if err = chownDatabase(dockerGroup.Gid); err != nil {
+				return err
+			}
 			mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
 			if err != nil {
 				return err
@@ -366,45 +372,56 @@ func disclaimer(forceYes *bool) error {
 }
 
 func checkGroups(forceYes *bool) (*user.Group, error) {
-	groupExist := false
 	dockerGroup, err := user.LookupGroup(crowdsecGroup)
 	if err == nil {
-		groupExist = true
+		return dockerGroup, nil
 	}
-	if !groupExist {
+	if !*forceYes {
-		if !*forceYes {
+		var answer bool
-			var answer bool
+		prompt := &survey.Confirm{
-			prompt := &survey.Confirm{
+			Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
-				Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
+			Default: true,
-				Default: true,
-			}
-			if err := survey.AskOne(prompt, &answer); err != nil {
-				return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
-			}
-			if !answer {
-				return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
-			}
 		}
-		groupAddCmd, err := exec.LookPath("groupadd")
+		if err := survey.AskOne(prompt, &answer); err != nil {
-		if err != nil {
+			return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
-			return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
 		}
-
+		if !answer {
-		groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
+			return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
-		if err := groupAdd.Run(); err != nil {
-			return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
-		}
-		dockerGroup, err = user.LookupGroup(crowdsecGroup)
-		if err != nil {
-			return dockerGroup, fmt.Errorf("unable to lookup '%s' group: %+v", dockerGroup, err)
 		}
 	}
-	intID, err := strconv.Atoi(dockerGroup.Gid)
+	groupAddCmd, err := exec.LookPath("groupadd")
 	if err != nil {
-		return dockerGroup, fmt.Errorf("unable to convert group ID to int: %s", err)
+		return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
 	}
-	if err := os.Chown(csConfig.DbConfig.DbPath, 0, intID); err != nil {
+
-		return dockerGroup, fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
+	groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
+	if err := groupAdd.Run(); err != nil {
+		return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
 	}
-	return dockerGroup, nil
+	return user.LookupGroup(crowdsecGroup)
+}
+
+func chownDatabase(gid string) error {
+	intID, err := strconv.Atoi(gid)
+	if err != nil {
+		return fmt.Errorf("unable to convert group ID to int: %s", err)
+	}
+	if stat, err := os.Stat(csConfig.DbConfig.DbPath); !os.IsNotExist(err) {
+		info := stat.Sys()
+		if err := os.Chown(csConfig.DbConfig.DbPath, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
+			return fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
+		}
+	}
+	if csConfig.DbConfig.Type == "sqlite" && csConfig.DbConfig.UseWal != nil && *csConfig.DbConfig.UseWal {
+		for _, ext := range []string{"-wal", "-shm"} {
+			file := csConfig.DbConfig.DbPath + ext
+			if stat, err := os.Stat(file); !os.IsNotExist(err) {
+				info := stat.Sys()
+				if err := os.Chown(file, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
+					return fmt.Errorf("unable to chown sqlite db file '%s': %s", file, err)
+				}
+			}
+		}
+	}
+	return nil
 }

cmd/crowdsec-cli/dashboard_unsupported.go

@@ -0,0 +1,22 @@
+//go:build !linux
+
+package main
+
+import (
+	"runtime"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func NewDashboardCmd() *cobra.Command {
+	var cmdDashboard = &cobra.Command{
+		Use:               "dashboard",
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			log.Infof("Dashboard command is disabled on %s", runtime.GOOS)
+		},
+	}
+
+	return cmdDashboard
+}

cmd/crowdsec-cli/decisions_import.go

@@ -232,7 +232,7 @@ func NewDecisionsImportCmd() *cobra.Command {
 		Short: "Import decisions from a file or pipe",
 		Long: "expected format:\n" +
 			"csv  : any of duration,reason,scope,type,value, with a header line\n" +
-			`json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`,
+			"json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`",
 		DisableAutoGenTag: true,
 		Example: `decisions.csv:
 duration,scope,value

cmd/crowdsec-cli/explain.go

@@ -21,9 +21,15 @@ func GetLineCountForFile(filepath string) (int, error) {
 	}
 	defer f.Close()
 	lc := 0
-	fs := bufio.NewScanner(f)
+	fs := bufio.NewReader(f)
-	for fs.Scan() {
+	for {
-		lc++
+		input, err := fs.ReadBytes('\n')
+		if len(input) > 1 {
+			lc++
+		}
+		if err != nil && err == io.EOF {
+			break
+		}
 	}
 	return lc, nil
 }
@@ -79,19 +85,6 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	fileInfo, _ := os.Stdin.Stat()
-
-	if logType == "" || (logLine == "" && logFile == "" && dsn == "") {
-		printHelp(cmd)
-		fmt.Println()
-		fmt.Printf("Please provide --type flag\n")
-		os.Exit(1)
-	}
-
-	if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
-		return fmt.Errorf("the option -f - is intended to work with pipes")
-	}
-
 	var f *os.File
 
 	// using empty string fallback to /tmp
@@ -99,6 +92,13 @@ func runExplain(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err)
 	}
+	defer func() {
+		if _, err := os.Stat(dir); !os.IsNotExist(err) {
+			if err := os.RemoveAll(dir); err != nil {
+				log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
+			}
+		}
+	}()
 	tmpFile := ""
 	// we create a  temporary log file if a log line/stdin has been provided
 	if logLine != "" || logFile == "-" {
@@ -121,13 +121,15 @@ func runExplain(cmd *cobra.Command, args []string) error {
 				if err != nil && err == io.EOF {
 					break
 				}
-				_, err = f.Write(input)
+				if len(input) > 1 {
-				if err != nil {
+					_, err = f.Write(input)
+				}
+				if err != nil || len(input) <= 1 {
 					errCount++
 				}
 			}
 			if errCount > 0 {
-				log.Warnf("Failed to write %d lines to tmp file", errCount)
+				log.Warnf("Failed to write %d lines to %s", errCount, tmpFile)
 			}
 		}
 		f.Close()
@@ -145,8 +147,12 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		if err != nil {
 			return err
 		}
+		log.Debugf("file %s has %d lines", absolutePath, lineCount)
+		if lineCount == 0 {
+			return fmt.Errorf("the log file is empty: %s", absolutePath)
+		}
 		if lineCount > 100 {
-			log.Warnf("The log file contains %d lines. This may take a lot of resources.", lineCount)
+			log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount)
 		}
 	}
 
@@ -166,12 +172,6 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("fail to run crowdsec for test: %v", err)
 	}
 
-	// rm the temporary log file if only a log line/stdin was provided
-	if tmpFile != "" {
-		if err := os.Remove(tmpFile); err != nil {
-			return fmt.Errorf("unable to remove tmp log file '%s': %+v", tmpFile, err)
-		}
-	}
 	parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
 	bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)
 
@@ -187,10 +187,6 @@ func runExplain(cmd *cobra.Command, args []string) error {
 
 	hubtest.DumpTree(*parserDump, *bucketStateDump, opts)
 
-	if err := os.RemoveAll(dir); err != nil {
-		return fmt.Errorf("unable to delete temporary directory '%s': %s", dir, err)
-	}
-
 	return nil
 }
 
@@ -210,6 +206,45 @@ tail -n 5 myfile.log | cscli explain --type nginx -f -
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
 		RunE:              runExplain,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			flags := cmd.Flags()
+
+			logFile, err := flags.GetString("file")
+			if err != nil {
+				return err
+			}
+
+			dsn, err := flags.GetString("dsn")
+			if err != nil {
+				return err
+			}
+
+			logLine, err := flags.GetString("log")
+			if err != nil {
+				return err
+			}
+
+			logType, err := flags.GetString("type")
+			if err != nil {
+				return err
+			}
+
+			if logLine == "" && logFile == "" && dsn == "" {
+				printHelp(cmd)
+				fmt.Println()
+				return fmt.Errorf("please provide --log, --file or --dsn flag")
+			}
+			if logType == "" {
+				printHelp(cmd)
+				fmt.Println()
+				return fmt.Errorf("please provide --type flag")
+			}
+			fileInfo, _ := os.Stdin.Stat()
+			if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
+				return fmt.Errorf("the option -f - is intended to work with pipes")
+			}
+			return nil
+		},
 	}
 
 	flags := cmdExplain.Flags()

cmd/crowdsec-cli/hub.go

@@ -66,7 +66,7 @@ func runHubList(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	err = listItems(color.Output, cwhub.ItemTypes, items)
+	err = listItems(color.Output, cwhub.ItemTypes, items, true)
 	if err != nil {
 		return err
 	}

cmd/crowdsec-cli/itemcommands.go

@@ -640,7 +640,7 @@ func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) err
 			return err
 		}
 
-		if err = listItems(color.Output, []string{it.name}, items); err != nil {
+		if err = listItems(color.Output, []string{it.name}, items, false); err != nil {
 			return err
 		}
 

cmd/crowdsec-cli/items.go

@@ -53,11 +53,19 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b
 	return items, nil
 }
 
-func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item) error {
+func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item, omitIfEmpty bool) error {
 	switch csConfig.Cscli.Output {
 	case "human":
+		nothingToDisplay := true
 		for _, itemType := range itemTypes {
+			if omitIfEmpty && len(items[itemType]) == 0 {
+				continue
+			}
 			listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
+			nothingToDisplay = false
+		}
+		if nothingToDisplay {
+			fmt.Println("No items to display")
 		}
 	case "json":
 		type itemHubStatus struct {
@@ -75,14 +83,15 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item
 			hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
 
 			for i, item := range items[itemType] {
-				status, emo := item.InstallStatus()
+				status := item.State.Text()
+				status_emo := item.State.Emoji()
 				hubStatus[itemType][i] = itemHubStatus{
 					Name:         item.Name,
 					LocalVersion: item.State.LocalVersion,
 					LocalPath:    item.State.LocalPath,
 					Description:  item.Description,
 					Status:       status,
-					UTF8Status:   fmt.Sprintf("%v  %s", emo, status),
+					UTF8Status:   fmt.Sprintf("%v  %s", status_emo, status),
 				}
 			}
 		}
@@ -107,10 +116,9 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item
 
 		for _, itemType := range itemTypes {
 			for _, item := range items[itemType] {
-				status, _ := item.InstallStatus()
 				row := []string{
 					item.Name,
-					status,
+					item.State.Text(),
 					item.State.LocalVersion,
 					item.Description,
 				}

cmd/crowdsec-cli/lapi.go

@@ -150,11 +150,11 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 		log.Fatalf("unable to marshal api credentials: %s", err)
 	}
 	if dumpFile != "" {
-		err = os.WriteFile(dumpFile, apiConfigDump, 0644)
+		err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
 		if err != nil {
 			log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
 		}
-		log.Printf("Local API credentials dumped to '%s'", dumpFile)
+		log.Printf("Local API credentials written to '%s'", dumpFile)
 	} else {
 		fmt.Printf("%s\n", string(apiConfigDump))
 	}
@@ -332,7 +332,7 @@ cscli lapi context detect crowdsecurity/sshd-logs
 			}
 
 			// to avoid all the log.Info from the loaders functions
-			log.SetLevel(log.ErrorLevel)
+			log.SetLevel(log.WarnLevel)
 
 			err = exprhelpers.Init(nil)
 			if err != nil {
@@ -499,13 +499,13 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
 
 	if node.Grok.RegexpName != "" {
 		grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName)
-		if err != nil {
+		// ignore error (parser does not exist?)
-			log.Warningf("Can't get subgrok: %s", err)
+		if err == nil {
-		}
+			for _, capturedField := range grokCompiled.Names() {
-		for _, capturedField := range grokCompiled.Names() {
+				fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
-			fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+				if !slices.Contains(ret, fieldName) {
-			if !slices.Contains(ret, fieldName) {
+					ret = append(ret, fieldName)
-				ret = append(ret, fieldName)
+				}
 			}
 		}
 	}
@@ -545,13 +545,13 @@ func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
 		}
 		if subnode.Grok.RegexpName != "" {
 			grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName)
-			if err != nil {
+			if err == nil {
-				log.Warningf("Can't get subgrok: %s", err)
+				// ignore error (parser does not exist?)
-			}
+				for _, capturedField := range grokCompiled.Names() {
-			for _, capturedField := range grokCompiled.Names() {
+					fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
-				fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+					if !slices.Contains(ret, fieldName) {
-				if !slices.Contains(ret, fieldName) {
+						ret = append(ret, fieldName)
-					ret = append(ret, fieldName)
+					}
 				}
 			}
 		}

cmd/crowdsec-cli/machines.go

@@ -30,9 +30,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
-var (
+const passwordLength = 64
-	passwordLength = 64
-)
 
 func generatePassword(length int) string {
 	upper := "ABCDEFGHIJKLMNOPQRSTUVWXY"
@@ -43,6 +41,7 @@ func generatePassword(length int) string {
 	charsetLength := len(charset)
 
 	buf := make([]byte, length)
+
 	for i := 0; i < length; i++ {
 		rInt, err := saferand.Int(saferand.Reader, big.NewInt(int64(charsetLength)))
 		if err != nil {
@@ -190,7 +189,6 @@ cscli machines add MyTestMachine --password MyPassword
 }
 
 func runMachinesAdd(cmd *cobra.Command, args []string) error {
-	var dumpFile string
 	var err error
 
 	flags := cmd.Flags()
@@ -200,7 +198,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	outputFile, err := flags.GetString("file")
+	dumpFile, err := flags.GetString("file")
 	if err != nil {
 		return err
 	}
@@ -220,7 +218,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	forceAdd, err := flags.GetBool("force")
+	force, err := flags.GetBool("force")
 	if err != nil {
 		return err
 	}
@@ -242,17 +240,28 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 	}
 
 	/*check if file already exists*/
-	if outputFile != "" {
+	if dumpFile == "" && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
-		dumpFile = outputFile
+		credFile := csConfig.API.Client.CredentialsFilePath
-	} else if csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
+		// use the default only if the file does not exist
-		dumpFile = csConfig.API.Client.CredentialsFilePath
+		_, err := os.Stat(credFile)
+		switch {
+		case os.IsNotExist(err) || force:
+			dumpFile = csConfig.API.Client.CredentialsFilePath
+		case err != nil:
+			return fmt.Errorf("unable to stat '%s': %s", credFile, err)
+		default:
+			return fmt.Errorf(`credentials file '%s' already exists: please remove it, use "--force" or specify a different file with "-f" ("-f -" for standard output)`, credFile)
+		}
+	}
+
+	if dumpFile == "" {
+		return fmt.Errorf(`please specify a file to dump credentials to, with -f ("-f -" for standard output)`)
 	}
 
 	// create a password if it's not specified by user
 	if machinePassword == "" && !interactive {
 		if !autoAdd {
-			printHelp(cmd)
+			return fmt.Errorf("please specify a password with --password or use --auto")
-			return nil
 		}
 		machinePassword = generatePassword(passwordLength)
 	} else if machinePassword == "" && interactive {
@@ -262,7 +271,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		survey.AskOne(qs, &machinePassword)
 	}
 	password := strfmt.Password(machinePassword)
-	_, err = dbClient.CreateMachine(&machineID, &password, "", true, forceAdd, types.PasswordAuthType)
+	_, err = dbClient.CreateMachine(&machineID, &password, "", true, force, types.PasswordAuthType)
 	if err != nil {
 		return fmt.Errorf("unable to create machine: %s", err)
 	}
@@ -291,7 +300,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		if err != nil {
 			return fmt.Errorf("write api credentials in '%s' failed: %s", dumpFile, err)
 		}
-		log.Printf("API credentials dumped to '%s'", dumpFile)
+		log.Printf("API credentials written to '%s'", dumpFile)
 	} else {
 		fmt.Printf("%s\n", string(apiConfigDump))
 	}

cmd/crowdsec-cli/support.go

@@ -140,7 +140,7 @@ func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
 		log.Warnf("could not collect %s list: %s", itemType, err)
 	}
 
-	if err := listItems(out, []string{itemType}, items); err != nil {
+	if err := listItems(out, []string{itemType}, items, false); err != nil {
 		log.Warnf("could not collect %s list: %s", itemType, err)
 	}
 	return out.Bytes()

cmd/crowdsec-cli/utils_table.go

@@ -18,8 +18,8 @@ func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	for _, item := range items {
-		status, emo := item.InstallStatus()
+		status := fmt.Sprintf("%v  %s", item.State.Emoji(), item.State.Text())
-		t.AddRow(item.Name, fmt.Sprintf("%v  %s", emo, status), item.State.LocalVersion, item.State.LocalPath)
+		t.AddRow(item.Name, status, item.State.LocalVersion, item.State.LocalPath)
 	}
 	renderTableTitle(out, title)
 	t.Render()

cmd/crowdsec/main.go

@@ -262,10 +262,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		return nil, errors.New("You must run at least the API Server or crowdsec")
 	}
 
-	if flags.TestMode && !cConfig.DisableAgent {
-		cConfig.Crowdsec.LintOnly = true
-	}
-
 	if flags.OneShotDSN != "" && flags.SingleFileType == "" {
 		return nil, errors.New("-dsn requires a -type argument")
 	}

cmd/notification-http/main.go

@@ -58,13 +58,12 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
 	if err != nil {
 		return nil, err
 	}
-
 	for headerName, headerValue := range cfg.Headers {
 		logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue))
 		request.Header.Add(headerName, headerValue)
 	}
 	logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
-	resp, err := client.Do(request)
+	resp, err := client.Do(request.WithContext(ctx))
 	if err != nil {
 		logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))
 		return nil, err

cmd/notification-sentinel/main.go

@@ -90,7 +90,7 @@ func (s *SentinelPlugin) Notify(ctx context.Context, notification *protobufs.Not
 	req.Header.Set("x-ms-date", now)
 
 	client := &http.Client{}
-	resp, err := client.Do(req)
+	resp, err := client.Do(req.WithContext(ctx))
 	if err != nil {
 		logger.Error("failed to send request", "error", err)
 		return &protobufs.Empty{}, err

cmd/notification-slack/main.go

@@ -38,10 +38,9 @@ func (n *Notify) Notify(ctx context.Context, notification *protobufs.Notificatio
 	if cfg.LogLevel != nil && *cfg.LogLevel != "" {
 		logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel))
 	}
-
 	logger.Info(fmt.Sprintf("found notify signal for %s config", notification.Name))
 	logger.Debug(fmt.Sprintf("posting to %s webhook, message %s", cfg.Webhook, notification.Text))
-	err := slack.PostWebhook(n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
+	err := slack.PostWebhookContext(ctx, n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
 		Text: notification.Text,
 	})
 	if err != nil {

cmd/notification-splunk/main.go

@@ -65,7 +65,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
 
 	req.Header.Add("Authorization", fmt.Sprintf("Splunk %s", cfg.Token))
 	logger.Debug(fmt.Sprintf("posting event %s to %s", string(data), req.URL))
-	resp, err := s.Client.Do(req)
+	resp, err := s.Client.Do(req.WithContext(ctx))
 	if err != nil {
 		return &protobufs.Empty{}, err
 	}

docker/docker_start.sh

@@ -202,7 +202,7 @@ if isfalse "$DISABLE_LOCAL_API"; then
             # if the db is persistent but the credentials are not, we need to
             # delete the old machine to generate new credentials
             cscli machines delete "$CUSTOM_HOSTNAME" >/dev/null 2>&1 || true
-            cscli machines add "$CUSTOM_HOSTNAME" --auto
+            cscli machines add "$CUSTOM_HOSTNAME" --auto --force
         fi
     fi
 

go.mod

@@ -70,6 +70,7 @@ require (
 	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/prom2json v1.3.0
 	github.com/r3labs/diff/v2 v2.14.1
+	github.com/sanity-io/litter v1.5.5
 	github.com/segmentio/kafka-go v0.4.45
 	github.com/shirou/gopsutil/v3 v3.23.5
 	github.com/sirupsen/logrus v1.9.3

go.sum

@@ -114,6 +114,7 @@ github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0
 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
 github.com/crowdsecurity/machineid v1.0.2/go.mod h1:XWUSlnS0R0+u/JK5ulidwlbceNT3ZOCKteoVQEn6Luo=
+github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -570,6 +571,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
@@ -609,6 +611,8 @@ github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OK
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo=
+github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/segmentio/kafka-go v0.4.45 h1:prqrZp1mMId4kI6pyPolkLsH6sWOUmDxmmucbL4WS6E=
 github.com/segmentio/kafka-go v0.4.45/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
@@ -646,6 +650,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

pkg/apiserver/apiserver.go

@@ -106,13 +106,13 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
 	var flushScheduler *gocron.Scheduler
 	dbClient, err := database.NewClient(config.DbConfig)
 	if err != nil {
-		return &APIServer{}, fmt.Errorf("unable to init database client: %w", err)
+		return nil, fmt.Errorf("unable to init database client: %w", err)
 	}
 
 	if config.DbConfig.Flush != nil {
 		flushScheduler, err = dbClient.StartFlushScheduler(config.DbConfig.Flush)
 		if err != nil {
-			return &APIServer{}, err
+			return nil, err
 		}
 	}
 
@@ -129,7 +129,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
 
 	if config.TrustedProxies != nil && config.UseForwardedForHeaders {
 		if err := router.SetTrustedProxies(*config.TrustedProxies); err != nil {
-			return &APIServer{}, fmt.Errorf("while setting trusted_proxies: %w", err)
+			return nil, fmt.Errorf("while setting trusted_proxies: %w", err)
 		}
 		router.ForwardedByClientIP = true
 	} else {
@@ -215,7 +215,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
 		log.Printf("Loading CAPI manager")
 		apiClient, err = NewAPIC(config.OnlineClient, dbClient, config.ConsoleConfig, config.CapiWhitelists)
 		if err != nil {
-			return &APIServer{}, err
+			return nil, err
 		}
 		log.Infof("CAPI manager configured successfully")
 		isMachineEnrolled = isEnrolled(apiClient.apiClient)
@@ -225,7 +225,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
 				log.Infof("Machine is enrolled in the console, Loading PAPI Client")
 				papiClient, err = NewPAPI(apiClient, dbClient, config.ConsoleConfig, *config.PapiLogLevel)
 				if err != nil {
-					return &APIServer{}, err
+					return nil, err
 				}
 				controller.DecisionDeleteChan = papiClient.Channels.DeleteDecisionChannel
 			} else {
@@ -241,7 +241,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
 	if trustedIPs, err := config.GetTrustedIPs(); err == nil {
 		controller.TrustedIPs = trustedIPs
 	} else {
-		return &APIServer{}, err
+		return nil, err
 	}
 
 	return &APIServer{

pkg/apiserver/apiserver_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 	"github.com/crowdsecurity/go-cs-lib/version"
 
 	middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
@@ -295,8 +296,8 @@ func TestWithWrongDBConfig(t *testing.T) {
 	config.API.Server.DbConfig.Type = "test"
 	apiServer, err := NewServer(config.API.Server)
 
-	assert.Equal(t, apiServer, &APIServer{})
+	cstest.RequireErrorContains(t, err, "unable to init database client: unknown database type 'test'")
-	assert.Equal(t, "unable to init database client: unknown database type 'test'", err.Error())
+	assert.Nil(t, apiServer)
 }
 
 func TestWithWrongFlushConfig(t *testing.T) {
@@ -305,8 +306,8 @@ func TestWithWrongFlushConfig(t *testing.T) {
 	config.API.Server.DbConfig.Flush.MaxItems = &maxItems
 	apiServer, err := NewServer(config.API.Server)
 
-	assert.Equal(t, apiServer, &APIServer{})
+	cstest.RequireErrorContains(t, err, "max_items can't be zero or negative number")
-	assert.Equal(t, "max_items can't be zero or negative number", err.Error())
+	assert.Nil(t, apiServer)
 }
 
 func TestUnknownPath(t *testing.T) {

pkg/apiserver/middlewares/v1/api_key.go

@@ -55,132 +55,110 @@ func HashSHA512(str string) string {
 	return hashStr
 }
 
+func (a *APIKey) authTLS(c *gin.Context, logger *log.Entry) *ent.Bouncer {
+	if a.TlsAuth == nil {
+		logger.Error("TLS Auth is not configured but client presented a certificate")
+		return nil
+	}
+
+	validCert, extractedCN, err := a.TlsAuth.ValidateCert(c)
+	if !validCert {
+		logger.Errorf("invalid client certificate: %s", err)
+		return nil
+	}
+	if err != nil {
+		logger.Error(err)
+		return nil
+	}
+
+	logger = logger.WithFields(log.Fields{
+		"cn": extractedCN,
+	})
+
+	bouncerName := fmt.Sprintf("%s@%s", extractedCN, c.ClientIP())
+	bouncer, err := a.DbClient.SelectBouncerByName(bouncerName)
+
+	//This is likely not the proper way, but isNotFound does not seem to work
+	if err != nil && strings.Contains(err.Error(), "bouncer not found") {
+		//Because we have a valid cert, automatically create the bouncer in the database if it does not exist
+		//Set a random API key, but it will never be used
+		apiKey, err := GenerateAPIKey(dummyAPIKeySize)
+		if err != nil {
+			logger.Errorf("error generating mock api key: %s", err)
+			return nil
+		}
+		logger.Infof("Creating bouncer %s", bouncerName)
+		bouncer, err = a.DbClient.CreateBouncer(bouncerName, c.ClientIP(), HashSHA512(apiKey), types.TlsAuthType)
+		if err != nil {
+			logger.Errorf("while creating bouncer db entry: %s", err)
+			return nil
+		}
+	} else if err != nil {
+		//error while selecting bouncer
+		logger.Errorf("while selecting bouncers: %s", err)
+		return nil
+	} else if bouncer.AuthType != types.TlsAuthType {
+		//bouncer was found in DB
+		logger.Errorf("bouncer isn't allowed to auth by TLS")
+		return nil
+	}
+	return bouncer
+}
+
+func (a *APIKey) authPlain(c *gin.Context, logger *log.Entry) *ent.Bouncer {
+	val, ok := c.Request.Header[APIKeyHeader]
+	if !ok {
+		logger.Errorf("API key not found")
+		return nil
+	}
+	hashStr := HashSHA512(val[0])
+
+	bouncer, err := a.DbClient.SelectBouncer(hashStr)
+	if err != nil {
+		logger.Errorf("while fetching bouncer info: %s", err)
+		return nil
+	}
+
+	if bouncer.AuthType != types.ApiKeyAuthType {
+		logger.Errorf("bouncer %s attempted to login using an API key but it is configured to auth with %s", bouncer.Name, bouncer.AuthType)
+		return nil
+	}
+
+	return bouncer
+}
+
 func (a *APIKey) MiddlewareFunc() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		var bouncer *ent.Bouncer
-		var err error
+
+		logger := log.WithFields(log.Fields{
+			"ip": c.ClientIP(),
+		})
 
 		if c.Request.TLS != nil && len(c.Request.TLS.PeerCertificates) > 0 {
-			if a.TlsAuth == nil {
+			bouncer = a.authTLS(c, logger)
-				log.WithField("ip", c.ClientIP()).Error("TLS Auth is not configured but client presented a certificate")
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
-			validCert, extractedCN, err := a.TlsAuth.ValidateCert(c)
-			if !validCert {
-				log.WithField("ip", c.ClientIP()).Errorf("invalid client certificate: %s", err)
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
-			if err != nil {
-				log.WithField("ip", c.ClientIP()).Error(err)
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
-			bouncerName := fmt.Sprintf("%s@%s", extractedCN, c.ClientIP())
-			bouncer, err = a.DbClient.SelectBouncerByName(bouncerName)
-			//This is likely not the proper way, but isNotFound does not seem to work
-			if err != nil && strings.Contains(err.Error(), "bouncer not found") {
-				//Because we have a valid cert, automatically create the bouncer in the database if it does not exist
-				//Set a random API key, but it will never be used
-				apiKey, err := GenerateAPIKey(dummyAPIKeySize)
-				if err != nil {
-					log.WithFields(log.Fields{
-						"ip": c.ClientIP(),
-						"cn": extractedCN,
-					}).Errorf("error generating mock api key: %s", err)
-					c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-					c.Abort()
-					return
-				}
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-					"cn": extractedCN,
-				}).Infof("Creating bouncer %s", bouncerName)
-				bouncer, err = a.DbClient.CreateBouncer(bouncerName, c.ClientIP(), HashSHA512(apiKey), types.TlsAuthType)
-				if err != nil {
-					log.WithFields(log.Fields{
-						"ip": c.ClientIP(),
-						"cn": extractedCN,
-					}).Errorf("creating bouncer db entry : %s", err)
-					c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-					c.Abort()
-					return
-				}
-			} else if err != nil {
-				//error while selecting bouncer
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-					"cn": extractedCN,
-				}).Errorf("while selecting bouncers: %s", err)
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			} else if bouncer.AuthType != types.TlsAuthType {
-				//bouncer was found in DB
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-					"cn": extractedCN,
-				}).Errorf("bouncer isn't allowed to auth by TLS")
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
 		} else {
-			//API Key Authentication
+			bouncer = a.authPlain(c, logger)
-			val, ok := c.Request.Header[APIKeyHeader]
-			if !ok {
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-				}).Errorf("API key not found")
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
-			hashStr := HashSHA512(val[0])
-			bouncer, err = a.DbClient.SelectBouncer(hashStr)
-			if err != nil {
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-				}).Errorf("while fetching bouncer info: %s", err)
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
-			if bouncer.AuthType != types.ApiKeyAuthType {
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-				}).Errorf("bouncer %s attempted to login using an API key but it is configured to auth with %s", bouncer.Name, bouncer.AuthType)
-				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-				c.Abort()
-				return
-			}
 		}
 
 		if bouncer == nil {
-			log.WithFields(log.Fields{
-				"ip": c.ClientIP(),
-			}).Errorf("bouncer not found")
 			c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
 			c.Abort()
 			return
 		}
 
-		//maybe we want to store the whole bouncer object in the context instead, this would avoid another db query
+		logger = logger.WithFields(log.Fields{
-		//in StreamDecision
+			"name": bouncer.Name,
+		})
+
+		// maybe we want to store the whole bouncer object in the context instead, this would avoid another db query
+		// in StreamDecision
 		c.Set("BOUNCER_NAME", bouncer.Name)
 		c.Set("BOUNCER_HASHED_KEY", bouncer.APIKey)
 
 		if bouncer.IPAddress == "" {
-			err = a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID)
+			if err := a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID); err != nil {
-			if err != nil {
+				logger.Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err)
-				log.WithFields(log.Fields{
-					"ip":   c.ClientIP(),
-					"name": bouncer.Name,
-				}).Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err)
 				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
 				c.Abort()
 				return
@@ -189,12 +167,8 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc {
 
 		if bouncer.IPAddress != c.ClientIP() && bouncer.IPAddress != "" {
 			log.Warningf("new IP address detected for bouncer '%s': %s (old: %s)", bouncer.Name, c.ClientIP(), bouncer.IPAddress)
-			err = a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID)
+			if err := a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID); err != nil {
-			if err != nil {
+				logger.Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err)
-				log.WithFields(log.Fields{
-					"ip":   c.ClientIP(),
-					"name": bouncer.Name,
-				}).Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err)
 				c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
 				c.Abort()
 				return
@@ -202,21 +176,14 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc {
 		}
 
 		useragent := strings.Split(c.Request.UserAgent(), "/")
-
 		if len(useragent) != 2 {
-			log.WithFields(log.Fields{
+			logger.Warningf("bad user agent '%s'", c.Request.UserAgent())
-				"ip":   c.ClientIP(),
-				"name": bouncer.Name,
-			}).Warningf("bad user agent '%s'", c.Request.UserAgent())
 			useragent = []string{c.Request.UserAgent(), "N/A"}
 		}
 
 		if bouncer.Version != useragent[1] || bouncer.Type != useragent[0] {
 			if err := a.DbClient.UpdateBouncerTypeAndVersion(useragent[0], useragent[1], bouncer.ID); err != nil {
-				log.WithFields(log.Fields{
+				logger.Errorf("failed to update bouncer version and type: %s", err)
-					"ip":   c.ClientIP(),
-					"name": bouncer.Name,
-				}).Errorf("failed to update bouncer version and type: %s", err)
 				c.JSON(http.StatusForbidden, gin.H{"message": "bad user agent"})
 				c.Abort()
 				return

pkg/apiserver/middlewares/v1/middlewares.go

@@ -14,7 +14,7 @@ func NewMiddlewares(dbClient *database.Client) (*Middlewares, error) {
 
 	ret.JWT, err = NewJWT(dbClient)
 	if err != nil {
-		return &Middlewares{}, err
+		return nil, err
 	}
 
 	ret.APIKey = NewAPIKey(dbClient)

pkg/csconfig/crowdsec_service.go

@@ -23,7 +23,6 @@ type CrowdsecServiceCfg struct {
 	BucketsRoutinesCount      int               `yaml:"buckets_routines"`
 	OutputRoutinesCount       int               `yaml:"output_routines"`
 	SimulationConfig          *SimulationConfig `yaml:"-"`
-	LintOnly                  bool              `yaml:"-"`                          // if set to true, exit after loading configs
 	BucketStateFile           string            `yaml:"state_input_file,omitempty"` // if we need to unserialize buckets at start
 	BucketStateDumpDir        string            `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
 	BucketsGCEnabled          bool              `yaml:"-"`                          // we need to garbage collect buckets when in forensic mode

pkg/csplugin/notifier.go

@@ -26,7 +26,7 @@ func (m *GRPCClient) Notify(ctx context.Context, notification *protobufs.Notific
 	done := make(chan error)
 	go func() {
 		_, err := m.client.Notify(
-			context.Background(), &protobufs.Notification{Text: notification.Text, Name: notification.Name},
+			ctx, &protobufs.Notification{Text: notification.Text, Name: notification.Name},
 		)
 		done <- err
 	}()

pkg/cwhub/hub.go

@@ -109,7 +109,7 @@ func (h *Hub) ItemStats() []string {
 		loaded += fmt.Sprintf("%d %s, ", len(h.GetItemMap(itemType)), itemType)
 
 		for _, item := range h.GetItemMap(itemType) {
-			if item.IsLocal() {
+			if item.State.IsLocal() {
 				local++
 			}
 

pkg/cwhub/item.go

@@ -55,6 +55,48 @@ type ItemState struct {
 	BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"`
 }
 
+// IsLocal returns true if the item has been create by a user (not downloaded from the hub).
+func (s *ItemState) IsLocal() bool {
+	return s.Installed && !s.Downloaded
+}
+
+// Text returns the status of the item as a string (eg. "enabled,update-available").
+func (s *ItemState) Text() string {
+	ret := "disabled"
+
+	if s.Installed {
+		ret = "enabled"
+	}
+
+	if s.IsLocal() {
+		ret += ",local"
+	}
+
+	if s.Tainted {
+		ret += ",tainted"
+	} else if !s.UpToDate && !s.IsLocal() {
+		ret += ",update-available"
+	}
+
+	return ret
+}
+
+// Emoji returns the status of the item as an emoji (eg. emoji.Warning).
+func (s *ItemState) Emoji() emoji.Emoji {
+	switch {
+	case s.IsLocal():
+		return emoji.House
+	case !s.Installed:
+		return emoji.Prohibited
+	case s.Tainted || (!s.UpToDate && !s.IsLocal()):
+		return emoji.Warning
+	case s.Installed:
+		return emoji.CheckMark
+	default:
+		return emoji.QuestionMark
+	}
+}
+
 // Item is created from an index file and enriched with local info.
 type Item struct {
 	hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods
@@ -111,11 +153,6 @@ func (i *Item) HasSubItems() bool {
 	return i.Type == COLLECTIONS
 }
 
-// IsLocal returns true if the item has been create by a user (not downloaded from the hub).
-func (i *Item) IsLocal() bool {
-	return i.State.Installed && !i.State.Downloaded
-}
-
 // MarshalJSON is used to prepare the output for "cscli ... inspect -o json".
 // It must not use a pointer receiver.
 func (i Item) MarshalJSON() ([]byte, error) {
@@ -143,7 +180,7 @@ func (i Item) MarshalJSON() ([]byte, error) {
 		UpToDate:             i.State.UpToDate,
 		Tainted:              i.State.Tainted,
 		BelongsToCollections: i.State.BelongsToCollections,
-		Local:                i.IsLocal(),
+		Local:                i.State.IsLocal(),
 	})
 }
 
@@ -159,7 +196,7 @@ func (i Item) MarshalYAML() (interface{}, error) {
 	}{
 		Alias: Alias(i),
 		State: i.State,
-		Local: i.IsLocal(),
+		Local: i.State.IsLocal(),
 	}, nil
 }
 
@@ -324,48 +361,6 @@ func (i *Item) descendants() ([]*Item, error) {
 	return ret, nil
 }
 
-// InstallStatus returns the status of the item as a string and an emoji
-// (eg. "enabled,update-available" and emoji.Warning).
-func (i *Item) InstallStatus() (string, emoji.Emoji) {
-	status := "disabled"
-	ok := false
-
-	if i.State.Installed {
-		ok = true
-		status = "enabled"
-	}
-
-	managed := true
-	if i.IsLocal() {
-		managed = false
-		status += ",local"
-	}
-
-	warning := false
-	if i.State.Tainted {
-		warning = true
-		status += ",tainted"
-	} else if !i.State.UpToDate && !i.IsLocal() {
-		warning = true
-		status += ",update-available"
-	}
-
-	emo := emoji.QuestionMark
-
-	switch {
-	case !managed:
-		emo = emoji.House
-	case !i.State.Installed:
-		emo = emoji.Prohibited
-	case warning:
-		emo = emoji.Warning
-	case ok:
-		emo = emoji.CheckMark
-	}
-
-	return status, emo
-}
-
 // versionStatus returns the status of the item version compared to the hub version.
 // semver requires the 'v' prefix.
 func (i *Item) versionStatus() int {

pkg/cwhub/item_test.go

@@ -23,7 +23,7 @@ func TestItemStatus(t *testing.T) {
 		item.State.Tainted = false
 		item.State.Downloaded = true
 
-		txt, _ := item.InstallStatus()
+		txt := item.State.Text()
 		require.Equal(t, "enabled,update-available", txt)
 
 		item.State.Installed = true
@@ -31,7 +31,7 @@ func TestItemStatus(t *testing.T) {
 		item.State.Tainted = false
 		item.State.Downloaded = false
 
-		txt, _ = item.InstallStatus()
+		txt = item.State.Text()
 		require.Equal(t, "enabled,local", txt)
 	}
 

pkg/cwhub/iteminstall.go

@@ -13,7 +13,7 @@ func (i *Item) enable() error {
 			return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
 		}
 
-		if i.IsLocal() {
+		if i.State.IsLocal() {
 			return fmt.Errorf("%s is local, won't enable", i.Name)
 		}
 

pkg/cwhub/itemremove.go

@@ -66,7 +66,7 @@ func (i *Item) disable(purge bool, force bool) (bool, error) {
 
 // Remove disables the item, optionally removing the downloaded content.
 func (i *Item) Remove(purge bool, force bool) (bool, error) {
-	if i.IsLocal() {
+	if i.State.IsLocal() {
 		log.Warningf("%s is a local item, please delete manually", i.Name)
 		return false, nil
 	}

pkg/cwhub/itemupgrade.go

@@ -20,7 +20,7 @@ import (
 func (i *Item) Upgrade(force bool) (bool, error) {
 	updated := false
 
-	if i.IsLocal() {
+	if i.State.IsLocal() {
 		log.Infof("not upgrading %s: local item", i.Name)
 		return false, nil
 	}
@@ -155,7 +155,7 @@ func (i *Item) fetch() ([]byte, error) {
 
 // download downloads the item from the hub and writes it to the hub directory.
 func (i *Item) download(overwrite bool) (string, error) {
-	if i.IsLocal() {
+	if i.State.IsLocal() {
 		return "", fmt.Errorf("%s is local, can't download", i.Name)
 	}
 	// if user didn't --force, don't overwrite local, tainted, up-to-date files

pkg/cwhub/sync.go

@@ -426,7 +426,7 @@ func (h *Hub) localSync() error {
 		case versionFuture:
 			warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
 		case versionUnknown:
-			if !item.IsLocal() {
+			if !item.State.IsLocal() {
 				warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
 			}
 		}

pkg/database/bouncers.go

@@ -13,7 +13,7 @@ import (
 func (c *Client) SelectBouncer(apiKeyHash string) (*ent.Bouncer, error) {
 	result, err := c.Ent.Bouncer.Query().Where(bouncer.APIKeyEQ(apiKeyHash)).First(c.CTX)
 	if err != nil {
-		return &ent.Bouncer{}, errors.Wrapf(QueryFail, "select bouncer: %s", err)
+		return nil, err
 	}
 
 	return result, nil
@@ -22,7 +22,7 @@ func (c *Client) SelectBouncer(apiKeyHash string) (*ent.Bouncer, error) {
 func (c *Client) SelectBouncerByName(bouncerName string) (*ent.Bouncer, error) {
 	result, err := c.Ent.Bouncer.Query().Where(bouncer.NameEQ(bouncerName)).First(c.CTX)
 	if err != nil {
-		return &ent.Bouncer{}, errors.Wrapf(QueryFail, "select bouncer: %s", err)
+		return nil, err
 	}
 
 	return result, nil
@@ -31,7 +31,7 @@ func (c *Client) SelectBouncerByName(bouncerName string) (*ent.Bouncer, error) {
 func (c *Client) ListBouncers() ([]*ent.Bouncer, error) {
 	result, err := c.Ent.Bouncer.Query().All(c.CTX)
 	if err != nil {
-		return []*ent.Bouncer{}, errors.Wrapf(QueryFail, "listing bouncer: %s", err)
+		return nil, errors.Wrapf(QueryFail, "listing bouncers: %s", err)
 	}
 	return result, nil
 }

pkg/database/machines.go

@@ -19,8 +19,8 @@ const CapiListsMachineID = types.ListOrigin
 func (c *Client) CreateMachine(machineID *string, password *strfmt.Password, ipAddress string, isValidated bool, force bool, authType string) (*ent.Machine, error) {
 	hashPassword, err := bcrypt.GenerateFromPassword([]byte(*password), bcrypt.DefaultCost)
 	if err != nil {
-		c.Log.Warningf("CreateMachine : %s", err)
+		c.Log.Warningf("CreateMachine: %s", err)
-		return nil, errors.Wrap(HashError, "")
+		return nil, HashError
 	}
 
 	machineExist, err := c.Ent.Machine.
@@ -78,7 +78,7 @@ func (c *Client) QueryMachineByID(machineID string) (*ent.Machine, error) {
 func (c *Client) ListMachines() ([]*ent.Machine, error) {
 	machines, err := c.Ent.Machine.Query().All(c.CTX)
 	if err != nil {
-		return []*ent.Machine{}, errors.Wrapf(QueryFail, "listing machines: %s", err)
+		return nil, errors.Wrapf(QueryFail, "listing machines: %s", err)
 	}
 	return machines, nil
 }
@@ -101,7 +101,7 @@ func (c *Client) QueryPendingMachine() ([]*ent.Machine, error) {
 	machines, err = c.Ent.Machine.Query().Where(machine.IsValidatedEQ(false)).All(c.CTX)
 	if err != nil {
 		c.Log.Warningf("QueryPendingMachine : %s", err)
-		return []*ent.Machine{}, errors.Wrapf(QueryFail, "querying pending machines: %s", err)
+		return nil, errors.Wrapf(QueryFail, "querying pending machines: %s", err)
 	}
 	return machines, nil
 }
@@ -190,12 +190,13 @@ func (c *Client) IsMachineRegistered(machineID string) (bool, error) {
 		return true, nil
 	}
 	if len(exist) > 1 {
-		return false, fmt.Errorf("More than one item with the same machineID in database")
+		return false, fmt.Errorf("more than one item with the same machineID in database")
 	}
 
 	return false, nil
 
 }
+
 func (c *Client) QueryLastValidatedHeartbeatLT(t time.Time) ([]*ent.Machine, error) {
 	return c.Ent.Machine.Query().Where(machine.LastHeartbeatLT(t), machine.IsValidatedEQ(true)).All(c.CTX)
 }

pkg/metabase/container.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"runtime"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -93,14 +92,6 @@ func (c *Container) Create() error {
 		Tty:   true,
 		Env:   env,
 	}
-	os := runtime.GOOS
-	switch os {
-	case "linux":
-	case "windows", "darwin":
-		return fmt.Errorf("mac and windows are not supported yet")
-	default:
-		return fmt.Errorf("OS '%s' is not supported", os)
-	}
 
 	log.Infof("creating container '%s'", c.Name)
 	resp, err := c.CLI.ContainerCreate(ctx, dockerConfig, hostConfig, nil, nil, c.Name)

test/bats/01_crowdsec.bats

@@ -227,3 +227,16 @@ teardown() {
     assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
+@test "crowdsec -t (error in acquisition file)" {
+    # we can verify the acquisition configuration without running crowdsec
+    ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
+    config_set "$ACQUIS_YAML" 'del(.filenames)'
+
+    rune -1 wait-for "${CROWDSEC}"
+    assert_stderr --partial "failed to configure datasource file: no filename or filenames configuration provided"
+
+    config_set "$ACQUIS_YAML" '.filenames=["file.log"]'
+    config_set "$ACQUIS_YAML" '.meh=3'
+    rune -1 wait-for "${CROWDSEC}"
+    assert_stderr --partial "field meh not found in type fileacquisition.FileConfiguration"
+}

test/bats/01_cscli.bats

@@ -108,6 +108,20 @@ teardown() {
     rune -0 cscli config show -o json
     rune -0 jq -c '.API.Client.Credentials | [.url,.login[0:32]]' <(output)
     assert_json '["http://127.0.0.1:8080/","githubciXXXXXXXXXXXXXXXXXXXXXXXX"]'
+
+    # pointer to boolean
+
+    rune -0 cscli config show --key Config.API.Client.InsecureSkipVerify
+    assert_output "&false"
+
+    # complex type
+    rune -0 cscli config show --key Config.PluginConfig
+    assert_output - <<-EOT
+	&csconfig.PluginCfg{
+	  User: "nobody",
+	  Group: "nogroup",
+	}
+	EOT
 }
 
 @test "cscli - required configuration paths" {

test/bats/05_config_yaml_local.bats

@@ -34,22 +34,22 @@ teardown() {
 @test "config.yaml.local - cscli (log_level)" {
     config_set '.common.log_level="warning"'
     rune -0 cscli config show --key Config.Common.LogLevel
-    assert_output "warning"
+    assert_output "&3"
 
     echo "{'common':{'log_level':'debug'}}" >"${CONFIG_YAML}.local"
     rune -0 cscli config show --key Config.Common.LogLevel
-    assert_output "debug"
+    assert_output "&5"
 }
 
 @test "config.yaml.local - cscli (log_level - with envvar)" {
     config_set '.common.log_level="warning"'
     rune -0 cscli config show --key Config.Common.LogLevel
-    assert_output "warning"
+    assert_output "&3"
 
     export CROWDSEC_LOG_LEVEL=debug
     echo "{'common':{'log_level':'${CROWDSEC_LOG_LEVEL}'}}" >"${CONFIG_YAML}.local"
     rune -0 cscli config show --key Config.Common.LogLevel
-    assert_output "debug"
+    assert_output "&5"
 }
 
 @test "config.yaml.local - crowdsec (listen_url)" {

test/bats/20_hub.bats

@@ -34,17 +34,19 @@ teardown() {
 
     # no items
     rune -0 cscli hub list
-    assert_output --regexp "APPSEC-CONFIGS.*APPSEC-RULES.*PARSERS.*POSTOVERFLOWS.*SCENARIOS.*COLLECTIONS.*"
+    assert_output "No items to display"
     rune -0 cscli hub list -o json
     assert_json '{appsec-configs:[],appsec-rules:[],parsers:[],scenarios:[],collections:[],postoverflows:[]}'
     rune -0 cscli hub list -o raw
     assert_output 'name,status,version,description,type'
 
-    # some items
+    # some items: with output=human, show only non-empty tables
     rune -0 cscli parsers install crowdsecurity/whitelists
     rune -0 cscli scenarios install crowdsecurity/telnet-bf
     rune -0 cscli hub list
-    assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*"
+    assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*"
+    refute_output --partial 'POSTOVERFLOWS'
+    refute_output --partial 'COLLECTIONS'
     rune -0 cscli hub list -o json
     rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output)
     rune -0 cscli hub list -o raw

test/bats/30_machines.bats

@@ -23,20 +23,24 @@ teardown() {
 
 #----------
 
-@test "can list machines as regular user" {
-    rune -0 cscli machines list
-}
-
 @test "we have exactly one machine" {
     rune -0 cscli machines list -o json
     rune -0 jq -c '[. | length, .[0].machineId[0:32], .[0].isValidated]' <(output)
     assert_output '[1,"githubciXXXXXXXXXXXXXXXXXXXXXXXX",true]'
 }
 
+@test "don't overwrite local credentials by default" {
+    rune -1 cscli machines add local -a -o json
+    rune -0 jq -r '.msg' <(stderr)
+    assert_output --partial 'already exists: please remove it, use "--force" or specify a different file with "-f"'
+    rune -0 cscli machines add local -a --force
+    assert_stderr --partial "Machine 'local' successfully added to the local API"
+}
+
 @test "add a new machine and delete it" {
     rune -0 cscli machines add -a -f /dev/null CiTestMachine -o human
     assert_stderr --partial "Machine 'CiTestMachine' successfully added to the local API"
-    assert_stderr --partial "API credentials dumped to '/dev/null'"
+    assert_stderr --partial "API credentials written to '/dev/null'"
 
     # we now have two machines
     rune -0 cscli machines list -o json
@@ -56,7 +60,7 @@ teardown() {
 @test "register, validate and then remove a machine" {
     rune -0 cscli lapi register --machine CiTestMachineRegister -f /dev/null -o human
     assert_stderr --partial "Successfully registered to Local API (LAPI)"
-    assert_stderr --partial "Local API credentials dumped to '/dev/null'"
+    assert_stderr --partial "Local API credentials written to '/dev/null'"
 
     # the machine is not validated yet
     rune -0 cscli machines list -o json

test/lib/config/config-global

@@ -61,12 +61,12 @@ make_init_data() {
     ./instance-db config-yaml
     ./instance-db setup
 
+    ./bin/preload-hub-items
+
     # when installed packages are always using sqlite, so no need to regenerate
     # local credz for sqlite
 
-    ./bin/preload-hub-items
+    [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto --force
-
-    [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto
 
     mkdir -p "$LOCAL_INIT_DIR"
 

test/lib/config/config-local

@@ -115,11 +115,12 @@ make_init_data() {
     ./instance-db config-yaml
     ./instance-db setup
 
-    "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
     "$CSCLI" --warning hub update
 
     ./bin/preload-hub-items
 
+    "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto --force
+
     mkdir -p "$LOCAL_INIT_DIR"
 
     ./instance-db dump "${LOCAL_INIT_DIR}/database"