diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index 0261eab9c..e316abbc6 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -110,7 +110,7 @@ func NewCapiRegisterCmd() *cobra.Command { if err != nil { return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err) } - log.Printf("Central API credentials dumped to '%s'", dumpFile) + log.Printf("Central API credentials written to '%s'", dumpFile) } else { fmt.Printf("%s\n", string(apiConfigDump)) } diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 93772d611..c4d09e687 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -46,7 +46,7 @@ func backupHub(dirPath string) error { } //for the local/tainted ones, we back up the full file - if v.State.Tainted || v.IsLocal() || !v.State.UpToDate { + if v.State.Tainted || v.State.IsLocal() || !v.State.UpToDate { //we need to backup stages for parsers if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS { fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) @@ -54,7 +54,7 @@ func backupHub(dirPath string) error { return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err) } } - clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.IsLocal(), v.State.UpToDate) + clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.State.IsLocal(), v.State.UpToDate) tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) if err = CopyFile(v.State.LocalPath, tfile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err) diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 56f628281..ab49836a9 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -183,7 +183,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { if csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" { apiConfigDumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath } - err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o644) + err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o600) if err != nil { return fmt.Errorf("write api credentials in '%s' failed: %s", apiConfigDumpFile, err) } diff --git a/cmd/crowdsec-cli/config_show.go b/cmd/crowdsec-cli/config_show.go index ca7051195..1fd795c87 100644 --- a/cmd/crowdsec-cli/config_show.go +++ b/cmd/crowdsec-cli/config_show.go @@ -7,6 +7,7 @@ import ( "text/template" "github.com/antonmedv/expr" + "github.com/sanity-io/litter" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" @@ -35,13 +36,13 @@ func showConfigKey(key string) error { switch csConfig.Cscli.Output { case "human", "raw": + // Don't use litter for strings, it adds quotes + // that we didn't have before switch output.(type) { case string: - fmt.Printf("%s\n", output) - case int: - fmt.Printf("%d\n", output) + fmt.Println(output) default: - fmt.Printf("%v\n", output) + litter.Dump(output) } case "json": data, err := json.MarshalIndent(output, "", " ") diff --git a/cmd/crowdsec-cli/dashboard.go b/cmd/crowdsec-cli/dashboard.go index 8eab614e9..a2d6c229a 100644 --- a/cmd/crowdsec-cli/dashboard.go +++ b/cmd/crowdsec-cli/dashboard.go @@ -1,3 +1,5 @@ +//go:build linux + package main import ( @@ -9,6 +11,7 @@ import ( "path/filepath" "strconv" "strings" + "syscall" "unicode" "github.com/AlecAivazis/survey/v2" @@ -136,6 +139,9 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password if err != nil { return err } + if err = chownDatabase(dockerGroup.Gid); err != nil { + return err + } mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID, metabaseImage) if err != nil { return err @@ -366,45 +372,56 @@ func disclaimer(forceYes *bool) error { } func checkGroups(forceYes *bool) (*user.Group, error) { - groupExist := false dockerGroup, err := user.LookupGroup(crowdsecGroup) if err == nil { - groupExist = true + return dockerGroup, nil } - if !groupExist { - if !*forceYes { - var answer bool - prompt := &survey.Confirm{ - Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup), - Default: true, - } - if err := survey.AskOne(prompt, &answer); err != nil { - return dockerGroup, fmt.Errorf("unable to ask to question: %s", err) - } - if !answer { - return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup) - } + if !*forceYes { + var answer bool + prompt := &survey.Confirm{ + Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup), + Default: true, } - groupAddCmd, err := exec.LookPath("groupadd") - if err != nil { - return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue") + if err := survey.AskOne(prompt, &answer); err != nil { + return dockerGroup, fmt.Errorf("unable to ask to question: %s", err) } - - groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}} - if err := groupAdd.Run(); err != nil { - return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err) - } - dockerGroup, err = user.LookupGroup(crowdsecGroup) - if err != nil { - return dockerGroup, fmt.Errorf("unable to lookup '%s' group: %+v", dockerGroup, err) + if !answer { + return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup) } } - intID, err := strconv.Atoi(dockerGroup.Gid) + groupAddCmd, err := exec.LookPath("groupadd") if err != nil { - return dockerGroup, fmt.Errorf("unable to convert group ID to int: %s", err) + return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue") } - if err := os.Chown(csConfig.DbConfig.DbPath, 0, intID); err != nil { - return dockerGroup, fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err) + + groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}} + if err := groupAdd.Run(); err != nil { + return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err) } - return dockerGroup, nil + return user.LookupGroup(crowdsecGroup) +} + +func chownDatabase(gid string) error { + intID, err := strconv.Atoi(gid) + if err != nil { + return fmt.Errorf("unable to convert group ID to int: %s", err) + } + if stat, err := os.Stat(csConfig.DbConfig.DbPath); !os.IsNotExist(err) { + info := stat.Sys() + if err := os.Chown(csConfig.DbConfig.DbPath, int(info.(*syscall.Stat_t).Uid), intID); err != nil { + return fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err) + } + } + if csConfig.DbConfig.Type == "sqlite" && csConfig.DbConfig.UseWal != nil && *csConfig.DbConfig.UseWal { + for _, ext := range []string{"-wal", "-shm"} { + file := csConfig.DbConfig.DbPath + ext + if stat, err := os.Stat(file); !os.IsNotExist(err) { + info := stat.Sys() + if err := os.Chown(file, int(info.(*syscall.Stat_t).Uid), intID); err != nil { + return fmt.Errorf("unable to chown sqlite db file '%s': %s", file, err) + } + } + } + } + return nil } diff --git a/cmd/crowdsec-cli/dashboard_unsupported.go b/cmd/crowdsec-cli/dashboard_unsupported.go new file mode 100644 index 000000000..f2325ea07 --- /dev/null +++ b/cmd/crowdsec-cli/dashboard_unsupported.go @@ -0,0 +1,22 @@ +//go:build !linux + +package main + +import ( + "runtime" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +func NewDashboardCmd() *cobra.Command { + var cmdDashboard = &cobra.Command{ + Use: "dashboard", + DisableAutoGenTag: true, + Run: func(cmd *cobra.Command, args []string) { + log.Infof("Dashboard command is disabled on %s", runtime.GOOS) + }, + } + + return cmdDashboard +} diff --git a/cmd/crowdsec-cli/decisions_import.go b/cmd/crowdsec-cli/decisions_import.go index 56fc37c87..31dbc0ead 100644 --- a/cmd/crowdsec-cli/decisions_import.go +++ b/cmd/crowdsec-cli/decisions_import.go @@ -232,7 +232,7 @@ func NewDecisionsImportCmd() *cobra.Command { Short: "Import decisions from a file or pipe", Long: "expected format:\n" + "csv : any of duration,reason,scope,type,value, with a header line\n" + - `json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`, + "json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`", DisableAutoGenTag: true, Example: `decisions.csv: duration,scope,value diff --git a/cmd/crowdsec-cli/explain.go b/cmd/crowdsec-cli/explain.go index 0c33a845e..1a9d24b85 100644 --- a/cmd/crowdsec-cli/explain.go +++ b/cmd/crowdsec-cli/explain.go @@ -21,9 +21,15 @@ func GetLineCountForFile(filepath string) (int, error) { } defer f.Close() lc := 0 - fs := bufio.NewScanner(f) - for fs.Scan() { - lc++ + fs := bufio.NewReader(f) + for { + input, err := fs.ReadBytes('\n') + if len(input) > 1 { + lc++ + } + if err != nil && err == io.EOF { + break + } } return lc, nil } @@ -79,19 +85,6 @@ func runExplain(cmd *cobra.Command, args []string) error { return err } - fileInfo, _ := os.Stdin.Stat() - - if logType == "" || (logLine == "" && logFile == "" && dsn == "") { - printHelp(cmd) - fmt.Println() - fmt.Printf("Please provide --type flag\n") - os.Exit(1) - } - - if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) { - return fmt.Errorf("the option -f - is intended to work with pipes") - } - var f *os.File // using empty string fallback to /tmp @@ -99,6 +92,13 @@ func runExplain(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err) } + defer func() { + if _, err := os.Stat(dir); !os.IsNotExist(err) { + if err := os.RemoveAll(dir); err != nil { + log.Errorf("unable to delete temporary directory '%s': %s", dir, err) + } + } + }() tmpFile := "" // we create a temporary log file if a log line/stdin has been provided if logLine != "" || logFile == "-" { @@ -121,13 +121,15 @@ func runExplain(cmd *cobra.Command, args []string) error { if err != nil && err == io.EOF { break } - _, err = f.Write(input) - if err != nil { + if len(input) > 1 { + _, err = f.Write(input) + } + if err != nil || len(input) <= 1 { errCount++ } } if errCount > 0 { - log.Warnf("Failed to write %d lines to tmp file", errCount) + log.Warnf("Failed to write %d lines to %s", errCount, tmpFile) } } f.Close() @@ -145,8 +147,12 @@ func runExplain(cmd *cobra.Command, args []string) error { if err != nil { return err } + log.Debugf("file %s has %d lines", absolutePath, lineCount) + if lineCount == 0 { + return fmt.Errorf("the log file is empty: %s", absolutePath) + } if lineCount > 100 { - log.Warnf("The log file contains %d lines. This may take a lot of resources.", lineCount) + log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount) } } @@ -166,12 +172,6 @@ func runExplain(cmd *cobra.Command, args []string) error { return fmt.Errorf("fail to run crowdsec for test: %v", err) } - // rm the temporary log file if only a log line/stdin was provided - if tmpFile != "" { - if err := os.Remove(tmpFile); err != nil { - return fmt.Errorf("unable to remove tmp log file '%s': %+v", tmpFile, err) - } - } parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName) bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName) @@ -187,10 +187,6 @@ func runExplain(cmd *cobra.Command, args []string) error { hubtest.DumpTree(*parserDump, *bucketStateDump, opts) - if err := os.RemoveAll(dir); err != nil { - return fmt.Errorf("unable to delete temporary directory '%s': %s", dir, err) - } - return nil } @@ -210,6 +206,45 @@ tail -n 5 myfile.log | cscli explain --type nginx -f - Args: cobra.ExactArgs(0), DisableAutoGenTag: true, RunE: runExplain, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + logFile, err := flags.GetString("file") + if err != nil { + return err + } + + dsn, err := flags.GetString("dsn") + if err != nil { + return err + } + + logLine, err := flags.GetString("log") + if err != nil { + return err + } + + logType, err := flags.GetString("type") + if err != nil { + return err + } + + if logLine == "" && logFile == "" && dsn == "" { + printHelp(cmd) + fmt.Println() + return fmt.Errorf("please provide --log, --file or --dsn flag") + } + if logType == "" { + printHelp(cmd) + fmt.Println() + return fmt.Errorf("please provide --type flag") + } + fileInfo, _ := os.Stdin.Stat() + if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) { + return fmt.Errorf("the option -f - is intended to work with pipes") + } + return nil + }, } flags := cmdExplain.Flags() diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 2d139cd90..0a48e0231 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -66,7 +66,7 @@ func runHubList(cmd *cobra.Command, args []string) error { } } - err = listItems(color.Output, cwhub.ItemTypes, items) + err = listItems(color.Output, cwhub.ItemTypes, items, true) if err != nil { return err } diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index d429719b5..8003486fd 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -640,7 +640,7 @@ func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) err return err } - if err = listItems(color.Output, []string{it.name}, items); err != nil { + if err = listItems(color.Output, []string{it.name}, items, false); err != nil { return err } diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index c77ff3f88..6a91ab1f5 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -53,11 +53,19 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b return items, nil } -func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item) error { +func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item, omitIfEmpty bool) error { switch csConfig.Cscli.Output { case "human": + nothingToDisplay := true for _, itemType := range itemTypes { + if omitIfEmpty && len(items[itemType]) == 0 { + continue + } listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType]) + nothingToDisplay = false + } + if nothingToDisplay { + fmt.Println("No items to display") } case "json": type itemHubStatus struct { @@ -75,14 +83,15 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item hubStatus[itemType] = make([]itemHubStatus, len(items[itemType])) for i, item := range items[itemType] { - status, emo := item.InstallStatus() + status := item.State.Text() + status_emo := item.State.Emoji() hubStatus[itemType][i] = itemHubStatus{ Name: item.Name, LocalVersion: item.State.LocalVersion, LocalPath: item.State.LocalPath, Description: item.Description, Status: status, - UTF8Status: fmt.Sprintf("%v %s", emo, status), + UTF8Status: fmt.Sprintf("%v %s", status_emo, status), } } } @@ -107,10 +116,9 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item for _, itemType := range itemTypes { for _, item := range items[itemType] { - status, _ := item.InstallStatus() row := []string{ item.Name, - status, + item.State.Text(), item.State.LocalVersion, item.Description, } diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index b2870cb20..9b6900a8f 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -150,11 +150,11 @@ func runLapiRegister(cmd *cobra.Command, args []string) error { log.Fatalf("unable to marshal api credentials: %s", err) } if dumpFile != "" { - err = os.WriteFile(dumpFile, apiConfigDump, 0644) + err = os.WriteFile(dumpFile, apiConfigDump, 0o600) if err != nil { log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err) } - log.Printf("Local API credentials dumped to '%s'", dumpFile) + log.Printf("Local API credentials written to '%s'", dumpFile) } else { fmt.Printf("%s\n", string(apiConfigDump)) } @@ -332,7 +332,7 @@ cscli lapi context detect crowdsecurity/sshd-logs } // to avoid all the log.Info from the loaders functions - log.SetLevel(log.ErrorLevel) + log.SetLevel(log.WarnLevel) err = exprhelpers.Init(nil) if err != nil { @@ -499,13 +499,13 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string { if node.Grok.RegexpName != "" { grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName) - if err != nil { - log.Warningf("Can't get subgrok: %s", err) - } - for _, capturedField := range grokCompiled.Names() { - fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) - if !slices.Contains(ret, fieldName) { - ret = append(ret, fieldName) + // ignore error (parser does not exist?) + if err == nil { + for _, capturedField := range grokCompiled.Names() { + fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) + if !slices.Contains(ret, fieldName) { + ret = append(ret, fieldName) + } } } } @@ -545,13 +545,13 @@ func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string { } if subnode.Grok.RegexpName != "" { grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName) - if err != nil { - log.Warningf("Can't get subgrok: %s", err) - } - for _, capturedField := range grokCompiled.Names() { - fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) - if !slices.Contains(ret, fieldName) { - ret = append(ret, fieldName) + if err == nil { + // ignore error (parser does not exist?) + for _, capturedField := range grokCompiled.Names() { + fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) + if !slices.Contains(ret, fieldName) { + ret = append(ret, fieldName) + } } } } diff --git a/cmd/crowdsec-cli/machines.go b/cmd/crowdsec-cli/machines.go index 6c97c0109..72e6579cc 100644 --- a/cmd/crowdsec-cli/machines.go +++ b/cmd/crowdsec-cli/machines.go @@ -30,9 +30,7 @@ import ( "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" ) -var ( - passwordLength = 64 -) +const passwordLength = 64 func generatePassword(length int) string { upper := "ABCDEFGHIJKLMNOPQRSTUVWXY" @@ -43,6 +41,7 @@ func generatePassword(length int) string { charsetLength := len(charset) buf := make([]byte, length) + for i := 0; i < length; i++ { rInt, err := saferand.Int(saferand.Reader, big.NewInt(int64(charsetLength))) if err != nil { @@ -190,7 +189,6 @@ cscli machines add MyTestMachine --password MyPassword } func runMachinesAdd(cmd *cobra.Command, args []string) error { - var dumpFile string var err error flags := cmd.Flags() @@ -200,7 +198,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error { return err } - outputFile, err := flags.GetString("file") + dumpFile, err := flags.GetString("file") if err != nil { return err } @@ -220,7 +218,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error { return err } - forceAdd, err := flags.GetBool("force") + force, err := flags.GetBool("force") if err != nil { return err } @@ -242,17 +240,28 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error { } /*check if file already exists*/ - if outputFile != "" { - dumpFile = outputFile - } else if csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" { - dumpFile = csConfig.API.Client.CredentialsFilePath + if dumpFile == "" && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" { + credFile := csConfig.API.Client.CredentialsFilePath + // use the default only if the file does not exist + _, err := os.Stat(credFile) + switch { + case os.IsNotExist(err) || force: + dumpFile = csConfig.API.Client.CredentialsFilePath + case err != nil: + return fmt.Errorf("unable to stat '%s': %s", credFile, err) + default: + return fmt.Errorf(`credentials file '%s' already exists: please remove it, use "--force" or specify a different file with "-f" ("-f -" for standard output)`, credFile) + } + } + + if dumpFile == "" { + return fmt.Errorf(`please specify a file to dump credentials to, with -f ("-f -" for standard output)`) } // create a password if it's not specified by user if machinePassword == "" && !interactive { if !autoAdd { - printHelp(cmd) - return nil + return fmt.Errorf("please specify a password with --password or use --auto") } machinePassword = generatePassword(passwordLength) } else if machinePassword == "" && interactive { @@ -262,7 +271,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error { survey.AskOne(qs, &machinePassword) } password := strfmt.Password(machinePassword) - _, err = dbClient.CreateMachine(&machineID, &password, "", true, forceAdd, types.PasswordAuthType) + _, err = dbClient.CreateMachine(&machineID, &password, "", true, force, types.PasswordAuthType) if err != nil { return fmt.Errorf("unable to create machine: %s", err) } @@ -291,7 +300,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("write api credentials in '%s' failed: %s", dumpFile, err) } - log.Printf("API credentials dumped to '%s'", dumpFile) + log.Printf("API credentials written to '%s'", dumpFile) } else { fmt.Printf("%s\n", string(apiConfigDump)) } diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 1470d37aa..6395ad6d7 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -140,7 +140,7 @@ func collectHubItems(hub *cwhub.Hub, itemType string) []byte { log.Warnf("could not collect %s list: %s", itemType, err) } - if err := listItems(out, []string{itemType}, items); err != nil { + if err := listItems(out, []string{itemType}, items, false); err != nil { log.Warnf("could not collect %s list: %s", itemType, err) } return out.Bytes() diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index 28b185a01..f3179e40f 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -18,8 +18,8 @@ func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) { t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) for _, item := range items { - status, emo := item.InstallStatus() - t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.State.LocalVersion, item.State.LocalPath) + status := fmt.Sprintf("%v %s", item.State.Emoji(), item.State.Text()) + t.AddRow(item.Name, status, item.State.LocalVersion, item.State.LocalPath) } renderTableTitle(out, title) t.Render() diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 8c7fb2991..362ed1869 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -262,10 +262,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo return nil, errors.New("You must run at least the API Server or crowdsec") } - if flags.TestMode && !cConfig.DisableAgent { - cConfig.Crowdsec.LintOnly = true - } - if flags.OneShotDSN != "" && flags.SingleFileType == "" { return nil, errors.New("-dsn requires a -type argument") } diff --git a/cmd/notification-http/main.go b/cmd/notification-http/main.go index f7908ddda..6d1da788e 100644 --- a/cmd/notification-http/main.go +++ b/cmd/notification-http/main.go @@ -58,13 +58,12 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific if err != nil { return nil, err } - for headerName, headerValue := range cfg.Headers { logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue)) request.Header.Add(headerName, headerValue) } logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text)) - resp, err := client.Do(request) + resp, err := client.Do(request.WithContext(ctx)) if err != nil { logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err)) return nil, err diff --git a/cmd/notification-sentinel/main.go b/cmd/notification-sentinel/main.go index 18eff1b05..c627f9271 100644 --- a/cmd/notification-sentinel/main.go +++ b/cmd/notification-sentinel/main.go @@ -90,7 +90,7 @@ func (s *SentinelPlugin) Notify(ctx context.Context, notification *protobufs.Not req.Header.Set("x-ms-date", now) client := &http.Client{} - resp, err := client.Do(req) + resp, err := client.Do(req.WithContext(ctx)) if err != nil { logger.Error("failed to send request", "error", err) return &protobufs.Empty{}, err diff --git a/cmd/notification-slack/main.go b/cmd/notification-slack/main.go index 901832381..373cd9527 100644 --- a/cmd/notification-slack/main.go +++ b/cmd/notification-slack/main.go @@ -38,10 +38,9 @@ func (n *Notify) Notify(ctx context.Context, notification *protobufs.Notificatio if cfg.LogLevel != nil && *cfg.LogLevel != "" { logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel)) } - logger.Info(fmt.Sprintf("found notify signal for %s config", notification.Name)) logger.Debug(fmt.Sprintf("posting to %s webhook, message %s", cfg.Webhook, notification.Text)) - err := slack.PostWebhook(n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{ + err := slack.PostWebhookContext(ctx, n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{ Text: notification.Text, }) if err != nil { diff --git a/cmd/notification-splunk/main.go b/cmd/notification-splunk/main.go index 826986877..b24aa538f 100644 --- a/cmd/notification-splunk/main.go +++ b/cmd/notification-splunk/main.go @@ -65,7 +65,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio req.Header.Add("Authorization", fmt.Sprintf("Splunk %s", cfg.Token)) logger.Debug(fmt.Sprintf("posting event %s to %s", string(data), req.URL)) - resp, err := s.Client.Do(req) + resp, err := s.Client.Do(req.WithContext(ctx)) if err != nil { return &protobufs.Empty{}, err } diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 8a3e55529..61d695f2c 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -202,7 +202,7 @@ if isfalse "$DISABLE_LOCAL_API"; then # if the db is persistent but the credentials are not, we need to # delete the old machine to generate new credentials cscli machines delete "$CUSTOM_HOSTNAME" >/dev/null 2>&1 || true - cscli machines add "$CUSTOM_HOSTNAME" --auto + cscli machines add "$CUSTOM_HOSTNAME" --auto --force fi fi diff --git a/go.mod b/go.mod index 68e7b6a4d..99454bb67 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ require ( github.com/prometheus/client_model v0.4.0 github.com/prometheus/prom2json v1.3.0 github.com/r3labs/diff/v2 v2.14.1 + github.com/sanity-io/litter v1.5.5 github.com/segmentio/kafka-go v0.4.45 github.com/shirou/gopsutil/v3 v3.23.5 github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index 10f864160..b7ce796f1 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,7 @@ github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= github.com/crowdsecurity/machineid v1.0.2/go.mod h1:XWUSlnS0R0+u/JK5ulidwlbceNT3ZOCKteoVQEn6Luo= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -570,6 +571,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= @@ -609,6 +611,8 @@ github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OK github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= +github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/segmentio/kafka-go v0.4.45 h1:prqrZp1mMId4kI6pyPolkLsH6sWOUmDxmmucbL4WS6E= github.com/segmentio/kafka-go v0.4.45/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= @@ -646,6 +650,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 070013298..cfeb13d27 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -106,13 +106,13 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { var flushScheduler *gocron.Scheduler dbClient, err := database.NewClient(config.DbConfig) if err != nil { - return &APIServer{}, fmt.Errorf("unable to init database client: %w", err) + return nil, fmt.Errorf("unable to init database client: %w", err) } if config.DbConfig.Flush != nil { flushScheduler, err = dbClient.StartFlushScheduler(config.DbConfig.Flush) if err != nil { - return &APIServer{}, err + return nil, err } } @@ -129,7 +129,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { if config.TrustedProxies != nil && config.UseForwardedForHeaders { if err := router.SetTrustedProxies(*config.TrustedProxies); err != nil { - return &APIServer{}, fmt.Errorf("while setting trusted_proxies: %w", err) + return nil, fmt.Errorf("while setting trusted_proxies: %w", err) } router.ForwardedByClientIP = true } else { @@ -215,7 +215,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { log.Printf("Loading CAPI manager") apiClient, err = NewAPIC(config.OnlineClient, dbClient, config.ConsoleConfig, config.CapiWhitelists) if err != nil { - return &APIServer{}, err + return nil, err } log.Infof("CAPI manager configured successfully") isMachineEnrolled = isEnrolled(apiClient.apiClient) @@ -225,7 +225,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { log.Infof("Machine is enrolled in the console, Loading PAPI Client") papiClient, err = NewPAPI(apiClient, dbClient, config.ConsoleConfig, *config.PapiLogLevel) if err != nil { - return &APIServer{}, err + return nil, err } controller.DecisionDeleteChan = papiClient.Channels.DeleteDecisionChannel } else { @@ -241,7 +241,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { if trustedIPs, err := config.GetTrustedIPs(); err == nil { controller.TrustedIPs = trustedIPs } else { - return &APIServer{}, err + return nil, err } return &APIServer{ diff --git a/pkg/apiserver/apiserver_test.go b/pkg/apiserver/apiserver_test.go index 435c9601a..6150c351b 100644 --- a/pkg/apiserver/apiserver_test.go +++ b/pkg/apiserver/apiserver_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/crowdsecurity/go-cs-lib/cstest" "github.com/crowdsecurity/go-cs-lib/version" middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1" @@ -295,8 +296,8 @@ func TestWithWrongDBConfig(t *testing.T) { config.API.Server.DbConfig.Type = "test" apiServer, err := NewServer(config.API.Server) - assert.Equal(t, apiServer, &APIServer{}) - assert.Equal(t, "unable to init database client: unknown database type 'test'", err.Error()) + cstest.RequireErrorContains(t, err, "unable to init database client: unknown database type 'test'") + assert.Nil(t, apiServer) } func TestWithWrongFlushConfig(t *testing.T) { @@ -305,8 +306,8 @@ func TestWithWrongFlushConfig(t *testing.T) { config.API.Server.DbConfig.Flush.MaxItems = &maxItems apiServer, err := NewServer(config.API.Server) - assert.Equal(t, apiServer, &APIServer{}) - assert.Equal(t, "max_items can't be zero or negative number", err.Error()) + cstest.RequireErrorContains(t, err, "max_items can't be zero or negative number") + assert.Nil(t, apiServer) } func TestUnknownPath(t *testing.T) { diff --git a/pkg/apiserver/middlewares/v1/api_key.go b/pkg/apiserver/middlewares/v1/api_key.go index 207f35fc4..1481a0145 100644 --- a/pkg/apiserver/middlewares/v1/api_key.go +++ b/pkg/apiserver/middlewares/v1/api_key.go @@ -55,132 +55,110 @@ func HashSHA512(str string) string { return hashStr } +func (a *APIKey) authTLS(c *gin.Context, logger *log.Entry) *ent.Bouncer { + if a.TlsAuth == nil { + logger.Error("TLS Auth is not configured but client presented a certificate") + return nil + } + + validCert, extractedCN, err := a.TlsAuth.ValidateCert(c) + if !validCert { + logger.Errorf("invalid client certificate: %s", err) + return nil + } + if err != nil { + logger.Error(err) + return nil + } + + logger = logger.WithFields(log.Fields{ + "cn": extractedCN, + }) + + bouncerName := fmt.Sprintf("%s@%s", extractedCN, c.ClientIP()) + bouncer, err := a.DbClient.SelectBouncerByName(bouncerName) + + //This is likely not the proper way, but isNotFound does not seem to work + if err != nil && strings.Contains(err.Error(), "bouncer not found") { + //Because we have a valid cert, automatically create the bouncer in the database if it does not exist + //Set a random API key, but it will never be used + apiKey, err := GenerateAPIKey(dummyAPIKeySize) + if err != nil { + logger.Errorf("error generating mock api key: %s", err) + return nil + } + logger.Infof("Creating bouncer %s", bouncerName) + bouncer, err = a.DbClient.CreateBouncer(bouncerName, c.ClientIP(), HashSHA512(apiKey), types.TlsAuthType) + if err != nil { + logger.Errorf("while creating bouncer db entry: %s", err) + return nil + } + } else if err != nil { + //error while selecting bouncer + logger.Errorf("while selecting bouncers: %s", err) + return nil + } else if bouncer.AuthType != types.TlsAuthType { + //bouncer was found in DB + logger.Errorf("bouncer isn't allowed to auth by TLS") + return nil + } + return bouncer +} + +func (a *APIKey) authPlain(c *gin.Context, logger *log.Entry) *ent.Bouncer { + val, ok := c.Request.Header[APIKeyHeader] + if !ok { + logger.Errorf("API key not found") + return nil + } + hashStr := HashSHA512(val[0]) + + bouncer, err := a.DbClient.SelectBouncer(hashStr) + if err != nil { + logger.Errorf("while fetching bouncer info: %s", err) + return nil + } + + if bouncer.AuthType != types.ApiKeyAuthType { + logger.Errorf("bouncer %s attempted to login using an API key but it is configured to auth with %s", bouncer.Name, bouncer.AuthType) + return nil + } + + return bouncer +} + func (a *APIKey) MiddlewareFunc() gin.HandlerFunc { return func(c *gin.Context) { var bouncer *ent.Bouncer - var err error + + logger := log.WithFields(log.Fields{ + "ip": c.ClientIP(), + }) if c.Request.TLS != nil && len(c.Request.TLS.PeerCertificates) > 0 { - if a.TlsAuth == nil { - log.WithField("ip", c.ClientIP()).Error("TLS Auth is not configured but client presented a certificate") - c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) - c.Abort() - return - } - validCert, extractedCN, err := a.TlsAuth.ValidateCert(c) - if !validCert { - log.WithField("ip", c.ClientIP()).Errorf("invalid client certificate: %s", err) - c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) - c.Abort() - return - } - if err != nil { - log.WithField("ip", c.ClientIP()).Error(err) - c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) - c.Abort() - return - } - bouncerName := fmt.Sprintf("%s@%s", extractedCN, c.ClientIP()) - bouncer, err = a.DbClient.SelectBouncerByName(bouncerName) - //This is likely not the proper way, but isNotFound does not seem to work - if err != nil && strings.Contains(err.Error(), "bouncer not found") { - //Because we have a valid cert, automatically create the bouncer in the database if it does not exist - //Set a random API key, but it will never be used - apiKey, err := GenerateAPIKey(dummyAPIKeySize) - if err != nil { - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - "cn": extractedCN, - }).Errorf("error generating mock api key: %s", err) - c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) - c.Abort() - return - } - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - "cn": extractedCN, - }).Infof("Creating bouncer %s", bouncerName) - bouncer, err = a.DbClient.CreateBouncer(bouncerName, c.ClientIP(), HashSHA512(apiKey), types.TlsAuthType) - if err != nil { - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - "cn": extractedCN, - }).Errorf("creating bouncer db entry : %s", err) - c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) - c.Abort() - return - } - } else if err != nil { - //error while selecting bouncer - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - "cn": extractedCN, - }).Errorf("while selecting bouncers: %s", err) - c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) - c.Abort() - return - } else if bouncer.AuthType != types.TlsAuthType { - //bouncer was found in DB - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - "cn": extractedCN, - }).Errorf("bouncer isn't allowed to auth by TLS") - c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) - c.Abort() - return - } + bouncer = a.authTLS(c, logger) } else { - //API Key Authentication - val, ok := c.Request.Header[APIKeyHeader] - if !ok { - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - }).Errorf("API key not found") - c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) - c.Abort() - return - } - hashStr := HashSHA512(val[0]) - bouncer, err = a.DbClient.SelectBouncer(hashStr) - if err != nil { - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - }).Errorf("while fetching bouncer info: %s", err) - c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) - c.Abort() - return - } - if bouncer.AuthType != types.ApiKeyAuthType { - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - }).Errorf("bouncer %s attempted to login using an API key but it is configured to auth with %s", bouncer.Name, bouncer.AuthType) - c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) - c.Abort() - return - } + bouncer = a.authPlain(c, logger) } if bouncer == nil { - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - }).Errorf("bouncer not found") c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) c.Abort() return } - //maybe we want to store the whole bouncer object in the context instead, this would avoid another db query - //in StreamDecision + logger = logger.WithFields(log.Fields{ + "name": bouncer.Name, + }) + + // maybe we want to store the whole bouncer object in the context instead, this would avoid another db query + // in StreamDecision c.Set("BOUNCER_NAME", bouncer.Name) c.Set("BOUNCER_HASHED_KEY", bouncer.APIKey) if bouncer.IPAddress == "" { - err = a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID) - if err != nil { - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - "name": bouncer.Name, - }).Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err) + if err := a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID); err != nil { + logger.Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err) c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) c.Abort() return @@ -189,12 +167,8 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc { if bouncer.IPAddress != c.ClientIP() && bouncer.IPAddress != "" { log.Warningf("new IP address detected for bouncer '%s': %s (old: %s)", bouncer.Name, c.ClientIP(), bouncer.IPAddress) - err = a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID) - if err != nil { - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - "name": bouncer.Name, - }).Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err) + if err := a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID); err != nil { + logger.Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err) c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) c.Abort() return @@ -202,21 +176,14 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc { } useragent := strings.Split(c.Request.UserAgent(), "/") - if len(useragent) != 2 { - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - "name": bouncer.Name, - }).Warningf("bad user agent '%s'", c.Request.UserAgent()) + logger.Warningf("bad user agent '%s'", c.Request.UserAgent()) useragent = []string{c.Request.UserAgent(), "N/A"} } if bouncer.Version != useragent[1] || bouncer.Type != useragent[0] { if err := a.DbClient.UpdateBouncerTypeAndVersion(useragent[0], useragent[1], bouncer.ID); err != nil { - log.WithFields(log.Fields{ - "ip": c.ClientIP(), - "name": bouncer.Name, - }).Errorf("failed to update bouncer version and type: %s", err) + logger.Errorf("failed to update bouncer version and type: %s", err) c.JSON(http.StatusForbidden, gin.H{"message": "bad user agent"}) c.Abort() return diff --git a/pkg/apiserver/middlewares/v1/middlewares.go b/pkg/apiserver/middlewares/v1/middlewares.go index 26879bd8e..ef2d93b92 100644 --- a/pkg/apiserver/middlewares/v1/middlewares.go +++ b/pkg/apiserver/middlewares/v1/middlewares.go @@ -14,7 +14,7 @@ func NewMiddlewares(dbClient *database.Client) (*Middlewares, error) { ret.JWT, err = NewJWT(dbClient) if err != nil { - return &Middlewares{}, err + return nil, err } ret.APIKey = NewAPIKey(dbClient) diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index dc226cfd6..c0b8b5a38 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -23,7 +23,6 @@ type CrowdsecServiceCfg struct { BucketsRoutinesCount int `yaml:"buckets_routines"` OutputRoutinesCount int `yaml:"output_routines"` SimulationConfig *SimulationConfig `yaml:"-"` - LintOnly bool `yaml:"-"` // if set to true, exit after loading configs BucketStateFile string `yaml:"state_input_file,omitempty"` // if we need to unserialize buckets at start BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode diff --git a/pkg/csplugin/notifier.go b/pkg/csplugin/notifier.go index 8ab1aa923..a4f5bbc0e 100644 --- a/pkg/csplugin/notifier.go +++ b/pkg/csplugin/notifier.go @@ -26,7 +26,7 @@ func (m *GRPCClient) Notify(ctx context.Context, notification *protobufs.Notific done := make(chan error) go func() { _, err := m.client.Notify( - context.Background(), &protobufs.Notification{Text: notification.Text, Name: notification.Name}, + ctx, &protobufs.Notification{Text: notification.Text, Name: notification.Name}, ) done <- err }() diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 18218bd87..bd623e1f9 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -109,7 +109,7 @@ func (h *Hub) ItemStats() []string { loaded += fmt.Sprintf("%d %s, ", len(h.GetItemMap(itemType)), itemType) for _, item := range h.GetItemMap(itemType) { - if item.IsLocal() { + if item.State.IsLocal() { local++ } diff --git a/pkg/cwhub/item.go b/pkg/cwhub/item.go index 3772d4768..4d588da3a 100644 --- a/pkg/cwhub/item.go +++ b/pkg/cwhub/item.go @@ -55,6 +55,48 @@ type ItemState struct { BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` } +// IsLocal returns true if the item has been create by a user (not downloaded from the hub). +func (s *ItemState) IsLocal() bool { + return s.Installed && !s.Downloaded +} + +// Text returns the status of the item as a string (eg. "enabled,update-available"). +func (s *ItemState) Text() string { + ret := "disabled" + + if s.Installed { + ret = "enabled" + } + + if s.IsLocal() { + ret += ",local" + } + + if s.Tainted { + ret += ",tainted" + } else if !s.UpToDate && !s.IsLocal() { + ret += ",update-available" + } + + return ret +} + +// Emoji returns the status of the item as an emoji (eg. emoji.Warning). +func (s *ItemState) Emoji() emoji.Emoji { + switch { + case s.IsLocal(): + return emoji.House + case !s.Installed: + return emoji.Prohibited + case s.Tainted || (!s.UpToDate && !s.IsLocal()): + return emoji.Warning + case s.Installed: + return emoji.CheckMark + default: + return emoji.QuestionMark + } +} + // Item is created from an index file and enriched with local info. type Item struct { hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods @@ -111,11 +153,6 @@ func (i *Item) HasSubItems() bool { return i.Type == COLLECTIONS } -// IsLocal returns true if the item has been create by a user (not downloaded from the hub). -func (i *Item) IsLocal() bool { - return i.State.Installed && !i.State.Downloaded -} - // MarshalJSON is used to prepare the output for "cscli ... inspect -o json". // It must not use a pointer receiver. func (i Item) MarshalJSON() ([]byte, error) { @@ -143,7 +180,7 @@ func (i Item) MarshalJSON() ([]byte, error) { UpToDate: i.State.UpToDate, Tainted: i.State.Tainted, BelongsToCollections: i.State.BelongsToCollections, - Local: i.IsLocal(), + Local: i.State.IsLocal(), }) } @@ -159,7 +196,7 @@ func (i Item) MarshalYAML() (interface{}, error) { }{ Alias: Alias(i), State: i.State, - Local: i.IsLocal(), + Local: i.State.IsLocal(), }, nil } @@ -324,48 +361,6 @@ func (i *Item) descendants() ([]*Item, error) { return ret, nil } -// InstallStatus returns the status of the item as a string and an emoji -// (eg. "enabled,update-available" and emoji.Warning). -func (i *Item) InstallStatus() (string, emoji.Emoji) { - status := "disabled" - ok := false - - if i.State.Installed { - ok = true - status = "enabled" - } - - managed := true - if i.IsLocal() { - managed = false - status += ",local" - } - - warning := false - if i.State.Tainted { - warning = true - status += ",tainted" - } else if !i.State.UpToDate && !i.IsLocal() { - warning = true - status += ",update-available" - } - - emo := emoji.QuestionMark - - switch { - case !managed: - emo = emoji.House - case !i.State.Installed: - emo = emoji.Prohibited - case warning: - emo = emoji.Warning - case ok: - emo = emoji.CheckMark - } - - return status, emo -} - // versionStatus returns the status of the item version compared to the hub version. // semver requires the 'v' prefix. func (i *Item) versionStatus() int { diff --git a/pkg/cwhub/item_test.go b/pkg/cwhub/item_test.go index 33a52e8e0..703bbb5cb 100644 --- a/pkg/cwhub/item_test.go +++ b/pkg/cwhub/item_test.go @@ -23,7 +23,7 @@ func TestItemStatus(t *testing.T) { item.State.Tainted = false item.State.Downloaded = true - txt, _ := item.InstallStatus() + txt := item.State.Text() require.Equal(t, "enabled,update-available", txt) item.State.Installed = true @@ -31,7 +31,7 @@ func TestItemStatus(t *testing.T) { item.State.Tainted = false item.State.Downloaded = false - txt, _ = item.InstallStatus() + txt = item.State.Text() require.Equal(t, "enabled,local", txt) } diff --git a/pkg/cwhub/iteminstall.go b/pkg/cwhub/iteminstall.go index 038e5471d..270d4c114 100644 --- a/pkg/cwhub/iteminstall.go +++ b/pkg/cwhub/iteminstall.go @@ -13,7 +13,7 @@ func (i *Item) enable() error { return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name) } - if i.IsLocal() { + if i.State.IsLocal() { return fmt.Errorf("%s is local, won't enable", i.Name) } diff --git a/pkg/cwhub/itemremove.go b/pkg/cwhub/itemremove.go index cba5a5904..a58bd3fa8 100644 --- a/pkg/cwhub/itemremove.go +++ b/pkg/cwhub/itemremove.go @@ -66,7 +66,7 @@ func (i *Item) disable(purge bool, force bool) (bool, error) { // Remove disables the item, optionally removing the downloaded content. func (i *Item) Remove(purge bool, force bool) (bool, error) { - if i.IsLocal() { + if i.State.IsLocal() { log.Warningf("%s is a local item, please delete manually", i.Name) return false, nil } diff --git a/pkg/cwhub/itemupgrade.go b/pkg/cwhub/itemupgrade.go index 5e7c6f71a..07c83b3c4 100644 --- a/pkg/cwhub/itemupgrade.go +++ b/pkg/cwhub/itemupgrade.go @@ -20,7 +20,7 @@ import ( func (i *Item) Upgrade(force bool) (bool, error) { updated := false - if i.IsLocal() { + if i.State.IsLocal() { log.Infof("not upgrading %s: local item", i.Name) return false, nil } @@ -155,7 +155,7 @@ func (i *Item) fetch() ([]byte, error) { // download downloads the item from the hub and writes it to the hub directory. func (i *Item) download(overwrite bool) (string, error) { - if i.IsLocal() { + if i.State.IsLocal() { return "", fmt.Errorf("%s is local, can't download", i.Name) } // if user didn't --force, don't overwrite local, tainted, up-to-date files diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index a5f839f0b..72f97b114 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -426,7 +426,7 @@ func (h *Hub) localSync() error { case versionFuture: warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version)) case versionUnknown: - if !item.IsLocal() { + if !item.State.IsLocal() { warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version)) } } diff --git a/pkg/database/bouncers.go b/pkg/database/bouncers.go index 804337ecc..496b9b6cc 100644 --- a/pkg/database/bouncers.go +++ b/pkg/database/bouncers.go @@ -13,7 +13,7 @@ import ( func (c *Client) SelectBouncer(apiKeyHash string) (*ent.Bouncer, error) { result, err := c.Ent.Bouncer.Query().Where(bouncer.APIKeyEQ(apiKeyHash)).First(c.CTX) if err != nil { - return &ent.Bouncer{}, errors.Wrapf(QueryFail, "select bouncer: %s", err) + return nil, err } return result, nil @@ -22,7 +22,7 @@ func (c *Client) SelectBouncer(apiKeyHash string) (*ent.Bouncer, error) { func (c *Client) SelectBouncerByName(bouncerName string) (*ent.Bouncer, error) { result, err := c.Ent.Bouncer.Query().Where(bouncer.NameEQ(bouncerName)).First(c.CTX) if err != nil { - return &ent.Bouncer{}, errors.Wrapf(QueryFail, "select bouncer: %s", err) + return nil, err } return result, nil @@ -31,7 +31,7 @@ func (c *Client) SelectBouncerByName(bouncerName string) (*ent.Bouncer, error) { func (c *Client) ListBouncers() ([]*ent.Bouncer, error) { result, err := c.Ent.Bouncer.Query().All(c.CTX) if err != nil { - return []*ent.Bouncer{}, errors.Wrapf(QueryFail, "listing bouncer: %s", err) + return nil, errors.Wrapf(QueryFail, "listing bouncers: %s", err) } return result, nil } diff --git a/pkg/database/machines.go b/pkg/database/machines.go index 98992d478..b9834e57e 100644 --- a/pkg/database/machines.go +++ b/pkg/database/machines.go @@ -19,8 +19,8 @@ const CapiListsMachineID = types.ListOrigin func (c *Client) CreateMachine(machineID *string, password *strfmt.Password, ipAddress string, isValidated bool, force bool, authType string) (*ent.Machine, error) { hashPassword, err := bcrypt.GenerateFromPassword([]byte(*password), bcrypt.DefaultCost) if err != nil { - c.Log.Warningf("CreateMachine : %s", err) - return nil, errors.Wrap(HashError, "") + c.Log.Warningf("CreateMachine: %s", err) + return nil, HashError } machineExist, err := c.Ent.Machine. @@ -78,7 +78,7 @@ func (c *Client) QueryMachineByID(machineID string) (*ent.Machine, error) { func (c *Client) ListMachines() ([]*ent.Machine, error) { machines, err := c.Ent.Machine.Query().All(c.CTX) if err != nil { - return []*ent.Machine{}, errors.Wrapf(QueryFail, "listing machines: %s", err) + return nil, errors.Wrapf(QueryFail, "listing machines: %s", err) } return machines, nil } @@ -101,7 +101,7 @@ func (c *Client) QueryPendingMachine() ([]*ent.Machine, error) { machines, err = c.Ent.Machine.Query().Where(machine.IsValidatedEQ(false)).All(c.CTX) if err != nil { c.Log.Warningf("QueryPendingMachine : %s", err) - return []*ent.Machine{}, errors.Wrapf(QueryFail, "querying pending machines: %s", err) + return nil, errors.Wrapf(QueryFail, "querying pending machines: %s", err) } return machines, nil } @@ -190,12 +190,13 @@ func (c *Client) IsMachineRegistered(machineID string) (bool, error) { return true, nil } if len(exist) > 1 { - return false, fmt.Errorf("More than one item with the same machineID in database") + return false, fmt.Errorf("more than one item with the same machineID in database") } return false, nil } + func (c *Client) QueryLastValidatedHeartbeatLT(t time.Time) ([]*ent.Machine, error) { return c.Ent.Machine.Query().Where(machine.LastHeartbeatLT(t), machine.IsValidatedEQ(true)).All(c.CTX) } diff --git a/pkg/metabase/container.go b/pkg/metabase/container.go index b3df7aca7..8b3dd4084 100644 --- a/pkg/metabase/container.go +++ b/pkg/metabase/container.go @@ -4,7 +4,6 @@ import ( "bufio" "context" "fmt" - "runtime" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -93,14 +92,6 @@ func (c *Container) Create() error { Tty: true, Env: env, } - os := runtime.GOOS - switch os { - case "linux": - case "windows", "darwin": - return fmt.Errorf("mac and windows are not supported yet") - default: - return fmt.Errorf("OS '%s' is not supported", os) - } log.Infof("creating container '%s'", c.Name) resp, err := c.CLI.ContainerCreate(ctx, dockerConfig, hostConfig, nil, nil, c.Name) diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index 7bcc35b1d..fb99d1ec3 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -227,3 +227,16 @@ teardown() { assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } +@test "crowdsec -t (error in acquisition file)" { + # we can verify the acquisition configuration without running crowdsec + ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') + config_set "$ACQUIS_YAML" 'del(.filenames)' + + rune -1 wait-for "${CROWDSEC}" + assert_stderr --partial "failed to configure datasource file: no filename or filenames configuration provided" + + config_set "$ACQUIS_YAML" '.filenames=["file.log"]' + config_set "$ACQUIS_YAML" '.meh=3' + rune -1 wait-for "${CROWDSEC}" + assert_stderr --partial "field meh not found in type fileacquisition.FileConfiguration" +} diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index a49df68bc..518590826 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -108,6 +108,20 @@ teardown() { rune -0 cscli config show -o json rune -0 jq -c '.API.Client.Credentials | [.url,.login[0:32]]' <(output) assert_json '["http://127.0.0.1:8080/","githubciXXXXXXXXXXXXXXXXXXXXXXXX"]' + + # pointer to boolean + + rune -0 cscli config show --key Config.API.Client.InsecureSkipVerify + assert_output "&false" + + # complex type + rune -0 cscli config show --key Config.PluginConfig + assert_output - <<-EOT + &csconfig.PluginCfg{ + User: "nobody", + Group: "nogroup", + } + EOT } @test "cscli - required configuration paths" { diff --git a/test/bats/05_config_yaml_local.bats b/test/bats/05_config_yaml_local.bats index 6f8b5c28b..b8b6da117 100644 --- a/test/bats/05_config_yaml_local.bats +++ b/test/bats/05_config_yaml_local.bats @@ -34,22 +34,22 @@ teardown() { @test "config.yaml.local - cscli (log_level)" { config_set '.common.log_level="warning"' rune -0 cscli config show --key Config.Common.LogLevel - assert_output "warning" + assert_output "&3" echo "{'common':{'log_level':'debug'}}" >"${CONFIG_YAML}.local" rune -0 cscli config show --key Config.Common.LogLevel - assert_output "debug" + assert_output "&5" } @test "config.yaml.local - cscli (log_level - with envvar)" { config_set '.common.log_level="warning"' rune -0 cscli config show --key Config.Common.LogLevel - assert_output "warning" + assert_output "&3" export CROWDSEC_LOG_LEVEL=debug echo "{'common':{'log_level':'${CROWDSEC_LOG_LEVEL}'}}" >"${CONFIG_YAML}.local" rune -0 cscli config show --key Config.Common.LogLevel - assert_output "debug" + assert_output "&5" } @test "config.yaml.local - crowdsec (listen_url)" { diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index 200b3180a..3558e8604 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -34,17 +34,19 @@ teardown() { # no items rune -0 cscli hub list - assert_output --regexp "APPSEC-CONFIGS.*APPSEC-RULES.*PARSERS.*POSTOVERFLOWS.*SCENARIOS.*COLLECTIONS.*" + assert_output "No items to display" rune -0 cscli hub list -o json assert_json '{appsec-configs:[],appsec-rules:[],parsers:[],scenarios:[],collections:[],postoverflows:[]}' rune -0 cscli hub list -o raw assert_output 'name,status,version,description,type' - # some items + # some items: with output=human, show only non-empty tables rune -0 cscli parsers install crowdsecurity/whitelists rune -0 cscli scenarios install crowdsecurity/telnet-bf rune -0 cscli hub list - assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*" + assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*" + refute_output --partial 'POSTOVERFLOWS' + refute_output --partial 'COLLECTIONS' rune -0 cscli hub list -o json rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output) rune -0 cscli hub list -o raw diff --git a/test/bats/30_machines.bats b/test/bats/30_machines.bats index d5ddf840f..f321b8f3f 100644 --- a/test/bats/30_machines.bats +++ b/test/bats/30_machines.bats @@ -23,20 +23,24 @@ teardown() { #---------- -@test "can list machines as regular user" { - rune -0 cscli machines list -} - @test "we have exactly one machine" { rune -0 cscli machines list -o json rune -0 jq -c '[. | length, .[0].machineId[0:32], .[0].isValidated]' <(output) assert_output '[1,"githubciXXXXXXXXXXXXXXXXXXXXXXXX",true]' } +@test "don't overwrite local credentials by default" { + rune -1 cscli machines add local -a -o json + rune -0 jq -r '.msg' <(stderr) + assert_output --partial 'already exists: please remove it, use "--force" or specify a different file with "-f"' + rune -0 cscli machines add local -a --force + assert_stderr --partial "Machine 'local' successfully added to the local API" +} + @test "add a new machine and delete it" { rune -0 cscli machines add -a -f /dev/null CiTestMachine -o human assert_stderr --partial "Machine 'CiTestMachine' successfully added to the local API" - assert_stderr --partial "API credentials dumped to '/dev/null'" + assert_stderr --partial "API credentials written to '/dev/null'" # we now have two machines rune -0 cscli machines list -o json @@ -56,7 +60,7 @@ teardown() { @test "register, validate and then remove a machine" { rune -0 cscli lapi register --machine CiTestMachineRegister -f /dev/null -o human assert_stderr --partial "Successfully registered to Local API (LAPI)" - assert_stderr --partial "Local API credentials dumped to '/dev/null'" + assert_stderr --partial "Local API credentials written to '/dev/null'" # the machine is not validated yet rune -0 cscli machines list -o json diff --git a/test/lib/config/config-global b/test/lib/config/config-global index 7814cefbb..22ac76df6 100755 --- a/test/lib/config/config-global +++ b/test/lib/config/config-global @@ -61,12 +61,12 @@ make_init_data() { ./instance-db config-yaml ./instance-db setup + ./bin/preload-hub-items + # when installed packages are always using sqlite, so no need to regenerate # local credz for sqlite - ./bin/preload-hub-items - - [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto + [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto --force mkdir -p "$LOCAL_INIT_DIR" diff --git a/test/lib/config/config-local b/test/lib/config/config-local index 0941484ff..e3b7bc685 100755 --- a/test/lib/config/config-local +++ b/test/lib/config/config-local @@ -115,11 +115,12 @@ make_init_data() { ./instance-db config-yaml ./instance-db setup - "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto "$CSCLI" --warning hub update ./bin/preload-hub-items + "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto --force + mkdir -p "$LOCAL_INIT_DIR" ./instance-db dump "${LOCAL_INIT_DIR}/database"