From 157589d31ec377bc20b371eda239181ceb1cd94f Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 12 Jan 2023 17:04:28 +0100 Subject: [PATCH] cscli explain: add crowdsec path option (#1983) --- cmd/crowdsec-cli/explain.go | 267 +++++++++++--------- tests/bats/01_base.bats | 5 + tests/bats/testdata/explain/explain-log.txt | 14 + 3 files changed, 172 insertions(+), 114 deletions(-) create mode 100644 tests/bats/testdata/explain/explain-log.txt diff --git a/cmd/crowdsec-cli/explain.go b/cmd/crowdsec-cli/explain.go index 09bb0474d..abbcd66a4 100644 --- a/cmd/crowdsec-cli/explain.go +++ b/cmd/crowdsec-cli/explain.go @@ -15,16 +15,149 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -func NewExplainCmd() *cobra.Command { - /* ---- HUB COMMAND */ - var logFile string - var dsn string - var logLine string - var logType string - var opts hubtest.DumpOpts - var err error +func runExplain(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() - var cmdExplain = &cobra.Command{ + logFile, err := flags.GetString("file") + if err != nil { + return err + } + + dsn, err := flags.GetString("dsn") + if err != nil { + return err + } + + logLine, err := flags.GetString("log") + if err != nil { + return err + } + + logType, err := flags.GetString("type") + if err != nil { + return err + } + + opts := hubtest.DumpOpts{} + + opts.Details, err = flags.GetBool("verbose") + if err != nil { + return err + } + + opts.SkipOk, err = flags.GetBool("failures") + if err != nil { + return err + } + + crowdsec, err := flags.GetString("crowdsec") + if err != nil { + return err + } + + fileInfo, _ := os.Stdin.Stat() + + if logType == "" || (logLine == "" && logFile == "" && dsn == "") { + printHelp(cmd) + fmt.Println() + fmt.Printf("Please provide --type flag\n") + os.Exit(1) + } + + if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) { + log.Fatal("-f - is intended to work with pipes.") + } + + var f *os.File + dir := os.TempDir() + + tmpFile := "" + // we create a temporary log file if a log line/stdin has been provided + if logLine != "" || logFile == "-" { + tmpFile = filepath.Join(dir, "cscli_test_tmp.log") + f, err = os.Create(tmpFile) + if err != nil { + log.Fatal(err) + } + + if logLine != "" { + _, err = f.WriteString(logLine) + if err != nil { + log.Fatal(err) + } + } else if logFile == "-" { + reader := bufio.NewReader(os.Stdin) + errCount := 0 + for { + input, err := reader.ReadBytes('\n') + if err != nil && err == io.EOF { + break + } + _, err = f.Write(input) + if err != nil { + errCount++ + } + } + if errCount > 0 { + log.Warnf("Failed to write %d lines to tmp file", errCount) + } + } + f.Close() + // this is the file that was going to be read by crowdsec anyway + logFile = tmpFile + } + + if logFile != "" { + absolutePath, err := filepath.Abs(logFile) + if err != nil { + log.Fatalf("unable to get absolute path of '%s', exiting", logFile) + } + dsn = fmt.Sprintf("file://%s", absolutePath) + lineCount := types.GetLineCountForFile(absolutePath) + if lineCount > 100 { + log.Warnf("log file contains %d lines. This may take lot of resources.", lineCount) + } + } + + if dsn == "" { + log.Fatal("no acquisition (--file or --dsn) provided, can't run cscli test.") + } + + cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", "./", "-no-api"} + crowdsecCmd := exec.Command(crowdsec, cmdArgs...) + crowdsecCmd.Dir = dir + output, err := crowdsecCmd.CombinedOutput() + if err != nil { + fmt.Println(string(output)) + log.Fatalf("fail to run crowdsec for test: %v", err) + } + + // rm the temporary log file if only a log line/stdin was provided + if tmpFile != "" { + if err := os.Remove(tmpFile); err != nil { + log.Fatalf("unable to remove tmp log file '%s': %+v", tmpFile, err) + } + } + parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName) + bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName) + + parserDump, err := hubtest.LoadParserDump(parserDumpFile) + if err != nil { + log.Fatalf("unable to load parser dump result: %s", err) + } + + bucketStateDump, err := hubtest.LoadBucketPourDump(bucketStateDumpFile) + if err != nil { + log.Fatalf("unable to load bucket dump result: %s", err) + } + + hubtest.DumpTree(*parserDump, *bucketStateDump, opts) + + return nil +} + +func NewExplainCmd() *cobra.Command { + cmdExplain := &cobra.Command{ Use: "explain", Short: "Explain log pipeline", Long: ` @@ -38,112 +171,18 @@ tail -n 5 myfile.log | cscli explain --type nginx -f - `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - fileInfo, _ := os.Stdin.Stat() - - if logType == "" || (logLine == "" && logFile == "" && dsn == "") { - printHelp(cmd) - fmt.Println() - fmt.Printf("Please provide --type flag\n") - os.Exit(1) - } - - if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) { - log.Fatal("-f - is intended to work with pipes.") - } - - var f *os.File - dir := os.TempDir() - - tmpFile := "" - // we create a temporary log file if a log line/stdin has been provided - if logLine != "" || logFile == "-" { - tmpFile = filepath.Join(dir, "cscli_test_tmp.log") - f, err = os.Create(tmpFile) - if err != nil { - log.Fatal(err) - } - - if logLine != "" { - _, err = f.WriteString(logLine) - if err != nil { - log.Fatal(err) - } - } else if logFile == "-" { - reader := bufio.NewReader(os.Stdin) - errCount := 0 - for { - input, err := reader.ReadBytes('\n') - if err != nil && err == io.EOF { - break - } - _, err = f.Write(input) - if err != nil { - errCount++ - } - } - if errCount > 0 { - log.Warnf("Failed to write %d lines to tmp file", errCount) - } - } - f.Close() - //this is the file that was going to be read by crowdsec anyway - logFile = tmpFile - } - - if logFile != "" { - absolutePath, err := filepath.Abs(logFile) - if err != nil { - log.Fatalf("unable to get absolute path of '%s', exiting", logFile) - } - dsn = fmt.Sprintf("file://%s", absolutePath) - lineCount := types.GetLineCountForFile(absolutePath) - if lineCount > 100 { - log.Warnf("log file contains %d lines. This may take lot of resources.", lineCount) - } - } - - if dsn == "" { - log.Fatal("no acquisition (--file or --dsn) provided, can't run cscli test.") - } - - cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", "./", "-no-api"} - crowdsecCmd := exec.Command("crowdsec", cmdArgs...) - crowdsecCmd.Dir = dir - output, err := crowdsecCmd.CombinedOutput() - if err != nil { - fmt.Println(string(output)) - log.Fatalf("fail to run crowdsec for test: %v", err) - } - - // rm the temporary log file if only a log line/stdin was provided - if tmpFile != "" { - if err := os.Remove(tmpFile); err != nil { - log.Fatalf("unable to remove tmp log file '%s': %+v", tmpFile, err) - } - } - parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName) - bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName) - - parserDump, err := hubtest.LoadParserDump(parserDumpFile) - if err != nil { - log.Fatalf("unable to load parser dump result: %s", err) - } - - bucketStateDump, err := hubtest.LoadBucketPourDump(bucketStateDumpFile) - if err != nil { - log.Fatalf("unable to load bucket dump result: %s", err) - } - - hubtest.DumpTree(*parserDump, *bucketStateDump, opts) - }, + RunE: runExplain, } - cmdExplain.PersistentFlags().StringVarP(&logFile, "file", "f", "", "Log file to test") - cmdExplain.PersistentFlags().StringVarP(&dsn, "dsn", "d", "", "DSN to test") - cmdExplain.PersistentFlags().StringVarP(&logLine, "log", "l", "", "Log line to test") - cmdExplain.PersistentFlags().StringVarP(&logType, "type", "t", "", "Type of the acquisition to test") - cmdExplain.PersistentFlags().BoolVarP(&opts.Details, "verbose", "v", false, "Display individual changes") - cmdExplain.PersistentFlags().BoolVar(&opts.SkipOk, "failures", false, "Only show failed lines") + + flags := cmdExplain.Flags() + + flags.StringP("file", "f", "", "Log file to test") + flags.StringP("dsn", "d", "", "DSN to test") + flags.StringP("log", "l", "", "Log line to test") + flags.StringP("type", "t", "", "Type of the acquisition to test") + flags.BoolP("verbose", "v", false, "Display individual changes") + flags.Bool("failures", false, "Only show failed lines") + flags.String("crowdsec", "crowdsec", "Path to crowdsec") return cmdExplain } diff --git a/tests/bats/01_base.bats b/tests/bats/01_base.bats index 9f37ccc6a..f0f382603 100644 --- a/tests/bats/01_base.bats +++ b/tests/bats/01_base.bats @@ -274,3 +274,8 @@ declare stderr run -0 cscli support dump -f "$BATS_TEST_TMPDIR"/dump.zip assert_file_exist "$BATS_TEST_TMPDIR"/dump.zip } + +@test "cscli explain" { + rune -0 cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog --crowdsec "$CROWDSEC" + assert_output - <"$BATS_TEST_DIRNAME"/testdata/explain/explain-log.txt +} diff --git a/tests/bats/testdata/explain/explain-log.txt b/tests/bats/testdata/explain/explain-log.txt new file mode 100644 index 000000000..828899348 --- /dev/null +++ b/tests/bats/testdata/explain/explain-log.txt @@ -0,0 +1,14 @@ +line: Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4 + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs (first_parser) + ├ s01-parse + | └ 🟢 crowdsecurity/sshd-logs (+8 ~1) + ├ s02-enrich + | ├ 🟢 crowdsecurity/dateparse-enrich (+2 ~1) + | └ 🟢 crowdsecurity/geoip-enrich (+10) + ├-------- parser success 🟢 + ├ Scenarios + ├ 🟢 crowdsecurity/ssh-bf + ├ 🟢 crowdsecurity/ssh-bf_user-enum + ├ 🟢 crowdsecurity/ssh-slow-bf + └ 🟢 crowdsecurity/ssh-slow-bf_user-enum