From 5c83695177cd4044a8cc953978103377b63607f0 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 12 Feb 2024 11:23:17 +0100 Subject: [PATCH] refact "cscli explain" (#2835) --- cmd/crowdsec-cli/explain.go | 186 +++++++++++++++--------------------- cmd/crowdsec-cli/main.go | 2 +- go.mod | 4 +- go.sum | 14 +-- 4 files changed, 83 insertions(+), 123 deletions(-) diff --git a/cmd/crowdsec-cli/explain.go b/cmd/crowdsec-cli/explain.go index d21c17049..ce323fd0c 100644 --- a/cmd/crowdsec-cli/explain.go +++ b/cmd/crowdsec-cli/explain.go @@ -16,33 +16,53 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/hubtest" ) -func GetLineCountForFile(filepath string) (int, error) { +func getLineCountForFile(filepath string) (int, error) { f, err := os.Open(filepath) if err != nil { return 0, err } defer f.Close() + lc := 0 fs := bufio.NewReader(f) + for { input, err := fs.ReadBytes('\n') if len(input) > 1 { lc++ } + if err != nil && err == io.EOF { break } } + return lc, nil } -type cliExplain struct{} - -func NewCLIExplain() *cliExplain { - return &cliExplain{} +type cliExplain struct { + cfg configGetter + flags struct { + logFile string + dsn string + logLine string + logType string + details bool + skipOk bool + onlySuccessfulParsers bool + noClean bool + crowdsec string + labels string + } } -func (cli cliExplain) NewCommand() *cobra.Command { +func NewCLIExplain(cfg configGetter) *cliExplain { + return &cliExplain{ + cfg: cfg, + } +} + +func (cli *cliExplain) NewCommand() *cobra.Command { cmd := &cobra.Command{ Use: "explain", Short: "Explain log pipeline", @@ -57,118 +77,50 @@ tail -n 5 myfile.log | cscli explain --type nginx -f - `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: cli.run, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - logFile, err := flags.GetString("file") - if err != nil { - return err - } - - dsn, err := flags.GetString("dsn") - if err != nil { - return err - } - - logLine, err := flags.GetString("log") - if err != nil { - return err - } - - logType, err := flags.GetString("type") - if err != nil { - return err - } - - if logLine == "" && logFile == "" && dsn == "" { - printHelp(cmd) - fmt.Println() - return fmt.Errorf("please provide --log, --file or --dsn flag") - } - if logType == "" { - printHelp(cmd) - fmt.Println() - return fmt.Errorf("please provide --type flag") - } + RunE: func(_ *cobra.Command, _ []string) error { + return cli.run() + }, + PersistentPreRunE: func(_ *cobra.Command, _ []string) error { fileInfo, _ := os.Stdin.Stat() - if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) { + if cli.flags.logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) { return fmt.Errorf("the option -f - is intended to work with pipes") } + return nil }, } flags := cmd.Flags() - flags.StringP("file", "f", "", "Log file to test") - flags.StringP("dsn", "d", "", "DSN to test") - flags.StringP("log", "l", "", "Log line to test") - flags.StringP("type", "t", "", "Type of the acquisition to test") - flags.String("labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)") - flags.BoolP("verbose", "v", false, "Display individual changes") - flags.Bool("failures", false, "Only show failed lines") - flags.Bool("only-successful-parsers", false, "Only show successful parsers") - flags.String("crowdsec", "crowdsec", "Path to crowdsec") - flags.Bool("no-clean", false, "Don't clean runtime environment after tests") + flags.StringVarP(&cli.flags.logFile, "file", "f", "", "Log file to test") + flags.StringVarP(&cli.flags.dsn, "dsn", "d", "", "DSN to test") + flags.StringVarP(&cli.flags.logLine, "log", "l", "", "Log line to test") + flags.StringVarP(&cli.flags.logType, "type", "t", "", "Type of the acquisition to test") + flags.StringVar(&cli.flags.labels, "labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)") + flags.BoolVarP(&cli.flags.details, "verbose", "v", false, "Display individual changes") + flags.BoolVar(&cli.flags.skipOk, "failures", false, "Only show failed lines") + flags.BoolVar(&cli.flags.onlySuccessfulParsers, "only-successful-parsers", false, "Only show successful parsers") + flags.StringVar(&cli.flags.crowdsec, "crowdsec", "crowdsec", "Path to crowdsec") + flags.BoolVar(&cli.flags.noClean, "no-clean", false, "Don't clean runtime environment after tests") + + cmd.MarkFlagRequired("type") + cmd.MarkFlagsOneRequired("log", "file", "dsn") return cmd } -func (cli cliExplain) run(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() +func (cli *cliExplain) run() error { + logFile := cli.flags.logFile + logLine := cli.flags.logLine + logType := cli.flags.logType + dsn := cli.flags.dsn + labels := cli.flags.labels + crowdsec := cli.flags.crowdsec - logFile, err := flags.GetString("file") - if err != nil { - return err - } - - dsn, err := flags.GetString("dsn") - if err != nil { - return err - } - - logLine, err := flags.GetString("log") - if err != nil { - return err - } - - logType, err := flags.GetString("type") - if err != nil { - return err - } - - opts := dumps.DumpOpts{} - - opts.Details, err = flags.GetBool("verbose") - if err != nil { - return err - } - - no_clean, err := flags.GetBool("no-clean") - if err != nil { - return err - } - - opts.SkipOk, err = flags.GetBool("failures") - if err != nil { - return err - } - - opts.ShowNotOkParsers, err = flags.GetBool("only-successful-parsers") - opts.ShowNotOkParsers = !opts.ShowNotOkParsers - if err != nil { - return err - } - - crowdsec, err := flags.GetString("crowdsec") - if err != nil { - return err - } - - labels, err := flags.GetString("labels") - if err != nil { - return err + opts := dumps.DumpOpts{ + Details: cli.flags.details, + SkipOk: cli.flags.skipOk, + ShowNotOkParsers: !cli.flags.onlySuccessfulParsers, } var f *os.File @@ -176,21 +128,25 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error { // using empty string fallback to /tmp dir, err := os.MkdirTemp("", "cscli_explain") if err != nil { - return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err) + return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %w", err) } + defer func() { - if no_clean { + if cli.flags.noClean { return } + if _, err := os.Stat(dir); !os.IsNotExist(err) { if err := os.RemoveAll(dir); err != nil { log.Errorf("unable to delete temporary directory '%s': %s", dir, err) } } }() + // we create a temporary log file if a log line/stdin has been provided if logLine != "" || logFile == "-" { tmpFile := filepath.Join(dir, "cscli_test_tmp.log") + f, err = os.Create(tmpFile) if err != nil { return err @@ -220,6 +176,7 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error { log.Warnf("Failed to write %d lines to %s", errCount, tmpFile) } } + f.Close() // this is the file that was going to be read by crowdsec anyway logFile = tmpFile @@ -230,15 +187,20 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("unable to get absolute path of '%s', exiting", logFile) } + dsn = fmt.Sprintf("file://%s", absolutePath) - lineCount, err := GetLineCountForFile(absolutePath) + + lineCount, err := getLineCountForFile(absolutePath) if err != nil { return err } + log.Debugf("file %s has %d lines", absolutePath, lineCount) + if lineCount == 0 { return fmt.Errorf("the log file is empty: %s", absolutePath) } + if lineCount > 100 { log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount) } @@ -249,15 +211,19 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error { } cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", dir, "-no-api"} + if labels != "" { log.Debugf("adding labels %s", labels) cmdArgs = append(cmdArgs, "-label", labels) } + crowdsecCmd := exec.Command(crowdsec, cmdArgs...) + output, err := crowdsecCmd.CombinedOutput() if err != nil { fmt.Println(string(output)) - return fmt.Errorf("fail to run crowdsec for test: %v", err) + + return fmt.Errorf("fail to run crowdsec for test: %w", err) } parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName) @@ -265,12 +231,12 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error { parserDump, err := dumps.LoadParserDump(parserDumpFile) if err != nil { - return fmt.Errorf("unable to load parser dump result: %s", err) + return fmt.Errorf("unable to load parser dump result: %w", err) } bucketStateDump, err := dumps.LoadBucketPourDump(bucketStateDumpFile) if err != nil { - return fmt.Errorf("unable to load bucket dump result: %s", err) + return fmt.Errorf("unable to load bucket dump result: %w", err) } dumps.DumpTree(*parserDump, *bucketStateDump, opts) diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 55fcacee3..439986235 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -244,7 +244,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall cmd.AddCommand(NewCLILapi(cli.cfg).NewCommand()) cmd.AddCommand(NewCompletionCmd()) cmd.AddCommand(NewConsoleCmd()) - cmd.AddCommand(NewCLIExplain().NewCommand()) + cmd.AddCommand(NewCLIExplain(cli.cfg).NewCommand()) cmd.AddCommand(NewCLIHubTest().NewCommand()) cmd.AddCommand(NewCLINotifications().NewCommand()) cmd.AddCommand(NewCLISupport().NewCommand()) diff --git a/go.mod b/go.mod index d61c191c1..e1da18387 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,7 @@ require ( github.com/shirou/gopsutil/v3 v3.23.5 github.com/sirupsen/logrus v1.9.3 github.com/slack-go/slack v0.12.2 - github.com/spf13/cobra v1.7.0 + github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v1.3.0 @@ -108,7 +108,7 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/corazawaf/libinjection-go v0.1.2 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/creack/pty v1.1.18 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/go.sum b/go.sum index f5f61594e..8fa202131 100644 --- a/go.sum +++ b/go.sum @@ -91,21 +91,17 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f h1:FkOB9aDw0xzDd14pTarGRLsUNAymONq3dc7zhvsXElg= -github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f/go.mod h1:TrU7Li+z2RHNrPy0TKJ6R65V6Yzpan2sTIRryJJyJso= github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 h1:hyrYw3h8clMcRL2u5ooZ3tmwnmJftmhb9Ws1MKmavvI= github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607/go.mod h1:br36fEqurGYZQGit+iDYsIzW0FF6VufMbDzyyLxEuPA= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= -github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= github.com/crowdsecurity/go-cs-lib v0.0.6 h1:Ef6MylXe0GaJE9vrfvxEdbHb31+JUP1os+murPz7Pos= github.com/crowdsecurity/go-cs-lib v0.0.6/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= @@ -640,8 +636,8 @@ github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -809,8 +805,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=