crowdsec/cmd/crowdsec-cli/explain.go

package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"github.com/crowdsecurity/crowdsec/pkg/dumps"
	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)

func getLineCountForFile(filepath string) (int, error) {
	f, err := os.Open(filepath)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	lc := 0
	fs := bufio.NewReader(f)

	for {
		input, err := fs.ReadBytes('\n')
		if len(input) > 1 {
			lc++
		}

		if err != nil && err == io.EOF {
			break
		}
	}

	return lc, nil
}

type cliExplain struct {
	cfg   configGetter
	flags struct {
		logFile               string
		dsn                   string
		logLine               string
		logType               string
		details               bool
		skipOk                bool
		onlySuccessfulParsers bool
		noClean               bool
		crowdsec              string
		labels                string
	}
}

func NewCLIExplain(cfg configGetter) *cliExplain {
	return &cliExplain{
		cfg: cfg,
	}
}

func (cli *cliExplain) NewCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "explain",
		Short: "Explain log pipeline",
		Long: `
Explain log pipeline 
		`,
		Example: `
cscli explain --file ./myfile.log --type nginx 
cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
cscli explain --dsn "file://myfile.log" --type nginx
tail -n 5 myfile.log | cscli explain --type nginx -f -
		`,
		Args:              cobra.ExactArgs(0),
		DisableAutoGenTag: true,
		RunE: func(_ *cobra.Command, _ []string) error {
			return cli.run()
		},
		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
			fileInfo, _ := os.Stdin.Stat()
			if cli.flags.logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
				return errors.New("the option -f - is intended to work with pipes")
			}

			return nil
		},
	}

	flags := cmd.Flags()

	flags.StringVarP(&cli.flags.logFile, "file", "f", "", "Log file to test")
	flags.StringVarP(&cli.flags.dsn, "dsn", "d", "", "DSN to test")
	flags.StringVarP(&cli.flags.logLine, "log", "l", "", "Log line to test")
	flags.StringVarP(&cli.flags.logType, "type", "t", "", "Type of the acquisition to test")
	flags.StringVar(&cli.flags.labels, "labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
	flags.BoolVarP(&cli.flags.details, "verbose", "v", false, "Display individual changes")
	flags.BoolVar(&cli.flags.skipOk, "failures", false, "Only show failed lines")
	flags.BoolVar(&cli.flags.onlySuccessfulParsers, "only-successful-parsers", false, "Only show successful parsers")
	flags.StringVar(&cli.flags.crowdsec, "crowdsec", "crowdsec", "Path to crowdsec")
	flags.BoolVar(&cli.flags.noClean, "no-clean", false, "Don't clean runtime environment after tests")

	cmd.MarkFlagRequired("type")
	cmd.MarkFlagsOneRequired("log", "file", "dsn")

	return cmd
}

func (cli *cliExplain) run() error {
	logFile := cli.flags.logFile
	logLine := cli.flags.logLine
	logType := cli.flags.logType
	dsn := cli.flags.dsn
	labels := cli.flags.labels
	crowdsec := cli.flags.crowdsec

	opts := dumps.DumpOpts{
		Details:          cli.flags.details,
		SkipOk:           cli.flags.skipOk,
		ShowNotOkParsers: !cli.flags.onlySuccessfulParsers,
	}

	var f *os.File

	// using empty string fallback to /tmp
	dir, err := os.MkdirTemp("", "cscli_explain")
	if err != nil {
		return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %w", err)
	}

	defer func() {
		if cli.flags.noClean {
			return
		}

		if _, err := os.Stat(dir); !os.IsNotExist(err) {
			if err := os.RemoveAll(dir); err != nil {
				log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
			}
		}
	}()

	// we create a  temporary log file if a log line/stdin has been provided
	if logLine != "" || logFile == "-" {
		tmpFile := filepath.Join(dir, "cscli_test_tmp.log")

		f, err = os.Create(tmpFile)
		if err != nil {
			return err
		}

		if logLine != "" {
			_, err = f.WriteString(logLine)
			if err != nil {
				return err
			}
		} else if logFile == "-" {
			reader := bufio.NewReader(os.Stdin)
			errCount := 0

			for {
				input, err := reader.ReadBytes('\n')
				if err != nil && errors.Is(err, io.EOF) {
					break
				}

				if len(input) > 1 {
					_, err = f.Write(input)
				}

				if err != nil || len(input) <= 1 {
					errCount++
				}
			}

			if errCount > 0 {
				log.Warnf("Failed to write %d lines to %s", errCount, tmpFile)
			}
		}

		f.Close()
		// this is the file that was going to be read by crowdsec anyway
		logFile = tmpFile
	}

	if logFile != "" {
		absolutePath, err := filepath.Abs(logFile)
		if err != nil {
			return fmt.Errorf("unable to get absolute path of '%s', exiting", logFile)
		}

		dsn = fmt.Sprintf("file://%s", absolutePath)

		lineCount, err := getLineCountForFile(absolutePath)
		if err != nil {
			return err
		}

		log.Debugf("file %s has %d lines", absolutePath, lineCount)

		if lineCount == 0 {
			return fmt.Errorf("the log file is empty: %s", absolutePath)
		}

		if lineCount > 100 {
			log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount)
		}
	}

	if dsn == "" {
		return errors.New("no acquisition (--file or --dsn) provided, can't run cscli test")
	}

	cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", dir, "-no-api"}

	if labels != "" {
		log.Debugf("adding labels %s", labels)
		cmdArgs = append(cmdArgs, "-label", labels)
	}

	crowdsecCmd := exec.Command(crowdsec, cmdArgs...)

	output, err := crowdsecCmd.CombinedOutput()
	if err != nil {
		fmt.Println(string(output))

		return fmt.Errorf("fail to run crowdsec for test: %w", err)
	}

	parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
	bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)

	parserDump, err := dumps.LoadParserDump(parserDumpFile)
	if err != nil {
		return fmt.Errorf("unable to load parser dump result: %w", err)
	}

	bucketStateDump, err := dumps.LoadBucketPourDump(bucketStateDumpFile)
	if err != nil {
		return fmt.Errorf("unable to load bucket dump result: %w", err)
	}

	dumps.DumpTree(*parserDump, *bucketStateDump, opts)

	return nil
}