package dumps import ( "errors" "fmt" "io" "os" "sort" "strings" "time" "github.com/fatih/color" diff "github.com/r3labs/diff/v2" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" "github.com/crowdsecurity/go-cs-lib/maptools" "github.com/crowdsecurity/crowdsec/pkg/emoji" "github.com/crowdsecurity/crowdsec/pkg/types" ) type ParserResult struct { Idx int Evt types.Event Success bool } type ParserResults map[string]map[string][]ParserResult type DumpOpts struct { Details bool SkipOk bool ShowNotOkParsers bool } func LoadParserDump(filepath string) (*ParserResults, error) { dumpData, err := os.Open(filepath) if err != nil { return nil, err } defer dumpData.Close() results, err := io.ReadAll(dumpData) if err != nil { return nil, err } pdump := ParserResults{} if err := yaml.Unmarshal(results, &pdump); err != nil { return nil, err } /* we know that some variables should always be set, let's check if they're present in last parser output of last stage */ stages := maptools.SortedKeys(pdump) var lastStage string // Loop over stages to find last successful one with at least one parser for i := len(stages) - 2; i >= 0; i-- { if len(pdump[stages[i]]) != 0 { lastStage = stages[i] break } } parsers := make([]string, 0, len(pdump[lastStage])) for k := range pdump[lastStage] { parsers = append(parsers, k) } sort.Strings(parsers) if len(parsers) == 0 { return nil, errors.New("no parser found. Please install the appropriate parser and retry") } lastParser := parsers[len(parsers)-1] for idx, result := range pdump[lastStage][lastParser] { if result.Evt.StrTime == "" { log.Warningf("Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode.", idx, len(pdump[lastStage][lastParser])) } else { log.Debugf("Line %d/%d has evt.StrTime set to '%s'", idx, len(pdump[lastStage][lastParser]), result.Evt.StrTime) } } return &pdump, nil } func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) { // note : we can use line -> time as the unique identifier (of acquisition) state := make(map[time.Time]map[string]map[string]ParserResult) assoc := make(map[time.Time]string, 0) parser_order := make(map[string][]string) for stage, parsers := range parserResults { // let's process parsers in the order according to idx parser_order[stage] = make([]string, len(parsers)) for pname, parser := range parsers { if len(parser) > 0 { parser_order[stage][parser[0].Idx-1] = pname } } for _, parser := range parser_order[stage] { results := parsers[parser] for _, parserRes := range results { evt := parserRes.Evt if _, ok := state[evt.Line.Time]; !ok { state[evt.Line.Time] = make(map[string]map[string]ParserResult) assoc[evt.Line.Time] = evt.Line.Raw } if _, ok := state[evt.Line.Time][stage]; !ok { state[evt.Line.Time][stage] = make(map[string]ParserResult) } state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success} } } } for bname, evtlist := range bucketPour { for _, evt := range evtlist { if evt.Line.Raw == "" { continue } // it might be bucket overflow being reprocessed, skip this if _, ok := state[evt.Line.Time]; !ok { state[evt.Line.Time] = make(map[string]map[string]ParserResult) assoc[evt.Line.Time] = evt.Line.Raw } // there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase // we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if _, ok := state[evt.Line.Time]["buckets"]; !ok { state[evt.Line.Time]["buckets"] = make(map[string]ParserResult) } state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true} } } yellow := color.New(color.FgYellow).SprintFunc() red := color.New(color.FgRed).SprintFunc() green := color.New(color.FgGreen).SprintFunc() whitelistReason := "" // get each line for tstamp, rawstr := range assoc { if opts.SkipOk { if _, ok := state[tstamp]["buckets"]["OK"]; ok { continue } } fmt.Printf("line: %s\n", rawstr) skeys := make([]string, 0, len(state[tstamp])) for k := range state[tstamp] { // there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase // we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if k == "buckets" { continue } skeys = append(skeys, k) } sort.Strings(skeys) // iterate stage var prevItem types.Event for _, stage := range skeys { parsers := state[tstamp][stage] sep := "├" presep := "|" fmt.Printf("\t%s %s\n", sep, stage) for idx, parser := range parser_order[stage] { res := parsers[parser].Success sep := "├" if idx == len(parser_order[stage])-1 { sep = "└" } created := 0 updated := 0 deleted := 0 whitelisted := false changeStr := "" detailsDisplay := "" if res { changelog, _ := diff.Diff(prevItem, parsers[parser].Evt) for _, change := range changelog { switch change.Type { case "create": created++ detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To)) case "update": detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To)) if change.Path[0] == "Whitelisted" && change.To == true { whitelisted = true if whitelistReason == "" { whitelistReason = parsers[parser].Evt.WhitelistReason } } updated++ case "delete": deleted++ detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, "."))) } } prevItem = parsers[parser].Evt } if created > 0 { changeStr += green(fmt.Sprintf("+%d", created)) } if updated > 0 { if len(changeStr) > 0 { changeStr += " " } changeStr += yellow(fmt.Sprintf("~%d", updated)) } if deleted > 0 { if len(changeStr) > 0 { changeStr += " " } changeStr += red(fmt.Sprintf("-%d", deleted)) } if whitelisted { if len(changeStr) > 0 { changeStr += " " } changeStr += red("[whitelisted]") } if changeStr == "" { changeStr = yellow("unchanged") } if res { fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr) if opts.Details { fmt.Print(detailsDisplay) } } else if opts.ShowNotOkParsers { fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser) } } } sep := "└" if len(state[tstamp]["buckets"]) > 0 { sep = "├" } // did the event enter the bucket pour phase ? if _, ok := state[tstamp]["buckets"]["OK"]; ok { fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle) } else if whitelistReason != "" { fmt.Printf("\t%s-------- parser success, ignored by whitelist (%s) %s\n", sep, whitelistReason, emoji.GreenCircle) } else { fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle) } // now print bucket info if len(state[tstamp]["buckets"]) > 0 { fmt.Printf("\t├ Scenarios\n") } bnames := make([]string, 0, len(state[tstamp]["buckets"])) for k := range state[tstamp]["buckets"] { // there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase // we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if k == "OK" { continue } bnames = append(bnames, k) } sort.Strings(bnames) for idx, bname := range bnames { sep := "├" if idx == len(bnames)-1 { sep = "└" } fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname) } fmt.Println() } }