2024-01-15 08:16:03 +00:00
package dumps
import (
2024-02-23 15:05:01 +00:00
"errors"
2024-01-15 08:16:03 +00:00
"fmt"
"io"
"os"
"sort"
"strings"
"time"
"github.com/fatih/color"
diff "github.com/r3labs/diff/v2"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
2024-02-23 15:05:01 +00:00
"github.com/crowdsecurity/go-cs-lib/maptools"
"github.com/crowdsecurity/crowdsec/pkg/emoji"
"github.com/crowdsecurity/crowdsec/pkg/types"
2024-01-15 08:16:03 +00:00
)
type ParserResult struct {
Idx int
Evt types . Event
Success bool
}
type ParserResults map [ string ] map [ string ] [ ] ParserResult
type DumpOpts struct {
Details bool
SkipOk bool
ShowNotOkParsers bool
}
func LoadParserDump ( filepath string ) ( * ParserResults , error ) {
dumpData , err := os . Open ( filepath )
if err != nil {
return nil , err
}
defer dumpData . Close ( )
results , err := io . ReadAll ( dumpData )
if err != nil {
return nil , err
}
pdump := ParserResults { }
if err := yaml . Unmarshal ( results , & pdump ) ; err != nil {
return nil , err
}
/ * we know that some variables should always be set ,
let ' s check if they ' re present in last parser output of last stage * /
stages := maptools . SortedKeys ( pdump )
var lastStage string
2024-02-23 15:05:01 +00:00
// Loop over stages to find last successful one with at least one parser
2024-01-15 08:16:03 +00:00
for i := len ( stages ) - 2 ; i >= 0 ; i -- {
if len ( pdump [ stages [ i ] ] ) != 0 {
lastStage = stages [ i ]
break
}
}
parsers := make ( [ ] string , 0 , len ( pdump [ lastStage ] ) )
for k := range pdump [ lastStage ] {
parsers = append ( parsers , k )
}
sort . Strings ( parsers )
if len ( parsers ) == 0 {
2024-02-23 15:05:01 +00:00
return nil , errors . New ( "no parser found. Please install the appropriate parser and retry" )
2024-01-15 08:16:03 +00:00
}
lastParser := parsers [ len ( parsers ) - 1 ]
for idx , result := range pdump [ lastStage ] [ lastParser ] {
if result . Evt . StrTime == "" {
log . Warningf ( "Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode." , idx , len ( pdump [ lastStage ] [ lastParser ] ) )
} else {
log . Debugf ( "Line %d/%d has evt.StrTime set to '%s'" , idx , len ( pdump [ lastStage ] [ lastParser ] ) , result . Evt . StrTime )
}
}
return & pdump , nil
}
func DumpTree ( parserResults ParserResults , bucketPour BucketPourInfo , opts DumpOpts ) {
2024-02-23 15:05:01 +00:00
// note : we can use line -> time as the unique identifier (of acquisition)
2024-01-15 08:16:03 +00:00
state := make ( map [ time . Time ] map [ string ] map [ string ] ParserResult )
assoc := make ( map [ time . Time ] string , 0 )
parser_order := make ( map [ string ] [ ] string )
for stage , parsers := range parserResults {
2024-02-23 15:05:01 +00:00
// let's process parsers in the order according to idx
2024-01-15 08:16:03 +00:00
parser_order [ stage ] = make ( [ ] string , len ( parsers ) )
2024-02-23 15:05:01 +00:00
2024-01-15 08:16:03 +00:00
for pname , parser := range parsers {
if len ( parser ) > 0 {
parser_order [ stage ] [ parser [ 0 ] . Idx - 1 ] = pname
}
}
for _ , parser := range parser_order [ stage ] {
results := parsers [ parser ]
for _ , parserRes := range results {
evt := parserRes . Evt
if _ , ok := state [ evt . Line . Time ] ; ! ok {
state [ evt . Line . Time ] = make ( map [ string ] map [ string ] ParserResult )
assoc [ evt . Line . Time ] = evt . Line . Raw
}
if _ , ok := state [ evt . Line . Time ] [ stage ] ; ! ok {
state [ evt . Line . Time ] [ stage ] = make ( map [ string ] ParserResult )
}
state [ evt . Line . Time ] [ stage ] [ parser ] = ParserResult { Evt : evt , Success : parserRes . Success }
}
}
}
for bname , evtlist := range bucketPour {
for _ , evt := range evtlist {
if evt . Line . Raw == "" {
continue
}
2024-02-23 15:05:01 +00:00
// it might be bucket overflow being reprocessed, skip this
2024-01-15 08:16:03 +00:00
if _ , ok := state [ evt . Line . Time ] ; ! ok {
state [ evt . Line . Time ] = make ( map [ string ] map [ string ] ParserResult )
assoc [ evt . Line . Time ] = evt . Line . Raw
}
2024-02-23 15:05:01 +00:00
// there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
// we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
2024-01-15 08:16:03 +00:00
if _ , ok := state [ evt . Line . Time ] [ "buckets" ] ; ! ok {
state [ evt . Line . Time ] [ "buckets" ] = make ( map [ string ] ParserResult )
}
state [ evt . Line . Time ] [ "buckets" ] [ bname ] = ParserResult { Success : true }
}
}
yellow := color . New ( color . FgYellow ) . SprintFunc ( )
red := color . New ( color . FgRed ) . SprintFunc ( )
green := color . New ( color . FgGreen ) . SprintFunc ( )
whitelistReason := ""
2024-02-23 15:05:01 +00:00
// get each line
2024-01-15 08:16:03 +00:00
for tstamp , rawstr := range assoc {
if opts . SkipOk {
if _ , ok := state [ tstamp ] [ "buckets" ] [ "OK" ] ; ok {
continue
}
}
fmt . Printf ( "line: %s\n" , rawstr )
skeys := make ( [ ] string , 0 , len ( state [ tstamp ] ) )
for k := range state [ tstamp ] {
2024-02-23 15:05:01 +00:00
// there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
// we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
2024-01-15 08:16:03 +00:00
if k == "buckets" {
continue
}
skeys = append ( skeys , k )
}
sort . Strings ( skeys )
// iterate stage
var prevItem types . Event
for _ , stage := range skeys {
parsers := state [ tstamp ] [ stage ]
sep := "├"
presep := "|"
fmt . Printf ( "\t%s %s\n" , sep , stage )
for idx , parser := range parser_order [ stage ] {
res := parsers [ parser ] . Success
sep := "├"
if idx == len ( parser_order [ stage ] ) - 1 {
sep = "└"
}
created := 0
updated := 0
deleted := 0
whitelisted := false
changeStr := ""
detailsDisplay := ""
if res {
changelog , _ := diff . Diff ( prevItem , parsers [ parser ] . Evt )
for _ , change := range changelog {
switch change . Type {
case "create" :
created ++
detailsDisplay += fmt . Sprintf ( "\t%s\t\t%s %s evt.%s : %s\n" , presep , sep , change . Type , strings . Join ( change . Path , "." ) , green ( change . To ) )
case "update" :
detailsDisplay += fmt . Sprintf ( "\t%s\t\t%s %s evt.%s : %s -> %s\n" , presep , sep , change . Type , strings . Join ( change . Path , "." ) , change . From , yellow ( change . To ) )
if change . Path [ 0 ] == "Whitelisted" && change . To == true {
whitelisted = true
if whitelistReason == "" {
whitelistReason = parsers [ parser ] . Evt . WhitelistReason
}
}
2024-02-23 15:05:01 +00:00
2024-01-15 08:16:03 +00:00
updated ++
case "delete" :
deleted ++
detailsDisplay += fmt . Sprintf ( "\t%s\t\t%s %s evt.%s\n" , presep , sep , change . Type , red ( strings . Join ( change . Path , "." ) ) )
}
}
prevItem = parsers [ parser ] . Evt
}
if created > 0 {
changeStr += green ( fmt . Sprintf ( "+%d" , created ) )
}
if updated > 0 {
if len ( changeStr ) > 0 {
changeStr += " "
}
changeStr += yellow ( fmt . Sprintf ( "~%d" , updated ) )
}
if deleted > 0 {
if len ( changeStr ) > 0 {
changeStr += " "
}
changeStr += red ( fmt . Sprintf ( "-%d" , deleted ) )
}
if whitelisted {
if len ( changeStr ) > 0 {
changeStr += " "
}
changeStr += red ( "[whitelisted]" )
}
if changeStr == "" {
changeStr = yellow ( "unchanged" )
}
if res {
fmt . Printf ( "\t%s\t%s %s %s (%s)\n" , presep , sep , emoji . GreenCircle , parser , changeStr )
if opts . Details {
fmt . Print ( detailsDisplay )
}
} else if opts . ShowNotOkParsers {
fmt . Printf ( "\t%s\t%s %s %s\n" , presep , sep , emoji . RedCircle , parser )
}
}
}
sep := "└"
if len ( state [ tstamp ] [ "buckets" ] ) > 0 {
sep = "├"
}
2024-02-23 15:05:01 +00:00
// did the event enter the bucket pour phase ?
2024-01-15 08:16:03 +00:00
if _ , ok := state [ tstamp ] [ "buckets" ] [ "OK" ] ; ok {
fmt . Printf ( "\t%s-------- parser success %s\n" , sep , emoji . GreenCircle )
} else if whitelistReason != "" {
fmt . Printf ( "\t%s-------- parser success, ignored by whitelist (%s) %s\n" , sep , whitelistReason , emoji . GreenCircle )
} else {
fmt . Printf ( "\t%s-------- parser failure %s\n" , sep , emoji . RedCircle )
}
2024-02-23 15:05:01 +00:00
// now print bucket info
2024-01-15 08:16:03 +00:00
if len ( state [ tstamp ] [ "buckets" ] ) > 0 {
fmt . Printf ( "\t├ Scenarios\n" )
}
bnames := make ( [ ] string , 0 , len ( state [ tstamp ] [ "buckets" ] ) )
for k := range state [ tstamp ] [ "buckets" ] {
2024-02-23 15:05:01 +00:00
// there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
// we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
2024-01-15 08:16:03 +00:00
if k == "OK" {
continue
}
bnames = append ( bnames , k )
}
sort . Strings ( bnames )
for idx , bname := range bnames {
sep := "├"
if idx == len ( bnames ) - 1 {
sep = "└"
}
fmt . Printf ( "\t\t%s %s %s\n" , sep , emoji . GreenCircle , bname )
}
fmt . Println ( )
}
}