diff --git a/cmd/crowdsec-cli/alerts.go b/cmd/crowdsec-cli/alerts.go index 1b7ba90df..4e80b3508 100644 --- a/cmd/crowdsec-cli/alerts.go +++ b/cmd/crowdsec-cli/alerts.go @@ -126,17 +126,17 @@ func DisplayOneAlert(alert *models.Alert, withDetail bool) error { if *alert.Source.Value != "" { scopeAndValue += ":" + *alert.Source.Value } - fmt.Printf(" - ID : %d\n", alert.ID) - fmt.Printf(" - Date : %s\n", alert.CreatedAt) - fmt.Printf(" - Machine : %s\n", alert.MachineID) - fmt.Printf(" - Simulation : %v\n", *alert.Simulated) - fmt.Printf(" - Reason : %s\n", *alert.Scenario) + fmt.Printf(" - ID : %d\n", alert.ID) + fmt.Printf(" - Date : %s\n", alert.CreatedAt) + fmt.Printf(" - Machine : %s\n", alert.MachineID) + fmt.Printf(" - Simulation : %v\n", *alert.Simulated) + fmt.Printf(" - Reason : %s\n", *alert.Scenario) fmt.Printf(" - Events Count : %d\n", *alert.EventsCount) - fmt.Printf(" - Scope:Value: %s\n", scopeAndValue) - fmt.Printf(" - Country : %s\n", alert.Source.Cn) - fmt.Printf(" - AS : %s\n", alert.Source.AsName) - fmt.Printf(" - Begin : %s\n", *alert.StartAt) - fmt.Printf(" - End : %s\n\n", *alert.StopAt) + fmt.Printf(" - Scope:Value : %s\n", scopeAndValue) + fmt.Printf(" - Country : %s\n", alert.Source.Cn) + fmt.Printf(" - AS : %s\n", alert.Source.AsName) + fmt.Printf(" - Begin : %s\n", *alert.StartAt) + fmt.Printf(" - End : %s\n\n", *alert.StopAt) foundActive := false table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"ID", "scope:value", "action", "expiration", "created_at"}) @@ -167,6 +167,28 @@ func DisplayOneAlert(alert *models.Alert, withDetail bool) error { table.Render() // Send output } + fmt.Printf("\n - Context :\n") + sort.Slice(alert.Meta, func(i, j int) bool { + return alert.Meta[i].Key < alert.Meta[j].Key + }) + table = tablewriter.NewWriter(os.Stdout) + table.SetHeader([]string{"Key", "Value"}) + for _, meta := range alert.Meta { + var valSlice []string + if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil { + log.Fatalf("unknown context value type '%s' : %s", meta.Value, err) + } + for _, value := range valSlice { + table.Append([]string{ + meta.Key, + value, + }) + } + } + table.SetAutoMergeCells(true) + table.SetRowLine(true) + table.Render() + if withDetail { fmt.Printf("\n - Events :\n") for _, event := range alert.Events { @@ -185,28 +207,6 @@ func DisplayOneAlert(alert *models.Alert, withDetail bool) error { table.Render() // Send output } - - fmt.Printf("\n - Context :\n") - sort.Slice(alert.Meta, func(i, j int) bool { - return alert.Meta[i].Key < alert.Meta[j].Key - }) - table = tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"Key", "Value"}) - for _, meta := range alert.Meta { - var valSlice []string - if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil { - log.Fatalf("unknown context value type '%s' : %s", meta.Value, err) - } - for _, value := range valSlice { - table.Append([]string{ - meta.Key, - value, - }) - } - } - table.SetAutoMergeCells(true) - table.SetRowLine(true) - table.Render() // Send output } } return nil diff --git a/cmd/crowdsec-cli/console.go b/cmd/crowdsec-cli/console.go index cd5de970b..fd299dc04 100644 --- a/cmd/crowdsec-cli/console.go +++ b/cmd/crowdsec-cli/console.go @@ -9,11 +9,15 @@ import ( "io/fs" "net/url" "os" + "sort" + "strings" "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/enescakir/emoji" "github.com/go-openapi/strfmt" @@ -315,15 +319,80 @@ Disable given information push to the central API.`, var detectAll bool cmdLabelDetect := &cobra.Command{ - Use: "status", - Short: "List label to send with alerts", + Use: "detect", + Short: "Detect available fields from the installed parsers", DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - // load all parsers - // + var err error + + // to avoid all the log.Info from the loaders functions + log.SetLevel(log.ErrorLevel) + + err = exprhelpers.Init() + if err != nil { + log.Fatalf("Failed to init expr helpers : %s", err) + } + + // Populate cwhub package tools + if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { + log.Fatalf("Failed to load hub index : %s", err) + } + + csParsers := parser.NewParsers() + if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil { + log.Fatalf("unable to load parsers: %s", err) + } + + fieldByParsers := make(map[string][]string) + for _, node := range csParsers.Nodes { + if !detectAll && !inSlice(node.Name, args) { + continue + } + if !detectAll { + args = removeFromSlice(node.Name, args) + } + fieldByParsers[node.Name] = make([]string, 0) + fieldByParsers[node.Name] = detectNode(node, *csParsers.Ctx) + + subNodeFields := detectSubNode(node, *csParsers.Ctx) + for _, field := range subNodeFields { + if !inSlice(field, fieldByParsers[node.Name]) { + fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field) + } + } + + } + + fmt.Printf("Acquisition :\n\n") + fmt.Printf(" - evt.Line.Module\n") + fmt.Printf(" - evt.Line.Raw\n") + fmt.Printf(" - evt.Line.Src\n") + fmt.Println() + + parsersKey := make([]string, 0) + for k := range fieldByParsers { + parsersKey = append(parsersKey, k) + } + sort.Strings(parsersKey) + + for _, k := range parsersKey { + fmt.Printf("%s :\n\n", k) + values := fieldByParsers[k] + sort.Strings(values) + for _, value := range values { + fmt.Printf(" - %s\n", value) + } + fmt.Println() + } + + if len(args) > 0 { + for _, parserNotFound := range args { + log.Errorf("parser '%s' not found, can't detect fields", parserNotFound) + } + } }, } - cmdLabelAdd.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser") + cmdLabelDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser") cmdLabel.AddCommand(cmdLabelDetect) var keysToDelete []string @@ -439,3 +508,124 @@ func SetConsoleOpts(args []string, wanted bool) { } } + +func detectStaticField(GrokStatics []types.ExtraField) []string { + ret := make([]string, 0) + for _, static := range GrokStatics { + if static.Parsed != "" { + fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + if static.Meta != "" { + fieldName := fmt.Sprintf("evt.Meta.%s", static.Meta) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + if static.TargetByName != "" { + fieldName := static.TargetByName + if !strings.HasPrefix(fieldName, "evt.") { + fieldName = "evt." + fieldName + } + if !inSlice(static.TargetByName, ret) { + ret = append(ret, static.TargetByName) + } + } + } + + return ret +} + +func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string { + var ret = make([]string, 0) + if node.Grok.RunTimeRegexp != nil { + for _, capturedField := range node.Grok.RunTimeRegexp.Names() { + fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + } + + if node.Grok.RegexpName != "" { + grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName) + if err != nil { + log.Warningf("Can't get subgrok: %s", err) + } + for _, capturedField := range grokCompiled.Names() { + fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + } + + if len(node.Grok.Statics) > 0 { + staticsField := detectStaticField(node.Grok.Statics) + for _, staticField := range staticsField { + if !inSlice(staticField, ret) { + ret = append(ret, staticField) + } + } + } + + if len(node.Statics) > 0 { + staticsField := detectStaticField(node.Statics) + for _, staticField := range staticsField { + if !inSlice(staticField, ret) { + ret = append(ret, staticField) + } + } + } + + return ret +} + +func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string { + var ret = make([]string, 0) + + for _, subnode := range node.LeavesNodes { + if subnode.Grok.RunTimeRegexp != nil { + for _, capturedField := range subnode.Grok.RunTimeRegexp.Names() { + fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + } + if subnode.Grok.RegexpName != "" { + grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName) + if err != nil { + log.Warningf("Can't get subgrok: %s", err) + } + for _, capturedField := range grokCompiled.Names() { + fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + } + + if len(subnode.Grok.Statics) > 0 { + staticsField := detectStaticField(subnode.Grok.Statics) + for _, staticField := range staticsField { + if !inSlice(staticField, ret) { + ret = append(ret, staticField) + } + } + } + + if len(subnode.Statics) > 0 { + staticsField := detectStaticField(subnode.Statics) + for _, staticField := range staticsField { + if !inSlice(staticField, ret) { + ret = append(ret, staticField) + } + } + } + } + + return ret +} diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 433f453c4..9030cafc3 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -31,7 +31,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { } // Start loading configs - csParsers := newParsers() + csParsers := parser.NewParsers() if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err) } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index bd0ca51b2..a71e75509 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -4,7 +4,6 @@ import ( "flag" "fmt" "os" - "sort" "strings" _ "net/http/pprof" @@ -69,45 +68,6 @@ type Flags struct { type labelsMap map[string]string -// Return new parsers -// nodes and povfwnodes are already initialized in parser.LoadStages -func newParsers() *parser.Parsers { - parsers := &parser.Parsers{ - Ctx: &parser.UnixParserCtx{}, - Povfwctx: &parser.UnixParserCtx{}, - StageFiles: make([]parser.Stagefile, 0), - PovfwStageFiles: make([]parser.Stagefile, 0), - } - for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} { - for _, hubParserItem := range cwhub.GetItemMap(itemType) { - if hubParserItem.Installed { - stagefile := parser.Stagefile{ - Filename: hubParserItem.LocalPath, - Stage: hubParserItem.Stage, - } - if itemType == cwhub.PARSERS { - parsers.StageFiles = append(parsers.StageFiles, stagefile) - } - if itemType == cwhub.PARSERS_OVFLW { - parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile) - } - } - } - } - if parsers.StageFiles != nil { - sort.Slice(parsers.StageFiles, func(i, j int) bool { - return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename - }) - } - if parsers.PovfwStageFiles != nil { - sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool { - return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename - }) - } - - return parsers -} - func LoadBuckets(cConfig *csconfig.Config) error { var ( diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 7faa3b4dc..62e65a59d 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -4,8 +4,10 @@ import ( "fmt" "io/ioutil" "path" + "sort" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/grokky" log "github.com/sirupsen/logrus" @@ -46,6 +48,45 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) { return &r, nil } +// Return new parsers +// nodes and povfwnodes are already initialized in LoadStages +func NewParsers() *Parsers { + parsers := &Parsers{ + Ctx: &UnixParserCtx{}, + Povfwctx: &UnixParserCtx{}, + StageFiles: make([]Stagefile, 0), + PovfwStageFiles: make([]Stagefile, 0), + } + for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} { + for _, hubParserItem := range cwhub.GetItemMap(itemType) { + if hubParserItem.Installed { + stagefile := Stagefile{ + Filename: hubParserItem.LocalPath, + Stage: hubParserItem.Stage, + } + if itemType == cwhub.PARSERS { + parsers.StageFiles = append(parsers.StageFiles, stagefile) + } + if itemType == cwhub.PARSERS_OVFLW { + parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile) + } + } + } + } + if parsers.StageFiles != nil { + sort.Slice(parsers.StageFiles, func(i, j int) bool { + return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename + }) + } + if parsers.PovfwStageFiles != nil { + sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool { + return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename + }) + } + + return parsers +} + func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { var err error