diff --git a/cmd/crowdsec/parse.go b/cmd/crowdsec/parse.go index e357e8436..db6455689 100644 --- a/cmd/crowdsec/parse.go +++ b/cmd/crowdsec/parse.go @@ -1,8 +1,6 @@ package main import ( - "errors" - "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -29,10 +27,9 @@ LOOP: globalParserHits.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc() /* parse the log using magic */ - parsed, error := parser.Parse(parserCTX, event, nodes) - if error != nil { - log.Errorf("failed parsing : %v\n", error) - return errors.New("parsing failed :/") + parsed, err := parser.Parse(parserCTX, event, nodes) + if err != nil { + log.Errorf("failed parsing : %v\n", err) } if !parsed.Process { globalParserHitsKo.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc() diff --git a/pkg/exprhelpers/exprlib.go b/pkg/exprhelpers/exprlib.go index 0c22cf33f..47396ebbf 100644 --- a/pkg/exprhelpers/exprlib.go +++ b/pkg/exprhelpers/exprlib.go @@ -44,6 +44,9 @@ func GetExprEnv(ctx map[string]interface{}) map[string]interface{} { "JsonExtract": JsonExtract, "JsonExtractUnescape": JsonExtractUnescape, "JsonExtractLib": JsonExtractLib, + "JsonExtractSlice": JsonExtractSlice, + "JsonExtractObject": JsonExtractObject, + "ToJsonString": ToJson, "File": File, "RegexpInFile": RegexpInFile, "Upper": Upper, diff --git a/pkg/exprhelpers/jsonextract.go b/pkg/exprhelpers/jsonextract.go index 1e3022df4..d1f749f8c 100644 --- a/pkg/exprhelpers/jsonextract.go +++ b/pkg/exprhelpers/jsonextract.go @@ -1,6 +1,8 @@ package exprhelpers import ( + "encoding/json" + "fmt" "strings" "github.com/buger/jsonparser" @@ -58,3 +60,80 @@ func JsonExtract(jsblob string, target string) string { log.Tracef("extract path %+v", fullpath) return JsonExtractLib(jsblob, fullpath...) } + +func jsonExtractType(jsblob string, target string, t jsonparser.ValueType) ([]byte, error) { + if !strings.HasPrefix(target, "[") { + target = strings.Replace(target, "[", ".[", -1) + } + fullpath := strings.Split(target, ".") + + log.Tracef("extract path %+v", fullpath) + + value, dataType, _, err := jsonparser.Get( + jsonparser.StringToBytes(jsblob), + fullpath..., + ) + + if err != nil { + if err == jsonparser.KeyPathNotFoundError { + log.Debugf("Key %+v doesn't exist", target) + return nil, fmt.Errorf("key %s does not exist", target) + } + log.Errorf("jsonExtractType : %s : %s", target, err) + return nil, fmt.Errorf("jsonExtractType: %s : %w", target, err) + } + + if dataType != t { + log.Errorf("jsonExtractType : expected type %s for target %s but found %s", t, target, dataType.String()) + return nil, fmt.Errorf("jsonExtractType: expected type %s for target %s but found %s", t, target, dataType.String()) + } + + return value, nil +} + +func JsonExtractSlice(jsblob string, target string) []interface{} { + + value, err := jsonExtractType(jsblob, target, jsonparser.Array) + + if err != nil { + log.Errorf("JsonExtractSlice : %s", err) + return nil + } + + s := make([]interface{}, 0) + + err = json.Unmarshal(value, &s) + if err != nil { + log.Errorf("JsonExtractSlice: could not convert '%s' to slice: %s", value, err) + return nil + } + return s +} + +func JsonExtractObject(jsblob string, target string) map[string]interface{} { + + value, err := jsonExtractType(jsblob, target, jsonparser.Object) + + if err != nil { + log.Errorf("JsonExtractObject: %s", err) + return nil + } + + s := make(map[string]interface{}) + + err = json.Unmarshal(value, &s) + if err != nil { + log.Errorf("JsonExtractObject: could not convert '%s' to map[string]interface{}: %s", value, err) + return nil + } + return s +} + +func ToJson(obj interface{}) string { + b, err := json.Marshal(obj) + if err != nil { + log.Errorf("ToJson : %s", err) + return "" + } + return string(b) +} diff --git a/pkg/exprhelpers/jsonextract_test.go b/pkg/exprhelpers/jsonextract_test.go index 1e3563348..30825478e 100644 --- a/pkg/exprhelpers/jsonextract_test.go +++ b/pkg/exprhelpers/jsonextract_test.go @@ -35,6 +35,12 @@ func TestJsonExtract(t *testing.T) { targetField: "non_existing_field", expectResult: "", }, + { + name: "extract subfield", + jsonBlob: `{"test" : {"a": "b"}}`, + targetField: "test.a", + expectResult: "b", + }, } for _, test := range tests { @@ -85,5 +91,158 @@ func TestJsonExtractUnescape(t *testing.T) { } log.Printf("test '%s' : OK", test.name) } - +} + +func TestJsonExtractSlice(t *testing.T) { + if err := Init(); err != nil { + log.Fatalf(err.Error()) + } + + err := FileInit(TestFolder, "test_data_re.txt", "regex") + if err != nil { + log.Fatalf(err.Error()) + } + + tests := []struct { + name string + jsonBlob string + targetField string + expectResult []interface{} + }{ + { + name: "try to extract a string as a slice", + jsonBlob: `{"test" : "1234"}`, + targetField: "test", + expectResult: nil, + }, + { + name: "basic json slice extract", + jsonBlob: `{"test" : ["1234"]}`, + targetField: "test", + expectResult: []interface{}{"1234"}, + }, + { + name: "extract with complex expression", + jsonBlob: `{"test": {"foo": [{"a":"b"}]}}`, + targetField: "test.foo", + expectResult: []interface{}{map[string]interface{}{"a": "b"}}, + }, + { + name: "extract non-existing key", + jsonBlob: `{"test: "11234"}`, + targetField: "foo", + expectResult: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := JsonExtractSlice(test.jsonBlob, test.targetField) + assert.Equal(t, test.expectResult, result) + }) + } +} + +func TestJsonExtractObject(t *testing.T) { + if err := Init(); err != nil { + log.Fatalf(err.Error()) + } + + err := FileInit(TestFolder, "test_data_re.txt", "regex") + if err != nil { + log.Fatalf(err.Error()) + } + + tests := []struct { + name string + jsonBlob string + targetField string + expectResult map[string]interface{} + }{ + { + name: "try to extract a string as an object", + jsonBlob: `{"test" : "1234"}`, + targetField: "test", + expectResult: nil, + }, + { + name: "basic json object extract", + jsonBlob: `{"test" : {"1234": {"foo": "bar"}}}`, + targetField: "test", + expectResult: map[string]interface{}{"1234": map[string]interface{}{"foo": "bar"}}, + }, + { + name: "extract with complex expression", + jsonBlob: `{"test": {"foo": [{"a":"b"}]}}`, + targetField: "test.foo[0]", + expectResult: map[string]interface{}{"a": "b"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := JsonExtractObject(test.jsonBlob, test.targetField) + assert.Equal(t, test.expectResult, result) + }) + } +} + +func TestToJson(t *testing.T) { + tests := []struct { + name string + obj interface{} + expectResult string + }{ + { + name: "convert int", + obj: 42, + expectResult: "42", + }, + { + name: "convert slice", + obj: []string{"foo", "bar"}, + expectResult: `["foo","bar"]`, + }, + { + name: "convert map", + obj: map[string]string{"foo": "bar"}, + expectResult: `{"foo":"bar"}`, + }, + { + name: "convert struct", + obj: struct{ Foo string }{"bar"}, + expectResult: `{"Foo":"bar"}`, + }, + { + name: "convert complex struct", + obj: struct { + Foo string + Bar struct { + Baz string + } + Bla []string + }{ + Foo: "bar", + Bar: struct { + Baz string + }{ + Baz: "baz", + }, + Bla: []string{"foo", "bar"}, + }, + expectResult: `{"Foo":"bar","Bar":{"Baz":"baz"},"Bla":["foo","bar"]}`, + }, + { + name: "convert invalid type", + obj: func() {}, + expectResult: "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := ToJson(test.obj) + assert.Equal(t, test.expectResult, result) + }) + } } diff --git a/pkg/parser/node.go b/pkg/parser/node.go index dedafec80..6cc23ca2e 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -272,7 +272,8 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri // if the grok succeed, process associated statics err := n.ProcessStatics(n.Grok.Statics, p) if err != nil { - clog.Fatalf("(%s) Failed to process statics : %v", n.rn, err) + clog.Errorf("(%s) Failed to process statics : %v", n.rn, err) + return false, err } } else { //grok failed, node failed @@ -337,7 +338,8 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri // if all else is good in whitelist, process node's statics err := n.ProcessStatics(n.Statics, p) if err != nil { - clog.Fatalf("Failed to process statics : %v", err) + clog.Errorf("Failed to process statics : %v", err) + return false, err } } else { clog.Tracef("! No node statics") diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index 5fc8fe88a..3ffeb1548 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -133,8 +133,12 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er value = out case int: value = strconv.Itoa(out) + case map[string]interface{}: + clog.Warnf("Expression returned a map, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string") + case []interface{}: + clog.Warnf("Expression returned a map, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string") default: - clog.Fatalf("unexpected return type for RunTimeValue : %T", output) + clog.Errorf("unexpected return type for RunTimeValue : %T", output) return errors.New("unexpected return type for RunTimeValue") } } @@ -309,7 +313,8 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) } ret, err := node.process(&event, ctx, cachedExprEnv) if err != nil { - clog.Fatalf("Error while processing node : %v", err) + clog.Errorf("Error while processing node : %v", err) + return event, err } clog.Tracef("node (%s) ret : %v", node.rn, ret) if ParseDump {