From 424215f2286391f394cb711eb0597790c4975603 Mon Sep 17 00:00:00 2001 From: Laurence Jones Date: Fri, 12 May 2023 08:43:01 +0100 Subject: [PATCH] Add ParseKV helper and rework UnmarshalJSON as a proper helper (#2184) --- pkg/exprhelpers/expr_lib.go | 14 +++++ pkg/exprhelpers/exprlib_test.go | 55 +++++++++++++++++++ pkg/exprhelpers/helpers.go | 40 ++++++++++++++ pkg/exprhelpers/jsonextract.go | 17 ++++++ pkg/exprhelpers/jsonextract_test.go | 67 ++++++++++++++++++++++- pkg/parser/runtime.go | 2 +- pkg/parser/tests/json-unmarshal/test.yaml | 8 ++- 7 files changed, 198 insertions(+), 5 deletions(-) diff --git a/pkg/exprhelpers/expr_lib.go b/pkg/exprhelpers/expr_lib.go index 0a8346d14..f4e1f4722 100644 --- a/pkg/exprhelpers/expr_lib.go +++ b/pkg/exprhelpers/expr_lib.go @@ -398,6 +398,20 @@ var exprFuncs = []exprCustomFunc{ new(func(string) string), }, }, + { + name: "UnmarshalJSON", + function: UnmarshalJSON, + signature: []interface{}{ + new(func(string, map[string]interface{}, string) error), + }, + }, + { + name: "ParseKV", + function: ParseKV, + signature: []interface{}{ + new(func(string, map[string]interface{}, string) error), + }, + }, { name: "Hostname", function: Hostname, diff --git a/pkg/exprhelpers/exprlib_test.go b/pkg/exprhelpers/exprlib_test.go index 1bb69dee9..f5e39e9d2 100644 --- a/pkg/exprhelpers/exprlib_test.go +++ b/pkg/exprhelpers/exprlib_test.go @@ -1360,3 +1360,58 @@ func TestB64Decode(t *testing.T) { }) } } + +func TestParseKv(t *testing.T) { + err := Init(nil) + require.NoError(t, err) + + tests := []struct { + name string + value string + expected map[string]string + expr string + expectedBuildErr bool + expectedRuntimeErr bool + }{ + { + name: "ParseKv() test: valid string", + value: "foo=bar", + expected: map[string]string{"foo": "bar"}, + expr: `ParseKV(value, out, "a")`, + }, + { + name: "ParseKv() test: valid string", + value: "foo=bar bar=foo", + expected: map[string]string{"foo": "bar", "bar": "foo"}, + expr: `ParseKV(value, out, "a")`, + }, + { + name: "ParseKv() test: valid string", + value: "foo=bar bar=foo foo=foo", + expected: map[string]string{"foo": "foo", "bar": "foo"}, + expr: `ParseKV(value, out, "a")`, + }, + { + name: "ParseKV() test: quoted string", + value: `foo="bar=toto"`, + expected: map[string]string{"foo": "bar=toto"}, + expr: `ParseKV(value, out, "a")`, + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + outMap := make(map[string]interface{}) + env := map[string]interface{}{ + "value": tc.value, + "out": outMap, + } + vm, err := expr.Compile(tc.expr, GetExprOptions(env)...) + assert.NoError(t, err) + _, err = expr.Run(vm, env) + assert.NoError(t, err) + assert.Equal(t, tc.expected, outMap["a"]) + }) + } +} diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 1e722f074..4a1404304 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -51,6 +51,8 @@ var dbClient *database.Client var exprFunctionOptions []expr.Option +var keyValuePattern = regexp.MustCompile(`\s*(?P[^=\s]+)\s*=\s*(?:"(?P[^"\\]*(?:\\.[^"\\]*)*)"|(?P[^=\s]+))`) + func GetExprOptions(ctx map[string]interface{}) []expr.Option { ret := []expr.Option{} ret = append(ret, exprFunctionOptions...) @@ -596,6 +598,44 @@ func B64Decode(params ...any) (any, error) { return string(decoded), nil } +func ParseKV(params ...any) (any, error) { + + blob := params[0].(string) + target := params[1].(map[string]interface{}) + prefix := params[2].(string) + + matches := keyValuePattern.FindAllStringSubmatch(blob, -1) + if matches == nil { + log.Errorf("could not find any key/value pair in line") + return nil, fmt.Errorf("invalid input format") + } + if _, ok := target[prefix]; !ok { + target[prefix] = make(map[string]string) + } else { + _, ok := target[prefix].(map[string]string) + if !ok { + log.Errorf("ParseKV: target is not a map[string]string") + return nil, fmt.Errorf("target is not a map[string]string") + } + } + for _, match := range matches { + key := "" + value := "" + for i, name := range keyValuePattern.SubexpNames() { + if name == "key" { + key = match[i] + } else if name == "quoted_value" && match[i] != "" { + value = match[i] + } else if name == "value" && match[i] != "" { + value = match[i] + } + } + target[prefix].(map[string]string)[key] = value + } + log.Tracef("unmarshaled KV: %+v", target[prefix]) + return nil, nil +} + func Hostname(params ...any) (any, error) { hostname, err := os.Hostname() if err != nil { diff --git a/pkg/exprhelpers/jsonextract.go b/pkg/exprhelpers/jsonextract.go index 12dbb9da8..f3a9ae78a 100644 --- a/pkg/exprhelpers/jsonextract.go +++ b/pkg/exprhelpers/jsonextract.go @@ -163,3 +163,20 @@ func ToJson(params ...any) (any, error) { } return string(b), nil } + +// Func UnmarshalJSON(jsonBlob []byte, target interface{}) error { +func UnmarshalJSON(params ...any) (any, error) { + jsonBlob := params[0].(string) + target := params[1].(map[string]interface{}) + key := params[2].(string) + + var out interface{} + + err := json.Unmarshal([]byte(jsonBlob), &out) + if err != nil { + log.Errorf("UnmarshalJSON : %s", err) + return "", nil + } + target[key] = out + return target, nil +} diff --git a/pkg/exprhelpers/jsonextract_test.go b/pkg/exprhelpers/jsonextract_test.go index 594087474..481c7d723 100644 --- a/pkg/exprhelpers/jsonextract_test.go +++ b/pkg/exprhelpers/jsonextract_test.go @@ -1,9 +1,10 @@ package exprhelpers import ( - "log" "testing" + log "github.com/sirupsen/logrus" + "github.com/antonmedv/expr" "github.com/stretchr/testify/assert" ) @@ -304,3 +305,67 @@ func TestToJson(t *testing.T) { }) } } + +func TestUnmarshalJSON(t *testing.T) { + err := Init(nil) + assert.NoError(t, err) + tests := []struct { + name string + json string + expectResult interface{} + expr string + }{ + { + name: "convert int", + json: "42", + expectResult: float64(42), + expr: "UnmarshalJSON(json, out, 'a')", + }, + { + name: "convert slice", + json: `["foo","bar"]`, + expectResult: []interface{}{"foo", "bar"}, + expr: "UnmarshalJSON(json, out, 'a')", + }, + { + name: "convert map", + json: `{"foo":"bar"}`, + expectResult: map[string]interface{}{"foo": "bar"}, + expr: "UnmarshalJSON(json, out, 'a')", + }, + { + name: "convert struct", + json: `{"Foo":"bar"}`, + expectResult: map[string]interface{}{"Foo": "bar"}, + expr: "UnmarshalJSON(json, out, 'a')", + }, + { + name: "convert complex struct", + json: `{"Foo":"bar","Bar":{"Baz":"baz"},"Bla":["foo","bar"]}`, + expectResult: map[string]interface{}{ + "Foo": "bar", + "Bar": map[string]interface{}{ + "Baz": "baz", + }, + "Bla": []interface{}{"foo", "bar"}, + }, + expr: "UnmarshalJSON(json, out, 'a')", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + outMap := make(map[string]interface{}) + env := map[string]interface{}{ + "json": test.json, + "out": outMap, + } + vm, err := expr.Compile(test.expr, GetExprOptions(env)...) + assert.NoError(t, err) + _, err = expr.Run(vm, env) + assert.NoError(t, err) + assert.Equal(t, test.expectResult, outMap["a"]) + }) + } + +} diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index 4541eafd9..5f47e0f5c 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -164,7 +164,7 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er processed = true clog.Debugf("+ Method %s('%s') returned %d entries to merge in .Enriched\n", static.Method, value, len(ret)) //Hackish check, but those methods do not return any data by design - if len(ret) == 0 && static.Method != "UnmarshalXML" && static.Method != "UnmarshalJSON" { + if len(ret) == 0 && static.Method != "UnmarshalJSON" { clog.Debugf("+ Method '%s' empty response on '%s'", static.Method, value) } for k, v := range ret { diff --git a/pkg/parser/tests/json-unmarshal/test.yaml b/pkg/parser/tests/json-unmarshal/test.yaml index 9d4e2e025..4b4154690 100644 --- a/pkg/parser/tests/json-unmarshal/test.yaml +++ b/pkg/parser/tests/json-unmarshal/test.yaml @@ -8,11 +8,13 @@ lines: #these are the results we expect from the parser results: - Unmarshaled: - foo: "bar" - pouet: 42 + JSON: + foo: "bar" + pouet: 42 Process: true Stage: s00-raw - - Unmarshaled: {} + - Unmarshaled: + JSON: {} Process: true Stage: s00-raw