Add ParseKV helper and rework UnmarshalJSON as a proper helper (#2184)

pkg/exprhelpers/expr_lib.go

@@ -398,6 +398,20 @@ var exprFuncs = []exprCustomFunc{
 			new(func(string) string),
 		},
 	},
+	{
+		name:     "UnmarshalJSON",
+		function: UnmarshalJSON,
+		signature: []interface{}{
+			new(func(string, map[string]interface{}, string) error),
+		},
+	},
+	{
+		name:     "ParseKV",
+		function: ParseKV,
+		signature: []interface{}{
+			new(func(string, map[string]interface{}, string) error),
+		},
+	},
 	{
 		name:     "Hostname",
 		function: Hostname,

pkg/exprhelpers/exprlib_test.go

@@ -1360,3 +1360,58 @@ func TestB64Decode(t *testing.T) {
 		})
 	}
 }
+
+func TestParseKv(t *testing.T) {
+	err := Init(nil)
+	require.NoError(t, err)
+
+	tests := []struct {
+		name               string
+		value              string
+		expected           map[string]string
+		expr               string
+		expectedBuildErr   bool
+		expectedRuntimeErr bool
+	}{
+		{
+			name:     "ParseKv() test: valid string",
+			value:    "foo=bar",
+			expected: map[string]string{"foo": "bar"},
+			expr:     `ParseKV(value, out, "a")`,
+		},
+		{
+			name:     "ParseKv() test: valid string",
+			value:    "foo=bar bar=foo",
+			expected: map[string]string{"foo": "bar", "bar": "foo"},
+			expr:     `ParseKV(value, out, "a")`,
+		},
+		{
+			name:     "ParseKv() test: valid string",
+			value:    "foo=bar bar=foo foo=foo",
+			expected: map[string]string{"foo": "foo", "bar": "foo"},
+			expr:     `ParseKV(value, out, "a")`,
+		},
+		{
+			name:     "ParseKV() test: quoted string",
+			value:    `foo="bar=toto"`,
+			expected: map[string]string{"foo": "bar=toto"},
+			expr:     `ParseKV(value, out, "a")`,
+		},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			outMap := make(map[string]interface{})
+			env := map[string]interface{}{
+				"value": tc.value,
+				"out":   outMap,
+			}
+			vm, err := expr.Compile(tc.expr, GetExprOptions(env)...)
+			assert.NoError(t, err)
+			_, err = expr.Run(vm, env)
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expected, outMap["a"])
+		})
+	}
+}

pkg/exprhelpers/helpers.go

@@ -51,6 +51,8 @@ var dbClient *database.Client
 
 var exprFunctionOptions []expr.Option
 
+var keyValuePattern = regexp.MustCompile(`\s*(?P<key>[^=\s]+)\s*=\s*(?:"(?P<quoted_value>[^"\\]*(?:\\.[^"\\]*)*)"|(?P<value>[^=\s]+))`)
+
 func GetExprOptions(ctx map[string]interface{}) []expr.Option {
 	ret := []expr.Option{}
 	ret = append(ret, exprFunctionOptions...)
@@ -596,6 +598,44 @@ func B64Decode(params ...any) (any, error) {
 	return string(decoded), nil
 }
 
+func ParseKV(params ...any) (any, error) {
+
+	blob := params[0].(string)
+	target := params[1].(map[string]interface{})
+	prefix := params[2].(string)
+
+	matches := keyValuePattern.FindAllStringSubmatch(blob, -1)
+	if matches == nil {
+		log.Errorf("could not find any key/value pair in line")
+		return nil, fmt.Errorf("invalid input format")
+	}
+	if _, ok := target[prefix]; !ok {
+		target[prefix] = make(map[string]string)
+	} else {
+		_, ok := target[prefix].(map[string]string)
+		if !ok {
+			log.Errorf("ParseKV: target is not a map[string]string")
+			return nil, fmt.Errorf("target is not a map[string]string")
+		}
+	}
+	for _, match := range matches {
+		key := ""
+		value := ""
+		for i, name := range keyValuePattern.SubexpNames() {
+			if name == "key" {
+				key = match[i]
+			} else if name == "quoted_value" && match[i] != "" {
+				value = match[i]
+			} else if name == "value" && match[i] != "" {
+				value = match[i]
+			}
+		}
+		target[prefix].(map[string]string)[key] = value
+	}
+	log.Tracef("unmarshaled KV: %+v", target[prefix])
+	return nil, nil
+}
+
 func Hostname(params ...any) (any, error) {
 	hostname, err := os.Hostname()
 	if err != nil {

pkg/exprhelpers/jsonextract.go

@@ -163,3 +163,20 @@ func ToJson(params ...any) (any, error) {
 	}
 	return string(b), nil
 }
+
+// Func UnmarshalJSON(jsonBlob []byte, target interface{}) error {
+func UnmarshalJSON(params ...any) (any, error) {
+	jsonBlob := params[0].(string)
+	target := params[1].(map[string]interface{})
+	key := params[2].(string)
+
+	var out interface{}
+
+	err := json.Unmarshal([]byte(jsonBlob), &out)
+	if err != nil {
+		log.Errorf("UnmarshalJSON : %s", err)
+		return "", nil
+	}
+	target[key] = out
+	return target, nil
+}

pkg/exprhelpers/jsonextract_test.go

@@ -1,9 +1,10 @@
 package exprhelpers
 
 import (
-	"log"
 	"testing"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/antonmedv/expr"
 	"github.com/stretchr/testify/assert"
 )
@@ -304,3 +305,67 @@ func TestToJson(t *testing.T) {
 		})
 	}
 }
+
+func TestUnmarshalJSON(t *testing.T) {
+	err := Init(nil)
+	assert.NoError(t, err)
+	tests := []struct {
+		name         string
+		json         string
+		expectResult interface{}
+		expr         string
+	}{
+		{
+			name:         "convert int",
+			json:         "42",
+			expectResult: float64(42),
+			expr:         "UnmarshalJSON(json, out, 'a')",
+		},
+		{
+			name:         "convert slice",
+			json:         `["foo","bar"]`,
+			expectResult: []interface{}{"foo", "bar"},
+			expr:         "UnmarshalJSON(json, out, 'a')",
+		},
+		{
+			name:         "convert map",
+			json:         `{"foo":"bar"}`,
+			expectResult: map[string]interface{}{"foo": "bar"},
+			expr:         "UnmarshalJSON(json, out, 'a')",
+		},
+		{
+			name:         "convert struct",
+			json:         `{"Foo":"bar"}`,
+			expectResult: map[string]interface{}{"Foo": "bar"},
+			expr:         "UnmarshalJSON(json, out, 'a')",
+		},
+		{
+			name: "convert complex struct",
+			json: `{"Foo":"bar","Bar":{"Baz":"baz"},"Bla":["foo","bar"]}`,
+			expectResult: map[string]interface{}{
+				"Foo": "bar",
+				"Bar": map[string]interface{}{
+					"Baz": "baz",
+				},
+				"Bla": []interface{}{"foo", "bar"},
+			},
+			expr: "UnmarshalJSON(json, out, 'a')",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			outMap := make(map[string]interface{})
+			env := map[string]interface{}{
+				"json": test.json,
+				"out":  outMap,
+			}
+			vm, err := expr.Compile(test.expr, GetExprOptions(env)...)
+			assert.NoError(t, err)
+			_, err = expr.Run(vm, env)
+			assert.NoError(t, err)
+			assert.Equal(t, test.expectResult, outMap["a"])
+		})
+	}
+
+}

pkg/parser/runtime.go

@@ -164,7 +164,7 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er
 				processed = true
 				clog.Debugf("+ Method %s('%s') returned %d entries to merge in .Enriched\n", static.Method, value, len(ret))
 				//Hackish check, but those methods do not return any data by design
-				if len(ret) == 0 && static.Method != "UnmarshalXML" && static.Method != "UnmarshalJSON" {
+				if len(ret) == 0 && static.Method != "UnmarshalJSON" {
 					clog.Debugf("+ Method '%s' empty response on '%s'", static.Method, value)
 				}
 				for k, v := range ret {

pkg/parser/tests/json-unmarshal/test.yaml

@@ -8,11 +8,13 @@ lines:
 #these are the results we expect from the parser
 results:
   - Unmarshaled:
-      foo: "bar"
-      pouet: 42
+      JSON:
+        foo: "bar"
+        pouet: 42
     Process: true
     Stage: s00-raw
-  - Unmarshaled: {}
+  - Unmarshaled:
+      JSON: {}
     Process: true
     Stage: s00-raw