Add ParseKV helper and rework UnmarshalJSON as a proper helper (#2184)
This commit is contained in:
parent
e1f5ed41df
commit
424215f228
|
@ -398,6 +398,20 @@ var exprFuncs = []exprCustomFunc{
|
||||||
new(func(string) string),
|
new(func(string) string),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "UnmarshalJSON",
|
||||||
|
function: UnmarshalJSON,
|
||||||
|
signature: []interface{}{
|
||||||
|
new(func(string, map[string]interface{}, string) error),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ParseKV",
|
||||||
|
function: ParseKV,
|
||||||
|
signature: []interface{}{
|
||||||
|
new(func(string, map[string]interface{}, string) error),
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "Hostname",
|
name: "Hostname",
|
||||||
function: Hostname,
|
function: Hostname,
|
||||||
|
|
|
@ -1360,3 +1360,58 @@ func TestB64Decode(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseKv(t *testing.T) {
|
||||||
|
err := Init(nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value string
|
||||||
|
expected map[string]string
|
||||||
|
expr string
|
||||||
|
expectedBuildErr bool
|
||||||
|
expectedRuntimeErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "ParseKv() test: valid string",
|
||||||
|
value: "foo=bar",
|
||||||
|
expected: map[string]string{"foo": "bar"},
|
||||||
|
expr: `ParseKV(value, out, "a")`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ParseKv() test: valid string",
|
||||||
|
value: "foo=bar bar=foo",
|
||||||
|
expected: map[string]string{"foo": "bar", "bar": "foo"},
|
||||||
|
expr: `ParseKV(value, out, "a")`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ParseKv() test: valid string",
|
||||||
|
value: "foo=bar bar=foo foo=foo",
|
||||||
|
expected: map[string]string{"foo": "foo", "bar": "foo"},
|
||||||
|
expr: `ParseKV(value, out, "a")`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ParseKV() test: quoted string",
|
||||||
|
value: `foo="bar=toto"`,
|
||||||
|
expected: map[string]string{"foo": "bar=toto"},
|
||||||
|
expr: `ParseKV(value, out, "a")`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
tc := tc
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
outMap := make(map[string]interface{})
|
||||||
|
env := map[string]interface{}{
|
||||||
|
"value": tc.value,
|
||||||
|
"out": outMap,
|
||||||
|
}
|
||||||
|
vm, err := expr.Compile(tc.expr, GetExprOptions(env)...)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
_, err = expr.Run(vm, env)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, tc.expected, outMap["a"])
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -51,6 +51,8 @@ var dbClient *database.Client
|
||||||
|
|
||||||
var exprFunctionOptions []expr.Option
|
var exprFunctionOptions []expr.Option
|
||||||
|
|
||||||
|
var keyValuePattern = regexp.MustCompile(`\s*(?P<key>[^=\s]+)\s*=\s*(?:"(?P<quoted_value>[^"\\]*(?:\\.[^"\\]*)*)"|(?P<value>[^=\s]+))`)
|
||||||
|
|
||||||
func GetExprOptions(ctx map[string]interface{}) []expr.Option {
|
func GetExprOptions(ctx map[string]interface{}) []expr.Option {
|
||||||
ret := []expr.Option{}
|
ret := []expr.Option{}
|
||||||
ret = append(ret, exprFunctionOptions...)
|
ret = append(ret, exprFunctionOptions...)
|
||||||
|
@ -596,6 +598,44 @@ func B64Decode(params ...any) (any, error) {
|
||||||
return string(decoded), nil
|
return string(decoded), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ParseKV(params ...any) (any, error) {
|
||||||
|
|
||||||
|
blob := params[0].(string)
|
||||||
|
target := params[1].(map[string]interface{})
|
||||||
|
prefix := params[2].(string)
|
||||||
|
|
||||||
|
matches := keyValuePattern.FindAllStringSubmatch(blob, -1)
|
||||||
|
if matches == nil {
|
||||||
|
log.Errorf("could not find any key/value pair in line")
|
||||||
|
return nil, fmt.Errorf("invalid input format")
|
||||||
|
}
|
||||||
|
if _, ok := target[prefix]; !ok {
|
||||||
|
target[prefix] = make(map[string]string)
|
||||||
|
} else {
|
||||||
|
_, ok := target[prefix].(map[string]string)
|
||||||
|
if !ok {
|
||||||
|
log.Errorf("ParseKV: target is not a map[string]string")
|
||||||
|
return nil, fmt.Errorf("target is not a map[string]string")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, match := range matches {
|
||||||
|
key := ""
|
||||||
|
value := ""
|
||||||
|
for i, name := range keyValuePattern.SubexpNames() {
|
||||||
|
if name == "key" {
|
||||||
|
key = match[i]
|
||||||
|
} else if name == "quoted_value" && match[i] != "" {
|
||||||
|
value = match[i]
|
||||||
|
} else if name == "value" && match[i] != "" {
|
||||||
|
value = match[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
target[prefix].(map[string]string)[key] = value
|
||||||
|
}
|
||||||
|
log.Tracef("unmarshaled KV: %+v", target[prefix])
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
func Hostname(params ...any) (any, error) {
|
func Hostname(params ...any) (any, error) {
|
||||||
hostname, err := os.Hostname()
|
hostname, err := os.Hostname()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -163,3 +163,20 @@ func ToJson(params ...any) (any, error) {
|
||||||
}
|
}
|
||||||
return string(b), nil
|
return string(b), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Func UnmarshalJSON(jsonBlob []byte, target interface{}) error {
|
||||||
|
func UnmarshalJSON(params ...any) (any, error) {
|
||||||
|
jsonBlob := params[0].(string)
|
||||||
|
target := params[1].(map[string]interface{})
|
||||||
|
key := params[2].(string)
|
||||||
|
|
||||||
|
var out interface{}
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(jsonBlob), &out)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("UnmarshalJSON : %s", err)
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
target[key] = out
|
||||||
|
return target, nil
|
||||||
|
}
|
||||||
|
|
|
@ -1,9 +1,10 @@
|
||||||
package exprhelpers
|
package exprhelpers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"log"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
"github.com/antonmedv/expr"
|
"github.com/antonmedv/expr"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
@ -304,3 +305,67 @@ func TestToJson(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestUnmarshalJSON(t *testing.T) {
|
||||||
|
err := Init(nil)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
json string
|
||||||
|
expectResult interface{}
|
||||||
|
expr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "convert int",
|
||||||
|
json: "42",
|
||||||
|
expectResult: float64(42),
|
||||||
|
expr: "UnmarshalJSON(json, out, 'a')",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "convert slice",
|
||||||
|
json: `["foo","bar"]`,
|
||||||
|
expectResult: []interface{}{"foo", "bar"},
|
||||||
|
expr: "UnmarshalJSON(json, out, 'a')",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "convert map",
|
||||||
|
json: `{"foo":"bar"}`,
|
||||||
|
expectResult: map[string]interface{}{"foo": "bar"},
|
||||||
|
expr: "UnmarshalJSON(json, out, 'a')",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "convert struct",
|
||||||
|
json: `{"Foo":"bar"}`,
|
||||||
|
expectResult: map[string]interface{}{"Foo": "bar"},
|
||||||
|
expr: "UnmarshalJSON(json, out, 'a')",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "convert complex struct",
|
||||||
|
json: `{"Foo":"bar","Bar":{"Baz":"baz"},"Bla":["foo","bar"]}`,
|
||||||
|
expectResult: map[string]interface{}{
|
||||||
|
"Foo": "bar",
|
||||||
|
"Bar": map[string]interface{}{
|
||||||
|
"Baz": "baz",
|
||||||
|
},
|
||||||
|
"Bla": []interface{}{"foo", "bar"},
|
||||||
|
},
|
||||||
|
expr: "UnmarshalJSON(json, out, 'a')",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
outMap := make(map[string]interface{})
|
||||||
|
env := map[string]interface{}{
|
||||||
|
"json": test.json,
|
||||||
|
"out": outMap,
|
||||||
|
}
|
||||||
|
vm, err := expr.Compile(test.expr, GetExprOptions(env)...)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
_, err = expr.Run(vm, env)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, test.expectResult, outMap["a"])
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
|
@ -164,7 +164,7 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er
|
||||||
processed = true
|
processed = true
|
||||||
clog.Debugf("+ Method %s('%s') returned %d entries to merge in .Enriched\n", static.Method, value, len(ret))
|
clog.Debugf("+ Method %s('%s') returned %d entries to merge in .Enriched\n", static.Method, value, len(ret))
|
||||||
//Hackish check, but those methods do not return any data by design
|
//Hackish check, but those methods do not return any data by design
|
||||||
if len(ret) == 0 && static.Method != "UnmarshalXML" && static.Method != "UnmarshalJSON" {
|
if len(ret) == 0 && static.Method != "UnmarshalJSON" {
|
||||||
clog.Debugf("+ Method '%s' empty response on '%s'", static.Method, value)
|
clog.Debugf("+ Method '%s' empty response on '%s'", static.Method, value)
|
||||||
}
|
}
|
||||||
for k, v := range ret {
|
for k, v := range ret {
|
||||||
|
|
|
@ -8,11 +8,13 @@ lines:
|
||||||
#these are the results we expect from the parser
|
#these are the results we expect from the parser
|
||||||
results:
|
results:
|
||||||
- Unmarshaled:
|
- Unmarshaled:
|
||||||
foo: "bar"
|
JSON:
|
||||||
pouet: 42
|
foo: "bar"
|
||||||
|
pouet: 42
|
||||||
Process: true
|
Process: true
|
||||||
Stage: s00-raw
|
Stage: s00-raw
|
||||||
- Unmarshaled: {}
|
- Unmarshaled:
|
||||||
|
JSON: {}
|
||||||
Process: true
|
Process: true
|
||||||
Stage: s00-raw
|
Stage: s00-raw
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue