Add more JSON expr helpers (#1576)
This commit is contained in:
parent
b7f1c5455f
commit
4b311684ab
|
@ -1,8 +1,6 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
@ -29,10 +27,9 @@ LOOP:
|
||||||
globalParserHits.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc()
|
globalParserHits.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc()
|
||||||
|
|
||||||
/* parse the log using magic */
|
/* parse the log using magic */
|
||||||
parsed, error := parser.Parse(parserCTX, event, nodes)
|
parsed, err := parser.Parse(parserCTX, event, nodes)
|
||||||
if error != nil {
|
if err != nil {
|
||||||
log.Errorf("failed parsing : %v\n", error)
|
log.Errorf("failed parsing : %v\n", err)
|
||||||
return errors.New("parsing failed :/")
|
|
||||||
}
|
}
|
||||||
if !parsed.Process {
|
if !parsed.Process {
|
||||||
globalParserHitsKo.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc()
|
globalParserHitsKo.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc()
|
||||||
|
|
|
@ -44,6 +44,9 @@ func GetExprEnv(ctx map[string]interface{}) map[string]interface{} {
|
||||||
"JsonExtract": JsonExtract,
|
"JsonExtract": JsonExtract,
|
||||||
"JsonExtractUnescape": JsonExtractUnescape,
|
"JsonExtractUnescape": JsonExtractUnescape,
|
||||||
"JsonExtractLib": JsonExtractLib,
|
"JsonExtractLib": JsonExtractLib,
|
||||||
|
"JsonExtractSlice": JsonExtractSlice,
|
||||||
|
"JsonExtractObject": JsonExtractObject,
|
||||||
|
"ToJsonString": ToJson,
|
||||||
"File": File,
|
"File": File,
|
||||||
"RegexpInFile": RegexpInFile,
|
"RegexpInFile": RegexpInFile,
|
||||||
"Upper": Upper,
|
"Upper": Upper,
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
package exprhelpers
|
package exprhelpers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/buger/jsonparser"
|
"github.com/buger/jsonparser"
|
||||||
|
@ -58,3 +60,80 @@ func JsonExtract(jsblob string, target string) string {
|
||||||
log.Tracef("extract path %+v", fullpath)
|
log.Tracef("extract path %+v", fullpath)
|
||||||
return JsonExtractLib(jsblob, fullpath...)
|
return JsonExtractLib(jsblob, fullpath...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func jsonExtractType(jsblob string, target string, t jsonparser.ValueType) ([]byte, error) {
|
||||||
|
if !strings.HasPrefix(target, "[") {
|
||||||
|
target = strings.Replace(target, "[", ".[", -1)
|
||||||
|
}
|
||||||
|
fullpath := strings.Split(target, ".")
|
||||||
|
|
||||||
|
log.Tracef("extract path %+v", fullpath)
|
||||||
|
|
||||||
|
value, dataType, _, err := jsonparser.Get(
|
||||||
|
jsonparser.StringToBytes(jsblob),
|
||||||
|
fullpath...,
|
||||||
|
)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
if err == jsonparser.KeyPathNotFoundError {
|
||||||
|
log.Debugf("Key %+v doesn't exist", target)
|
||||||
|
return nil, fmt.Errorf("key %s does not exist", target)
|
||||||
|
}
|
||||||
|
log.Errorf("jsonExtractType : %s : %s", target, err)
|
||||||
|
return nil, fmt.Errorf("jsonExtractType: %s : %w", target, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if dataType != t {
|
||||||
|
log.Errorf("jsonExtractType : expected type %s for target %s but found %s", t, target, dataType.String())
|
||||||
|
return nil, fmt.Errorf("jsonExtractType: expected type %s for target %s but found %s", t, target, dataType.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func JsonExtractSlice(jsblob string, target string) []interface{} {
|
||||||
|
|
||||||
|
value, err := jsonExtractType(jsblob, target, jsonparser.Array)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("JsonExtractSlice : %s", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s := make([]interface{}, 0)
|
||||||
|
|
||||||
|
err = json.Unmarshal(value, &s)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("JsonExtractSlice: could not convert '%s' to slice: %s", value, err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func JsonExtractObject(jsblob string, target string) map[string]interface{} {
|
||||||
|
|
||||||
|
value, err := jsonExtractType(jsblob, target, jsonparser.Object)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("JsonExtractObject: %s", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s := make(map[string]interface{})
|
||||||
|
|
||||||
|
err = json.Unmarshal(value, &s)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("JsonExtractObject: could not convert '%s' to map[string]interface{}: %s", value, err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func ToJson(obj interface{}) string {
|
||||||
|
b, err := json.Marshal(obj)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("ToJson : %s", err)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
|
@ -35,6 +35,12 @@ func TestJsonExtract(t *testing.T) {
|
||||||
targetField: "non_existing_field",
|
targetField: "non_existing_field",
|
||||||
expectResult: "",
|
expectResult: "",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "extract subfield",
|
||||||
|
jsonBlob: `{"test" : {"a": "b"}}`,
|
||||||
|
targetField: "test.a",
|
||||||
|
expectResult: "b",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
|
@ -85,5 +91,158 @@ func TestJsonExtractUnescape(t *testing.T) {
|
||||||
}
|
}
|
||||||
log.Printf("test '%s' : OK", test.name)
|
log.Printf("test '%s' : OK", test.name)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJsonExtractSlice(t *testing.T) {
|
||||||
|
if err := Init(); err != nil {
|
||||||
|
log.Fatalf(err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
err := FileInit(TestFolder, "test_data_re.txt", "regex")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf(err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
jsonBlob string
|
||||||
|
targetField string
|
||||||
|
expectResult []interface{}
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "try to extract a string as a slice",
|
||||||
|
jsonBlob: `{"test" : "1234"}`,
|
||||||
|
targetField: "test",
|
||||||
|
expectResult: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "basic json slice extract",
|
||||||
|
jsonBlob: `{"test" : ["1234"]}`,
|
||||||
|
targetField: "test",
|
||||||
|
expectResult: []interface{}{"1234"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "extract with complex expression",
|
||||||
|
jsonBlob: `{"test": {"foo": [{"a":"b"}]}}`,
|
||||||
|
targetField: "test.foo",
|
||||||
|
expectResult: []interface{}{map[string]interface{}{"a": "b"}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "extract non-existing key",
|
||||||
|
jsonBlob: `{"test: "11234"}`,
|
||||||
|
targetField: "foo",
|
||||||
|
expectResult: nil,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
result := JsonExtractSlice(test.jsonBlob, test.targetField)
|
||||||
|
assert.Equal(t, test.expectResult, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJsonExtractObject(t *testing.T) {
|
||||||
|
if err := Init(); err != nil {
|
||||||
|
log.Fatalf(err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
err := FileInit(TestFolder, "test_data_re.txt", "regex")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf(err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
jsonBlob string
|
||||||
|
targetField string
|
||||||
|
expectResult map[string]interface{}
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "try to extract a string as an object",
|
||||||
|
jsonBlob: `{"test" : "1234"}`,
|
||||||
|
targetField: "test",
|
||||||
|
expectResult: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "basic json object extract",
|
||||||
|
jsonBlob: `{"test" : {"1234": {"foo": "bar"}}}`,
|
||||||
|
targetField: "test",
|
||||||
|
expectResult: map[string]interface{}{"1234": map[string]interface{}{"foo": "bar"}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "extract with complex expression",
|
||||||
|
jsonBlob: `{"test": {"foo": [{"a":"b"}]}}`,
|
||||||
|
targetField: "test.foo[0]",
|
||||||
|
expectResult: map[string]interface{}{"a": "b"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
result := JsonExtractObject(test.jsonBlob, test.targetField)
|
||||||
|
assert.Equal(t, test.expectResult, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestToJson(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
obj interface{}
|
||||||
|
expectResult string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "convert int",
|
||||||
|
obj: 42,
|
||||||
|
expectResult: "42",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "convert slice",
|
||||||
|
obj: []string{"foo", "bar"},
|
||||||
|
expectResult: `["foo","bar"]`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "convert map",
|
||||||
|
obj: map[string]string{"foo": "bar"},
|
||||||
|
expectResult: `{"foo":"bar"}`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "convert struct",
|
||||||
|
obj: struct{ Foo string }{"bar"},
|
||||||
|
expectResult: `{"Foo":"bar"}`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "convert complex struct",
|
||||||
|
obj: struct {
|
||||||
|
Foo string
|
||||||
|
Bar struct {
|
||||||
|
Baz string
|
||||||
|
}
|
||||||
|
Bla []string
|
||||||
|
}{
|
||||||
|
Foo: "bar",
|
||||||
|
Bar: struct {
|
||||||
|
Baz string
|
||||||
|
}{
|
||||||
|
Baz: "baz",
|
||||||
|
},
|
||||||
|
Bla: []string{"foo", "bar"},
|
||||||
|
},
|
||||||
|
expectResult: `{"Foo":"bar","Bar":{"Baz":"baz"},"Bla":["foo","bar"]}`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "convert invalid type",
|
||||||
|
obj: func() {},
|
||||||
|
expectResult: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
result := ToJson(test.obj)
|
||||||
|
assert.Equal(t, test.expectResult, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -272,7 +272,8 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
// if the grok succeed, process associated statics
|
// if the grok succeed, process associated statics
|
||||||
err := n.ProcessStatics(n.Grok.Statics, p)
|
err := n.ProcessStatics(n.Grok.Statics, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
clog.Fatalf("(%s) Failed to process statics : %v", n.rn, err)
|
clog.Errorf("(%s) Failed to process statics : %v", n.rn, err)
|
||||||
|
return false, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
//grok failed, node failed
|
//grok failed, node failed
|
||||||
|
@ -337,7 +338,8 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
// if all else is good in whitelist, process node's statics
|
// if all else is good in whitelist, process node's statics
|
||||||
err := n.ProcessStatics(n.Statics, p)
|
err := n.ProcessStatics(n.Statics, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
clog.Fatalf("Failed to process statics : %v", err)
|
clog.Errorf("Failed to process statics : %v", err)
|
||||||
|
return false, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
clog.Tracef("! No node statics")
|
clog.Tracef("! No node statics")
|
||||||
|
|
|
@ -133,8 +133,12 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er
|
||||||
value = out
|
value = out
|
||||||
case int:
|
case int:
|
||||||
value = strconv.Itoa(out)
|
value = strconv.Itoa(out)
|
||||||
|
case map[string]interface{}:
|
||||||
|
clog.Warnf("Expression returned a map, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string")
|
||||||
|
case []interface{}:
|
||||||
|
clog.Warnf("Expression returned a map, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string")
|
||||||
default:
|
default:
|
||||||
clog.Fatalf("unexpected return type for RunTimeValue : %T", output)
|
clog.Errorf("unexpected return type for RunTimeValue : %T", output)
|
||||||
return errors.New("unexpected return type for RunTimeValue")
|
return errors.New("unexpected return type for RunTimeValue")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -309,7 +313,8 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
|
||||||
}
|
}
|
||||||
ret, err := node.process(&event, ctx, cachedExprEnv)
|
ret, err := node.process(&event, ctx, cachedExprEnv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
clog.Fatalf("Error while processing node : %v", err)
|
clog.Errorf("Error while processing node : %v", err)
|
||||||
|
return event, err
|
||||||
}
|
}
|
||||||
clog.Tracef("node (%s) ret : %v", node.rn, ret)
|
clog.Tracef("node (%s) ret : %v", node.rn, ret)
|
||||||
if ParseDump {
|
if ParseDump {
|
||||||
|
|
Loading…
Reference in a new issue