From 32f32b41c71f9e8bff25db33286245db59a0b3b3 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Fri, 22 May 2020 18:12:33 +0200 Subject: [PATCH] add json support via expr helpers --- go.mod | 1 + go.sum | 2 ++ pkg/exprhelpers/exprlib.go | 2 +- pkg/exprhelpers/jsonextract.go | 32 +++++++++++++++++++ .../tests/base-json-extract/base-grok.yaml | 14 ++++++++ .../tests/base-json-extract/base-grok2.yaml | 14 ++++++++ .../tests/base-json-extract/parsers.yaml | 4 +++ pkg/parser/tests/base-json-extract/test.yaml | 16 ++++++++++ 8 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 pkg/exprhelpers/jsonextract.go create mode 100644 pkg/parser/tests/base-json-extract/base-grok.yaml create mode 100644 pkg/parser/tests/base-json-extract/base-grok2.yaml create mode 100644 pkg/parser/tests/base-json-extract/parsers.yaml create mode 100644 pkg/parser/tests/base-json-extract/test.yaml diff --git a/go.mod b/go.mod index f8f7c04d9..95bf7db39 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/Microsoft/go-winio v0.4.14 // indirect github.com/antonmedv/expr v1.8.2 + github.com/buger/jsonparser v1.0.0 github.com/containerd/containerd v1.3.4 // indirect github.com/davecgh/go-spew v1.1.1 github.com/denisbrodbeck/machineid v1.0.1 diff --git a/go.sum b/go.sum index 3a0b4d69a..374bcc68b 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/buger/jsonparser v1.0.0 h1:etJTGF5ESxjI0Ic2UaLQs2LQQpa8G9ykQScukbh4L8A= +github.com/buger/jsonparser v1.0.0/go.mod h1:tgcrVJ81GPSF0mz+0nu1Xaz0fazGPrmmJfJtxjbHhUQ= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= diff --git a/pkg/exprhelpers/exprlib.go b/pkg/exprhelpers/exprlib.go index debb8a15d..b1b01651f 100644 --- a/pkg/exprhelpers/exprlib.go +++ b/pkg/exprhelpers/exprlib.go @@ -17,7 +17,7 @@ func Atof(x string) float64 { func GetExprEnv(ctx map[string]interface{}) map[string]interface{} { - var ExprLib = map[string]interface{}{"Atof": Atof} + var ExprLib = map[string]interface{}{"Atof": Atof, "JsonExtract": JsonExtract, "JsonExtractLib": JsonExtractLib} for k, v := range ctx { ExprLib[k] = v } diff --git a/pkg/exprhelpers/jsonextract.go b/pkg/exprhelpers/jsonextract.go new file mode 100644 index 000000000..bda6dc736 --- /dev/null +++ b/pkg/exprhelpers/jsonextract.go @@ -0,0 +1,32 @@ +package exprhelpers + +import ( + "strings" + + "github.com/buger/jsonparser" + + log "github.com/sirupsen/logrus" +) + +func JsonExtractLib(jsblob string, target ...string) string { + value, dataType, _, err := jsonparser.Get( + jsonparser.StringToBytes(jsblob), + target..., + ) + + if err != nil { + log.Errorf("jsonExtractLib : %s", err) + return "" + } + if dataType == jsonparser.NotExist { + log.Debugf("%+v doesn't exist", target) + return "" + } + strvalue := string(value) + return strvalue +} + +func JsonExtract(jsblob string, target string) string { + fullpath := strings.Split(target, ".") + return JsonExtractLib(jsblob, fullpath...) +} diff --git a/pkg/parser/tests/base-json-extract/base-grok.yaml b/pkg/parser/tests/base-json-extract/base-grok.yaml new file mode 100644 index 000000000..a9bd5f129 --- /dev/null +++ b/pkg/parser/tests/base-json-extract/base-grok.yaml @@ -0,0 +1,14 @@ +filter: "evt.Line.Labels.type == 'json-1'" +debug: true +onsuccess: next_stage +name: tests/base-json-extract +statics: + - parsed: message + expression: JsonExtract(evt.Line.Raw, "log") + - meta: other_field + expression: JsonExtract(evt.Line.Raw, "testfield") + - meta: program + expression: evt.Line.Labels.progrname + + + diff --git a/pkg/parser/tests/base-json-extract/base-grok2.yaml b/pkg/parser/tests/base-json-extract/base-grok2.yaml new file mode 100644 index 000000000..3c2868788 --- /dev/null +++ b/pkg/parser/tests/base-json-extract/base-grok2.yaml @@ -0,0 +1,14 @@ +filter: "evt.Meta.program == 'my_test_prog'" +debug: true +onsuccess: next_stage +name: tests/base-grok +pattern_syntax: + MYCAP: ".*" +nodes: + - grok: + pattern: ^xxheader %{MYCAP:extracted_value} trailing stuff$ + apply_on: message +statics: + - meta: log_type + value: parsed_testlog + diff --git a/pkg/parser/tests/base-json-extract/parsers.yaml b/pkg/parser/tests/base-json-extract/parsers.yaml new file mode 100644 index 000000000..4471ad81b --- /dev/null +++ b/pkg/parser/tests/base-json-extract/parsers.yaml @@ -0,0 +1,4 @@ + - filename: {{.TestDirectory}}/base-grok.yaml + stage: s00-raw + - filename: {{.TestDirectory}}/base-grok2.yaml + stage: s01-parse \ No newline at end of file diff --git a/pkg/parser/tests/base-json-extract/test.yaml b/pkg/parser/tests/base-json-extract/test.yaml new file mode 100644 index 000000000..22e1c3aa3 --- /dev/null +++ b/pkg/parser/tests/base-json-extract/test.yaml @@ -0,0 +1,16 @@ +#these are the events we input into parser +lines: + - Line: + Labels: + type: json-1 + progrname: my_test_prog + Raw: '{"testfield": "some stuff", "log": "xxheader VALUE1 trailing stuff"}' +results: + - Meta: + other_field: some stuff + program: my_test_prog + Parsed: + message: xxheader VALUE1 trailing stuff + extracted_value: VALUE1 + Process: true +