beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Add more JSON expr helpers (#1576)

Browse source
This commit is contained in:
blotus 2022-06-08 12:15:29 +02:00 committed by GitHub
parent b7f1c5455f
commit 4b311684ab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 256 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
cmd/crowdsec/parse.go
View file

@ -1,8 +1,6 @@
package main
import (
"errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
@ -29,10 +27,9 @@ LOOP:
globalParserHits.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc()
/* parse the log using magic */
parsed, error := parser.Parse(parserCTX, event, nodes)
if error != nil {
log.Errorf("failed parsing : %v\n", error)
return errors.New("parsing failed :/")
parsed, err := parser.Parse(parserCTX, event, nodes)
if err != nil {
log.Errorf("failed parsing : %v\n", err)
}
if !parsed.Process {
globalParserHitsKo.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc()

3
pkg/exprhelpers/exprlib.go
View file

@ -44,6 +44,9 @@ func GetExprEnv(ctx map[string]interface{}) map[string]interface{} {
"JsonExtract": JsonExtract,
"JsonExtractUnescape": JsonExtractUnescape,
"JsonExtractLib": JsonExtractLib,
"JsonExtractSlice": JsonExtractSlice,
"JsonExtractObject": JsonExtractObject,
"ToJsonString": ToJson,
"File": File,
"RegexpInFile": RegexpInFile,
"Upper": Upper,

79
pkg/exprhelpers/jsonextract.go
View file

@ -1,6 +1,8 @@
package exprhelpers
import (
"encoding/json"
"fmt"
"strings"
"github.com/buger/jsonparser"
@ -58,3 +60,80 @@ func JsonExtract(jsblob string, target string) string {
log.Tracef("extract path %+v", fullpath)
return JsonExtractLib(jsblob, fullpath...)
}
func jsonExtractType(jsblob string, target string, t jsonparser.ValueType) ([]byte, error) {
if !strings.HasPrefix(target, "[") {
target = strings.Replace(target, "[", ".[", -1)
}
fullpath := strings.Split(target, ".")
log.Tracef("extract path %+v", fullpath)
value, dataType, _, err := jsonparser.Get(
jsonparser.StringToBytes(jsblob),
fullpath...,
)
if err != nil {
if err == jsonparser.KeyPathNotFoundError {
log.Debugf("Key %+v doesn't exist", target)
return nil, fmt.Errorf("key %s does not exist", target)
}
log.Errorf("jsonExtractType : %s : %s", target, err)
return nil, fmt.Errorf("jsonExtractType: %s : %w", target, err)
}
if dataType != t {
log.Errorf("jsonExtractType : expected type %s for target %s but found %s", t, target, dataType.String())
return nil, fmt.Errorf("jsonExtractType: expected type %s for target %s but found %s", t, target, dataType.String())
}
return value, nil
}
func JsonExtractSlice(jsblob string, target string) []interface{} {
value, err := jsonExtractType(jsblob, target, jsonparser.Array)
if err != nil {
log.Errorf("JsonExtractSlice : %s", err)
return nil
}
s := make([]interface{}, 0)
err = json.Unmarshal(value, &s)
if err != nil {
log.Errorf("JsonExtractSlice: could not convert '%s' to slice: %s", value, err)
return nil
}
return s
}
func JsonExtractObject(jsblob string, target string) map[string]interface{} {
value, err := jsonExtractType(jsblob, target, jsonparser.Object)
if err != nil {
log.Errorf("JsonExtractObject: %s", err)
return nil
}
s := make(map[string]interface{})
err = json.Unmarshal(value, &s)
if err != nil {
log.Errorf("JsonExtractObject: could not convert '%s' to map[string]interface{}: %s", value, err)
return nil
}
return s
}
func ToJson(obj interface{}) string {
b, err := json.Marshal(obj)
if err != nil {
log.Errorf("ToJson : %s", err)
return ""
}
return string(b)
}

161
pkg/exprhelpers/jsonextract_test.go
View file

@ -35,6 +35,12 @@ func TestJsonExtract(t *testing.T) {
targetField: "non_existing_field",
expectResult: "",
},
{
name: "extract subfield",
jsonBlob: `{"test" : {"a": "b"}}`,
targetField: "test.a",
expectResult: "b",
},
}
for _, test := range tests {
@ -85,5 +91,158 @@ func TestJsonExtractUnescape(t *testing.T) {
}
log.Printf("test '%s' : OK", test.name)
}
}
func TestJsonExtractSlice(t *testing.T) {
if err := Init(); err != nil {
log.Fatalf(err.Error())
}
err := FileInit(TestFolder, "test_data_re.txt", "regex")
if err != nil {
log.Fatalf(err.Error())
}
tests := []struct {
name string
jsonBlob string
targetField string
expectResult []interface{}
}{
{
name: "try to extract a string as a slice",
jsonBlob: `{"test" : "1234"}`,
targetField: "test",
expectResult: nil,
},
{
name: "basic json slice extract",
jsonBlob: `{"test" : ["1234"]}`,
targetField: "test",
expectResult: []interface{}{"1234"},
},
{
name: "extract with complex expression",
jsonBlob: `{"test": {"foo": [{"a":"b"}]}}`,
targetField: "test.foo",
expectResult: []interface{}{map[string]interface{}{"a": "b"}},
},
{
name: "extract non-existing key",
jsonBlob: `{"test: "11234"}`,
targetField: "foo",
expectResult: nil,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result := JsonExtractSlice(test.jsonBlob, test.targetField)
assert.Equal(t, test.expectResult, result)
})
}
}
func TestJsonExtractObject(t *testing.T) {
if err := Init(); err != nil {
log.Fatalf(err.Error())
}
err := FileInit(TestFolder, "test_data_re.txt", "regex")
if err != nil {
log.Fatalf(err.Error())
}
tests := []struct {
name string
jsonBlob string
targetField string
expectResult map[string]interface{}
}{
{
name: "try to extract a string as an object",
jsonBlob: `{"test" : "1234"}`,
targetField: "test",
expectResult: nil,
},
{
name: "basic json object extract",
jsonBlob: `{"test" : {"1234": {"foo": "bar"}}}`,
targetField: "test",
expectResult: map[string]interface{}{"1234": map[string]interface{}{"foo": "bar"}},
},
{
name: "extract with complex expression",
jsonBlob: `{"test": {"foo": [{"a":"b"}]}}`,
targetField: "test.foo[0]",
expectResult: map[string]interface{}{"a": "b"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result := JsonExtractObject(test.jsonBlob, test.targetField)
assert.Equal(t, test.expectResult, result)
})
}
}
func TestToJson(t *testing.T) {
tests := []struct {
name string
obj interface{}
expectResult string
}{
{
name: "convert int",
obj: 42,
expectResult: "42",
},
{
name: "convert slice",
obj: []string{"foo", "bar"},
expectResult: `["foo","bar"]`,
},
{
name: "convert map",
obj: map[string]string{"foo": "bar"},
expectResult: `{"foo":"bar"}`,
},
{
name: "convert struct",
obj: struct{ Foo string }{"bar"},
expectResult: `{"Foo":"bar"}`,
},
{
name: "convert complex struct",
obj: struct {
Foo string
Bar struct {
Baz string
}
Bla []string
}{
Foo: "bar",
Bar: struct {
Baz string
}{
Baz: "baz",
},
Bla: []string{"foo", "bar"},
},
expectResult: `{"Foo":"bar","Bar":{"Baz":"baz"},"Bla":["foo","bar"]}`,
},
{
name: "convert invalid type",
obj: func() {},
expectResult: "",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result := ToJson(test.obj)
assert.Equal(t, test.expectResult, result)
})
}
}

6
pkg/parser/node.go
View file

@ -272,7 +272,8 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
// if the grok succeed, process associated statics
err := n.ProcessStatics(n.Grok.Statics, p)
if err != nil {
clog.Fatalf("(%s) Failed to process statics : %v", n.rn, err)
clog.Errorf("(%s) Failed to process statics : %v", n.rn, err)
return false, err
}
} else {
//grok failed, node failed
@ -337,7 +338,8 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
// if all else is good in whitelist, process node's statics
err := n.ProcessStatics(n.Statics, p)
if err != nil {
clog.Fatalf("Failed to process statics : %v", err)
clog.Errorf("Failed to process statics : %v", err)
return false, err
}
} else {
clog.Tracef("! No node statics")

9
pkg/parser/runtime.go
View file

@ -133,8 +133,12 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er
value = out
case int:
value = strconv.Itoa(out)
case map[string]interface{}:
clog.Warnf("Expression returned a map, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string")
case []interface{}:
clog.Warnf("Expression returned a map, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string")
default:
clog.Fatalf("unexpected return type for RunTimeValue : %T", output)
clog.Errorf("unexpected return type for RunTimeValue : %T", output)
return errors.New("unexpected return type for RunTimeValue")
}
}
@ -309,7 +313,8 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
}
ret, err := node.process(&event, ctx, cachedExprEnv)
if err != nil {
clog.Fatalf("Error while processing node : %v", err)
clog.Errorf("Error while processing node : %v", err)
return event, err
}
clog.Tracef("node (%s) ret : %v", node.rn, ret)
if ParseDump {