beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Pattern syntax consistence (#675)

Browse source 
* fix #667

* improved error message

* mark the compability, ordered pattern_syntax will be tagged as 'version 2'

* fix tests + add tests to check grok subpattern dependencies
This commit is contained in:
Thibault "bui" Koechlin 2021-03-10 18:27:21 +01:00 committed by GitHub
parent 534c535490
commit 0981aa98d8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 24 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pkg/cwversion/version.go
View file

@ -27,7 +27,7 @@ var (
BuildDate string // = "I don't remember exactly"
Tag string // = "dev"
GoVersion string // = "1.13"
Constraint_parser = ">= 1.0, < 2.0"
Constraint_parser = ">= 1.0, <= 2.0"
Constraint_scenario = ">= 1.0, < 3.0"
Constraint_api = "v1"
Constraint_acquis = ">= 1.0, < 2.0"

13
pkg/parser/node.go
View file

@ -6,6 +6,7 @@ import (
"strings"
"github.com/antonmedv/expr"
yaml "gopkg.in/yaml.v2"
"github.com/antonmedv/expr/vm"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
@ -48,7 +49,8 @@ type Node struct {
/* If the node is actually a leaf, it can have : grok, enrich, statics */
//pattern_syntax are named grok patterns that are re-utilised over several grok patterns
SubGroks map[string]string `yaml:"pattern_syntax,omitempty"`
SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"`
//Holds a grok pattern
Grok types.GrokPattern `yaml:"grok,omitempty"`
//Statics can be present in any type of node and is executed last
@ -402,13 +404,14 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx []EnricherCtx) error {
}
/* handle pattern_syntax and groks */
for node, pattern := range n.SubGroks {
n.Logger.Tracef("Adding subpattern '%s' : '%s'", node, pattern)
if err := pctx.Grok.Add(node, pattern); err != nil {
n.Logger.Errorf("Unable to compile subpattern %s : %v", node, err)
for _, pattern := range n.SubGroks {
n.Logger.Tracef("Adding subpattern '%s' : '%s'", pattern.Key, pattern.Value)
if err := pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil {
n.Logger.Errorf("Unable to compile subpattern %s : %v", pattern.Key, err)
return err
}
}
/* load grok by name or compile in-place */
if n.Grok.RegexpName != "" {
n.Logger.Tracef("+ Regexp Compilation '%s'", n.Grok.RegexpName)

20
pkg/parser/node_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"github.com/crowdsecurity/crowdsec/pkg/types"
yaml "gopkg.in/yaml.v2"
)
func TestParserConfigs(t *testing.T) {
@ -25,20 +26,27 @@ func TestParserConfigs(t *testing.T) {
//empty node
{&Node{Debug: true, Stage: "s00", Filter: "true"}, false, false},
//bad subgrok
{&Node{Debug: true, Stage: "s00", SubGroks: map[string]string{"FOOBAR": "[a-$"}}, false, true},
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-$")}}}, false, true},
//valid node with grok pattern
{&Node{Debug: true, Stage: "s00", SubGroks: map[string]string{"FOOBAR": "[a-z]"}, Grok: types.GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true},
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: types.GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true},
//bad node success
{&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false},
//ok node success
{&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true},
//valid node with grok sub-pattern used by name
{&Node{Debug: true, Stage: "s00", SubGroks: map[string]string{"FOOBARx": "[a-z] %{DATA:lol}$"}, Grok: types.GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true},
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: types.GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true},
//node with unexisting grok pattern
{&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true},
//bad grok pattern
//{&Node{Debug: true, Grok: []GrokPattern{ GrokPattern{}, }}, false},
//node with grok pattern dependencies
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{
{Key: string("SUBGROK"), Value: string("[a-z]")},
{Key: string("MYGROK"), Value: string("[a-z]%{SUBGROK}")},
}, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true},
//node with broken grok pattern dependencies
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{
{Key: string("SUBGROKBIS"), Value: string("[a-z]%{MYGROKBIS}")},
{Key: string("MYGROKBIS"), Value: string("[a-z]")},
}, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true},
}
for idx := range CfgTests {
err := CfgTests[idx].NodeCfg.compile(pctx, []EnricherCtx{})

2
pkg/parser/stage.go
View file

@ -90,7 +90,7 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx []EnricherCtx)
log.Fatalf("Failed to check version : %s", err)
}
if !ok {
log.Errorf("%s doesn't satisfy parser format %s, skip", node.FormatVersion, cwversion.Constraint_parser)
log.Errorf("%s : %s doesn't satisfy parser format %s, skip", node.Name, node.FormatVersion, cwversion.Constraint_parser)
continue
}