Pattern syntax consistence (#675)
* fix #667 * improved error message * mark the compability, ordered pattern_syntax will be tagged as 'version 2' * fix tests + add tests to check grok subpattern dependencies
This commit is contained in:
parent
534c535490
commit
0981aa98d8
|
@ -27,7 +27,7 @@ var (
|
||||||
BuildDate string // = "I don't remember exactly"
|
BuildDate string // = "I don't remember exactly"
|
||||||
Tag string // = "dev"
|
Tag string // = "dev"
|
||||||
GoVersion string // = "1.13"
|
GoVersion string // = "1.13"
|
||||||
Constraint_parser = ">= 1.0, < 2.0"
|
Constraint_parser = ">= 1.0, <= 2.0"
|
||||||
Constraint_scenario = ">= 1.0, < 3.0"
|
Constraint_scenario = ">= 1.0, < 3.0"
|
||||||
Constraint_api = "v1"
|
Constraint_api = "v1"
|
||||||
Constraint_acquis = ">= 1.0, < 2.0"
|
Constraint_acquis = ">= 1.0, < 2.0"
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/antonmedv/expr"
|
"github.com/antonmedv/expr"
|
||||||
|
yaml "gopkg.in/yaml.v2"
|
||||||
|
|
||||||
"github.com/antonmedv/expr/vm"
|
"github.com/antonmedv/expr/vm"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||||
|
@ -48,7 +49,8 @@ type Node struct {
|
||||||
|
|
||||||
/* If the node is actually a leaf, it can have : grok, enrich, statics */
|
/* If the node is actually a leaf, it can have : grok, enrich, statics */
|
||||||
//pattern_syntax are named grok patterns that are re-utilised over several grok patterns
|
//pattern_syntax are named grok patterns that are re-utilised over several grok patterns
|
||||||
SubGroks map[string]string `yaml:"pattern_syntax,omitempty"`
|
SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"`
|
||||||
|
|
||||||
//Holds a grok pattern
|
//Holds a grok pattern
|
||||||
Grok types.GrokPattern `yaml:"grok,omitempty"`
|
Grok types.GrokPattern `yaml:"grok,omitempty"`
|
||||||
//Statics can be present in any type of node and is executed last
|
//Statics can be present in any type of node and is executed last
|
||||||
|
@ -402,13 +404,14 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx []EnricherCtx) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
/* handle pattern_syntax and groks */
|
/* handle pattern_syntax and groks */
|
||||||
for node, pattern := range n.SubGroks {
|
for _, pattern := range n.SubGroks {
|
||||||
n.Logger.Tracef("Adding subpattern '%s' : '%s'", node, pattern)
|
n.Logger.Tracef("Adding subpattern '%s' : '%s'", pattern.Key, pattern.Value)
|
||||||
if err := pctx.Grok.Add(node, pattern); err != nil {
|
if err := pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil {
|
||||||
n.Logger.Errorf("Unable to compile subpattern %s : %v", node, err)
|
n.Logger.Errorf("Unable to compile subpattern %s : %v", pattern.Key, err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* load grok by name or compile in-place */
|
/* load grok by name or compile in-place */
|
||||||
if n.Grok.RegexpName != "" {
|
if n.Grok.RegexpName != "" {
|
||||||
n.Logger.Tracef("+ Regexp Compilation '%s'", n.Grok.RegexpName)
|
n.Logger.Tracef("+ Regexp Compilation '%s'", n.Grok.RegexpName)
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
yaml "gopkg.in/yaml.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParserConfigs(t *testing.T) {
|
func TestParserConfigs(t *testing.T) {
|
||||||
|
@ -25,20 +26,27 @@ func TestParserConfigs(t *testing.T) {
|
||||||
//empty node
|
//empty node
|
||||||
{&Node{Debug: true, Stage: "s00", Filter: "true"}, false, false},
|
{&Node{Debug: true, Stage: "s00", Filter: "true"}, false, false},
|
||||||
//bad subgrok
|
//bad subgrok
|
||||||
{&Node{Debug: true, Stage: "s00", SubGroks: map[string]string{"FOOBAR": "[a-$"}}, false, true},
|
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-$")}}}, false, true},
|
||||||
//valid node with grok pattern
|
//valid node with grok pattern
|
||||||
{&Node{Debug: true, Stage: "s00", SubGroks: map[string]string{"FOOBAR": "[a-z]"}, Grok: types.GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true},
|
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: types.GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true},
|
||||||
//bad node success
|
//bad node success
|
||||||
{&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false},
|
{&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false},
|
||||||
//ok node success
|
//ok node success
|
||||||
{&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true},
|
{&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true},
|
||||||
//valid node with grok sub-pattern used by name
|
//valid node with grok sub-pattern used by name
|
||||||
{&Node{Debug: true, Stage: "s00", SubGroks: map[string]string{"FOOBARx": "[a-z] %{DATA:lol}$"}, Grok: types.GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true},
|
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: types.GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true},
|
||||||
//node with unexisting grok pattern
|
//node with unexisting grok pattern
|
||||||
{&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true},
|
{&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true},
|
||||||
|
//node with grok pattern dependencies
|
||||||
//bad grok pattern
|
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{
|
||||||
//{&Node{Debug: true, Grok: []GrokPattern{ GrokPattern{}, }}, false},
|
{Key: string("SUBGROK"), Value: string("[a-z]")},
|
||||||
|
{Key: string("MYGROK"), Value: string("[a-z]%{SUBGROK}")},
|
||||||
|
}, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true},
|
||||||
|
//node with broken grok pattern dependencies
|
||||||
|
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{
|
||||||
|
{Key: string("SUBGROKBIS"), Value: string("[a-z]%{MYGROKBIS}")},
|
||||||
|
{Key: string("MYGROKBIS"), Value: string("[a-z]")},
|
||||||
|
}, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true},
|
||||||
}
|
}
|
||||||
for idx := range CfgTests {
|
for idx := range CfgTests {
|
||||||
err := CfgTests[idx].NodeCfg.compile(pctx, []EnricherCtx{})
|
err := CfgTests[idx].NodeCfg.compile(pctx, []EnricherCtx{})
|
||||||
|
|
|
@ -90,7 +90,7 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx []EnricherCtx)
|
||||||
log.Fatalf("Failed to check version : %s", err)
|
log.Fatalf("Failed to check version : %s", err)
|
||||||
}
|
}
|
||||||
if !ok {
|
if !ok {
|
||||||
log.Errorf("%s doesn't satisfy parser format %s, skip", node.FormatVersion, cwversion.Constraint_parser)
|
log.Errorf("%s : %s doesn't satisfy parser format %s, skip", node.Name, node.FormatVersion, cwversion.Constraint_parser)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue