beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

fix stacktrace when mmdb file are not present (#935)

Browse source 
* fix stacktrace when mmdb file are not present
This commit is contained in:
AlteredCoder 2021-09-09 16:27:30 +02:00 committed by GitHub
parent 9b5cb6abc9
commit 5ae69aa293
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 181 additions and 150 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pkg/exprhelpers/visitor.go
View file

@ -124,7 +124,7 @@ func (e *ExprDebugger) Run(logger *logrus.Entry, filterResult bool, exprEnv map[
if err != nil {
logger.Errorf("unable to print debug expression for '%s': %s", expression.Str, err)
}
logger.Debugf(" %s = '%s'", expression.Str, debug)
logger.Debugf(" %s = '%v'", expression.Str, debug)
}
}

130
pkg/parser/enrich.go
View file

@ -1,8 +1,6 @@
package parser
import (
"time"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
)
@ -12,94 +10,62 @@ type EnrichFunc func(string, *types.Event, interface{}) (map[string]string, erro
type InitFunc func(map[string]string) (interface{}, error)
type EnricherCtx struct {
Funcs map[string]EnrichFunc
Init InitFunc
Registered map[string]*Enricher
}
type Enricher struct {
Name string
Path string //path to .so ?
RuntimeCtx interface{} //the internal context of plugin, given back over every call
initiated bool
InitFunc InitFunc
EnrichFunc EnrichFunc
Ctx interface{}
}
/* mimic plugin loading */
func Loadplugin(path string) ([]EnricherCtx, error) {
var err error
func Loadplugin(path string) (EnricherCtx, error) {
enricherCtx := EnricherCtx{}
enricherCtx.Registered = make(map[string]*Enricher)
c := EnricherCtx{}
c.Name = path
c.Path = path
/* we don't want to deal with plugin loading for now :p */
c.Funcs = map[string]EnrichFunc{
"GeoIpASN": GeoIpASN,
"GeoIpCity": GeoIpCity,
"reverse_dns": reverse_dns,
"ParseDate": ParseDate,
"IpToRange": IpToRange,
enricherConfig := map[string]string{"datadir": path}
EnrichersList := []*Enricher{
{
Name: "GeoIpCity",
InitFunc: GeoIPCityInit,
EnrichFunc: GeoIpCity,
},
{
Name: "GeoIpASN",
InitFunc: GeoIPASNInit,
EnrichFunc: GeoIpASN,
},
{
Name: "IpToRange",
InitFunc: IpToRangeInit,
EnrichFunc: IpToRange,
},
{
Name: "reverse_dns",
InitFunc: reverseDNSInit,
EnrichFunc: reverse_dns,
},
{
Name: "ParseDate",
InitFunc: parseDateInit,
EnrichFunc: ParseDate,
},
}
c.Init = GeoIpInit
c.RuntimeCtx, err = c.Init(map[string]string{"datadir": path})
if err != nil {
log.Warningf("load (fake) plugin load : %v", err)
c.initiated = false
}
c.initiated = true
return []EnricherCtx{c}, nil
}
func GenDateParse(date string) (string, time.Time) {
var (
layouts = [...]string{
time.RFC3339,
"02/Jan/2006:15:04:05 -0700",
"Mon Jan 2 15:04:05 2006",
"02-Jan-2006 15:04:05 europe/paris",
"01/02/2006 15:04:05",
"2006-01-02 15:04:05.999999999 -0700 MST",
"Jan 2 15:04:05",
"Mon Jan 02 15:04:05.000000 2006",
"2006-01-02T15:04:05Z07:00",
"2006/01/02",
"2006/01/02 15:04",
"2006-01-02",
"2006-01-02 15:04",
"2006/01/02 15:04:05",
"2006-01-02 15:04:05",
}
)
for _, dateFormat := range layouts {
t, err := time.Parse(dateFormat, date)
if err == nil && !t.IsZero() {
//if the year isn't set, set it to current date :)
if t.Year() == 0 {
t = t.AddDate(time.Now().Year(), 0, 0)
}
retstr, err := t.MarshalText()
if err != nil {
log.Warningf("Failed marshaling '%v'", t)
continue
}
return string(retstr), t
for _, enricher := range EnrichersList {
log.Debugf("Initiating enricher '%s'", enricher.Name)
pluginCtx, err := enricher.InitFunc(enricherConfig)
if err != nil {
log.Errorf("unable to register plugin '%s': %v", enricher.Name, err)
continue
}
enricher.Ctx = pluginCtx
log.Infof("Successfully registered enricher '%s'", enricher.Name)
enricherCtx.Registered[enricher.Name] = enricher
}
now := time.Now()
retstr, err := now.MarshalText()
if err != nil {
log.Warningf("Failed marshaling current time")
return "", time.Time{}
}
return string(retstr), now
}
func ParseDate(in string, p *types.Event, x interface{}) (map[string]string, error) {
var ret map[string]string = make(map[string]string)
tstr, tbin := GenDateParse(in)
if !tbin.IsZero() {
ret["MarshaledTime"] = string(tstr)
return ret, nil
}
return nil, nil
return enricherCtx, nil
}

70
pkg/parser/enrich_date.go Normal file
View file

@ -0,0 +1,70 @@
package parser
import (
"time"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
)
func GenDateParse(date string) (string, time.Time) {
var (
layouts = [...]string{
time.RFC3339,
"02/Jan/2006:15:04:05 -0700",
"Mon Jan 2 15:04:05 2006",
"02-Jan-2006 15:04:05 europe/paris",
"01/02/2006 15:04:05",
"2006-01-02 15:04:05.999999999 -0700 MST",
"Jan 2 15:04:05",
"Mon Jan 02 15:04:05.000000 2006",
"2006-01-02T15:04:05Z07:00",
"2006/01/02",
"2006/01/02 15:04",
"2006-01-02",
"2006-01-02 15:04",
"2006/01/02 15:04:05",
"2006-01-02 15:04:05",
}
)
for _, dateFormat := range layouts {
t, err := time.Parse(dateFormat, date)
if err == nil && !t.IsZero() {
//if the year isn't set, set it to current date :)
if t.Year() == 0 {
t = t.AddDate(time.Now().Year(), 0, 0)
}
retstr, err := t.MarshalText()
if err != nil {
log.Warningf("Failed marshaling '%v'", t)
continue
}
return string(retstr), t
}
}
now := time.Now()
retstr, err := now.MarshalText()
if err != nil {
log.Warningf("Failed marshaling current time")
return "", time.Time{}
}
return string(retstr), now
}
func ParseDate(in string, p *types.Event, x interface{}) (map[string]string, error) {
var ret map[string]string = make(map[string]string)
tstr, tbin := GenDateParse(in)
if !tbin.IsZero() {
ret["MarshaledTime"] = string(tstr)
return ret, nil
}
return nil, nil
}
func parseDateInit(cfg map[string]string) (interface{}, error) {
return nil, nil
}

4
pkg/parser/enrich_dns.go
View file

@ -25,3 +25,7 @@ func reverse_dns(field string, p *types.Event, ctx interface{}) (map[string]stri
ret["reverse_dns"] = rets[0]
return ret, nil
}
func reverseDNSInit(cfg map[string]string) (interface{}, error) {
return nil, nil
}

55
pkg/parser/enrich_geoip.go
View file

@ -12,12 +12,6 @@ import (
"github.com/oschwald/maxminddb-golang"
)
type GeoIpEnricherCtx struct {
dbc *geoip2.Reader
dba *geoip2.Reader
dbraw *maxminddb.Reader
}
func IpToRange(field string, p *types.Event, ctx interface{}) (map[string]string, error) {
var dummy interface{}
ret := make(map[string]string)
@ -30,7 +24,7 @@ func IpToRange(field string, p *types.Event, ctx interface{}) (map[string]string
log.Infof("Can't parse ip %s, no range enrich", field)
return nil, nil
}
net, ok, err := ctx.(GeoIpEnricherCtx).dbraw.LookupNetwork(ip, &dummy)
net, ok, err := ctx.(*maxminddb.Reader).LookupNetwork(ip, &dummy)
if err != nil {
log.Errorf("Failed to fetch network for %s : %v", ip.String(), err)
return nil, nil
@ -54,14 +48,16 @@ func GeoIpASN(field string, p *types.Event, ctx interface{}) (map[string]string,
log.Infof("Can't parse ip %s, no ASN enrich", ip)
return nil, nil
}
record, err := ctx.(GeoIpEnricherCtx).dba.ASN(ip)
record, err := ctx.(*geoip2.Reader).ASN(ip)
if err != nil {
log.Errorf("Unable to enrich ip '%s'", field)
return nil, nil
}
ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
ret["ASNOrg"] = record.AutonomousSystemOrganization
log.Tracef("geoip ASN %s -> %s, %s", field, ret["ASNNumber"], ret["ASNOrg"])
return ret, nil
}
@ -75,7 +71,7 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}) (map[string]string
log.Infof("Can't parse ip %s, no City enrich", ip)
return nil, nil
}
record, err := ctx.(GeoIpEnricherCtx).dbc.City(ip)
record, err := ctx.(*geoip2.Reader).City(ip)
if err != nil {
log.Debugf("Unable to enrich ip '%s'", ip)
return nil, nil
@ -90,27 +86,32 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}) (map[string]string
return ret, nil
}
/* All plugins must export an Init function */
func GeoIpInit(cfg map[string]string) (interface{}, error) {
var ctx GeoIpEnricherCtx
var err error
ctx.dbc, err = geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
ctx.dba, err = geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
func GeoIPCityInit(cfg map[string]string) (interface{}, error) {
dbCityReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
ctx.dbraw, err = maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return ctx, nil
return dbCityReader, nil
}
func GeoIPASNInit(cfg map[string]string) (interface{}, error) {
dbASReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return dbASReader, nil
}
func IpToRangeInit(cfg map[string]string) (interface{}, error) {
ipToRangeReader, err := maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return ipToRangeReader, nil
}

17
pkg/parser/node.go
View file

@ -47,7 +47,7 @@ type Node struct {
//If node has leafs, execute all of them until one asks for a 'break'
LeavesNodes []Node `yaml:"nodes,omitempty"`
//Flag used to describe when to 'break' or return an 'error'
EnrichFunctions []EnricherCtx
EnrichFunctions EnricherCtx
/* If the node is actually a leaf, it can have : grok, enrich, statics */
//pattern_syntax are named grok patterns that are re-utilised over several grok patterns
@ -62,7 +62,7 @@ type Node struct {
Data []*types.DataSource `yaml:"data,omitempty"`
}
func (n *Node) validate(pctx *UnixParserCtx, ectx []EnricherCtx) error {
func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
//stage is being set automagically
if n.Stage == "" {
@ -91,15 +91,8 @@ func (n *Node) validate(pctx *UnixParserCtx, ectx []EnricherCtx) error {
if static.ExpValue == "" {
return fmt.Errorf("static %d : when method is set, expression must be present", idx)
}
method_found := false
for _, enricherCtx := range ectx {
if _, ok := enricherCtx.Funcs[static.Method]; ok && enricherCtx.initiated {
method_found = true
break
}
}
if !method_found {
return fmt.Errorf("the method '%s' doesn't exist or the plugin has not been initialized", static.Method)
if _, ok := ectx.Registered[static.Method]; !ok {
log.Warningf("the method '%s' doesn't exist or the plugin has not been initialized", static.Method)
}
} else {
if static.Meta == "" && static.Parsed == "" && static.TargetByName == "" {
@ -363,7 +356,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
return NodeState, nil
}
func (n *Node) compile(pctx *UnixParserCtx, ectx []EnricherCtx) error {
func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
var err error
var valid bool

4
pkg/parser/node_test.go
View file

@ -49,7 +49,7 @@ func TestParserConfigs(t *testing.T) {
}, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true},
}
for idx := range CfgTests {
err := CfgTests[idx].NodeCfg.compile(pctx, []EnricherCtx{})
err := CfgTests[idx].NodeCfg.compile(pctx, EnricherCtx{})
if CfgTests[idx].Compiles == true && err != nil {
t.Fatalf("Compile: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
}
@ -57,7 +57,7 @@ func TestParserConfigs(t *testing.T) {
t.Fatalf("Compile: (%d/%d) expected errror", idx+1, len(CfgTests))
}
err = CfgTests[idx].NodeCfg.validate(pctx, []EnricherCtx{})
err = CfgTests[idx].NodeCfg.validate(pctx, EnricherCtx{})
if CfgTests[idx].Valid == true && err != nil {
t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
}

8
pkg/parser/parsing_test.go
View file

@ -89,7 +89,7 @@ func BenchmarkParser(t *testing.B) {
}
}
func testOneParser(pctx *UnixParserCtx, ectx []EnricherCtx, dir string, b *testing.B) error {
func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing.B) error {
var (
err error
@ -139,11 +139,11 @@ func testOneParser(pctx *UnixParserCtx, ectx []EnricherCtx, dir string, b *testi
}
//prepTests is going to do the initialisation of parser : it's going to load enrichment plugins and load the patterns. This is done here so that we don't redo it for each test
func prepTests() (*UnixParserCtx, []EnricherCtx, error) {
func prepTests() (*UnixParserCtx, EnricherCtx, error) {
var (
err error
pctx *UnixParserCtx
ectx []EnricherCtx
ectx EnricherCtx
)
err = exprhelpers.Init()
@ -166,7 +166,7 @@ func prepTests() (*UnixParserCtx, []EnricherCtx, error) {
// Init the parser
pctx, err = Init(map[string]interface{}{"patterns": cfgdir + string("/patterns/"), "data": "./tests/"})
if err != nil {
return nil, nil, fmt.Errorf("failed to initialize parser : %v", err)
return nil, ectx, fmt.Errorf("failed to initialize parser : %v", err)
}
return pctx, ectx, nil
}

37
pkg/parser/runtime.go
View file

@ -143,29 +143,26 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er
if static.Method != "" {
processed := false
/*still way too hackish, but : inject all the results in enriched, and */
for _, x := range n.EnrichFunctions {
if fptr, ok := x.Funcs[static.Method]; ok && x.initiated {
clog.Tracef("Found method '%s'", static.Method)
ret, err := fptr(value, event, x.RuntimeCtx)
if err != nil {
clog.Fatalf("plugin function error : %v", err)
}
processed = true
clog.Debugf("+ Method %s('%s') returned %d entries to merge in .Enriched\n", static.Method, value, len(ret))
if len(ret) == 0 {
clog.Debugf("+ Method '%s' empty response on '%s'", static.Method, value)
}
for k, v := range ret {
clog.Debugf("\t.Enriched[%s] = '%s'\n", k, v)
event.Enriched[k] = v
}
break
} else {
clog.Warningf("method '%s' doesn't exist or plugin not initialized", static.Method)
if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok {
clog.Tracef("Found method '%s'", static.Method)
ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx)
if err != nil {
clog.Errorf("method '%s' returned an error : %v", static.Method, err)
}
processed = true
clog.Debugf("+ Method %s('%s') returned %d entries to merge in .Enriched\n", static.Method, value, len(ret))
if len(ret) == 0 {
clog.Debugf("+ Method '%s' empty response on '%s'", static.Method, value)
}
for k, v := range ret {
clog.Debugf("\t.Enriched[%s] = '%s'\n", k, v)
event.Enriched[k] = v
}
} else {
clog.Debugf("method '%s' doesn't exist or plugin not initialized", static.Method)
}
if !processed {
clog.Warningf("method '%s' doesn't exist", static.Method)
clog.Debugf("method '%s' doesn't exist", static.Method)
}
} else if static.Parsed != "" {
clog.Debugf(".Parsed[%s] = '%s'", static.Parsed, value)

2
pkg/parser/stage.go
View file

@ -37,7 +37,7 @@ type Stagefile struct {
Stage string `yaml:"stage"`
}
func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx []EnricherCtx) ([]Node, error) {
func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ([]Node, error) {
var nodes []Node
tmpstages := make(map[string]bool)
pctx.Stages = []string{}

2
pkg/parser/unix_parser.go
View file

@ -25,7 +25,7 @@ type Parsers struct {
PovfwStageFiles []Stagefile
Nodes []Node
Povfwnodes []Node
EnricherCtx []EnricherCtx
EnricherCtx EnricherCtx
}
func Init(c map[string]interface{}) (*UnixParserCtx, error) {