fix stacktrace when mmdb file are not present (#935)

* fix stacktrace when mmdb file are not present

pkg/exprhelpers/visitor.go

@@ -124,7 +124,7 @@ func (e *ExprDebugger) Run(logger *logrus.Entry, filterResult bool, exprEnv map[
 		if err != nil {
 			logger.Errorf("unable to print debug expression for '%s': %s", expression.Str, err)
 		}
-		logger.Debugf("       %s = '%s'", expression.Str, debug)
+		logger.Debugf("       %s = '%v'", expression.Str, debug)
 	}
 }
 

pkg/parser/enrich.go

@@ -1,8 +1,6 @@
 package parser
 
 import (
-	"time"
-
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	log "github.com/sirupsen/logrus"
 )
@@ -12,94 +10,62 @@ type EnrichFunc func(string, *types.Event, interface{}) (map[string]string, erro
 type InitFunc func(map[string]string) (interface{}, error)
 
 type EnricherCtx struct {
-	Funcs      map[string]EnrichFunc
+	Registered map[string]*Enricher
-	Init       InitFunc
+}
+
+type Enricher struct {
 	Name       string
-	Path       string      //path to .so ?
+	InitFunc   InitFunc
-	RuntimeCtx interface{} //the internal context of plugin, given back over every call
+	EnrichFunc EnrichFunc
-	initiated  bool
+	Ctx        interface{}
 }
 
 /* mimic plugin loading */
-func Loadplugin(path string) ([]EnricherCtx, error) {
+func Loadplugin(path string) (EnricherCtx, error) {
-	var err error
+	enricherCtx := EnricherCtx{}
+	enricherCtx.Registered = make(map[string]*Enricher)
 
-	c := EnricherCtx{}
+	enricherConfig := map[string]string{"datadir": path}
-	c.Name = path
+
-	c.Path = path
+	EnrichersList := []*Enricher{
-	/* we don't want to deal with plugin loading for now :p */
+		{
-	c.Funcs = map[string]EnrichFunc{
+			Name:       "GeoIpCity",
-		"GeoIpASN":    GeoIpASN,
+			InitFunc:   GeoIPCityInit,
-		"GeoIpCity":   GeoIpCity,
+			EnrichFunc: GeoIpCity,
-		"reverse_dns": reverse_dns,
+		},
-		"ParseDate":   ParseDate,
+		{
-		"IpToRange":   IpToRange,
+			Name:       "GeoIpASN",
+			InitFunc:   GeoIPASNInit,
+			EnrichFunc: GeoIpASN,
+		},
+		{
+			Name:       "IpToRange",
+			InitFunc:   IpToRangeInit,
+			EnrichFunc: IpToRange,
+		},
+		{
+			Name:       "reverse_dns",
+			InitFunc:   reverseDNSInit,
+			EnrichFunc: reverse_dns,
+		},
+		{
+			Name:       "ParseDate",
+			InitFunc:   parseDateInit,
+			EnrichFunc: ParseDate,
+		},
 	}
-	c.Init = GeoIpInit
 
-	c.RuntimeCtx, err = c.Init(map[string]string{"datadir": path})
+	for _, enricher := range EnrichersList {
-	if err != nil {
+		log.Debugf("Initiating enricher '%s'", enricher.Name)
-		log.Warningf("load (fake) plugin load : %v", err)
+		pluginCtx, err := enricher.InitFunc(enricherConfig)
-		c.initiated = false
+		if err != nil {
-	}
+			log.Errorf("unable to register plugin '%s': %v", enricher.Name, err)
-	c.initiated = true
+			continue
-	return []EnricherCtx{c}, nil
-}
-
-func GenDateParse(date string) (string, time.Time) {
-	var (
-		layouts = [...]string{
-			time.RFC3339,
-			"02/Jan/2006:15:04:05 -0700",
-			"Mon Jan 2 15:04:05 2006",
-			"02-Jan-2006 15:04:05 europe/paris",
-			"01/02/2006 15:04:05",
-			"2006-01-02 15:04:05.999999999 -0700 MST",
-			"Jan  2 15:04:05",
-			"Mon Jan 02 15:04:05.000000 2006",
-			"2006-01-02T15:04:05Z07:00",
-			"2006/01/02",
-			"2006/01/02 15:04",
-			"2006-01-02",
-			"2006-01-02 15:04",
-			"2006/01/02 15:04:05",
-			"2006-01-02 15:04:05",
-		}
-	)
-
-	for _, dateFormat := range layouts {
-		t, err := time.Parse(dateFormat, date)
-		if err == nil && !t.IsZero() {
-			//if the year isn't set, set it to current date :)
-			if t.Year() == 0 {
-				t = t.AddDate(time.Now().Year(), 0, 0)
-			}
-			retstr, err := t.MarshalText()
-			if err != nil {
-				log.Warningf("Failed marshaling '%v'", t)
-				continue
-			}
-			return string(retstr), t
 		}
+		enricher.Ctx = pluginCtx
+		log.Infof("Successfully registered enricher '%s'", enricher.Name)
+		enricherCtx.Registered[enricher.Name] = enricher
 	}
 
-	now := time.Now()
+	return enricherCtx, nil
-	retstr, err := now.MarshalText()
-	if err != nil {
-		log.Warningf("Failed marshaling current time")
-		return "", time.Time{}
-	}
-	return string(retstr), now
-}
-
-func ParseDate(in string, p *types.Event, x interface{}) (map[string]string, error) {
-
-	var ret map[string]string = make(map[string]string)
-	tstr, tbin := GenDateParse(in)
-	if !tbin.IsZero() {
-		ret["MarshaledTime"] = string(tstr)
-		return ret, nil
-	}
-
-	return nil, nil
 }

pkg/parser/enrich_date.go

@@ -0,0 +1,70 @@
+package parser
+
+import (
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	log "github.com/sirupsen/logrus"
+)
+
+func GenDateParse(date string) (string, time.Time) {
+	var (
+		layouts = [...]string{
+			time.RFC3339,
+			"02/Jan/2006:15:04:05 -0700",
+			"Mon Jan 2 15:04:05 2006",
+			"02-Jan-2006 15:04:05 europe/paris",
+			"01/02/2006 15:04:05",
+			"2006-01-02 15:04:05.999999999 -0700 MST",
+			"Jan  2 15:04:05",
+			"Mon Jan 02 15:04:05.000000 2006",
+			"2006-01-02T15:04:05Z07:00",
+			"2006/01/02",
+			"2006/01/02 15:04",
+			"2006-01-02",
+			"2006-01-02 15:04",
+			"2006/01/02 15:04:05",
+			"2006-01-02 15:04:05",
+		}
+	)
+
+	for _, dateFormat := range layouts {
+		t, err := time.Parse(dateFormat, date)
+		if err == nil && !t.IsZero() {
+			//if the year isn't set, set it to current date :)
+			if t.Year() == 0 {
+				t = t.AddDate(time.Now().Year(), 0, 0)
+			}
+			retstr, err := t.MarshalText()
+			if err != nil {
+				log.Warningf("Failed marshaling '%v'", t)
+				continue
+			}
+			return string(retstr), t
+		}
+	}
+
+	now := time.Now()
+	retstr, err := now.MarshalText()
+	if err != nil {
+		log.Warningf("Failed marshaling current time")
+		return "", time.Time{}
+	}
+	return string(retstr), now
+}
+
+func ParseDate(in string, p *types.Event, x interface{}) (map[string]string, error) {
+
+	var ret map[string]string = make(map[string]string)
+	tstr, tbin := GenDateParse(in)
+	if !tbin.IsZero() {
+		ret["MarshaledTime"] = string(tstr)
+		return ret, nil
+	}
+
+	return nil, nil
+}
+
+func parseDateInit(cfg map[string]string) (interface{}, error) {
+	return nil, nil
+}

pkg/parser/enrich_dns.go

@@ -25,3 +25,7 @@ func reverse_dns(field string, p *types.Event, ctx interface{}) (map[string]stri
 	ret["reverse_dns"] = rets[0]
 	return ret, nil
 }
+
+func reverseDNSInit(cfg map[string]string) (interface{}, error) {
+	return nil, nil
+}

pkg/parser/enrich_geoip.go

@@ -12,12 +12,6 @@ import (
 	"github.com/oschwald/maxminddb-golang"
 )
 
-type GeoIpEnricherCtx struct {
-	dbc   *geoip2.Reader
-	dba   *geoip2.Reader
-	dbraw *maxminddb.Reader
-}
-
 func IpToRange(field string, p *types.Event, ctx interface{}) (map[string]string, error) {
 	var dummy interface{}
 	ret := make(map[string]string)
@@ -30,7 +24,7 @@ func IpToRange(field string, p *types.Event, ctx interface{}) (map[string]string
 		log.Infof("Can't parse ip %s, no range enrich", field)
 		return nil, nil
 	}
-	net, ok, err := ctx.(GeoIpEnricherCtx).dbraw.LookupNetwork(ip, &dummy)
+	net, ok, err := ctx.(*maxminddb.Reader).LookupNetwork(ip, &dummy)
 	if err != nil {
 		log.Errorf("Failed to fetch network for %s : %v", ip.String(), err)
 		return nil, nil
@@ -54,14 +48,16 @@ func GeoIpASN(field string, p *types.Event, ctx interface{}) (map[string]string,
 		log.Infof("Can't parse ip %s, no ASN enrich", ip)
 		return nil, nil
 	}
-	record, err := ctx.(GeoIpEnricherCtx).dba.ASN(ip)
+	record, err := ctx.(*geoip2.Reader).ASN(ip)
 	if err != nil {
 		log.Errorf("Unable to enrich ip '%s'", field)
 		return nil, nil
 	}
 	ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
 	ret["ASNOrg"] = record.AutonomousSystemOrganization
+
 	log.Tracef("geoip ASN %s -> %s, %s", field, ret["ASNNumber"], ret["ASNOrg"])
+
 	return ret, nil
 }
 
@@ -75,7 +71,7 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}) (map[string]string
 		log.Infof("Can't parse ip %s, no City enrich", ip)
 		return nil, nil
 	}
-	record, err := ctx.(GeoIpEnricherCtx).dbc.City(ip)
+	record, err := ctx.(*geoip2.Reader).City(ip)
 	if err != nil {
 		log.Debugf("Unable to enrich ip '%s'", ip)
 		return nil, nil
@@ -90,27 +86,32 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}) (map[string]string
 	return ret, nil
 }
 
-/* All plugins must export an Init function */
+func GeoIPCityInit(cfg map[string]string) (interface{}, error) {
-func GeoIpInit(cfg map[string]string) (interface{}, error) {
+	dbCityReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
-	var ctx GeoIpEnricherCtx
-	var err error
-
-	ctx.dbc, err = geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
-	if err != nil {
-		log.Debugf("couldn't open geoip : %v", err)
-		return nil, err
-	}
-	ctx.dba, err = geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
 	if err != nil {
 		log.Debugf("couldn't open geoip : %v", err)
 		return nil, err
 	}
 
-	ctx.dbraw, err = maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
+	return dbCityReader, nil
-	if err != nil {
+}
-		log.Debugf("couldn't open geoip : %v", err)
+
-		return nil, err
+func GeoIPASNInit(cfg map[string]string) (interface{}, error) {
-	}
+	dbASReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
-
+	if err != nil {
-	return ctx, nil
+		log.Debugf("couldn't open geoip : %v", err)
+		return nil, err
+	}
+
+	return dbASReader, nil
+}
+
+func IpToRangeInit(cfg map[string]string) (interface{}, error) {
+	ipToRangeReader, err := maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
+	if err != nil {
+		log.Debugf("couldn't open geoip : %v", err)
+		return nil, err
+	}
+
+	return ipToRangeReader, nil
 }

pkg/parser/node.go

@@ -47,7 +47,7 @@ type Node struct {
 	//If node has leafs, execute all of them until one asks for a 'break'
 	LeavesNodes []Node `yaml:"nodes,omitempty"`
 	//Flag used to describe when to 'break' or return an 'error'
-	EnrichFunctions []EnricherCtx
+	EnrichFunctions EnricherCtx
 
 	/* If the node is actually a leaf, it can have : grok, enrich, statics */
 	//pattern_syntax are named grok patterns that are re-utilised over several grok patterns
@@ -62,7 +62,7 @@ type Node struct {
 	Data      []*types.DataSource `yaml:"data,omitempty"`
 }
 
-func (n *Node) validate(pctx *UnixParserCtx, ectx []EnricherCtx) error {
+func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
 
 	//stage is being set automagically
 	if n.Stage == "" {
@@ -91,15 +91,8 @@ func (n *Node) validate(pctx *UnixParserCtx, ectx []EnricherCtx) error {
 			if static.ExpValue == "" {
 				return fmt.Errorf("static %d : when method is set, expression must be present", idx)
 			}
-			method_found := false
+			if _, ok := ectx.Registered[static.Method]; !ok {
-			for _, enricherCtx := range ectx {
+				log.Warningf("the method '%s' doesn't exist or the plugin has not been initialized", static.Method)
-				if _, ok := enricherCtx.Funcs[static.Method]; ok && enricherCtx.initiated {
-					method_found = true
-					break
-				}
-			}
-			if !method_found {
-				return fmt.Errorf("the method '%s' doesn't exist or the plugin has not been initialized", static.Method)
 			}
 		} else {
 			if static.Meta == "" && static.Parsed == "" && static.TargetByName == "" {
@@ -363,7 +356,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
 	return NodeState, nil
 }
 
-func (n *Node) compile(pctx *UnixParserCtx, ectx []EnricherCtx) error {
+func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
 	var err error
 	var valid bool
 

pkg/parser/node_test.go

@@ -49,7 +49,7 @@ func TestParserConfigs(t *testing.T) {
 		}, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true},
 	}
 	for idx := range CfgTests {
-		err := CfgTests[idx].NodeCfg.compile(pctx, []EnricherCtx{})
+		err := CfgTests[idx].NodeCfg.compile(pctx, EnricherCtx{})
 		if CfgTests[idx].Compiles == true && err != nil {
 			t.Fatalf("Compile: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
 		}
@@ -57,7 +57,7 @@ func TestParserConfigs(t *testing.T) {
 			t.Fatalf("Compile: (%d/%d) expected errror", idx+1, len(CfgTests))
 		}
 
-		err = CfgTests[idx].NodeCfg.validate(pctx, []EnricherCtx{})
+		err = CfgTests[idx].NodeCfg.validate(pctx, EnricherCtx{})
 		if CfgTests[idx].Valid == true && err != nil {
 			t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
 		}

pkg/parser/parsing_test.go

@@ -89,7 +89,7 @@ func BenchmarkParser(t *testing.B) {
 	}
 }
 
-func testOneParser(pctx *UnixParserCtx, ectx []EnricherCtx, dir string, b *testing.B) error {
+func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing.B) error {
 
 	var (
 		err    error
@@ -139,11 +139,11 @@ func testOneParser(pctx *UnixParserCtx, ectx []EnricherCtx, dir string, b *testi
 }
 
 //prepTests is going to do the initialisation of parser : it's going to load enrichment plugins and load the patterns. This is done here so that we don't redo it for each test
-func prepTests() (*UnixParserCtx, []EnricherCtx, error) {
+func prepTests() (*UnixParserCtx, EnricherCtx, error) {
 	var (
 		err  error
 		pctx *UnixParserCtx
-		ectx []EnricherCtx
+		ectx EnricherCtx
 	)
 
 	err = exprhelpers.Init()
@@ -166,7 +166,7 @@ func prepTests() (*UnixParserCtx, []EnricherCtx, error) {
 	// Init the parser
 	pctx, err = Init(map[string]interface{}{"patterns": cfgdir + string("/patterns/"), "data": "./tests/"})
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to initialize parser : %v", err)
+		return nil, ectx, fmt.Errorf("failed to initialize parser : %v", err)
 	}
 	return pctx, ectx, nil
 }

pkg/parser/runtime.go

@@ -143,29 +143,26 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er
 		if static.Method != "" {
 			processed := false
 			/*still way too hackish, but : inject all the results in enriched, and */
-			for _, x := range n.EnrichFunctions {
+			if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok {
-				if fptr, ok := x.Funcs[static.Method]; ok && x.initiated {
+				clog.Tracef("Found method '%s'", static.Method)
-					clog.Tracef("Found method '%s'", static.Method)
+				ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx)
-					ret, err := fptr(value, event, x.RuntimeCtx)
+				if err != nil {
-					if err != nil {
+					clog.Errorf("method '%s' returned an error : %v", static.Method, err)
-						clog.Fatalf("plugin function error : %v", err)
-					}
-					processed = true
-					clog.Debugf("+ Method %s('%s') returned %d entries to merge in .Enriched\n", static.Method, value, len(ret))
-					if len(ret) == 0 {
-						clog.Debugf("+ Method '%s' empty response on '%s'", static.Method, value)
-					}
-					for k, v := range ret {
-						clog.Debugf("\t.Enriched[%s] = '%s'\n", k, v)
-						event.Enriched[k] = v
-					}
-					break
-				} else {
-					clog.Warningf("method '%s' doesn't exist or plugin not initialized", static.Method)
 				}
+				processed = true
+				clog.Debugf("+ Method %s('%s') returned %d entries to merge in .Enriched\n", static.Method, value, len(ret))
+				if len(ret) == 0 {
+					clog.Debugf("+ Method '%s' empty response on '%s'", static.Method, value)
+				}
+				for k, v := range ret {
+					clog.Debugf("\t.Enriched[%s] = '%s'\n", k, v)
+					event.Enriched[k] = v
+				}
+			} else {
+				clog.Debugf("method '%s' doesn't exist or plugin not initialized", static.Method)
 			}
 			if !processed {
-				clog.Warningf("method '%s' doesn't exist", static.Method)
+				clog.Debugf("method '%s' doesn't exist", static.Method)
 			}
 		} else if static.Parsed != "" {
 			clog.Debugf(".Parsed[%s] = '%s'", static.Parsed, value)

pkg/parser/stage.go

@@ -37,7 +37,7 @@ type Stagefile struct {
 	Stage    string `yaml:"stage"`
 }
 
-func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx []EnricherCtx) ([]Node, error) {
+func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ([]Node, error) {
 	var nodes []Node
 	tmpstages := make(map[string]bool)
 	pctx.Stages = []string{}

pkg/parser/unix_parser.go

@@ -25,7 +25,7 @@ type Parsers struct {
 	PovfwStageFiles []Stagefile
 	Nodes           []Node
 	Povfwnodes      []Node
-	EnricherCtx     []EnricherCtx
+	EnricherCtx     EnricherCtx
 }
 
 func Init(c map[string]interface{}) (*UnixParserCtx, error) {