re-enable postoverflows (#117)

* re-enable postoverflows

* debug

* yoloooo

* remove debug

* remove error print

* fix test

* fix leakybucket test

* fix

Co-authored-by: AlteredCoder <AlteredCoder>

cmd/crowdsec/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
 	"github.com/crowdsecurity/crowdsec/pkg/outputs"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
@@ -282,6 +283,11 @@ func main() {
 		go runTachymeter(cConfig.HTTPListen)
 	}
 
+	err = exprhelpers.Init()
+	if err != nil {
+		log.Fatalf("Failed to init expr helpers : %s", err)
+	}
+
 	// Start loading configs
 	if err := LoadParsers(cConfig); err != nil {
 		log.Fatalf("Failed to load parsers: %s", err)

cmd/crowdsec/output.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"fmt"
+
 	log "github.com/sirupsen/logrus"
 
 	"time"
@@ -40,6 +42,12 @@ LOOP:
 				input <- event
 			}
 
+			/* process post overflow parser nodes */
+			event, err := parser.Parse(poctx, event, ponodes)
+			if err != nil {
+				return fmt.Errorf("postoverflow failed : %s", err)
+			}
+
 			if event.Overflow.Scenario == "" && event.Overflow.MapKey != "" {
 				//log.Infof("Deleting expired entry %s", event.Overflow.MapKey)
 				buckets.Bucket_map.Delete(event.Overflow.MapKey)

pkg/exprhelpers/exprlib.go

@@ -52,6 +52,7 @@ func Init() error {
 }
 
 func FileInit(fileFolder string, filename string, fileType string) error {
+	log.Printf("init (folder:%s) (file:%s) (type:%s)", fileFolder, filename, fileType)
 	filepath := path.Join(fileFolder, filename)
 	file, err := os.Open(filepath)
 	if err != nil {
@@ -87,7 +88,7 @@ func File(filename string) []string {
 	if _, ok := dataFile[filename]; ok {
 		return dataFile[filename]
 	}
-	log.Errorf("file '%s' not found for expr library", filename)
+	log.Errorf("file '%s' (type:string) not found in expr library", filename)
 	return []string{}
 }
 
@@ -99,7 +100,7 @@ func RegexpInFile(data string, filename string) bool {
 			}
 		}
 	} else {
-		log.Errorf("file '%s' not found for expr library", filename)
+		log.Errorf("file '%s' (type:regexp) not found in expr library", filename)
 	}
 	return false
 }

pkg/leakybucket/buckets_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/davecgh/go-spew/spew"
@@ -25,6 +26,10 @@ type TestFile struct {
 func TestBucket(t *testing.T) {
 
 	var envSetting = os.Getenv("TEST_ONLY")
+	err := exprhelpers.Init()
+	if err != nil {
+		log.Fatalf("exprhelpers init failed: %s", err)
+	}
 
 	if envSetting != "" {
 		if err := testOneBucket(t, envSetting); err != nil {

pkg/leakybucket/manager.go

@@ -112,10 +112,6 @@ func LoadBuckets(files []string, dataFolder string) ([]BucketFactory, chan types
 	)
 
 	var seed namegenerator.Generator = namegenerator.NewNameGenerator(time.Now().UTC().UnixNano())
-	err := exprhelpers.Init()
-	if err != nil {
-		return nil, nil, err
-	}
 
 	response = make(chan types.Event, 1)
 	for _, f := range files {

pkg/parser/node.go

@@ -137,7 +137,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
 		NodeState = true
 		clog.Debugf("eval(TRUE) '%s'", n.Filter)
 	} else {
-		clog.Tracef("Node has not filter, enter")
+		clog.Debugf("Node has not filter, enter")
 		NodeState = true
 	}
 
@@ -177,7 +177,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
 		clog.Debugf("no ip in event, cidr/ip whitelists not checked")
 	}
 	/* run whitelist expression tests anyway */
-	for _, e := range n.Whitelist.B_Exprs {
+	for eidx, e := range n.Whitelist.B_Exprs {
 		output, err := expr.Run(e, exprhelpers.GetExprEnv(map[string]interface{}{"evt": p}))
 		if err != nil {
 			clog.Warningf("failed to run whitelist expr : %v", err)
@@ -192,6 +192,8 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
 				p.Whitelisted = true
 				set = true
 			}
+		default:
+			log.Errorf("unexpected type %t (%v) while running '%s'", output, output, n.Whitelist.Exprs[eidx])
 		}
 	}
 	if set {

pkg/parser/parsing_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/davecgh/go-spew/spew"
 	log "github.com/sirupsen/logrus"
@@ -139,6 +140,10 @@ func testOneParser(pctx *UnixParserCtx, dir string, b *testing.B) error {
 func prepTests() (*UnixParserCtx, error) {
 	var pctx *UnixParserCtx
 	var p UnixParser
+	err := exprhelpers.Init()
+	if err != nil {
+		log.Fatalf("exprhelpers init failed: %s", err)
+	}
 
 	//Load enrichment
 	datadir := "../../data/"

pkg/parser/stage.go

@@ -43,10 +43,6 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx) ([]Node, error) {
 	tmpstages := make(map[string]bool)
 	pctx.Stages = []string{}
 
-	err := exprhelpers.Init()
-	if err != nil {
-		return nil, err
-	}
 	for _, stageFile := range stageFiles {
 		if !strings.HasSuffix(stageFile.Filename, ".yaml") {
 			log.Warningf("skip non yaml : %s", stageFile.Filename)