crowdsec/pkg/parser/stage.go

136 lines
3.8 KiB
Go
Raw Normal View History

2020-05-15 09:39:16 +00:00
package parser
/*
This file contains
- the runtime definition of parser
- the compilation/parsing routines of parser configuration
*/
import (
2022-11-29 08:16:07 +00:00
"errors"
2020-05-15 09:39:16 +00:00
"fmt"
"io"
_ "net/http/pprof"
"os"
"sort"
"strings"
"time"
"github.com/goombaio/namegenerator"
log "github.com/sirupsen/logrus"
2020-05-15 09:39:16 +00:00
yaml "gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
2020-05-15 09:39:16 +00:00
)
var seed namegenerator.Generator = namegenerator.NewNameGenerator(time.Now().UTC().UnixNano())
/*
identify generic component to alter maps, smartfilters ? (static, conditional static etc.)
*/
type Stagefile struct {
Filename string `yaml:"filename"`
Stage string `yaml:"stage"`
}
func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ([]Node, error) {
2020-05-15 09:39:16 +00:00
var nodes []Node
tmpstages := make(map[string]bool)
2020-05-24 10:41:45 +00:00
pctx.Stages = []string{}
2020-05-15 09:39:16 +00:00
for _, stageFile := range stageFiles {
2022-06-22 15:01:27 +00:00
if !strings.HasSuffix(stageFile.Filename, ".yaml") && !strings.HasSuffix(stageFile.Filename, ".yml") {
2020-05-15 09:39:16 +00:00
log.Warningf("skip non yaml : %s", stageFile.Filename)
continue
}
log.Debugf("loading parser file '%s'", stageFile)
2020-05-15 09:39:16 +00:00
st, err := os.Stat(stageFile.Filename)
if err != nil {
return nil, fmt.Errorf("failed to stat %s : %v", stageFile, err)
2020-05-15 09:39:16 +00:00
}
if st.IsDir() {
continue
}
yamlFile, err := os.Open(stageFile.Filename)
if err != nil {
return nil, fmt.Errorf("can't access parsing configuration file %s : %s", stageFile.Filename, err)
2020-05-15 09:39:16 +00:00
}
//process the yaml
dec := yaml.NewDecoder(yamlFile)
dec.SetStrict(true)
nodesCount := 0
for {
node := Node{}
node.OnSuccess = "continue" //default behavior is to continue
2020-05-15 09:39:16 +00:00
err = dec.Decode(&node)
if err != nil {
2022-11-29 08:16:07 +00:00
if errors.Is(err, io.EOF) {
2020-05-15 09:39:16 +00:00
log.Tracef("End of yaml file")
break
}
log.Fatalf("Error decoding parsing configuration file '%s': %v", stageFile.Filename, err)
}
//check for empty bucket
if node.Name == "" && node.Description == "" && node.Author == "" {
2023-09-12 09:04:56 +00:00
log.Infof("Node in %s has no name, author or description. Skipping.", stageFile.Filename)
2020-05-15 09:39:16 +00:00
continue
}
//check compat
if node.FormatVersion == "" {
log.Tracef("no version in %s, assuming '1.0'", node.Name)
2020-05-15 09:39:16 +00:00
node.FormatVersion = "1.0"
}
ok, err := cwversion.Satisfies(node.FormatVersion, cwversion.Constraint_parser)
2020-05-15 09:39:16 +00:00
if err != nil {
log.Fatalf("Failed to check version : %s", err)
}
if !ok {
log.Errorf("%s : %s doesn't satisfy parser format %s, skip", node.Name, node.FormatVersion, cwversion.Constraint_parser)
2020-05-15 09:39:16 +00:00
continue
}
node.Stage = stageFile.Stage
if _, ok := tmpstages[stageFile.Stage]; !ok {
tmpstages[stageFile.Stage] = true
}
//compile the node : grok pattern and expression
err = node.compile(pctx, ectx)
2020-05-15 09:39:16 +00:00
if err != nil {
if node.Name != "" {
return nil, fmt.Errorf("failed to compile node '%s' in '%s' : %s", node.Name, stageFile.Filename, err)
2020-05-15 09:39:16 +00:00
}
return nil, fmt.Errorf("failed to compile node in '%s' : %s", stageFile.Filename, err)
2020-05-15 09:39:16 +00:00
}
/* if the stage is empty, the node is empty, it's a trailing entry in users yaml file */
if node.Stage == "" {
continue
}
2020-05-27 14:31:08 +00:00
for _, data := range node.Data {
err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type)
if err != nil {
log.Error(err)
}
if data.Type == "regexp" { //cache only makes sense for regexp
exprhelpers.RegexpCacheInit(data.DestPath, *data)
2020-05-27 14:31:08 +00:00
}
}
2020-05-15 09:39:16 +00:00
nodes = append(nodes, node)
nodesCount++
}
log.WithFields(log.Fields{"file": stageFile.Filename, "stage": stageFile.Stage}).Infof("Loaded %d parser nodes", nodesCount)
2020-05-15 09:39:16 +00:00
}
for k := range tmpstages {
pctx.Stages = append(pctx.Stages, k)
}
sort.Strings(pctx.Stages)
log.Infof("Loaded %d nodes from %d stages", len(nodes), len(pctx.Stages))
2020-05-15 09:39:16 +00:00
return nodes, nil
}