crowdsec/pkg/parser/unix_parser.go

161 lines
4.3 KiB
Go
Raw Permalink Normal View History

2020-05-15 09:39:16 +00:00
package parser
import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/grokky"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
2020-05-15 09:39:16 +00:00
)
type UnixParserCtx struct {
2020-05-27 14:31:08 +00:00
Grok grokky.Host
Stages []string
Profiling bool
DataFolder string
2020-05-15 09:39:16 +00:00
}
type Parsers struct {
Ctx *UnixParserCtx
Povfwctx *UnixParserCtx
StageFiles []Stagefile
PovfwStageFiles []Stagefile
Nodes []Node
Povfwnodes []Node
EnricherCtx EnricherCtx
2020-05-15 09:39:16 +00:00
}
func Init(c map[string]interface{}) (*UnixParserCtx, error) {
2020-05-15 09:39:16 +00:00
r := UnixParserCtx{}
r.Grok = grokky.NewBase()
r.Grok.UseRe2 = fflag.Re2GrokSupport.IsEnabled()
files, err := os.ReadDir(c["patterns"].(string))
2020-05-15 09:39:16 +00:00
if err != nil {
return nil, err
}
2020-05-27 14:31:08 +00:00
r.DataFolder = c["data"].(string)
2020-05-15 09:39:16 +00:00
for _, f := range files {
if strings.Contains(f.Name(), ".") {
continue
}
if err := r.Grok.AddFromFile(filepath.Join(c["patterns"].(string), f.Name())); err != nil {
2020-05-15 09:39:16 +00:00
log.Errorf("failed to load pattern %s : %v", f.Name(), err)
return nil, err
}
}
log.Debugf("Loaded %d pattern files", len(files))
2020-05-15 09:39:16 +00:00
return &r, nil
}
// Return new parsers
// nodes and povfwnodes are already initialized in parser.LoadStages
func NewParsers(hub *cwhub.Hub) *Parsers {
parsers := &Parsers{
Ctx: &UnixParserCtx{},
Povfwctx: &UnixParserCtx{},
StageFiles: make([]Stagefile, 0),
PovfwStageFiles: make([]Stagefile, 0),
}
for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} {
for _, hubParserItem := range hub.GetItemMap(itemType) {
if hubParserItem.State.Installed {
stagefile := Stagefile{
Filename: hubParserItem.State.LocalPath,
Stage: hubParserItem.Stage,
}
if itemType == cwhub.PARSERS {
parsers.StageFiles = append(parsers.StageFiles, stagefile)
}
if itemType == cwhub.POSTOVERFLOWS {
parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
}
}
}
}
if parsers.StageFiles != nil {
sort.Slice(parsers.StageFiles, func(i, j int) bool {
return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
})
}
if parsers.PovfwStageFiles != nil {
sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
})
}
return parsers
}
func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
var err error
patternsDir := cConfig.ConfigPaths.PatternDir
log.Infof("Loading grok library %s", patternsDir)
/* load base regexps for two grok parsers */
parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
"data": cConfig.ConfigPaths.DataDir})
if err != nil {
return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
}
parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
"data": cConfig.ConfigPaths.DataDir})
if err != nil {
return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
}
/*
Load enrichers
*/
log.Infof("Loading enrich plugins")
parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir)
if err != nil {
return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
}
/*
Load the actual parsers
*/
log.Infof("Loading parsers from %d files", len(parsers.StageFiles))
parsers.Nodes, err = LoadStages(parsers.StageFiles, parsers.Ctx, parsers.EnricherCtx)
if err != nil {
return parsers, fmt.Errorf("failed to load parser config : %v", err)
}
if len(parsers.PovfwStageFiles) > 0 {
log.Infof("Loading postoverflow parsers")
parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.Povfwctx, parsers.EnricherCtx)
} else {
log.Infof("No postoverflow parsers to load")
parsers.Povfwnodes = []Node{}
}
if err != nil {
return parsers, fmt.Errorf("failed to load postoverflow config : %v", err)
}
if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
parsers.Ctx.Profiling = true
parsers.Povfwctx.Profiling = true
}
/*
Reset CTX grok to reduce memory footprint after we compile all the patterns
*/
parsers.Ctx.Grok = grokky.Host{}
parsers.Povfwctx.Grok = grokky.Host{}
parsers.StageFiles = []Stagefile{}
parsers.PovfwStageFiles = []Stagefile{}
return parsers, nil
}