From 9180ac7be9060340cd6242d066a1cce0c1dc5cf4 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 15 Jun 2023 22:51:57 +0200 Subject: [PATCH] wip --- pkg/acquisition/modules/waf/waf.go | 34 +++----- pkg/waf/waf.go | 134 ++++++++++++++--------------- pkg/waf/waf_expr_lib.go | 4 +- pkg/waf/waf_rules_collection.go | 35 ++++++++ 4 files changed, 117 insertions(+), 90 deletions(-) create mode 100644 pkg/waf/waf_rules_collection.go diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 883e1c3af..ad72d582c 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -117,24 +117,32 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { Handler: w.mux, } - crowdsecWafConfig := waf.NewWafConfig() + ruleLoader := waf.NewWafRuleLoader() - err = crowdsecWafConfig.LoadWafRules() + rulesCollections, err := ruleLoader.LoadWafRules() if err != nil { return fmt.Errorf("cannot load WAF rules: %w", err) } var inBandRules string - for _, rule := range crowdsecWafConfig.InbandRules { + var outOfBandRules string - inBandRules += rule.String() + "\n" + //spew.Dump(rulesCollections) + + for _, collection := range rulesCollections { + if !collection.OutOfBand { + inBandRules += collection.String() + "\n" + } else { + outOfBandRules += collection.String() + "\n" + } } + w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n"))) //w.logger.Infof("Loading rules %+v", inBandRules) - fs := os.DirFS(crowdsecWafConfig.Datadir) + fs := os.DirFS(ruleLoader.Datadir) //in-band waf : kill on sight inbandwaf, err := coraza.NewWAF( @@ -143,24 +151,11 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { WithDirectives(inBandRules).WithRootFS(fs), ) - //for _, rule := range inbandwaf.GetWAF().Rules.GetRules() { - // w.logger.Infof("Action for Rule %d: %+v ", rule.ID(), rule.GetActions()) - //} - - //betterwaf := experimental.ToBetterWAFEngine(inbandwaf) - - //spew.Dump(betterwaf.Waf.Rules) - if err != nil { return errors.Wrap(err, "Cannot create WAF") } w.inBandWaf = inbandwaf - var outOfBandRules string - for _, rule := range crowdsecWafConfig.OutOfBandRules { - outOfBandRules += rule.String() + "\n" - } - w.logger.Infof("Loading %d out-of-band rules", len(strings.Split(outOfBandRules, "\n"))) //out-of-band waf : log only outofbandwaf, err := coraza.NewWAF( @@ -173,9 +168,6 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { return errors.Wrap(err, "Cannot create WAF") } w.outOfBandWaf = outofbandwaf - //log.Printf("IB -> %s", spew.Sdump(w.inBandWaf)) - - //We don´t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec w.mux.HandleFunc(w.config.Path, w.wafHandler) return nil diff --git a/pkg/waf/waf.go b/pkg/waf/waf.go index 32ed9c268..97bfc997c 100644 --- a/pkg/waf/waf.go +++ b/pkg/waf/waf.go @@ -1,6 +1,7 @@ package waf import ( + "fmt" "os" "path/filepath" "strings" @@ -10,6 +11,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/davecgh/go-spew/spew" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) @@ -27,26 +29,25 @@ type CompiledHook struct { Apply []*vm.Program `yaml:"-"` } -type WafRule struct { +/*type WafConfig struct { + InbandRules []WafRule + OutOfBandRules []WafRule + Datadir string + logger *log.Entry +}*/ + +// This represents one "waf-rule" config +type WafConfig struct { SecLangFilesRules []string `yaml:"seclang_files_rules"` SecLangRules []string `yaml:"seclang_rules"` OnLoad []Hook `yaml:"on_load"` PreEval []Hook `yaml:"pre_eval"` OnMatch []Hook `yaml:"on_match"` - - CompiledOnLoad []CompiledHook `yaml:"-"` - CompiledPreEval []CompiledHook `yaml:"-"` - CompiledOnMatch []CompiledHook `yaml:"-"` - - MergedRules []string `yaml:"-"` - OutOfBand bool `yaml:"-"` } -type WafConfig struct { - InbandRules []WafRule - OutOfBandRules []WafRule - Datadir string - logger *log.Entry +type WafRuleLoader struct { + logger *log.Entry + Datadir string } func buildHook(hook Hook) (CompiledHook, error) { @@ -54,48 +55,56 @@ func buildHook(hook Hook) (CompiledHook, error) { if hook.Filter != "" { program, err := expr.Compile(hook.Filter) //FIXME: opts if err != nil { - log.Errorf("unable to compile filter %s : %s", hook.Filter, err) - return CompiledHook{}, err + return CompiledHook{}, fmt.Errorf("unable to compile filter %s : %w", hook.Filter, err) } compiledHook.Filter = program } for _, apply := range hook.Apply { program, err := expr.Compile(apply, GetExprWAFOptions(map[string]interface{}{ - "InBandRules": []WafRule{}, - "OutOfBandRules": []WafRule{}, + "rules": &WafRulesCollection{}, })...) if err != nil { - log.Errorf("unable to compile apply %s : %s", apply, err) - return CompiledHook{}, err + return CompiledHook{}, fmt.Errorf("unable to compile apply %s : %w", apply, err) } compiledHook.Apply = append(compiledHook.Apply, program) } return compiledHook, nil } -func (w *WafConfig) LoadWafRules() error { - var files []string +func (w *WafRuleLoader) LoadWafRules() ([]*WafRulesCollection, error) { + var wafRulesFiles []string for _, hubWafRuleItem := range cwhub.GetItemMap(cwhub.WAF_RULES) { if hubWafRuleItem.Installed { - files = append(files, hubWafRuleItem.LocalPath) + wafRulesFiles = append(wafRulesFiles, hubWafRuleItem.LocalPath) } } - w.logger.Infof("Loading %d waf files", len(files)) - for _, file := range files { - fileContent, err := os.ReadFile(file) //FIXME: actually read from datadir + if len(wafRulesFiles) == 0 { + return nil, fmt.Errorf("no waf rules found in hub") + } + + w.logger.Infof("Loading %d waf files", len(wafRulesFiles)) + wafRulesCollections := []*WafRulesCollection{} + for _, wafRulesFile := range wafRulesFiles { + + fileContent, err := os.ReadFile(wafRulesFile) if err != nil { - w.logger.Errorf("unable to read file %s : %s", file, err) + w.logger.Errorf("unable to read file %s : %s", wafRulesFile, err) continue } - wafRule := WafRule{} - err = yaml.Unmarshal(fileContent, &wafRule) + wafConfig := WafConfig{} + err = yaml.Unmarshal(fileContent, &wafConfig) if err != nil { - w.logger.Errorf("unable to unmarshal file %s : %s", file, err) + w.logger.Errorf("unable to unmarshal file %s : %s", wafRulesFile, err) continue } - if wafRule.SecLangFilesRules != nil { - for _, rulesFile := range wafRule.SecLangFilesRules { + + spew.Dump(wafConfig) + + collection := &WafRulesCollection{} + + if wafConfig.SecLangFilesRules != nil { + for _, rulesFile := range wafConfig.SecLangFilesRules { fullPath := filepath.Join(w.Datadir, rulesFile) c, err := os.ReadFile(fullPath) if err != nil { @@ -109,77 +118,72 @@ func (w *WafConfig) LoadWafRules() error { if strings.TrimSpace(line) == "" { continue } - wafRule.MergedRules = append(wafRule.MergedRules, line) + collection.Rules = append(collection.Rules, WafRule{RawRule: line}) } } } - if wafRule.SecLangRules != nil { - wafRule.MergedRules = append(wafRule.MergedRules, wafRule.SecLangRules...) + + if wafConfig.SecLangRules != nil { + for _, rule := range wafConfig.SecLangRules { + collection.Rules = append(collection.Rules, WafRule{RawRule: rule}) + } } + //TODO: add our own format + //compile hooks - for _, hook := range wafRule.OnLoad { + for _, hook := range wafConfig.OnLoad { compiledHook, err := buildHook(hook) if err != nil { - w.logger.Errorf("unable to build hook %s : %s", hook.Filter, err) + w.logger.Errorf("unable to build on_load hook %s : %s", hook.Filter, err) continue } - wafRule.CompiledOnLoad = append(wafRule.CompiledOnLoad, compiledHook) + collection.CompiledOnLoad = append(collection.CompiledOnLoad, compiledHook) } - for _, hook := range wafRule.PreEval { + for _, hook := range wafConfig.PreEval { compiledHook, err := buildHook(hook) if err != nil { - w.logger.Errorf("unable to build hook %s : %s", hook.Filter, err) + w.logger.Errorf("unable to build pre_eval hook %s : %s", hook.Filter, err) continue } - wafRule.CompiledPreEval = append(wafRule.CompiledPreEval, compiledHook) + collection.CompiledPreEval = append(collection.CompiledPreEval, compiledHook) } - for _, hook := range wafRule.OnMatch { + for _, hook := range wafConfig.OnMatch { compiledHook, err := buildHook(hook) if err != nil { - w.logger.Errorf("unable to build hook %s : %s", hook.Filter, err) + w.logger.Errorf("unable to build on_match hook %s : %s", hook.Filter, err) continue } - wafRule.CompiledOnMatch = append(wafRule.CompiledOnMatch, compiledHook) + collection.CompiledOnMatch = append(collection.CompiledOnMatch, compiledHook) } //Run the on_load hooks - - if len(wafRule.CompiledOnLoad) > 0 { - w.logger.Infof("Running %d on_load hooks", len(wafRule.CompiledOnLoad)) - for hookIdx, onLoadHook := range wafRule.CompiledOnLoad { + if len(collection.CompiledOnLoad) > 0 { + w.logger.Infof("Running %d on_load hooks", len(collection.CompiledOnLoad)) + for hookIdx, onLoadHook := range collection.CompiledOnLoad { //Ignore filter for on load ? if onLoadHook.Apply != nil { for exprIdx, applyExpr := range onLoadHook.Apply { _, err := expr.Run(applyExpr, map[string]interface{}{ - "InBandRules": []WafRule{}, - "OutOfBandRules": []WafRule{}, + "rules": collection, }) if err != nil { - w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafRule.OnLoad[hookIdx].Apply[exprIdx], err) + w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafConfig.OnLoad[hookIdx].Apply[exprIdx], err) continue } } } } } - - if wafRule.MergedRules != nil { - if wafRule.OutOfBand { - w.OutOfBandRules = append(w.OutOfBandRules, wafRule) - } else { - w.InbandRules = append(w.InbandRules, wafRule) - } - } else { - w.logger.Warnf("no rules found in file %s ??", file) - } + wafRulesCollections = append(wafRulesCollections, collection) } - return nil + + return wafRulesCollections, nil } -func NewWafConfig() *WafConfig { +func NewWafRuleLoader() *WafRuleLoader { //FIXME: find a better way to get the datadir clog := log.New() if err := types.ConfigureLogger(clog); err != nil { @@ -192,9 +196,5 @@ func NewWafConfig() *WafConfig { initWafHelpers() - return &WafConfig{Datadir: csconfig.DataDir, logger: logger} -} - -func (w *WafRule) String() string { - return strings.Join(w.MergedRules, "\n") + return &WafRuleLoader{Datadir: csconfig.DataDir, logger: logger} } diff --git a/pkg/waf/waf_expr_lib.go b/pkg/waf/waf_expr_lib.go index 79d693458..33dd1f4e4 100644 --- a/pkg/waf/waf_expr_lib.go +++ b/pkg/waf/waf_expr_lib.go @@ -9,7 +9,7 @@ type exprCustomFunc struct { } var exprFuncs = []exprCustomFunc{ - { + /*{ name: "SetRulesToInband", function: SetRulesToInband, signature: []interface{}{ @@ -22,5 +22,5 @@ var exprFuncs = []exprCustomFunc{ signature: []interface{}{ new(func() error), }, - }, + },*/ } diff --git a/pkg/waf/waf_rules_collection.go b/pkg/waf/waf_rules_collection.go new file mode 100644 index 000000000..3ce130e56 --- /dev/null +++ b/pkg/waf/waf_rules_collection.go @@ -0,0 +1,35 @@ +package waf + +import "strings" + +type WafRule struct { + RawRule string +} + +// This is the "compiled" state of a WafConfig +type WafRulesCollection struct { + Rules []WafRule + CompiledOnLoad []CompiledHook `yaml:"-"` + CompiledPreEval []CompiledHook `yaml:"-"` + CompiledOnMatch []CompiledHook `yaml:"-"` + OutOfBand bool +} + +func (w *WafRulesCollection) SetInBand() error { + w.OutOfBand = false + return nil +} + +func (w *WafRulesCollection) SetOutOfBand() error { + w.OutOfBand = true + return nil +} + +func (w *WafRulesCollection) String() string { + //return strings.Join(w.Rules, "\n") + var rules []string + for _, rule := range w.Rules { + rules = append(rules, rule.RawRule) + } + return strings.Join(rules, "\n") +}