From 51f70e47e31ed35edde8992cac6ff7a9e1307b70 Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Wed, 13 Dec 2023 17:45:56 +0100 Subject: [PATCH] Minor improvements to hubtest and appsec component (#2656) --- cmd/crowdsec-cli/hubtest.go | 18 ++++++-- go.mod | 2 +- go.sum | 2 + pkg/acquisition/modules/appsec/appsec.go | 4 +- .../modules/appsec/appsec_runner.go | 9 +++- .../modules/appsec/bodyprocessors/raw.go | 45 +++++++++++++++++++ pkg/appsec/appsec_rule/modsecurity.go | 8 ++-- pkg/appsec/appsec_rules_collection.go | 2 +- pkg/appsec/request.go | 4 +- pkg/hubtest/hubtest.go | 21 +++++++++ pkg/hubtest/hubtest_item.go | 12 ++++- pkg/hubtest/nucleirunner.go | 5 +++ wizard.sh | 2 + 13 files changed, 119 insertions(+), 15 deletions(-) create mode 100644 pkg/acquisition/modules/appsec/bodyprocessors/raw.go diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index d448c7cd2..813f423c9 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "text/template" "github.com/AlecAivazis/survey/v2" "github.com/enescakir/emoji" @@ -100,6 +101,10 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath) } + if isAppsecTest { + logType = "appsec" + } + if logType == "" { return fmt.Errorf("please provide a type (--type) for the test") } @@ -115,17 +120,24 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios //create empty nuclei template file nucleiFileName := fmt.Sprintf("%s.yaml", testName) nucleiFilePath := filepath.Join(testPath, nucleiFileName) - nucleiFile, err := os.Create(nucleiFilePath) + nucleiFile, err := os.OpenFile(nucleiFilePath, os.O_RDWR|os.O_CREATE, 0755) if err != nil { return err } + + ntpl := template.Must(template.New("nuclei").Parse(hubtest.TemplateNucleiFile)) + if ntpl == nil { + return fmt.Errorf("unable to parse nuclei template") + } + ntpl.ExecuteTemplate(nucleiFile, "nuclei", struct{ TestName string }{TestName: testName}) nucleiFile.Close() - configFileData.AppsecRules = []string{"your_rule_here.yaml"} + configFileData.AppsecRules = []string{"./appsec-rules//your_rule_here.yaml"} configFileData.NucleiTemplate = nucleiFileName fmt.Println() fmt.Printf(" Test name : %s\n", testName) fmt.Printf(" Test path : %s\n", testPath) - fmt.Printf(" Nuclei Template : %s\n", nucleiFileName) + fmt.Printf(" Config File : %s\n", configFilePath) + fmt.Printf(" Nuclei Template : %s\n", nucleiFilePath) } else { // create empty log file logFileName := fmt.Sprintf("%s.log", testName) diff --git a/go.mod b/go.mod index 222803086..82a8b501b 100644 --- a/go.mod +++ b/go.mod @@ -91,7 +91,7 @@ require ( ) require ( - github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879 + github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f golang.org/x/text v0.14.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools/v3 v3.5.0 diff --git a/go.sum b/go.sum index ddfcb1bcb..d5f126ace 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879 h1:dhAc0AelASC3BbfuLURJeai1LYgFNgpMds0KPd9whbo= github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f h1:FkOB9aDw0xzDd14pTarGRLsUNAymONq3dc7zhvsXElg= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f/go.mod h1:TrU7Li+z2RHNrPy0TKJ6R65V6Yzpan2sTIRryJJyJso= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index 2ae0ad939..49830eb85 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -337,7 +337,7 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { // parse the request only once parsedRequest, err := appsec.NewParsedRequestFromRequest(r) if err != nil { - log.Errorf("%s", err) + w.logger.Errorf("%s", err) rw.WriteHeader(http.StatusInternalServerError) return } @@ -358,7 +358,7 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { } appsecResponse := w.AppsecRuntime.GenerateResponse(response, logger) - + logger.Debugf("Response: %+v", appsecResponse) rw.WriteHeader(appsecResponse.HTTPStatus) body, err := json.Marshal(BodyResponse{Action: appsecResponse.Action}) if err != nil { diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index 0287d2fa1..3d6d27cf3 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -13,6 +13,8 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" + + _ "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec/bodyprocessors" ) // that's the runtime structure of the Application security engine as seen from the acquis @@ -190,6 +192,9 @@ func (r *AppsecRunner) processRequest(tx appsec.ExtendedTransaction, request *ap } func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error { + if len(r.AppsecRuntime.InBandRules) == 0 { + return nil + } tx := appsec.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID) r.AppsecRuntime.InBandTx = tx err := r.processRequest(tx, request) @@ -197,7 +202,9 @@ func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error { } func (r *AppsecRunner) ProcessOutOfBandRules(request *appsec.ParsedRequest) error { - r.logger.Debugf("Processing out of band rules") + if len(r.AppsecRuntime.OutOfBandRules) == 0 { + return nil + } tx := appsec.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID) r.AppsecRuntime.OutOfBandTx = tx err := r.processRequest(tx, request) diff --git a/pkg/acquisition/modules/appsec/bodyprocessors/raw.go b/pkg/acquisition/modules/appsec/bodyprocessors/raw.go new file mode 100644 index 000000000..e2e23eb57 --- /dev/null +++ b/pkg/acquisition/modules/appsec/bodyprocessors/raw.go @@ -0,0 +1,45 @@ +package bodyprocessors + +import ( + "io" + "strconv" + "strings" + + "github.com/crowdsecurity/coraza/v3/experimental/plugins" + "github.com/crowdsecurity/coraza/v3/experimental/plugins/plugintypes" +) + +type rawBodyProcessor struct { +} + +type setterInterface interface { + Set(string) +} + +func (*rawBodyProcessor) ProcessRequest(reader io.Reader, v plugintypes.TransactionVariables, options plugintypes.BodyProcessorOptions) error { + buf := new(strings.Builder) + if _, err := io.Copy(buf, reader); err != nil { + return err + } + + b := buf.String() + + v.RequestBody().(setterInterface).Set(b) + v.RequestBodyLength().(setterInterface).Set(strconv.Itoa(len(b))) + return nil +} + +func (*rawBodyProcessor) ProcessResponse(reader io.Reader, v plugintypes.TransactionVariables, options plugintypes.BodyProcessorOptions) error { + return nil +} + +var ( + _ plugintypes.BodyProcessor = &rawBodyProcessor{} +) + +//nolint:gochecknoinits //Coraza recommends to use init() for registering plugins +func init() { + plugins.RegisterBodyProcessor("raw", func() plugintypes.BodyProcessor { + return &rawBodyProcessor{} + }) +} diff --git a/pkg/appsec/appsec_rule/modsecurity.go b/pkg/appsec/appsec_rule/modsecurity.go index 760c697cc..1698953df 100644 --- a/pkg/appsec/appsec_rule/modsecurity.go +++ b/pkg/appsec/appsec_rule/modsecurity.go @@ -15,10 +15,12 @@ var zonesMap map[string]string = map[string]string{ "ARGS_NAMES": "ARGS_GET_NAMES", "BODY_ARGS": "ARGS_POST", "BODY_ARGS_NAMES": "ARGS_POST_NAMES", + "HEADERS_NAMES": "REQUEST_HEADERS_NAMES", "HEADERS": "REQUEST_HEADERS", "METHOD": "REQUEST_METHOD", "PROTOCOL": "REQUEST_PROTOCOL", "URI": "REQUEST_URI", + "RAW_BODY": "REQUEST_BODY", } var transformMap map[string]string = map[string]string{ @@ -31,7 +33,7 @@ var transformMap map[string]string = map[string]string{ var matchMap map[string]string = map[string]string{ "regex": "@rx", - "equal": "@streq", + "equals": "@streq", "startsWith": "@beginsWith", "endsWith": "@endsWith", "contains": "@contains", @@ -39,8 +41,8 @@ var matchMap map[string]string = map[string]string{ "libinjectionXSS": "@detectXSS", "gt": "@gt", "lt": "@lt", - "ge": "@ge", - "le": "@le", + "gte": "@ge", + "lte": "@le", } var bodyTypeMatch map[string]string = map[string]string{ diff --git a/pkg/appsec/appsec_rules_collection.go b/pkg/appsec/appsec_rules_collection.go index 4ccc63989..f6a135cae 100644 --- a/pkg/appsec/appsec_rules_collection.go +++ b/pkg/appsec/appsec_rules_collection.go @@ -104,7 +104,7 @@ func LoadCollection(pattern string, logger *log.Entry) ([]AppsecCollection, erro for _, rule := range appsecRule.Rules { strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name) if err != nil { - logger.Errorf("unable to convert rule %s : %s", rule.Name, err) + logger.Errorf("unable to convert rule %s : %s", appsecRule.Name, err) return nil, err } logger.Debugf("Adding rule %s", strRule) diff --git a/pkg/appsec/request.go b/pkg/appsec/request.go index 9979caf90..3ac2f83cf 100644 --- a/pkg/appsec/request.go +++ b/pkg/appsec/request.go @@ -269,10 +269,10 @@ func (r *ReqDumpFilter) ToJSON() error { // Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { var err error - body := make([]byte, 0) + body := make([]byte, r.ContentLength) if r.Body != nil { - body, err = io.ReadAll(r.Body) + _, err = io.ReadFull(r.Body, body) if err != nil { return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err) } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index b9bde4e2c..70c6d3537 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -32,6 +32,27 @@ const ( templateProfileFile = "template_profiles.yaml" templateAcquisFile = "template_acquis.yaml" templateAppsecProfilePath = "template_appsec-profile.yaml" + TemplateNucleiFile = `id: {{.TestName}} +info: + name: {{.TestName}} + author: crowdsec + severity: info + description: {{.TestName}} testing + tags: appsec-testing +http: +#this is a dummy request, edit the request(s) to match your needs + - raw: + - | + GET /test HTTP/1.1 + Host: {{"{{"}}Hostname{{"}}"}} + + cookie-reuse: true +#test will fail because we won't match http status + matchers: + - type: status + status: + - 403 +` ) func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecTest bool) (HubTest, error) { diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 05be6803d..c74bdb8f3 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -540,6 +540,8 @@ func (t *HubTestItem) Clean() error { func (t *HubTestItem) RunWithNucleiTemplate() error { + crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", t.RuntimePath) + testPath := filepath.Join(t.HubTestPath, t.Name) if _, err := os.Stat(testPath); os.IsNotExist(err) { return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath) @@ -550,7 +552,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { } //machine add - cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"} + cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--force", "--auto"} cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) output, err := cscliRegisterCmd.CombinedOutput() @@ -581,6 +583,13 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { //wait for the appsec port to be available if _, err := IsAlive(DefaultAppsecHost); err != nil { + crowdsecLog, err2 := os.ReadFile(crowdsecLogFile) + if err2 != nil { + log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err) + } else { + log.Errorf("crowdsec log file '%s'", crowdsecLogFile) + log.Errorf("%s\n", string(crowdsecLog)) + } return fmt.Errorf("appsec is down: %s", err) } @@ -605,7 +614,6 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { } err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, DefaultNucleiTarget) - crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", nucleiConfig.OutputDir) if t.Config.ExpectedNucleiFailure { if err != nil && errors.Is(err, NucleiTemplateFail) { log.Infof("Appsec test %s failed as expected", t.Name) diff --git a/pkg/hubtest/nucleirunner.go b/pkg/hubtest/nucleirunner.go index e3c6af73c..52af60cab 100644 --- a/pkg/hubtest/nucleirunner.go +++ b/pkg/hubtest/nucleirunner.go @@ -36,6 +36,8 @@ func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string, args = append(args, nc.CmdLineOptions...) cmd := exec.Command(nc.Path, args...) + log.Debugf("Running Nuclei command: '%s'", cmd.String()) + var out bytes.Buffer var outErr bytes.Buffer @@ -59,6 +61,9 @@ func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string, log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json") return err } else if len(out.String()) == 0 { + log.Warningf("Stdout saved to %s", outputPrefix+"_stdout.txt") + log.Warningf("Stderr saved to %s", outputPrefix+"_stderr.txt") + log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json") //No stdout means no finding, it means our test failed return NucleiTemplateFail } diff --git a/wizard.sh b/wizard.sh index 7df4e6646..da15b2aa3 100755 --- a/wizard.sh +++ b/wizard.sh @@ -414,6 +414,8 @@ install_crowdsec() { mkdir -p "${CROWDSEC_CONFIG_PATH}/postoverflows" || exit mkdir -p "${CROWDSEC_CONFIG_PATH}/collections" || exit mkdir -p "${CROWDSEC_CONFIG_PATH}/patterns" || exit + mkdir -p "${CROWDSEC_CONFIG_PATH}/appsec-configs" || exit + mkdir -p "${CROWDSEC_CONFIG_PATH}/appsec-rules" || exit mkdir -p "${CROWDSEC_CONSOLE_DIR}" || exit #tmp