Minor improvements to hubtest and appsec component (#2656)

This commit is contained in:
Thibault "bui" Koechlin 2023-12-13 17:45:56 +01:00 committed by GitHub
parent 12d9fba4b3
commit 51f70e47e3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 119 additions and 15 deletions

View file

@ -7,6 +7,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"text/template"
"github.com/AlecAivazis/survey/v2" "github.com/AlecAivazis/survey/v2"
"github.com/enescakir/emoji" "github.com/enescakir/emoji"
@ -100,6 +101,10 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath) return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath)
} }
if isAppsecTest {
logType = "appsec"
}
if logType == "" { if logType == "" {
return fmt.Errorf("please provide a type (--type) for the test") return fmt.Errorf("please provide a type (--type) for the test")
} }
@ -115,17 +120,24 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
//create empty nuclei template file //create empty nuclei template file
nucleiFileName := fmt.Sprintf("%s.yaml", testName) nucleiFileName := fmt.Sprintf("%s.yaml", testName)
nucleiFilePath := filepath.Join(testPath, nucleiFileName) nucleiFilePath := filepath.Join(testPath, nucleiFileName)
nucleiFile, err := os.Create(nucleiFilePath) nucleiFile, err := os.OpenFile(nucleiFilePath, os.O_RDWR|os.O_CREATE, 0755)
if err != nil { if err != nil {
return err return err
} }
ntpl := template.Must(template.New("nuclei").Parse(hubtest.TemplateNucleiFile))
if ntpl == nil {
return fmt.Errorf("unable to parse nuclei template")
}
ntpl.ExecuteTemplate(nucleiFile, "nuclei", struct{ TestName string }{TestName: testName})
nucleiFile.Close() nucleiFile.Close()
configFileData.AppsecRules = []string{"your_rule_here.yaml"} configFileData.AppsecRules = []string{"./appsec-rules/<author>/your_rule_here.yaml"}
configFileData.NucleiTemplate = nucleiFileName configFileData.NucleiTemplate = nucleiFileName
fmt.Println() fmt.Println()
fmt.Printf(" Test name : %s\n", testName) fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath) fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Nuclei Template : %s\n", nucleiFileName) fmt.Printf(" Config File : %s\n", configFilePath)
fmt.Printf(" Nuclei Template : %s\n", nucleiFilePath)
} else { } else {
// create empty log file // create empty log file
logFileName := fmt.Sprintf("%s.log", testName) logFileName := fmt.Sprintf("%s.log", testName)

2
go.mod
View file

@ -91,7 +91,7 @@ require (
) )
require ( require (
github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879 github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f
golang.org/x/text v0.14.0 golang.org/x/text v0.14.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
gotest.tools/v3 v3.5.0 gotest.tools/v3 v3.5.0

2
go.sum
View file

@ -100,6 +100,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879 h1:dhAc0AelASC3BbfuLURJeai1LYgFNgpMds0KPd9whbo= github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879 h1:dhAc0AelASC3BbfuLURJeai1LYgFNgpMds0KPd9whbo=
github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI=
github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f h1:FkOB9aDw0xzDd14pTarGRLsUNAymONq3dc7zhvsXElg=
github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f/go.mod h1:TrU7Li+z2RHNrPy0TKJ6R65V6Yzpan2sTIRryJJyJso=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=

View file

@ -337,7 +337,7 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) {
// parse the request only once // parse the request only once
parsedRequest, err := appsec.NewParsedRequestFromRequest(r) parsedRequest, err := appsec.NewParsedRequestFromRequest(r)
if err != nil { if err != nil {
log.Errorf("%s", err) w.logger.Errorf("%s", err)
rw.WriteHeader(http.StatusInternalServerError) rw.WriteHeader(http.StatusInternalServerError)
return return
} }
@ -358,7 +358,7 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) {
} }
appsecResponse := w.AppsecRuntime.GenerateResponse(response, logger) appsecResponse := w.AppsecRuntime.GenerateResponse(response, logger)
logger.Debugf("Response: %+v", appsecResponse)
rw.WriteHeader(appsecResponse.HTTPStatus) rw.WriteHeader(appsecResponse.HTTPStatus)
body, err := json.Marshal(BodyResponse{Action: appsecResponse.Action}) body, err := json.Marshal(BodyResponse{Action: appsecResponse.Action})
if err != nil { if err != nil {

View file

@ -13,6 +13,8 @@ import (
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2" "gopkg.in/tomb.v2"
_ "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec/bodyprocessors"
) )
// that's the runtime structure of the Application security engine as seen from the acquis // that's the runtime structure of the Application security engine as seen from the acquis
@ -190,6 +192,9 @@ func (r *AppsecRunner) processRequest(tx appsec.ExtendedTransaction, request *ap
} }
func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error { func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error {
if len(r.AppsecRuntime.InBandRules) == 0 {
return nil
}
tx := appsec.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID) tx := appsec.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID)
r.AppsecRuntime.InBandTx = tx r.AppsecRuntime.InBandTx = tx
err := r.processRequest(tx, request) err := r.processRequest(tx, request)
@ -197,7 +202,9 @@ func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error {
} }
func (r *AppsecRunner) ProcessOutOfBandRules(request *appsec.ParsedRequest) error { func (r *AppsecRunner) ProcessOutOfBandRules(request *appsec.ParsedRequest) error {
r.logger.Debugf("Processing out of band rules") if len(r.AppsecRuntime.OutOfBandRules) == 0 {
return nil
}
tx := appsec.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID) tx := appsec.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID)
r.AppsecRuntime.OutOfBandTx = tx r.AppsecRuntime.OutOfBandTx = tx
err := r.processRequest(tx, request) err := r.processRequest(tx, request)

View file

@ -0,0 +1,45 @@
package bodyprocessors
import (
"io"
"strconv"
"strings"
"github.com/crowdsecurity/coraza/v3/experimental/plugins"
"github.com/crowdsecurity/coraza/v3/experimental/plugins/plugintypes"
)
type rawBodyProcessor struct {
}
type setterInterface interface {
Set(string)
}
func (*rawBodyProcessor) ProcessRequest(reader io.Reader, v plugintypes.TransactionVariables, options plugintypes.BodyProcessorOptions) error {
buf := new(strings.Builder)
if _, err := io.Copy(buf, reader); err != nil {
return err
}
b := buf.String()
v.RequestBody().(setterInterface).Set(b)
v.RequestBodyLength().(setterInterface).Set(strconv.Itoa(len(b)))
return nil
}
func (*rawBodyProcessor) ProcessResponse(reader io.Reader, v plugintypes.TransactionVariables, options plugintypes.BodyProcessorOptions) error {
return nil
}
var (
_ plugintypes.BodyProcessor = &rawBodyProcessor{}
)
//nolint:gochecknoinits //Coraza recommends to use init() for registering plugins
func init() {
plugins.RegisterBodyProcessor("raw", func() plugintypes.BodyProcessor {
return &rawBodyProcessor{}
})
}

View file

@ -15,10 +15,12 @@ var zonesMap map[string]string = map[string]string{
"ARGS_NAMES": "ARGS_GET_NAMES", "ARGS_NAMES": "ARGS_GET_NAMES",
"BODY_ARGS": "ARGS_POST", "BODY_ARGS": "ARGS_POST",
"BODY_ARGS_NAMES": "ARGS_POST_NAMES", "BODY_ARGS_NAMES": "ARGS_POST_NAMES",
"HEADERS_NAMES": "REQUEST_HEADERS_NAMES",
"HEADERS": "REQUEST_HEADERS", "HEADERS": "REQUEST_HEADERS",
"METHOD": "REQUEST_METHOD", "METHOD": "REQUEST_METHOD",
"PROTOCOL": "REQUEST_PROTOCOL", "PROTOCOL": "REQUEST_PROTOCOL",
"URI": "REQUEST_URI", "URI": "REQUEST_URI",
"RAW_BODY": "REQUEST_BODY",
} }
var transformMap map[string]string = map[string]string{ var transformMap map[string]string = map[string]string{
@ -31,7 +33,7 @@ var transformMap map[string]string = map[string]string{
var matchMap map[string]string = map[string]string{ var matchMap map[string]string = map[string]string{
"regex": "@rx", "regex": "@rx",
"equal": "@streq", "equals": "@streq",
"startsWith": "@beginsWith", "startsWith": "@beginsWith",
"endsWith": "@endsWith", "endsWith": "@endsWith",
"contains": "@contains", "contains": "@contains",
@ -39,8 +41,8 @@ var matchMap map[string]string = map[string]string{
"libinjectionXSS": "@detectXSS", "libinjectionXSS": "@detectXSS",
"gt": "@gt", "gt": "@gt",
"lt": "@lt", "lt": "@lt",
"ge": "@ge", "gte": "@ge",
"le": "@le", "lte": "@le",
} }
var bodyTypeMatch map[string]string = map[string]string{ var bodyTypeMatch map[string]string = map[string]string{

View file

@ -104,7 +104,7 @@ func LoadCollection(pattern string, logger *log.Entry) ([]AppsecCollection, erro
for _, rule := range appsecRule.Rules { for _, rule := range appsecRule.Rules {
strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name) strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name)
if err != nil { if err != nil {
logger.Errorf("unable to convert rule %s : %s", rule.Name, err) logger.Errorf("unable to convert rule %s : %s", appsecRule.Name, err)
return nil, err return nil, err
} }
logger.Debugf("Adding rule %s", strRule) logger.Debugf("Adding rule %s", strRule)

View file

@ -269,10 +269,10 @@ func (r *ReqDumpFilter) ToJSON() error {
// Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine // Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine
func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) {
var err error var err error
body := make([]byte, 0) body := make([]byte, r.ContentLength)
if r.Body != nil { if r.Body != nil {
body, err = io.ReadAll(r.Body) _, err = io.ReadFull(r.Body, body)
if err != nil { if err != nil {
return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err) return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err)
} }

View file

@ -32,6 +32,27 @@ const (
templateProfileFile = "template_profiles.yaml" templateProfileFile = "template_profiles.yaml"
templateAcquisFile = "template_acquis.yaml" templateAcquisFile = "template_acquis.yaml"
templateAppsecProfilePath = "template_appsec-profile.yaml" templateAppsecProfilePath = "template_appsec-profile.yaml"
TemplateNucleiFile = `id: {{.TestName}}
info:
name: {{.TestName}}
author: crowdsec
severity: info
description: {{.TestName}} testing
tags: appsec-testing
http:
#this is a dummy request, edit the request(s) to match your needs
- raw:
- |
GET /test HTTP/1.1
Host: {{"{{"}}Hostname{{"}}"}}
cookie-reuse: true
#test will fail because we won't match http status
matchers:
- type: status
status:
- 403
`
) )
func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecTest bool) (HubTest, error) { func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecTest bool) (HubTest, error) {

View file

@ -540,6 +540,8 @@ func (t *HubTestItem) Clean() error {
func (t *HubTestItem) RunWithNucleiTemplate() error { func (t *HubTestItem) RunWithNucleiTemplate() error {
crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", t.RuntimePath)
testPath := filepath.Join(t.HubTestPath, t.Name) testPath := filepath.Join(t.HubTestPath, t.Name)
if _, err := os.Stat(testPath); os.IsNotExist(err) { if _, err := os.Stat(testPath); os.IsNotExist(err) {
return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath) return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath)
@ -550,7 +552,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
} }
//machine add //machine add
cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"} cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--force", "--auto"}
cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...)
output, err := cscliRegisterCmd.CombinedOutput() output, err := cscliRegisterCmd.CombinedOutput()
@ -581,6 +583,13 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
//wait for the appsec port to be available //wait for the appsec port to be available
if _, err := IsAlive(DefaultAppsecHost); err != nil { if _, err := IsAlive(DefaultAppsecHost); err != nil {
crowdsecLog, err2 := os.ReadFile(crowdsecLogFile)
if err2 != nil {
log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err)
} else {
log.Errorf("crowdsec log file '%s'", crowdsecLogFile)
log.Errorf("%s\n", string(crowdsecLog))
}
return fmt.Errorf("appsec is down: %s", err) return fmt.Errorf("appsec is down: %s", err)
} }
@ -605,7 +614,6 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
} }
err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, DefaultNucleiTarget) err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, DefaultNucleiTarget)
crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", nucleiConfig.OutputDir)
if t.Config.ExpectedNucleiFailure { if t.Config.ExpectedNucleiFailure {
if err != nil && errors.Is(err, NucleiTemplateFail) { if err != nil && errors.Is(err, NucleiTemplateFail) {
log.Infof("Appsec test %s failed as expected", t.Name) log.Infof("Appsec test %s failed as expected", t.Name)

View file

@ -36,6 +36,8 @@ func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string,
args = append(args, nc.CmdLineOptions...) args = append(args, nc.CmdLineOptions...)
cmd := exec.Command(nc.Path, args...) cmd := exec.Command(nc.Path, args...)
log.Debugf("Running Nuclei command: '%s'", cmd.String())
var out bytes.Buffer var out bytes.Buffer
var outErr bytes.Buffer var outErr bytes.Buffer
@ -59,6 +61,9 @@ func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string,
log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json") log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json")
return err return err
} else if len(out.String()) == 0 { } else if len(out.String()) == 0 {
log.Warningf("Stdout saved to %s", outputPrefix+"_stdout.txt")
log.Warningf("Stderr saved to %s", outputPrefix+"_stderr.txt")
log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json")
//No stdout means no finding, it means our test failed //No stdout means no finding, it means our test failed
return NucleiTemplateFail return NucleiTemplateFail
} }

View file

@ -414,6 +414,8 @@ install_crowdsec() {
mkdir -p "${CROWDSEC_CONFIG_PATH}/postoverflows" || exit mkdir -p "${CROWDSEC_CONFIG_PATH}/postoverflows" || exit
mkdir -p "${CROWDSEC_CONFIG_PATH}/collections" || exit mkdir -p "${CROWDSEC_CONFIG_PATH}/collections" || exit
mkdir -p "${CROWDSEC_CONFIG_PATH}/patterns" || exit mkdir -p "${CROWDSEC_CONFIG_PATH}/patterns" || exit
mkdir -p "${CROWDSEC_CONFIG_PATH}/appsec-configs" || exit
mkdir -p "${CROWDSEC_CONFIG_PATH}/appsec-rules" || exit
mkdir -p "${CROWDSEC_CONSOLE_DIR}" || exit mkdir -p "${CROWDSEC_CONSOLE_DIR}" || exit
#tmp #tmp