beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Improvement to run hubtest for appsec in docker (#2660)

Browse source
This commit is contained in:
AlteredCoder 2023-12-14 16:05:16 +01:00 committed by GitHub
parent 89f704ef18
commit a941576acc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 37 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -46,7 +46,7 @@ COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
ENTRYPOINT /bin/bash docker_start.sh
ENTRYPOINT /bin/bash /docker_start.sh
FROM slim as plugins

8
cmd/crowdsec-cli/hubtest.go
View file

@ -222,7 +222,8 @@ func (cli cliHubTest) NewRunCmd() *cobra.Command {
var noClean bool
var runAll bool
var forceClean bool
var NucleiTargetHost string
var AppSecHost string
var cmd = &cobra.Command{
Use: "run",
Short: "run [test_name]",
@ -232,7 +233,8 @@ func (cli cliHubTest) NewRunCmd() *cobra.Command {
printHelp(cmd)
return fmt.Errorf("please provide test to run or --all flag")
}
hubPtr.NucleiTargetHost = NucleiTargetHost
hubPtr.AppSecHost = AppSecHost
if runAll {
if err := hubPtr.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err)
@ -373,6 +375,8 @@ func (cli cliHubTest) NewRunCmd() *cobra.Command {
cmd.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmd.Flags().StringVar(&NucleiTargetHost, "target", hubtest.DefaultNucleiTarget, "Target for AppSec Test")
cmd.Flags().StringVar(&AppSecHost, "host", hubtest.DefaultAppsecHost, "Address to expose AppSec for hubtest")
cmd.Flags().BoolVar(&runAll, "all", false, "Run all tests")
return cmd

6
pkg/appsec/request.go
View file

@ -269,7 +269,11 @@ func (r *ReqDumpFilter) ToJSON() error {
// Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine
func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) {
var err error
body := make([]byte, r.ContentLength)
contentLength := r.ContentLength
if contentLength < 0 {
contentLength = 0
}
body := make([]byte, contentLength)
if r.Body != nil {
_, err = io.ReadFull(r.Body, body)

9
pkg/hubtest/hubtest.go
View file

@ -22,8 +22,11 @@ type HubTest struct {
TemplateSimulationPath string
TemplateAcquisPath string
TemplateAppsecProfilePath string
HubIndex *cwhub.Hub
Tests []*HubTestItem
NucleiTargetHost string
AppSecHost string
HubIndex *cwhub.Hub
Tests []*HubTestItem
}
const (
@ -106,6 +109,8 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT
TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile),
TemplateAppsecProfilePath: filepath.Join(HubTestPath, templateAppsecProfilePath),
TemplateAcquisPath: filepath.Join(HubTestPath, templateAcquisFile),
NucleiTargetHost: DefaultNucleiTarget,
AppSecHost: DefaultAppsecHost,
HubIndex: hub,
}, nil
}

21
pkg/hubtest/hubtest_item.go
View file

@ -73,6 +73,9 @@ type HubTestItem struct {
ScenarioAssert *ScenarioAssert
CustomItemsLocation []string
NucleiTargetHost string
AppSecHost string
}
const (
@ -152,6 +155,8 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
ScenarioAssert: ScenarioAssert,
ParserAssert: ParserAssert,
CustomItemsLocation: []string{hubTest.HubPath, testPath},
NucleiTargetHost: hubTest.NucleiTargetHost,
AppSecHost: hubTest.AppSecHost,
}, nil
}
@ -315,7 +320,7 @@ func (t *HubTestItem) InstallHub() error {
// install appsec-rules in runtime environment
for _, appsecrule := range t.Config.AppsecRules {
log.Infof("adding rule '%s'", appsecrule)
log.Debugf("adding rule '%s'", appsecrule)
if appsecrule == "" {
continue
}
@ -582,7 +587,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
crowdsecDaemon.Start()
//wait for the appsec port to be available
if _, err := IsAlive(DefaultAppsecHost); err != nil {
if _, err := IsAlive(t.AppSecHost); err != nil {
crowdsecLog, err2 := os.ReadFile(crowdsecLogFile)
if err2 != nil {
log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err)
@ -594,9 +599,9 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
}
// check if the target is available
nucleiTargetParsedURL, err := url.Parse(DefaultNucleiTarget)
nucleiTargetParsedURL, err := url.Parse(t.NucleiTargetHost)
if err != nil {
return fmt.Errorf("unable to parse target '%s': %s", DefaultNucleiTarget, err)
return fmt.Errorf("unable to parse target '%s': %s", t.NucleiTargetHost, err)
}
nucleiTargetHost := nucleiTargetParsedURL.Host
if _, err := IsAlive(nucleiTargetHost); err != nil {
@ -613,9 +618,9 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
},
}
err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, DefaultNucleiTarget)
err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, t.NucleiTargetHost)
if t.Config.ExpectedNucleiFailure {
if err != nil && errors.Is(err, NucleiTemplateFail) {
if err != nil && errors.Is(err, ErrNucleiTemplateFail) {
log.Infof("Appsec test %s failed as expected", t.Name)
t.Success = true
} else {
@ -905,12 +910,12 @@ func (t *HubTestItem) Run() error {
//if it's an appsec rule test, we need acquis and appsec profile
if len(t.Config.AppsecRules) > 0 {
// copy template acquis file to runtime folder
log.Infof("copying %s to %s", t.TemplateAcquisPath, t.RuntimeAcquisFilePath)
log.Debugf("copying %s to %s", t.TemplateAcquisPath, t.RuntimeAcquisFilePath)
if err = Copy(t.TemplateAcquisPath, t.RuntimeAcquisFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAcquisPath, t.RuntimeAcquisFilePath, err)
}
log.Infof("copying %s to %s", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml"))
log.Debugf("copying %s to %s", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml"))
// copy template appsec-config file to runtime folder
if err = Copy(t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml")); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml"), err)

12
pkg/hubtest/nucleirunner.go
View file

@ -6,7 +6,6 @@ import (
"fmt"
"os"
"os/exec"
"strings"
"time"
log "github.com/sirupsen/logrus"
@ -18,16 +17,13 @@ type NucleiConfig struct {
CmdLineOptions []string `yaml:"cmdline_options"`
}
var NucleiTemplateFail = errors.New("Nuclei template failed")
var ErrNucleiTemplateFail = errors.New("nuclei template failed")
func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string, target string) error {
tstamp := time.Now().Unix()
//templatePath is the full path to the template, we just want the name ie. "sqli-random-test"
tmp := strings.Split(templatePath, "/")
template := strings.Split(tmp[len(tmp)-1], ".")[0]
outputPrefix := fmt.Sprintf("%s/%s_%s-%d", nc.OutputDir, testName, template, tstamp)
outputPrefix := fmt.Sprintf("%s/%s-%d", nc.OutputDir, testName, tstamp)
// CVE-2023-34362_CVE-2023-34362-1702562399_stderr.txt
args := []string{
"-u", target,
"-t", templatePath,
@ -65,7 +61,7 @@ func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string,
log.Warningf("Stderr saved to %s", outputPrefix+"_stderr.txt")
log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json")
//No stdout means no finding, it means our test failed
return NucleiTemplateFail
return ErrNucleiTemplateFail
}
return nil
}

2
pkg/hubtest/utils.go
View file

@ -116,7 +116,7 @@ func IsAlive(target string) (bool, error) {
for {
conn, err := net.Dial("tcp", target)
if err == nil {
log.Debugf("appsec is up after %s", time.Since(start))
log.Debugf("'%s' is up after %s", target, time.Since(start))
conn.Close()
return true, nil
}