Improvement to run hubtest for appsec in docker (#2660)
This commit is contained in:
parent
89f704ef18
commit
a941576acc
|
@ -46,7 +46,7 @@ COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
|
||||||
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
|
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
|
||||||
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
|
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
|
||||||
|
|
||||||
ENTRYPOINT /bin/bash docker_start.sh
|
ENTRYPOINT /bin/bash /docker_start.sh
|
||||||
|
|
||||||
FROM slim as plugins
|
FROM slim as plugins
|
||||||
|
|
||||||
|
|
|
@ -222,7 +222,8 @@ func (cli cliHubTest) NewRunCmd() *cobra.Command {
|
||||||
var noClean bool
|
var noClean bool
|
||||||
var runAll bool
|
var runAll bool
|
||||||
var forceClean bool
|
var forceClean bool
|
||||||
|
var NucleiTargetHost string
|
||||||
|
var AppSecHost string
|
||||||
var cmd = &cobra.Command{
|
var cmd = &cobra.Command{
|
||||||
Use: "run",
|
Use: "run",
|
||||||
Short: "run [test_name]",
|
Short: "run [test_name]",
|
||||||
|
@ -232,7 +233,8 @@ func (cli cliHubTest) NewRunCmd() *cobra.Command {
|
||||||
printHelp(cmd)
|
printHelp(cmd)
|
||||||
return fmt.Errorf("please provide test to run or --all flag")
|
return fmt.Errorf("please provide test to run or --all flag")
|
||||||
}
|
}
|
||||||
|
hubPtr.NucleiTargetHost = NucleiTargetHost
|
||||||
|
hubPtr.AppSecHost = AppSecHost
|
||||||
if runAll {
|
if runAll {
|
||||||
if err := hubPtr.LoadAllTests(); err != nil {
|
if err := hubPtr.LoadAllTests(); err != nil {
|
||||||
return fmt.Errorf("unable to load all tests: %+v", err)
|
return fmt.Errorf("unable to load all tests: %+v", err)
|
||||||
|
@ -373,6 +375,8 @@ func (cli cliHubTest) NewRunCmd() *cobra.Command {
|
||||||
|
|
||||||
cmd.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
|
cmd.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
|
||||||
cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
|
cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
|
||||||
|
cmd.Flags().StringVar(&NucleiTargetHost, "target", hubtest.DefaultNucleiTarget, "Target for AppSec Test")
|
||||||
|
cmd.Flags().StringVar(&AppSecHost, "host", hubtest.DefaultAppsecHost, "Address to expose AppSec for hubtest")
|
||||||
cmd.Flags().BoolVar(&runAll, "all", false, "Run all tests")
|
cmd.Flags().BoolVar(&runAll, "all", false, "Run all tests")
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
|
|
|
@ -269,7 +269,11 @@ func (r *ReqDumpFilter) ToJSON() error {
|
||||||
// Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine
|
// Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine
|
||||||
func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) {
|
func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) {
|
||||||
var err error
|
var err error
|
||||||
body := make([]byte, r.ContentLength)
|
contentLength := r.ContentLength
|
||||||
|
if contentLength < 0 {
|
||||||
|
contentLength = 0
|
||||||
|
}
|
||||||
|
body := make([]byte, contentLength)
|
||||||
|
|
||||||
if r.Body != nil {
|
if r.Body != nil {
|
||||||
_, err = io.ReadFull(r.Body, body)
|
_, err = io.ReadFull(r.Body, body)
|
||||||
|
|
|
@ -22,6 +22,9 @@ type HubTest struct {
|
||||||
TemplateSimulationPath string
|
TemplateSimulationPath string
|
||||||
TemplateAcquisPath string
|
TemplateAcquisPath string
|
||||||
TemplateAppsecProfilePath string
|
TemplateAppsecProfilePath string
|
||||||
|
NucleiTargetHost string
|
||||||
|
AppSecHost string
|
||||||
|
|
||||||
HubIndex *cwhub.Hub
|
HubIndex *cwhub.Hub
|
||||||
Tests []*HubTestItem
|
Tests []*HubTestItem
|
||||||
}
|
}
|
||||||
|
@ -106,6 +109,8 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT
|
||||||
TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile),
|
TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile),
|
||||||
TemplateAppsecProfilePath: filepath.Join(HubTestPath, templateAppsecProfilePath),
|
TemplateAppsecProfilePath: filepath.Join(HubTestPath, templateAppsecProfilePath),
|
||||||
TemplateAcquisPath: filepath.Join(HubTestPath, templateAcquisFile),
|
TemplateAcquisPath: filepath.Join(HubTestPath, templateAcquisFile),
|
||||||
|
NucleiTargetHost: DefaultNucleiTarget,
|
||||||
|
AppSecHost: DefaultAppsecHost,
|
||||||
HubIndex: hub,
|
HubIndex: hub,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -73,6 +73,9 @@ type HubTestItem struct {
|
||||||
ScenarioAssert *ScenarioAssert
|
ScenarioAssert *ScenarioAssert
|
||||||
|
|
||||||
CustomItemsLocation []string
|
CustomItemsLocation []string
|
||||||
|
|
||||||
|
NucleiTargetHost string
|
||||||
|
AppSecHost string
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -152,6 +155,8 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
|
||||||
ScenarioAssert: ScenarioAssert,
|
ScenarioAssert: ScenarioAssert,
|
||||||
ParserAssert: ParserAssert,
|
ParserAssert: ParserAssert,
|
||||||
CustomItemsLocation: []string{hubTest.HubPath, testPath},
|
CustomItemsLocation: []string{hubTest.HubPath, testPath},
|
||||||
|
NucleiTargetHost: hubTest.NucleiTargetHost,
|
||||||
|
AppSecHost: hubTest.AppSecHost,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -315,7 +320,7 @@ func (t *HubTestItem) InstallHub() error {
|
||||||
|
|
||||||
// install appsec-rules in runtime environment
|
// install appsec-rules in runtime environment
|
||||||
for _, appsecrule := range t.Config.AppsecRules {
|
for _, appsecrule := range t.Config.AppsecRules {
|
||||||
log.Infof("adding rule '%s'", appsecrule)
|
log.Debugf("adding rule '%s'", appsecrule)
|
||||||
if appsecrule == "" {
|
if appsecrule == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -582,7 +587,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
|
||||||
crowdsecDaemon.Start()
|
crowdsecDaemon.Start()
|
||||||
|
|
||||||
//wait for the appsec port to be available
|
//wait for the appsec port to be available
|
||||||
if _, err := IsAlive(DefaultAppsecHost); err != nil {
|
if _, err := IsAlive(t.AppSecHost); err != nil {
|
||||||
crowdsecLog, err2 := os.ReadFile(crowdsecLogFile)
|
crowdsecLog, err2 := os.ReadFile(crowdsecLogFile)
|
||||||
if err2 != nil {
|
if err2 != nil {
|
||||||
log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err)
|
log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err)
|
||||||
|
@ -594,9 +599,9 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// check if the target is available
|
// check if the target is available
|
||||||
nucleiTargetParsedURL, err := url.Parse(DefaultNucleiTarget)
|
nucleiTargetParsedURL, err := url.Parse(t.NucleiTargetHost)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to parse target '%s': %s", DefaultNucleiTarget, err)
|
return fmt.Errorf("unable to parse target '%s': %s", t.NucleiTargetHost, err)
|
||||||
}
|
}
|
||||||
nucleiTargetHost := nucleiTargetParsedURL.Host
|
nucleiTargetHost := nucleiTargetParsedURL.Host
|
||||||
if _, err := IsAlive(nucleiTargetHost); err != nil {
|
if _, err := IsAlive(nucleiTargetHost); err != nil {
|
||||||
|
@ -613,9 +618,9 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, DefaultNucleiTarget)
|
err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, t.NucleiTargetHost)
|
||||||
if t.Config.ExpectedNucleiFailure {
|
if t.Config.ExpectedNucleiFailure {
|
||||||
if err != nil && errors.Is(err, NucleiTemplateFail) {
|
if err != nil && errors.Is(err, ErrNucleiTemplateFail) {
|
||||||
log.Infof("Appsec test %s failed as expected", t.Name)
|
log.Infof("Appsec test %s failed as expected", t.Name)
|
||||||
t.Success = true
|
t.Success = true
|
||||||
} else {
|
} else {
|
||||||
|
@ -905,12 +910,12 @@ func (t *HubTestItem) Run() error {
|
||||||
//if it's an appsec rule test, we need acquis and appsec profile
|
//if it's an appsec rule test, we need acquis and appsec profile
|
||||||
if len(t.Config.AppsecRules) > 0 {
|
if len(t.Config.AppsecRules) > 0 {
|
||||||
// copy template acquis file to runtime folder
|
// copy template acquis file to runtime folder
|
||||||
log.Infof("copying %s to %s", t.TemplateAcquisPath, t.RuntimeAcquisFilePath)
|
log.Debugf("copying %s to %s", t.TemplateAcquisPath, t.RuntimeAcquisFilePath)
|
||||||
if err = Copy(t.TemplateAcquisPath, t.RuntimeAcquisFilePath); err != nil {
|
if err = Copy(t.TemplateAcquisPath, t.RuntimeAcquisFilePath); err != nil {
|
||||||
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAcquisPath, t.RuntimeAcquisFilePath, err)
|
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAcquisPath, t.RuntimeAcquisFilePath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("copying %s to %s", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml"))
|
log.Debugf("copying %s to %s", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml"))
|
||||||
// copy template appsec-config file to runtime folder
|
// copy template appsec-config file to runtime folder
|
||||||
if err = Copy(t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml")); err != nil {
|
if err = Copy(t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml")); err != nil {
|
||||||
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml"), err)
|
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml"), err)
|
||||||
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
@ -18,16 +17,13 @@ type NucleiConfig struct {
|
||||||
CmdLineOptions []string `yaml:"cmdline_options"`
|
CmdLineOptions []string `yaml:"cmdline_options"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var NucleiTemplateFail = errors.New("Nuclei template failed")
|
var ErrNucleiTemplateFail = errors.New("nuclei template failed")
|
||||||
|
|
||||||
func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string, target string) error {
|
func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string, target string) error {
|
||||||
tstamp := time.Now().Unix()
|
tstamp := time.Now().Unix()
|
||||||
//templatePath is the full path to the template, we just want the name ie. "sqli-random-test"
|
|
||||||
tmp := strings.Split(templatePath, "/")
|
|
||||||
template := strings.Split(tmp[len(tmp)-1], ".")[0]
|
|
||||||
|
|
||||||
outputPrefix := fmt.Sprintf("%s/%s_%s-%d", nc.OutputDir, testName, template, tstamp)
|
|
||||||
|
|
||||||
|
outputPrefix := fmt.Sprintf("%s/%s-%d", nc.OutputDir, testName, tstamp)
|
||||||
|
// CVE-2023-34362_CVE-2023-34362-1702562399_stderr.txt
|
||||||
args := []string{
|
args := []string{
|
||||||
"-u", target,
|
"-u", target,
|
||||||
"-t", templatePath,
|
"-t", templatePath,
|
||||||
|
@ -65,7 +61,7 @@ func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string,
|
||||||
log.Warningf("Stderr saved to %s", outputPrefix+"_stderr.txt")
|
log.Warningf("Stderr saved to %s", outputPrefix+"_stderr.txt")
|
||||||
log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json")
|
log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json")
|
||||||
//No stdout means no finding, it means our test failed
|
//No stdout means no finding, it means our test failed
|
||||||
return NucleiTemplateFail
|
return ErrNucleiTemplateFail
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -116,7 +116,7 @@ func IsAlive(target string) (bool, error) {
|
||||||
for {
|
for {
|
||||||
conn, err := net.Dial("tcp", target)
|
conn, err := net.Dial("tcp", target)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
log.Debugf("appsec is up after %s", time.Since(start))
|
log.Debugf("'%s' is up after %s", target, time.Since(start))
|
||||||
conn.Close()
|
conn.Close()
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue