diff --git a/cmd/crowdsec-cli/config.go b/cmd/crowdsec-cli/config.go index 92d2f6248..2e0a48f04 100644 --- a/cmd/crowdsec-cli/config.go +++ b/cmd/crowdsec-cli/config.go @@ -3,6 +3,8 @@ package main import ( "fmt" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" @@ -18,6 +20,8 @@ type cliConfig struct { InstallFolder string BackendPluginFolder string `yaml:"backend_folder"` DataFolder string `yaml:"data_folder"` + SimulationCfgPath string `yaml:"simulation_path,omitempty"` + SimulationCfg *csconfig.SimulationConfig } func NewConfigCmd() *cobra.Command { diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index aa86a6f20..9811d96f2 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -63,6 +63,8 @@ func initConfig() { cwhub.Cfgdir = config.configFolder cwhub.Hubdir = config.HubFolder config.configured = true + config.SimulationCfg = csConfig.SimulationCfg + config.SimulationCfgPath = csConfig.SimulationCfgPath } func main() { @@ -141,7 +143,7 @@ API interaction: rootCmd.AddCommand(NewBackupCmd()) rootCmd.AddCommand(NewDashboardCmd()) rootCmd.AddCommand(NewInspectCmd()) - + rootCmd.AddCommand(NewSimulationCmds()) if err := rootCmd.Execute(); err != nil { log.Fatalf("While executing root command : %s", err) } diff --git a/cmd/crowdsec-cli/simulation.go b/cmd/crowdsec-cli/simulation.go new file mode 100644 index 000000000..fbdf7d5df --- /dev/null +++ b/cmd/crowdsec-cli/simulation.go @@ -0,0 +1,222 @@ +package main + +import ( + "fmt" + "io/ioutil" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" +) + +func addToExclusion(name string) error { + config.SimulationCfg.Exclusions = append(config.SimulationCfg.Exclusions, name) + return nil +} + +func removeFromExclusion(name string) error { + index := indexOf(name, config.SimulationCfg.Exclusions) + + // Remove element from the slice + config.SimulationCfg.Exclusions[index] = config.SimulationCfg.Exclusions[len(config.SimulationCfg.Exclusions)-1] + config.SimulationCfg.Exclusions[len(config.SimulationCfg.Exclusions)-1] = "" + config.SimulationCfg.Exclusions = config.SimulationCfg.Exclusions[:len(config.SimulationCfg.Exclusions)-1] + + return nil +} + +func enableGlobalSimulation() error { + config.SimulationCfg.Simulation = true + config.SimulationCfg.Exclusions = []string{} + + if err := dumpSimulationFile(); err != nil { + log.Fatalf("unable to dump simulation file: %s", err.Error()) + } + + log.Printf("global simulation: enabled") + + return nil +} + +func dumpSimulationFile() error { + newConfigSim, err := yaml.Marshal(config.SimulationCfg) + if err != nil { + return fmt.Errorf("unable to marshal simulation configuration: %s", err) + } + err = ioutil.WriteFile(config.SimulationCfgPath, newConfigSim, 0644) + if err != nil { + return fmt.Errorf("write simulation config in '%s' : %s", config.SimulationCfgPath, err) + } + + return nil +} + +func disableGlobalSimulation() error { + config.SimulationCfg.Simulation = false + config.SimulationCfg.Exclusions = []string{} + newConfigSim, err := yaml.Marshal(config.SimulationCfg) + if err != nil { + return fmt.Errorf("unable to marshal new simulation configuration: %s", err) + } + err = ioutil.WriteFile(config.SimulationCfgPath, newConfigSim, 0644) + if err != nil { + return fmt.Errorf("unable to write new simulation config in '%s' : %s", config.SimulationCfgPath, err) + } + + log.Printf("global simulation: disabled") + return nil +} + +func simulationStatus() error { + if config.SimulationCfg == nil { + log.Printf("global simulation: disabled (configuration file is missing)") + return nil + } + if config.SimulationCfg.Simulation { + log.Println("global simulation: enabled") + if len(config.SimulationCfg.Exclusions) > 0 { + log.Println("Scenarios not in simulation mode :") + for _, scenario := range config.SimulationCfg.Exclusions { + log.Printf(" - %s", scenario) + } + } + } else { + log.Println("global simulation: disabled") + if len(config.SimulationCfg.Exclusions) > 0 { + log.Println("Scenarios in simulation mode :") + for _, scenario := range config.SimulationCfg.Exclusions { + log.Printf(" - %s", scenario) + } + } + } + return nil +} + +func NewSimulationCmds() *cobra.Command { + var cmdSimulation = &cobra.Command{ + Use: "simulation enable|disable [scenario_name]", + Short: "", + Long: ``, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if !config.configured { + return fmt.Errorf("you must configure cli before using simulation") + } + return nil + }, + } + cmdSimulation.Flags().SortFlags = false + cmdSimulation.PersistentFlags().SortFlags = false + + var cmdSimulationEnable = &cobra.Command{ + Use: "enable [scenario_name]", + Short: "Enable the simulation, globally or on specified scenarios", + Long: ``, + Example: `cscli simulation enable`, + Run: func(cmd *cobra.Command, args []string) { + if err := cwhub.GetHubIdx(); err != nil { + log.Fatalf("failed to get Hub index : %v", err) + } + + if len(args) > 0 { + for _, scenario := range args { + var v cwhub.Item + var ok bool + if _, ok = cwhub.HubIdx[cwhub.SCENARIOS]; ok { + if v, ok = cwhub.HubIdx[cwhub.SCENARIOS][scenario]; !ok { + log.Errorf("'%s' isn't present in hub index", scenario) + continue + } + if !v.Installed { + log.Warningf("'%s' isn't enabled", scenario) + } + } + isExcluded := inSlice(scenario, config.SimulationCfg.Exclusions) + if config.SimulationCfg.Simulation && !isExcluded { + log.Warningf("global simulation is already enabled") + continue + } + if !config.SimulationCfg.Simulation && isExcluded { + log.Warningf("simulation for '%s' already enabled", scenario) + continue + } + if config.SimulationCfg.Simulation && isExcluded { + if err := removeFromExclusion(scenario); err != nil { + log.Fatalf(err.Error()) + } + log.Printf("simulation enabled for '%s'", scenario) + continue + } + if err := addToExclusion(scenario); err != nil { + log.Fatalf(err.Error()) + } + log.Printf("simulation mode for '%s' enabled", scenario) + } + if err := dumpSimulationFile(); err != nil { + log.Fatalf("simulation enable: %s", err.Error()) + } + } else { + if err := enableGlobalSimulation(); err != nil { + log.Fatalf("unable to enable global simulation mode : %s", err.Error()) + } + } + }, + } + cmdSimulation.AddCommand(cmdSimulationEnable) + + var cmdSimulationDisable = &cobra.Command{ + Use: "disable [scenario_name]", + Short: "Disable the simulation mode. Disable only specified scenarios", + Long: ``, + Example: `cscli simulation disable`, + Run: func(cmd *cobra.Command, args []string) { + if len(args) > 0 { + for _, scenario := range args { + isExcluded := inSlice(scenario, config.SimulationCfg.Exclusions) + if !config.SimulationCfg.Simulation && !isExcluded { + log.Warningf("%s isn't in simulation mode", scenario) + continue + } + if !config.SimulationCfg.Simulation && isExcluded { + if err := removeFromExclusion(scenario); err != nil { + log.Fatalf(err.Error()) + } + log.Printf("simulation mode for '%s' disabled", scenario) + continue + } + if isExcluded { + log.Warningf("simulation mode is enabled but is already disable for '%s'", scenario) + continue + } + if err := addToExclusion(scenario); err != nil { + log.Fatalf(err.Error()) + } + log.Printf("simulation mode for '%s' disabled", scenario) + } + if err := dumpSimulationFile(); err != nil { + log.Fatalf("simulation disable: %s", err.Error()) + } + } else { + if err := disableGlobalSimulation(); err != nil { + log.Fatalf("unable to disable global simulation mode : %s", err.Error()) + } + } + }, + } + cmdSimulation.AddCommand(cmdSimulationDisable) + + var cmdSimulationStatus = &cobra.Command{ + Use: "status", + Short: "Show simulation mode status", + Long: ``, + Example: `cscli simulation status`, + Run: func(cmd *cobra.Command, args []string) { + if err := simulationStatus(); err != nil { + log.Fatalf(err.Error()) + } + }, + } + cmdSimulation.AddCommand(cmdSimulationStatus) + + return cmdSimulation +} diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go new file mode 100644 index 000000000..032a90a5c --- /dev/null +++ b/cmd/crowdsec-cli/utils.go @@ -0,0 +1,19 @@ +package main + +func inSlice(s string, slice []string) bool { + for _, str := range slice { + if s == str { + return true + } + } + return false +} + +func indexOf(s string, slice []string) int { + for i, elem := range slice { + if s == elem { + return i + } + } + return -1 +} diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index f4c8d2334..5ff0f64dd 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -33,6 +33,10 @@ LOOP: log.Infof("Done shutdown down output") break LOOP case event := <-overflow: + //if global simulation -> everything is simulation unless told otherwise + if cConfig.SimulationCfg != nil && cConfig.SimulationCfg.Simulation { + event.Overflow.Simulation = true + } if cConfig.Profiling { start = time.Now() } @@ -47,6 +51,14 @@ LOOP: if err != nil { return fmt.Errorf("postoverflow failed : %s", err) } + //check scenarios in simulation + if cConfig.SimulationCfg != nil { + for _, scenario_name := range cConfig.SimulationCfg.Exclusions { + if event.Overflow.Scenario == scenario_name { + event.Overflow.Simulation = !event.Overflow.Simulation + } + } + } if event.Overflow.Scenario == "" && event.Overflow.MapKey != "" { //log.Infof("Deleting expired entry %s", event.Overflow.MapKey) diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index bb1c61a73..f1f6f2a32 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -45,6 +45,11 @@ func reloadHandler(sig os.Signal) error { if err := leaky.ShutdownAllBuckets(buckets); err != nil { log.Fatalf("while shutting down routines : %s", err) } + //reload the simulation state + if err := cConfig.LoadSimulation(); err != nil { + log.Errorf("reload error (simulation) : %s", err) + } + //reload all and start processing again :) if err := LoadParsers(cConfig); err != nil { log.Fatalf("Failed to load parsers: %s", err) diff --git a/config/prod.yaml b/config/prod.yaml index 73c48f085..0a2edc640 100644 --- a/config/prod.yaml +++ b/config/prod.yaml @@ -4,6 +4,7 @@ config_dir: ${CFG} pid_dir: ${PID} log_dir: /var/log/ cscli_dir: ${CFG}/cscli +simulation_path: ${CFG}/simulation.yaml log_mode: file log_level: info profiling: false diff --git a/config/simulation.yaml b/config/simulation.yaml new file mode 100644 index 000000000..e9c689993 --- /dev/null +++ b/config/simulation.yaml @@ -0,0 +1,4 @@ +simulation: off +# exclusions: +# - crowdsecurity/ssh-bf + \ No newline at end of file diff --git a/go.mod b/go.mod index 95bf7db39..12ca23037 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/opencontainers/image-spec v1.0.1 // indirect github.com/oschwald/geoip2-golang v1.4.0 github.com/oschwald/maxminddb-golang v1.6.0 + github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.5.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index 823d79696..4f92a3061 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -15,29 +15,36 @@ import ( "gopkg.in/yaml.v2" ) +type SimulationConfig struct { + Simulation bool `yaml:"simulation"` + Exclusions []string `yaml:"exclusions,omitempty"` +} + // CrowdSec is the structure of the crowdsec configuration type CrowdSec struct { - WorkingFolder string `yaml:"working_dir,omitempty"` - DataFolder string `yaml:"data_dir,omitempty"` - ConfigFolder string `yaml:"config_dir,omitempty"` - AcquisitionFile string `yaml:"acquis_path,omitempty"` - SingleFile string //for forensic mode - SingleFileLabel string //for forensic mode - PIDFolder string `yaml:"pid_dir,omitempty"` - LogFolder string `yaml:"log_dir,omitempty"` - LogMode string `yaml:"log_mode,omitempty"` //like file, syslog or stdout ? - LogLevel log.Level `yaml:"log_level,omitempty"` //trace,debug,info,warning,error - Daemonize bool `yaml:"daemon,omitempty"` //true -> go background - Profiling bool `yaml:"profiling,omitempty"` //true -> enable runtime profiling - APIMode bool `yaml:"apimode,omitempty"` //true -> enable api push - CsCliFolder string `yaml:"cscli_dir"` //cscli folder - NbParsers int `yaml:"parser_routines"` //the number of go routines to start for parsing - Linter bool - Prometheus bool - HTTPListen string `yaml:"http_listen,omitempty"` - RestoreMode string - DumpBuckets bool - OutputConfig *outputs.OutputFactory `yaml:"plugin"` + WorkingFolder string `yaml:"working_dir,omitempty"` + DataFolder string `yaml:"data_dir,omitempty"` + ConfigFolder string `yaml:"config_dir,omitempty"` + AcquisitionFile string `yaml:"acquis_path,omitempty"` + SingleFile string //for forensic mode + SingleFileLabel string //for forensic mode + PIDFolder string `yaml:"pid_dir,omitempty"` + LogFolder string `yaml:"log_dir,omitempty"` + LogMode string `yaml:"log_mode,omitempty"` //like file, syslog or stdout ? + LogLevel log.Level `yaml:"log_level,omitempty"` //trace,debug,info,warning,error + Daemonize bool `yaml:"daemon,omitempty"` //true -> go background + Profiling bool `yaml:"profiling,omitempty"` //true -> enable runtime profiling + APIMode bool `yaml:"apimode,omitempty"` //true -> enable api push + CsCliFolder string `yaml:"cscli_dir"` //cscli folder + NbParsers int `yaml:"parser_routines"` //the number of go routines to start for parsing + SimulationCfgPath string `yaml:"simulation_path,omitempty"` + SimulationCfg *SimulationConfig + Linter bool + Prometheus bool + HTTPListen string `yaml:"http_listen,omitempty"` + RestoreMode string + DumpBuckets bool + OutputConfig *outputs.OutputFactory `yaml:"plugin"` } // NewCrowdSecConfig create a new crowdsec configuration with default configuration @@ -59,6 +66,21 @@ func NewCrowdSecConfig() *CrowdSec { } } +func (c *CrowdSec) LoadSimulation() error { + if c.SimulationCfgPath != "" { + rcfg, err := ioutil.ReadFile(c.SimulationCfgPath) + if err != nil { + return fmt.Errorf("while reading '%s' : %s", c.SimulationCfgPath, err) + } + simCfg := SimulationConfig{} + if err := yaml.UnmarshalStrict(rcfg, &simCfg); err != nil { + return fmt.Errorf("while parsing '%s' : %s", c.SimulationCfgPath, err) + } + c.SimulationCfg = &simCfg + } + return nil +} + func (c *CrowdSec) GetCliConfig(configFile *string) error { /*overriden by cfg file*/ if *configFile != "" { @@ -73,8 +95,10 @@ func (c *CrowdSec) GetCliConfig(configFile *string) error { c.AcquisitionFile = filepath.Clean(c.ConfigFolder + "/acquis.yaml") } } + if err := c.LoadSimulation(); err != nil { + return fmt.Errorf("loading simulation config : %s", err) + } return nil - } // GetOPT return flags parsed from command line @@ -111,18 +135,8 @@ func (c *CrowdSec) GetOPT() error { c.SingleFileLabel = *catFileType } - /*overriden by cfg file*/ - if *configFile != "" { - rcfg, err := ioutil.ReadFile(*configFile) - if err != nil { - return fmt.Errorf("read '%s' : %s", *configFile, err) - } - if err := yaml.UnmarshalStrict(rcfg, c); err != nil { - return fmt.Errorf("parse '%s' : %s", *configFile, err) - } - if c.AcquisitionFile == "" { - c.AcquisitionFile = filepath.Clean(c.ConfigFolder + "/acquis.yaml") - } + if err := c.GetCliConfig(configFile); err != nil { + log.Fatalf("Error while loading configuration : %s", err) } if *AcquisitionFile != "" { diff --git a/pkg/outputs/ouputs.go b/pkg/outputs/ouputs.go index 40ed67055..e2392ff2b 100644 --- a/pkg/outputs/ouputs.go +++ b/pkg/outputs/ouputs.go @@ -45,13 +45,17 @@ func OvflwToOrder(sig types.SignalOccurence, prof types.Profile) (*types.BanOrde var ordr types.BanOrder var warn error + if sig.Simulation { + log.Debugf("signal for '%s' is whitelisted", sig.Source_ip) + ordr.MeasureType = "simulation:" + } //Identify remediation type if prof.Remediation.Ban { - ordr.MeasureType = "ban" + ordr.MeasureType += "ban" } else if prof.Remediation.Slow { - ordr.MeasureType = "slow" + ordr.MeasureType += "slow" } else if prof.Remediation.Captcha { - ordr.MeasureType = "captcha" + ordr.MeasureType += "captcha" } else { /*if the profil has no remediation, no order */ return nil, nil, fmt.Errorf("no remediation") diff --git a/pkg/types/signal_occurence.go b/pkg/types/signal_occurence.go index 91f0c467a..f7f259281 100644 --- a/pkg/types/signal_occurence.go +++ b/pkg/types/signal_occurence.go @@ -38,6 +38,7 @@ type SignalOccurence struct { Capacity int `json:"capacity,omitempty"` Leak_speed time.Duration `json:"leak_speed,omitempty"` Whitelisted bool `gorm:"-"` + Simulation bool `gorm:"-"` Reprocess bool //Reprocess, when true, will make the overflow being processed again as a fresh log would Labels map[string]string `gorm:"-"` } diff --git a/scripts/test_env.sh b/scripts/test_env.sh index c3b3bacd7..48efadd8d 100755 --- a/scripts/test_env.sh +++ b/scripts/test_env.sh @@ -73,6 +73,7 @@ create_arbo() { copy_files() { cp "./config/profiles.yaml" "$CONFIG_DIR" cp "./config/dev.yaml" "$BASE" + cp "./config/simulation.yaml" "$CONFIG_DIR" cp "./cmd/crowdsec/crowdsec" "$BASE" cp "./cmd/crowdsec-cli/cscli" "$BASE" cp -r "./config/patterns" "$CONFIG_DIR"