refact "cscli decisions" (#2804)

* refact "cscli decisions"
* CI: relax mysql test timing
* lint
This commit is contained in:
mmetc 2024-02-01 22:36:21 +01:00 committed by GitHub
parent f5fbe4a200
commit 4160bb8102
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 106 additions and 51 deletions

View file

@ -176,7 +176,7 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
flags.StringVar(&metabaseImage, "metabase-image", metabaseImage, "Metabase image to use")
flags.StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
flags.BoolVarP(&forceYes, "yes", "y", false, "force yes")
//flags.StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
// flags.StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
flags.StringVar(&metabasePassword, "password", "", "metabase password")
return cmd
@ -443,6 +443,7 @@ func checkGroups(forceYes *bool) (*user.Group, error) {
func (cli *cliDashboard) chownDatabase(gid string) error {
cfg := cli.cfg()
intID, err := strconv.Atoi(gid)
if err != nil {
return fmt.Errorf("unable to convert group ID to int: %s", err)
}

View file

@ -25,7 +25,7 @@ import (
var Client *apiclient.ApiClient
func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
/*here we cheat a bit : to make it more readable for the user, we dedup some entries*/
spamLimit := make(map[string]bool)
skipped := 0
@ -49,7 +49,8 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
alertItem.Decisions = newDecisions
}
if csConfig.Cscli.Output == "raw" {
switch cli.cfg().Cscli.Output {
case "raw":
csvwriter := csv.NewWriter(os.Stdout)
header := []string{"id", "source", "ip", "reason", "action", "country", "as", "events_count", "expiration", "simulated", "alert_id"}
@ -89,21 +90,24 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
}
csvwriter.Flush()
} else if csConfig.Cscli.Output == "json" {
case "json":
if *alerts == nil {
// avoid returning "null" in `json"
// could be cleaner if we used slice of alerts directly
fmt.Println("[]")
return nil
}
x, _ := json.MarshalIndent(alerts, "", " ")
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "human" {
case "human":
if len(*alerts) == 0 {
fmt.Println("No active decisions")
return nil
}
decisionsTable(color.Output, alerts, printMachine)
cli.decisionsTable(color.Output, alerts, printMachine)
if skipped > 0 {
fmt.Printf("%d duplicated entries skipped\n", skipped)
}
@ -113,13 +117,17 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
}
type cliDecisions struct {}
func NewCLIDecisions() *cliDecisions {
return &cliDecisions{}
type cliDecisions struct {
cfg configGetter
}
func (cli cliDecisions) NewCommand() *cobra.Command {
func NewCLIDecisions(getconfig configGetter) *cliDecisions {
return &cliDecisions{
cfg: getconfig,
}
}
func (cli *cliDecisions) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "decisions [action]",
Short: "Manage decisions",
@ -130,16 +138,17 @@ func (cli cliDecisions) NewCommand() *cobra.Command {
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
if err := csConfig.LoadAPIClient(); err != nil {
cfg := cli.cfg()
if err := cfg.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
}
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
password := strfmt.Password(cfg.API.Client.Credentials.Password)
apiurl, err := url.Parse(cfg.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("parsing api url %s: %w", csConfig.API.Client.Credentials.URL, err)
return fmt.Errorf("parsing api url %s: %w", cfg.API.Client.Credentials.URL, err)
}
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
MachineID: cfg.API.Client.Credentials.Login,
Password: password,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiurl,
@ -152,15 +161,15 @@ func (cli cliDecisions) NewCommand() *cobra.Command {
},
}
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewAddCmd())
cmd.AddCommand(cli.NewDeleteCmd())
cmd.AddCommand(cli.NewImportCmd())
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newAddCmd())
cmd.AddCommand(cli.newDeleteCmd())
cmd.AddCommand(cli.newImportCmd())
return cmd
}
func (cli cliDecisions) NewListCmd() *cobra.Command {
func (cli *cliDecisions) newListCmd() *cobra.Command {
var filter = apiclient.AlertsListOpts{
ValueEquals: new(string),
ScopeEquals: new(string),
@ -262,7 +271,7 @@ cscli decisions list -t ban
return fmt.Errorf("unable to retrieve decisions: %w", err)
}
err = DecisionsToTable(alerts, printMachine)
err = cli.decisionsToTable(alerts, printMachine)
if err != nil {
return fmt.Errorf("unable to print decisions: %w", err)
}
@ -289,7 +298,7 @@ cscli decisions list -t ban
return cmd
}
func (cli cliDecisions) NewAddCmd() *cobra.Command {
func (cli *cliDecisions) newAddCmd() *cobra.Command {
var (
addIP string
addRange string
@ -325,7 +334,7 @@ cscli decisions add --scope username --value foobar
createdAt := time.Now().UTC().Format(time.RFC3339)
/*take care of shorthand options*/
if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
if err = manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
return err
}
@ -341,7 +350,7 @@ cscli decisions add --scope username --value foobar
}
if addReason == "" {
addReason = fmt.Sprintf("manual '%s' from '%s'", addType, csConfig.API.Client.Credentials.Login)
addReason = fmt.Sprintf("manual '%s' from '%s'", addType, cli.cfg().API.Client.Credentials.Login)
}
decision := models.Decision{
Duration: &addDuration,
@ -400,7 +409,7 @@ cscli decisions add --scope username --value foobar
return cmd
}
func (cli cliDecisions) NewDeleteCmd() *cobra.Command {
func (cli *cliDecisions) newDeleteCmd() *cobra.Command {
var delFilter = apiclient.DecisionsDeleteOpts{
ScopeEquals: new(string),
ValueEquals: new(string),

View file

@ -67,7 +67,7 @@ func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
}
func (cli cliDecisions) runImport(cmd *cobra.Command, args []string) error {
func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
input, err := flags.GetString("input")
@ -236,7 +236,7 @@ func (cli cliDecisions) runImport(cmd *cobra.Command, args []string) error {
}
func (cli cliDecisions) NewImportCmd() *cobra.Command {
func (cli *cliDecisions) newImportCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "import [options]",
Short: "Import decisions from a file or pipe",

View file

@ -8,13 +8,15 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/models"
)
func decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachine bool) {
func (cli *cliDecisions) decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachine bool) {
t := newTable(out)
t.SetRowLines(false)
header := []string{"ID", "Source", "Scope:Value", "Reason", "Action", "Country", "AS", "Events", "expiration", "Alert ID"}
if printMachine {
header = append(header, "Machine")
}
t.SetHeaders(header...)
for _, alertItem := range *alerts {
@ -22,6 +24,7 @@ func decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachin
if *alertItem.Simulated {
*decisionItem.Type = fmt.Sprintf("(simul)%s", *decisionItem.Type)
}
row := []string{
strconv.Itoa(int(decisionItem.ID)),
*decisionItem.Origin,
@ -42,5 +45,6 @@ func decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachin
t.AddRow(row...)
}
}
t.Render()
}

View file

@ -18,6 +18,7 @@ func (p *MachinePassword) Set(v string) error {
if len(v) > 72 {
return errors.New("password too long (max 72 characters)")
}
*p = MachinePassword(v)
return nil

View file

@ -45,6 +45,7 @@ func generatePassword(length int) string {
if err != nil {
log.Fatalf("failed getting data from prng for password generation : %s", err)
}
buf[i] = charset[rInt.Int64()]
}
@ -59,12 +60,14 @@ func generateIDPrefix() (string, error) {
if err == nil {
return prefix, nil
}
log.Debugf("failed to get machine-id with usual files: %s", err)
bID, err := uuid.NewRandom()
if err == nil {
return bID.String(), nil
}
return "", fmt.Errorf("generating machine id: %w", err)
}
@ -75,11 +78,14 @@ func generateID(prefix string) (string, error) {
if prefix == "" {
prefix, err = generateIDPrefix()
}
if err != nil {
return "", err
}
prefix = strings.ReplaceAll(prefix, "-", "")[:32]
suffix := generatePassword(16)
return prefix + suffix, nil
}
@ -289,6 +295,7 @@ func (cli *cliMachines) add(args []string, machinePassword string, dumpFile stri
if !autoAdd {
return fmt.Errorf("please specify a password with --password or use --auto")
}
machinePassword = generatePassword(passwordLength)
} else if machinePassword == "" && interactive {
qs := &survey.Password{
@ -328,10 +335,10 @@ func (cli *cliMachines) add(args []string, machinePassword string, dumpFile stri
}
if dumpFile != "" && dumpFile != "-" {
err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
if err != nil {
if err = os.WriteFile(dumpFile, apiConfigDump, 0o600); err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %s", dumpFile, err)
}
fmt.Fprintf(os.Stderr, "API credentials written to '%s'.\n", dumpFile)
} else {
fmt.Print(string(apiConfigDump))
@ -359,11 +366,11 @@ func (cli *cliMachines) deleteValid(cmd *cobra.Command, args []string, toComplet
func (cli *cliMachines) delete(machines []string) error {
for _, machineID := range machines {
err := cli.db.DeleteWatcher(machineID)
if err != nil {
if err := cli.db.DeleteWatcher(machineID); err != nil {
log.Errorf("unable to delete machine '%s': %s", machineID, err)
return nil
}
log.Infof("machine '%s' deleted successfully", machineID)
}
@ -473,6 +480,7 @@ func (cli *cliMachines) validate(machineID string) error {
if err := cli.db.ValidateMachine(machineID); err != nil {
return fmt.Errorf("unable to validate machine '%s': %s", machineID, err)
}
log.Infof("machine '%s' validated successfully", machineID)
return nil

View file

@ -157,8 +157,8 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
cmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning")
cmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
cmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
cmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github")
if err := cmd.PersistentFlags().MarkHidden("branch"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}
@ -197,7 +197,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
cmd.AddCommand(NewCLIHub(getconfig).NewCommand())
cmd.AddCommand(NewMetricsCmd())
cmd.AddCommand(NewCLIDashboard(getconfig).NewCommand())
cmd.AddCommand(NewCLIDecisions().NewCommand())
cmd.AddCommand(NewCLIDecisions(getconfig).NewCommand())
cmd.AddCommand(NewCLIAlerts().NewCommand())
cmd.AddCommand(NewCLISimulation(getconfig).NewCommand())
cmd.AddCommand(NewCLIBouncers(getconfig).NewCommand())

View file

@ -32,7 +32,7 @@ func (cli *cliPapi) NewCommand() *cobra.Command {
Short: "Manage interaction with Polling API (PAPI)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
@ -59,7 +59,7 @@ func (cli *cliPapi) NewStatusCmd() *cobra.Command {
Short: "Get status of the Polling API",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
var err error
cfg := cli.cfg()
dbClient, err = database.NewClient(cfg.DbConfig)
@ -111,7 +111,7 @@ func (cli *cliPapi) NewSyncCmd() *cobra.Command {
Short: "Sync with the Polling API, pulling all non-expired orders for the instance",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
var err error
cfg := cli.cfg()
t := tomb.Tomb{}

View file

@ -211,14 +211,17 @@ func (cli *cliSimulation) enableGlobalSimulation() error {
func (cli *cliSimulation) dumpSimulationFile() error {
cfg := cli.cfg()
newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal simulation configuration: %s", err)
}
err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
if err != nil {
return fmt.Errorf("write simulation config in '%s' failed: %s", cfg.ConfigPaths.SimulationFilePath, err)
}
log.Debugf("updated simulation file %s", cfg.ConfigPaths.SimulationFilePath)
return nil
@ -230,16 +233,19 @@ func (cli *cliSimulation) disableGlobalSimulation() error {
*cfg.Cscli.SimulationConfig.Simulation = false
cfg.Cscli.SimulationConfig.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
}
err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
if err != nil {
return fmt.Errorf("unable to write new simulation config in '%s' : %s", cfg.ConfigPaths.SimulationFilePath, err)
return fmt.Errorf("unable to write new simulation config in '%s': %s", cfg.ConfigPaths.SimulationFilePath, err)
}
log.Printf("global simulation: disabled")
return nil
}
@ -249,10 +255,13 @@ func (cli *cliSimulation) status() {
log.Printf("global simulation: disabled (configuration file is missing)")
return
}
if *cfg.Cscli.SimulationConfig.Simulation {
log.Println("global simulation: enabled")
if len(cfg.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios not in simulation mode :")
for _, scenario := range cfg.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}

View file

@ -76,9 +76,10 @@ func collectMetrics() ([]byte, []byte, error) {
if err != nil {
return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
}
client := &http.Client{}
resp, err := client.Do(req)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %s", err)
}
@ -100,17 +101,20 @@ func collectVersion() []byte {
func collectFeatures() []byte {
log.Info("Collecting feature flags")
enabledFeatures := fflag.Crowdsec.GetEnabledFeatures()
w := bytes.NewBuffer(nil)
for _, k := range enabledFeatures {
fmt.Fprintf(w, "%s\n", k)
}
return w.Bytes()
}
func collectOSInfo() ([]byte, error) {
log.Info("Collecting OS info")
info, err := osinfo.GetOSInfo()
if err != nil {
@ -133,6 +137,7 @@ func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
var err error
out := bytes.NewBuffer(nil)
log.Infof("Collecting %s list", itemType)
items := make(map[string][]*cwhub.Item)
@ -144,26 +149,33 @@ func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
if err := listItems(out, []string{itemType}, items, false); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
return out.Bytes()
}
func collectBouncers(dbClient *database.Client) ([]byte, error) {
out := bytes.NewBuffer(nil)
bouncers, err := dbClient.ListBouncers()
if err != nil {
return nil, fmt.Errorf("unable to list bouncers: %s", err)
}
getBouncersTable(out, bouncers)
return out.Bytes(), nil
}
func collectAgents(dbClient *database.Client) ([]byte, error) {
out := bytes.NewBuffer(nil)
machines, err := dbClient.ListMachines()
if err != nil {
return nil, fmt.Errorf("unable to list machines: %s", err)
}
getAgentsTable(out, machines)
return out.Bytes(), nil
}
@ -171,12 +183,14 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
return []byte("No agent credentials found, are we LAPI ?")
}
pwd := strfmt.Password(password)
apiurl, err := url.Parse(endpoint)
pwd := strfmt.Password(password)
apiurl, err := url.Parse(endpoint)
if err != nil {
return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
}
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
@ -189,6 +203,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
if err != nil {
return []byte(fmt.Sprintf("could not init client: %s", err))
}
t := models.WatcherAuthRequest{
MachineID: &login,
Password: &pwd,
@ -205,6 +220,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
func collectCrowdsecConfig() []byte {
log.Info("Collecting crowdsec config")
config, err := os.ReadFile(*csConfig.FilePath)
if err != nil {
return []byte(fmt.Sprintf("could not read config file: %s", err))
@ -217,15 +233,18 @@ func collectCrowdsecConfig() []byte {
func collectCrowdsecProfile() []byte {
log.Info("Collecting crowdsec profile")
config, err := os.ReadFile(csConfig.API.Server.ProfilesPath)
if err != nil {
return []byte(fmt.Sprintf("could not read profile file: %s", err))
}
return config
}
func collectAcquisitionConfig() map[string][]byte {
log.Info("Collecting acquisition config")
ret := make(map[string][]byte)
for _, filename := range csConfig.Crowdsec.AcquisitionFiles {
@ -287,7 +306,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
Run: func(_ *cobra.Command, _ []string) {
var err error
var skipHub, skipDB, skipCAPI, skipLAPI, skipAgent bool
infos := map[string][]byte{
@ -307,13 +326,13 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_AGENTS_PATH] = []byte(err.Error())
}
if err := csConfig.LoadAPIServer(true); err != nil {
if err = csConfig.LoadAPIServer(true); err != nil {
log.Warnf("could not load LAPI, skipping CAPI check")
skipLAPI = true
infos[SUPPORT_CAPI_STATUS_PATH] = []byte(err.Error())
}
if err := csConfig.LoadCrowdsec(); err != nil {
if err = csConfig.LoadCrowdsec(); err != nil {
log.Warnf("could not load agent config, skipping crowdsec config check")
skipAgent = true
}
@ -399,7 +418,6 @@ cscli support dump -f /tmp/crowdsec-support.zip
}
if !skipAgent {
acquis := collectAcquisitionConfig()
for filename, content := range acquis {

View file

@ -25,6 +25,7 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
return fmt.Errorf("%s isn't a valid range", *ipRange)
}
}
if *ip != "" {
ipRepr := net.ParseIP(*ip)
if ipRepr == nil {
@ -32,7 +33,7 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
}
}
//avoid confusion on scope (ip vs Ip and range vs Range)
// avoid confusion on scope (ip vs Ip and range vs Range)
switch strings.ToLower(*scope) {
case "ip":
*scope = types.Ip
@ -43,6 +44,7 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
case "as":
*scope = types.AS
}
return nil
}

View file

@ -178,6 +178,7 @@ func (l *LocalApiClientCfg) Load() error {
func (lapiCfg *LocalApiServerCfg) GetTrustedIPs() ([]net.IPNet, error) {
trustedIPs := make([]net.IPNet, 0)
for _, ip := range lapiCfg.TrustedIPs {
cidr := toValidCIDR(ip)
@ -265,7 +266,7 @@ func (c *Config) LoadAPIServer(inCli bool) error {
return fmt.Errorf("no listen_uri specified")
}
//inherit log level from common, then api->server
// inherit log level from common, then api->server
var logLevel log.Level
if c.API.Server.LogLevel != nil {
logLevel = *c.API.Server.LogLevel

View file

@ -25,7 +25,7 @@ var globalConfig = Config{}
// Config contains top-level defaults -> overridden by configuration file -> overridden by CLI flags
type Config struct {
//just a path to ourselves :p
// just a path to ourselves :p
FilePath *string `yaml:"-"`
Self []byte `yaml:"-"`
Common *CommonCfg `yaml:"common,omitempty"`
@ -44,10 +44,12 @@ type Config struct {
func NewConfig(configFile string, disableAgent bool, disableAPI bool, inCli bool) (*Config, string, error) {
patcher := yamlpatch.NewPatcher(configFile, ".local")
patcher.SetQuiet(inCli)
fcontent, err := patcher.MergedPatchContent()
if err != nil {
return nil, "", err
}
configData := csstring.StrictExpand(string(fcontent), os.LookupEnv)
cfg := Config{
FilePath: &configFile,

View file

@ -19,7 +19,7 @@ teardown_file() {
setup() {
load "../lib/setup.sh"
if is_db_mysql; then sleep 0.3; fi
if is_db_mysql; then sleep 0.5; fi
}
api() {