From e46ca38cbbb616940a6e33d5abecdf285eb84e0c Mon Sep 17 00:00:00 2001 From: blotus Date: Thu, 18 Aug 2022 11:54:01 +0200 Subject: [PATCH] add `cscli support dump` (#1634) --- cmd/crowdsec-cli/bouncers.go | 107 ++++---- cmd/crowdsec-cli/collections.go | 3 +- cmd/crowdsec-cli/hub.go | 3 +- cmd/crowdsec-cli/machines.go | 108 ++++---- cmd/crowdsec-cli/main.go | 3 +- cmd/crowdsec-cli/metrics.go | 67 ++--- cmd/crowdsec-cli/parsers.go | 3 +- cmd/crowdsec-cli/postoverflows.go | 3 +- cmd/crowdsec-cli/scenarios.go | 3 +- cmd/crowdsec-cli/support.go | 393 ++++++++++++++++++++++++++++++ cmd/crowdsec-cli/utils.go | 14 +- go.mod | 1 + go.sum | 2 + tests/bats/01_base.bats | 2 +- tests/bats/04_nocapi.bats | 2 +- 15 files changed, 576 insertions(+), 138 deletions(-) create mode 100644 cmd/crowdsec-cli/support.go diff --git a/cmd/crowdsec-cli/bouncers.go b/cmd/crowdsec-cli/bouncers.go index 94f1dbace..afe5d2f9b 100644 --- a/cmd/crowdsec-cli/bouncers.go +++ b/cmd/crowdsec-cli/bouncers.go @@ -1,10 +1,10 @@ package main import ( + "bytes" "encoding/csv" "encoding/json" "fmt" - "os" "time" middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1" @@ -12,6 +12,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/enescakir/emoji" "github.com/olekukonko/tablewriter" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -20,6 +21,60 @@ var keyIP string var keyLength int var key string +func getBouncers(dbClient *database.Client) ([]byte, error) { + bouncers, err := dbClient.ListBouncers() + w := bytes.NewBuffer(nil) + if err != nil { + return nil, fmt.Errorf("unable to list bouncers: %s", err) + } + if csConfig.Cscli.Output == "human" { + + table := tablewriter.NewWriter(w) + table.SetCenterSeparator("") + table.SetColumnSeparator("") + + table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) + table.SetAlignment(tablewriter.ALIGN_LEFT) + table.SetHeader([]string{"Name", "IP Address", "Valid", "Last API pull", "Type", "Version", "Auth Type"}) + for _, b := range bouncers { + var revoked string + if !b.Revoked { + revoked = emoji.CheckMark.String() + } else { + revoked = emoji.Prohibited.String() + } + table.Append([]string{b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType}) + } + table.Render() + } else if csConfig.Cscli.Output == "json" { + x, err := json.MarshalIndent(bouncers, "", " ") + if err != nil { + return nil, errors.Wrap(err, "failed to unmarshal") + } + return x, nil + } else if csConfig.Cscli.Output == "raw" { + csvwriter := csv.NewWriter(w) + err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"}) + if err != nil { + return nil, errors.Wrap(err, "failed to write raw header") + } + for _, b := range bouncers { + var revoked string + if !b.Revoked { + revoked = "validated" + } else { + revoked = "pending" + } + err := csvwriter.Write([]string{b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType}) + if err != nil { + return nil, errors.Wrap(err, "failed to write raw") + } + } + csvwriter.Flush() + } + return w.Bytes(), nil +} + func NewBouncersCmd() *cobra.Command { /* ---- DECISIONS COMMAND */ var cmdBouncers = &cobra.Command{ @@ -54,55 +109,11 @@ Note: This command requires database direct access, so is intended to be run on Args: cobra.ExactArgs(0), DisableAutoGenTag: true, Run: func(cmd *cobra.Command, arg []string) { - blockers, err := dbClient.ListBouncers() + bouncers, err := getBouncers(dbClient) if err != nil { - log.Errorf("unable to list blockers: %s", err) - } - if csConfig.Cscli.Output == "human" { - - table := tablewriter.NewWriter(os.Stdout) - table.SetCenterSeparator("") - table.SetColumnSeparator("") - - table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) - table.SetAlignment(tablewriter.ALIGN_LEFT) - table.SetHeader([]string{"Name", "IP Address", "Valid", "Last API pull", "Type", "Version", "Auth Type"}) - for _, b := range blockers { - var revoked string - if !b.Revoked { - revoked = emoji.CheckMark.String() - } else { - revoked = emoji.Prohibited.String() - } - table.Append([]string{b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType}) - } - table.Render() - } else if csConfig.Cscli.Output == "json" { - x, err := json.MarshalIndent(blockers, "", " ") - if err != nil { - log.Fatalf("failed to unmarshal") - } - fmt.Printf("%s", string(x)) - } else if csConfig.Cscli.Output == "raw" { - csvwriter := csv.NewWriter(os.Stdout) - err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"}) - if err != nil { - log.Fatalf("failed to write raw header: %s", err) - } - for _, b := range blockers { - var revoked string - if !b.Revoked { - revoked = "validated" - } else { - revoked = "pending" - } - err := csvwriter.Write([]string{b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType}) - if err != nil { - log.Fatalf("failed to write raw: %s", err) - } - } - csvwriter.Flush() + log.Fatalf("unable to list bouncers: %s", err) } + fmt.Printf("%s", bouncers) }, } cmdBouncers.AddCommand(cmdBouncersList) diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go index ae16f481c..33c845004 100644 --- a/cmd/crowdsec-cli/collections.go +++ b/cmd/crowdsec-cli/collections.go @@ -173,7 +173,8 @@ func NewCollectionsCmd() *cobra.Command { Args: cobra.ExactArgs(0), DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - ListItems([]string{cwhub.COLLECTIONS}, args, false, true, all) + items := ListItems([]string{cwhub.COLLECTIONS}, args, false, true, all) + fmt.Printf("%s\n", string(items)) }, } cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 32edda4ce..e6419bccf 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -56,9 +56,10 @@ cscli hub update # Download list of available configurations from the hub log.Info(v) } cwhub.DisplaySummary() - ListItems([]string{ + items := ListItems([]string{ cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW, }, args, true, false, all) + fmt.Printf("%s\n", items) }, } cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") diff --git a/cmd/crowdsec-cli/machines.go b/cmd/crowdsec-cli/machines.go index 1a37d5d85..05dede05c 100644 --- a/cmd/crowdsec-cli/machines.go +++ b/cmd/crowdsec-cli/machines.go @@ -1,13 +1,13 @@ package main import ( + "bytes" saferand "crypto/rand" "encoding/csv" "encoding/json" "fmt" "io/ioutil" "math/big" - "os" "strings" "time" @@ -109,6 +109,61 @@ func displayLastHeartBeat(m *ent.Machine, fancy bool) string { return hbDisplay } +func getAgents(dbClient *database.Client) ([]byte, error) { + w := bytes.NewBuffer(nil) + machines, err := dbClient.ListMachines() + if err != nil { + return nil, fmt.Errorf("unable to list machines: %s", err) + } + if csConfig.Cscli.Output == "human" { + table := tablewriter.NewWriter(w) + table.SetCenterSeparator("") + table.SetColumnSeparator("") + + table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) + table.SetAlignment(tablewriter.ALIGN_LEFT) + table.SetHeader([]string{"Name", "IP Address", "Last Update", "Status", "Version", "Auth Type", "Last Heartbeat"}) + for _, w := range machines { + var validated string + if w.IsValidated { + validated = emoji.CheckMark.String() + } else { + validated = emoji.Prohibited.String() + } + table.Append([]string{w.MachineId, w.IpAddress, w.UpdatedAt.Format(time.RFC3339), validated, w.Version, w.AuthType, displayLastHeartBeat(w, true)}) + } + table.Render() + } else if csConfig.Cscli.Output == "json" { + x, err := json.MarshalIndent(machines, "", " ") + if err != nil { + log.Fatalf("failed to unmarshal") + } + return x, nil + } else if csConfig.Cscli.Output == "raw" { + csvwriter := csv.NewWriter(w) + err := csvwriter.Write([]string{"machine_id", "ip_address", "updated_at", "validated", "version", "auth_type", "last_heartbeat"}) + if err != nil { + log.Fatalf("failed to write header: %s", err) + } + for _, w := range machines { + var validated string + if w.IsValidated { + validated = "true" + } else { + validated = "false" + } + err := csvwriter.Write([]string{w.MachineId, w.IpAddress, w.UpdatedAt.Format(time.RFC3339), validated, w.Version, w.AuthType, displayLastHeartBeat(w, false)}) + if err != nil { + log.Fatalf("failed to write raw output : %s", err) + } + } + csvwriter.Flush() + } else { + log.Errorf("unknown output '%s'", csConfig.Cscli.Output) + } + return w.Bytes(), nil +} + func NewMachinesCmd() *cobra.Command { /* ---- DECISIONS COMMAND */ var cmdMachines = &cobra.Command{ @@ -149,56 +204,11 @@ Note: This command requires database direct access, so is intended to be run on } }, Run: func(cmd *cobra.Command, args []string) { - machines, err := dbClient.ListMachines() + agents, err := getAgents(dbClient) if err != nil { - log.Errorf("unable to list machines: %s", err) - } - if csConfig.Cscli.Output == "human" { - table := tablewriter.NewWriter(os.Stdout) - table.SetCenterSeparator("") - table.SetColumnSeparator("") - - table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) - table.SetAlignment(tablewriter.ALIGN_LEFT) - table.SetHeader([]string{"Name", "IP Address", "Last Update", "Status", "Version", "Auth Type", "Last Heartbeat"}) - for _, w := range machines { - var validated string - if w.IsValidated { - validated = emoji.CheckMark.String() - } else { - validated = emoji.Prohibited.String() - } - table.Append([]string{w.MachineId, w.IpAddress, w.UpdatedAt.Format(time.RFC3339), validated, w.Version, w.AuthType, displayLastHeartBeat(w, true)}) - } - table.Render() - } else if csConfig.Cscli.Output == "json" { - x, err := json.MarshalIndent(machines, "", " ") - if err != nil { - log.Fatalf("failed to unmarshal") - } - fmt.Printf("%s", string(x)) - } else if csConfig.Cscli.Output == "raw" { - csvwriter := csv.NewWriter(os.Stdout) - err := csvwriter.Write([]string{"machine_id", "ip_address", "updated_at", "validated", "version", "auth_type", "last_heartbeat"}) - if err != nil { - log.Fatalf("failed to write header: %s", err) - } - for _, w := range machines { - var validated string - if w.IsValidated { - validated = "true" - } else { - validated = "false" - } - err := csvwriter.Write([]string{w.MachineId, w.IpAddress, w.UpdatedAt.Format(time.RFC3339), validated, w.Version, w.AuthType, displayLastHeartBeat(w, false)}) - if err != nil { - log.Fatalf("failed to write raw output : %s", err) - } - } - csvwriter.Flush() - } else { - log.Errorf("unknown output '%s'", csConfig.Cscli.Output) + log.Fatalf("unable to list machines: %s", err) } + fmt.Printf("%s\n", agents) }, } cmdMachines.AddCommand(cmdMachinesList) diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 10243057d..acb8c9379 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -93,7 +93,7 @@ func initConfig() { var validArgs = []string{ "scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines", "metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard", - "config", "completion", "version", "console", "notifications", + "config", "completion", "version", "console", "notifications", "support", } func prepender(filename string) string { @@ -200,6 +200,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewExplainCmd()) rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewNotificationsCmd()) + rootCmd.AddCommand(NewSupportCmd()) if err := rootCmd.Execute(); err != nil { if bincoverTesting != "" { diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index d8710df9a..e017df20d 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/json" "fmt" "net/http" @@ -89,7 +90,7 @@ func metricsToTable(table *tablewriter.Table, stats map[string]map[string]int, k } /*This is a complete rip from prom2json*/ -func ShowPrometheus(url string) { +func FormatPrometheusMetric(url string, formatType string) ([]byte, error) { mfChan := make(chan *dto.MetricFamily, 1024) // Start with the DefaultTransport for sane defaults. @@ -99,7 +100,6 @@ func ShowPrometheus(url string) { transport.DisableKeepAlives = true // Timeout early if the server doesn't even return the headers. transport.ResponseHeaderTimeout = time.Minute - go func() { defer types.CatchPanic("crowdsec/ShowPrometheus") err := prom2json.FetchMetricFamilies(url, mfChan, transport) @@ -283,42 +283,45 @@ func ShowPrometheus(url string) { } } - if csConfig.Cscli.Output == "human" { - acquisTable := tablewriter.NewWriter(os.Stdout) + ret := bytes.NewBuffer(nil) + + if formatType == "human" { + + acquisTable := tablewriter.NewWriter(ret) acquisTable.SetHeader([]string{"Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket"}) keys := []string{"reads", "parsed", "unparsed", "pour"} if err := metricsToTable(acquisTable, acquis_stats, keys); err != nil { log.Warningf("while collecting acquis stats : %s", err) } - bucketsTable := tablewriter.NewWriter(os.Stdout) + bucketsTable := tablewriter.NewWriter(ret) bucketsTable.SetHeader([]string{"Bucket", "Current Count", "Overflows", "Instantiated", "Poured", "Expired"}) keys = []string{"curr_count", "overflow", "instanciation", "pour", "underflow"} if err := metricsToTable(bucketsTable, buckets_stats, keys); err != nil { log.Warningf("while collecting acquis stats : %s", err) } - parsersTable := tablewriter.NewWriter(os.Stdout) + parsersTable := tablewriter.NewWriter(ret) parsersTable.SetHeader([]string{"Parsers", "Hits", "Parsed", "Unparsed"}) keys = []string{"hits", "parsed", "unparsed"} if err := metricsToTable(parsersTable, parsers_stats, keys); err != nil { log.Warningf("while collecting acquis stats : %s", err) } - lapiMachinesTable := tablewriter.NewWriter(os.Stdout) + lapiMachinesTable := tablewriter.NewWriter(ret) lapiMachinesTable.SetHeader([]string{"Machine", "Route", "Method", "Hits"}) if err := lapiMetricsToTable(lapiMachinesTable, lapi_machine_stats); err != nil { log.Warningf("while collecting machine lapi stats : %s", err) } //lapiMetricsToTable - lapiBouncersTable := tablewriter.NewWriter(os.Stdout) + lapiBouncersTable := tablewriter.NewWriter(ret) lapiBouncersTable.SetHeader([]string{"Bouncer", "Route", "Method", "Hits"}) if err := lapiMetricsToTable(lapiBouncersTable, lapi_bouncer_stats); err != nil { log.Warningf("while collecting bouncer lapi stats : %s", err) } - lapiDecisionsTable := tablewriter.NewWriter(os.Stdout) + lapiDecisionsTable := tablewriter.NewWriter(ret) lapiDecisionsTable.SetHeader([]string{"Bouncer", "Empty answers", "Non-empty answers"}) for bouncer, hits := range lapi_decisions_stats { row := []string{} @@ -329,7 +332,7 @@ func ShowPrometheus(url string) { } /*unfortunately, we can't reuse metricsToTable as the structure is too different :/*/ - lapiTable := tablewriter.NewWriter(os.Stdout) + lapiTable := tablewriter.NewWriter(ret) lapiTable.SetHeader([]string{"Route", "Method", "Hits"}) sortedKeys := []string{} for akey := range lapi_stats { @@ -352,7 +355,7 @@ func ShowPrometheus(url string) { } } - decisionsTable := tablewriter.NewWriter(os.Stdout) + decisionsTable := tablewriter.NewWriter(ret) decisionsTable.SetHeader([]string{"Reason", "Origin", "Action", "Count"}) for reason, origins := range decisions_stats { for origin, actions := range origins { @@ -367,7 +370,7 @@ func ShowPrometheus(url string) { } } - alertsTable := tablewriter.NewWriter(os.Stdout) + alertsTable := tablewriter.NewWriter(ret) alertsTable.SetHeader([]string{"Reason", "Count"}) for scenario, hits := range alerts_stats { row := []string{} @@ -377,71 +380,75 @@ func ShowPrometheus(url string) { } if bucketsTable.NumLines() > 0 { - log.Printf("Buckets Metrics:") + fmt.Fprintf(ret, "Buckets Metrics:\n") bucketsTable.SetAlignment(tablewriter.ALIGN_LEFT) bucketsTable.Render() } if acquisTable.NumLines() > 0 { - log.Printf("Acquisition Metrics:") + fmt.Fprintf(ret, "Acquisition Metrics:\n") acquisTable.SetAlignment(tablewriter.ALIGN_LEFT) acquisTable.Render() } if parsersTable.NumLines() > 0 { - log.Printf("Parser Metrics:") + fmt.Fprintf(ret, "Parser Metrics:\n") parsersTable.SetAlignment(tablewriter.ALIGN_LEFT) parsersTable.Render() } if lapiTable.NumLines() > 0 { - log.Printf("Local Api Metrics:") + fmt.Fprintf(ret, "Local Api Metrics:\n") lapiTable.SetAlignment(tablewriter.ALIGN_LEFT) lapiTable.Render() } if lapiMachinesTable.NumLines() > 0 { - log.Printf("Local Api Machines Metrics:") + fmt.Fprintf(ret, "Local Api Machines Metrics:\n") lapiMachinesTable.SetAlignment(tablewriter.ALIGN_LEFT) lapiMachinesTable.Render() } if lapiBouncersTable.NumLines() > 0 { - log.Printf("Local Api Bouncers Metrics:") + fmt.Fprintf(ret, "Local Api Bouncers Metrics:\n") lapiBouncersTable.SetAlignment(tablewriter.ALIGN_LEFT) lapiBouncersTable.Render() } if lapiDecisionsTable.NumLines() > 0 { - log.Printf("Local Api Bouncers Decisions:") + fmt.Fprintf(ret, "Local Api Bouncers Decisions:\n") lapiDecisionsTable.SetAlignment(tablewriter.ALIGN_LEFT) lapiDecisionsTable.Render() } if decisionsTable.NumLines() > 0 { - log.Printf("Local Api Decisions:") + fmt.Fprintf(ret, "Local Api Decisions:\n") decisionsTable.SetAlignment(tablewriter.ALIGN_LEFT) decisionsTable.Render() } if alertsTable.NumLines() > 0 { - log.Printf("Local Api Alerts:") + fmt.Fprintf(ret, "Local Api Alerts:\n") alertsTable.SetAlignment(tablewriter.ALIGN_LEFT) alertsTable.Render() } - } else if csConfig.Cscli.Output == "json" { + } else if formatType == "json" { for _, val := range []interface{}{acquis_stats, parsers_stats, buckets_stats, lapi_stats, lapi_bouncer_stats, lapi_machine_stats, lapi_decisions_stats, decisions_stats, alerts_stats} { x, err := json.MarshalIndent(val, "", " ") if err != nil { - log.Fatalf("failed to unmarshal metrics : %v", err) + return nil, fmt.Errorf("failed to unmarshal metrics : %v", err) } - fmt.Printf("%s\n", string(x)) + ret.Write(x) } - } else if csConfig.Cscli.Output == "raw" { + return ret.Bytes(), nil + + } else if formatType == "raw" { for _, val := range []interface{}{acquis_stats, parsers_stats, buckets_stats, lapi_stats, lapi_bouncer_stats, lapi_machine_stats, lapi_decisions_stats, decisions_stats, alerts_stats} { x, err := yaml.Marshal(val) if err != nil { - log.Fatalf("failed to unmarshal metrics : %v", err) + return nil, fmt.Errorf("failed to unmarshal metrics : %v", err) } - fmt.Printf("%s\n", string(x)) + ret.Write(x) } + return ret.Bytes(), nil } + return ret.Bytes(), nil } var noUnit bool @@ -472,7 +479,11 @@ func NewMetricsCmd() *cobra.Command { os.Exit(1) } - ShowPrometheus(prometheusURL + "/metrics") + metrics, err := FormatPrometheusMetric(prometheusURL+"/metrics", csConfig.Cscli.Output) + if err != nil { + log.Fatalf("could not fetch prometheus metrics: %s", err) + } + fmt.Printf("%s", metrics) }, } cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://:/metrics)") diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go index a760008e8..30c789f63 100644 --- a/cmd/crowdsec-cli/parsers.go +++ b/cmd/crowdsec-cli/parsers.go @@ -164,7 +164,8 @@ cscli parsers remove crowdsecurity/sshd-logs cscli parser list crowdsecurity/xxx`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - ListItems([]string{cwhub.PARSERS}, args, false, true, all) + items := ListItems([]string{cwhub.PARSERS}, args, false, true, all) + fmt.Printf("%s\n", items) }, } cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go index 6d725b2f6..f3efdc89c 100644 --- a/cmd/crowdsec-cli/postoverflows.go +++ b/cmd/crowdsec-cli/postoverflows.go @@ -162,7 +162,8 @@ func NewPostOverflowsCmd() *cobra.Command { cscli postoverflows list crowdsecurity/xxx`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - ListItems([]string{cwhub.PARSERS_OVFLW}, args, false, true, all) + items := ListItems([]string{cwhub.PARSERS_OVFLW}, args, false, true, all) + fmt.Printf("%s\n", items) }, } cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index 9d7edb79f..c0ed30fe7 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -166,7 +166,8 @@ cscli scenarios remove crowdsecurity/ssh-bf cscli scenarios list crowdsecurity/xxx`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - ListItems([]string{cwhub.SCENARIOS}, args, false, true, all) + items := ListItems([]string{cwhub.SCENARIOS}, args, false, true, all) + fmt.Printf("%s\n", items) }, } cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go new file mode 100644 index 000000000..49b6fdd41 --- /dev/null +++ b/cmd/crowdsec-cli/support.go @@ -0,0 +1,393 @@ +package main + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "path/filepath" + "regexp" + "strings" + + "github.com/blackfireio/osinfo" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/crowdsec/pkg/database" + "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/go-openapi/strfmt" + log "github.com/sirupsen/logrus" + + "github.com/spf13/cobra" +) + +const ( + SUPPORT_METRICS_HUMAN_PATH = "metrics/metrics.human" + SUPPORT_METRICS_PROMETHEUS_PATH = "metrics/metrics.prometheus" + SUPPORT_VERSION_PATH = "version.txt" + SUPPORT_OS_INFO_PATH = "osinfo.txt" + SUPPORT_PARSERS_PATH = "hub/parsers.txt" + SUPPORT_SCENARIOS_PATH = "hub/scenarios.txt" + SUPPORT_COLLECTIONS_PATH = "hub/collections.txt" + SUPPORT_POSTOVERFLOWS_PATH = "hub/postoverflows.txt" + SUPPORT_BOUNCERS_PATH = "lapi/bouncers.txt" + SUPPORT_AGENTS_PATH = "lapi/agents.txt" + SUPPORT_CROWDSEC_CONFIG_PATH = "config/crowdsec.yaml" + SUPPORT_LAPI_STATUS_PATH = "lapi_status.txt" + SUPPORT_CAPI_STATUS_PATH = "capi_status.txt" + SUPPORT_ACQUISITION_CONFIG_BASE_PATH = "config/acquis/" + SUPPORT_CROWDSEC_PROFILE_PATH = "config/profiles.yaml" +) + +func collectMetrics() ([]byte, []byte, error) { + log.Info("Collecting prometheus metrics") + err := csConfig.LoadPrometheus() + if err != nil { + return nil, nil, err + } + + if csConfig.Cscli.PrometheusUrl == "" { + log.Warn("No Prometheus URL configured, metrics will not be collected") + return nil, nil, fmt.Errorf("prometheus_uri is not set") + } + + humanMetrics, err := FormatPrometheusMetric(csConfig.Cscli.PrometheusUrl+"/metrics", "human") + + if err != nil { + return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err) + } + + req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil) + if err != nil { + return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err) + } + client := &http.Client{} + resp, err := client.Do(req) + + if err != nil { + return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %s", err) + } + + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, nil, fmt.Errorf("could not read metrics from prometheus endpoint: %s", err) + } + + return humanMetrics, body, nil +} + +func collectVersion() []byte { + log.Info("Collecting version") + return []byte(cwversion.ShowStr()) +} + +func collectOSInfo() ([]byte, error) { + log.Info("Collecting OS info") + info, err := osinfo.GetOSInfo() + + if err != nil { + return nil, err + } + + w := bytes.NewBuffer(nil) + w.WriteString(fmt.Sprintf("Architecture: %s\n", info.Architecture)) + w.WriteString(fmt.Sprintf("Family: %s\n", info.Family)) + w.WriteString(fmt.Sprintf("ID: %s\n", info.ID)) + w.WriteString(fmt.Sprintf("Name: %s\n", info.Name)) + w.WriteString(fmt.Sprintf("Codename: %s\n", info.Codename)) + w.WriteString(fmt.Sprintf("Version: %s\n", info.Version)) + w.WriteString(fmt.Sprintf("Build: %s\n", info.Build)) + + return w.Bytes(), nil +} + +func initHub() error { + if err := csConfig.LoadHub(); err != nil { + return fmt.Errorf("cannot load hub: %s", err) + } + if csConfig.Hub == nil { + return fmt.Errorf("hub not configured") + } + + if err := cwhub.SetHubBranch(); err != nil { + return fmt.Errorf("cannot set hub branch: %s", err) + } + + if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { + return fmt.Errorf("no hub index found: %s", err) + } + return nil +} + +func collectHubItems(itemType string) []byte { + log.Infof("Collecting %s list", itemType) + items := ListItems([]string{itemType}, []string{}, false, true, all) + return items +} + +func collectBouncers(dbClient *database.Client) ([]byte, error) { + return getBouncers(dbClient) +} + +func collectAgents(dbClient *database.Client) ([]byte, error) { + return getAgents(dbClient) +} + +func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte { + if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil { + return []byte("No agent credentials found, are we LAPI ?") + } + pwd := strfmt.Password(password) + apiurl, err := url.Parse(endpoint) + + if err != nil { + return []byte(fmt.Sprintf("cannot parse API URL: %s", err.Error())) + } + scenarios, err := cwhub.GetInstalledScenariosAsString() + if err != nil { + return []byte(fmt.Sprintf("could not collect scenarios: %s", err.Error())) + } + + Client, err = apiclient.NewDefaultClient(apiurl, + prefix, + fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + nil) + if err != nil { + return []byte(fmt.Sprintf("could not init client: %s", err.Error())) + } + t := models.WatcherAuthRequest{ + MachineID: &login, + Password: &pwd, + Scenarios: scenarios, + } + + _, err = Client.Auth.AuthenticateWatcher(context.Background(), t) + if err != nil { + return []byte(fmt.Sprintf("Could not authenticate to API: %s", err)) + } else { + return []byte("Successfully authenticated to LAPI") + } +} + +func collectCrowdsecConfig() []byte { + log.Info("Collecting crowdsec config") + config, err := ioutil.ReadFile(*csConfig.FilePath) + if err != nil { + return []byte(fmt.Sprintf("could not read config file: %s", err)) + } + + r := regexp.MustCompile(`(\s+password:|\s+user:|\s+host:)\s+.*`) + + return r.ReplaceAll(config, []byte("$1 ****REDACTED****")) +} + +func collectCrowdsecProfile() []byte { + log.Info("Collecting crowdsec profile") + config, err := ioutil.ReadFile(csConfig.API.Server.ProfilesPath) + if err != nil { + return []byte(fmt.Sprintf("could not read profile file: %s", err)) + } + return config +} + +func collectAcquisitionConfig() map[string][]byte { + log.Info("Collecting acquisition config") + ret := make(map[string][]byte) + + for _, filename := range csConfig.Crowdsec.AcquisitionFiles { + fileContent, err := ioutil.ReadFile(filename) + if err != nil { + ret[filename] = []byte(fmt.Sprintf("could not read file: %s", err)) + } else { + ret[filename] = fileContent + } + } + + return ret +} + +func NewSupportCmd() *cobra.Command { + var cmdSupport = &cobra.Command{ + Use: "support [action]", + Short: "Provide commands to help during support", + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + var outFile string + + cmdDump := &cobra.Command{ + Use: "dump", + Short: "Dump all your configuration to a zip file for easier support", + Long: `Dump the following informations: +- Crowdsec version +- OS version +- Installed collections list +- Installed parsers list +- Installed scenarios list +- Installed postoverflows list +- Bouncers list +- Machines list +- CAPI status +- LAPI status +- Crowdsec config (sensitive information like username and password are redacted) +- Crowdsec metrics`, + Example: `cscli support dump +cscli support dump -f /tmp/crowdsec-support.zip +`, + Args: cobra.NoArgs, + DisableAutoGenTag: true, + Run: func(cmd *cobra.Command, args []string) { + var err error + var skipHub, skipDB, skipCAPI, skipLAPI, skipAgent bool + infos := map[string][]byte{ + SUPPORT_VERSION_PATH: collectVersion(), + } + + if outFile == "" { + outFile = "/tmp/crowdsec-support.zip" + } + + dbClient, err = database.NewClient(csConfig.DbConfig) + + if err != nil { + log.Warnf("Could not connect to database: %s", err) + skipDB = true + infos[SUPPORT_BOUNCERS_PATH] = []byte(err.Error()) + infos[SUPPORT_AGENTS_PATH] = []byte(err.Error()) + } + + if err := csConfig.LoadAPIServer(); err != nil { + log.Warnf("could not load LAPI, skipping CAPI check") + skipLAPI = true + infos[SUPPORT_CAPI_STATUS_PATH] = []byte(err.Error()) + } + + if err := csConfig.LoadCrowdsec(); err != nil { + log.Warnf("could not load agent config, skipping crowdsec config check") + skipAgent = true + } + + err = initHub() + + if err != nil { + log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected") + skipHub = true + infos[SUPPORT_PARSERS_PATH] = []byte(err.Error()) + infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error()) + infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error()) + infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error()) + } + + if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil { + log.Warn("no agent credentials found, skipping LAPI connectivity check") + if _, ok := infos[SUPPORT_LAPI_STATUS_PATH]; ok { + infos[SUPPORT_LAPI_STATUS_PATH] = append(infos[SUPPORT_LAPI_STATUS_PATH], []byte("\nNo LAPI credentials found")...) + } + skipLAPI = true + } + + if csConfig.API.Server == nil || csConfig.API.Server.OnlineClient.Credentials == nil { + log.Warn("no CAPI credentials found, skipping CAPI connectivity check") + skipCAPI = true + } + + infos[SUPPORT_METRICS_HUMAN_PATH], infos[SUPPORT_METRICS_PROMETHEUS_PATH], err = collectMetrics() + if err != nil { + log.Warnf("could not collect prometheus metrics information: %s", err) + infos[SUPPORT_METRICS_HUMAN_PATH] = []byte(err.Error()) + infos[SUPPORT_METRICS_PROMETHEUS_PATH] = []byte(err.Error()) + } + + infos[SUPPORT_OS_INFO_PATH], err = collectOSInfo() + + if err != nil { + log.Warnf("could not collect OS information: %s", err) + infos[SUPPORT_OS_INFO_PATH] = []byte(err.Error()) + } + + infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig() + + if !skipHub { + infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS) + infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS) + infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW) + infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS) + } + + if !skipDB { + infos[SUPPORT_BOUNCERS_PATH], err = collectBouncers(dbClient) + if err != nil { + log.Warnf("could not collect bouncers information: %s", err) + infos[SUPPORT_BOUNCERS_PATH] = []byte(err.Error()) + } + + infos[SUPPORT_AGENTS_PATH], err = collectAgents(dbClient) + if err != nil { + log.Warnf("could not collect agents information: %s", err) + infos[SUPPORT_AGENTS_PATH] = []byte(err.Error()) + } + } + + if !skipCAPI { + log.Info("Collecting CAPI status") + infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login, + csConfig.API.Server.OnlineClient.Credentials.Password, + csConfig.API.Server.OnlineClient.Credentials.URL, + CAPIURLPrefix) + } + + if !skipLAPI { + log.Info("Collection LAPI status") + infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login, + csConfig.API.Client.Credentials.Password, + csConfig.API.Client.Credentials.URL, + LAPIURLPrefix) + infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile() + } + + if !skipAgent { + + acquis := collectAcquisitionConfig() + + for filename, content := range acquis { + fname := strings.ReplaceAll(filename, string(filepath.Separator), "___") + infos[SUPPORT_ACQUISITION_CONFIG_BASE_PATH+fname] = content + } + } + + w := bytes.NewBuffer(nil) + zipWriter := zip.NewWriter(w) + + for filename, data := range infos { + fw, err := zipWriter.Create(filename) + if err != nil { + log.Errorf("Could not add zip entry for %s: %s", filename, err) + continue + } + fw.Write(data) + } + err = zipWriter.Close() + if err != nil { + log.Fatalf("could not finalize zip file: %s", err) + } + err = ioutil.WriteFile(outFile, w.Bytes(), 0600) + if err != nil { + log.Fatalf("could not write zip file to %s: %s", outFile, err) + } + log.Infof("Written zip file to %s", outFile) + }, + } + cmdDump.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to") + cmdSupport.AddCommand(cmdDump) + + return cmdSupport +} diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 3f503e16d..6175adf37 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/csv" "encoding/json" "fmt" @@ -165,7 +166,7 @@ func compInstalledItems(itemType string, args []string, toComplete string) ([]st return comp, cobra.ShellCompDirectiveNoFileComp } -func ListItems(itemTypes []string, args []string, showType bool, showHeader bool, all bool) { +func ListItems(itemTypes []string, args []string, showType bool, showHeader bool, all bool) []byte { var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus) @@ -177,6 +178,8 @@ func ListItems(itemTypes []string, args []string, showType bool, showHeader bool hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all) } + w := bytes.NewBuffer(nil) + if csConfig.Cscli.Output == "human" { for _, itemType := range itemTypes { var statuses []cwhub.ItemHubStatus @@ -185,8 +188,8 @@ func ListItems(itemTypes []string, args []string, showType bool, showHeader bool log.Errorf("unknown item type: %s", itemType) continue } - fmt.Println(strings.ToUpper(itemType)) - table := tablewriter.NewWriter(os.Stdout) + fmt.Fprintf(w, "%s\n", strings.ToUpper(itemType)) + table := tablewriter.NewWriter(w) table.SetCenterSeparator("") table.SetColumnSeparator("") table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) @@ -202,9 +205,9 @@ func ListItems(itemTypes []string, args []string, showType bool, showHeader bool if err != nil { log.Fatalf("failed to unmarshal") } - fmt.Printf("%s", string(x)) + w.Write(x) } else if csConfig.Cscli.Output == "raw" { - csvwriter := csv.NewWriter(os.Stdout) + csvwriter := csv.NewWriter(w) if showHeader { header := []string{"name", "status", "version", "description"} if showType { @@ -244,6 +247,7 @@ func ListItems(itemTypes []string, args []string, showType bool, showHeader bool } csvwriter.Flush() } + return w.Bytes() } func InspectItem(name string, objecitemType string) { diff --git a/go.mod b/go.mod index 33ac270ea..15f57aa4a 100644 --- a/go.mod +++ b/go.mod @@ -85,6 +85,7 @@ require ( github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blackfireio/osinfo v1.0.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/containerd/containerd v1.6.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index 62bd8ef81..c2b01714d 100644 --- a/go.sum +++ b/go.sum @@ -105,6 +105,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blackfireio/osinfo v1.0.3 h1:Yk2t2GTPjBcESv6nDSWZKO87bGMQgO+Hi9OoXPpxX8c= +github.com/blackfireio/osinfo v1.0.3/go.mod h1:Pd987poVNmd5Wsx6PRPw4+w7kLlf9iJxoRKPtPAjOrA= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU= diff --git a/tests/bats/01_base.bats b/tests/bats/01_base.bats index 77b51847e..4e1ddd9e3 100644 --- a/tests/bats/01_base.bats +++ b/tests/bats/01_base.bats @@ -215,8 +215,8 @@ declare stderr run -0 --separate-stderr cscli metrics assert_output --partial "ROUTE" assert_output --partial '/v1/watchers/login' + assert_output --partial "Local Api Metrics:" - assert_stderr --partial "Local Api Metrics:" } @test "'cscli completion' with or without configuration file" { diff --git a/tests/bats/04_nocapi.bats b/tests/bats/04_nocapi.bats index c68cda8d5..f3ef5c44a 100644 --- a/tests/bats/04_nocapi.bats +++ b/tests/bats/04_nocapi.bats @@ -77,6 +77,6 @@ teardown() { run -0 --separate-stderr cscli metrics assert_output --partial "ROUTE" assert_output --partial '/v1/watchers/login' + assert_output --partial "Local Api Metrics:" - assert_stderr --partial "Local Api Metrics:" }