diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go index c7a3e2c56..8dbb41fca 100644 --- a/cmd/crowdsec-cli/collections.go +++ b/cmd/crowdsec-cli/collections.go @@ -254,6 +254,11 @@ func runCollectionsInspect(cmd *cobra.Command, args []string) error { return err } + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + for _, name := range args { if err = InspectItem(name, cwhub.COLLECTIONS, noMetrics); err != nil { return err @@ -292,8 +297,9 @@ func runCollectionsList(cmd *cobra.Command, args []string) error { return err } - // XXX: will happily ignore missing collections - ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all) + if err = ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all); err != nil { + return err + } return nil } diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 7b37ec4c7..bd0f6f77b 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -50,7 +50,7 @@ func runHubList(cmd *cobra.Command, args []string) error { return err } - if err := require.Hub(csConfig); err != nil { + if err = require.Hub(csConfig); err != nil { return err } @@ -62,9 +62,12 @@ func runHubList(cmd *cobra.Command, args []string) error { cwhub.DisplaySummary() - ListItems(color.Output, []string{ + err = ListItems(color.Output, []string{ cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW, - }, args, true, false, all) + }, nil, true, false, all) + if err != nil { + return err + } return nil } diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go new file mode 100644 index 000000000..1eb8e39e8 --- /dev/null +++ b/cmd/crowdsec-cli/item_metrics.go @@ -0,0 +1,245 @@ +package main + +import ( + "fmt" + "math" + "net/http" + "strconv" + "strings" + "time" + + "github.com/fatih/color" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/prom2json" + log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/go-cs-lib/trace" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func ShowMetrics(hubItem *cwhub.Item) { + switch hubItem.Type { + case cwhub.PARSERS: + metrics := GetParserMetric(prometheusURL, hubItem.Name) + parserMetricsTable(color.Output, hubItem.Name, metrics) + case cwhub.SCENARIOS: + metrics := GetScenarioMetric(prometheusURL, hubItem.Name) + scenarioMetricsTable(color.Output, hubItem.Name, metrics) + case cwhub.COLLECTIONS: + for _, item := range hubItem.Parsers { + metrics := GetParserMetric(prometheusURL, item) + parserMetricsTable(color.Output, item, metrics) + } + for _, item := range hubItem.Scenarios { + metrics := GetScenarioMetric(prometheusURL, item) + scenarioMetricsTable(color.Output, item, metrics) + } + for _, item := range hubItem.Collections { + hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item) + if hubItem == nil { + log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name) + } + ShowMetrics(hubItem) + } + default: + log.Errorf("item of type '%s' is unknown", hubItem.Type) + } +} + +// GetParserMetric is a complete rip from prom2json +func GetParserMetric(url string, itemName string) map[string]map[string]int { + stats := make(map[string]map[string]int) + + result := GetPrometheusMetric(url) + for idx, fam := range result { + if !strings.HasPrefix(fam.Name, "cs_") { + continue + } + log.Tracef("round %d", idx) + for _, m := range fam.Metrics { + metric, ok := m.(prom2json.Metric) + if !ok { + log.Debugf("failed to convert metric to prom2json.Metric") + continue + } + name, ok := metric.Labels["name"] + if !ok { + log.Debugf("no name in Metric %v", metric.Labels) + } + if name != itemName { + continue + } + source, ok := metric.Labels["source"] + if !ok { + log.Debugf("no source in Metric %v", metric.Labels) + } else { + if srctype, ok := metric.Labels["type"]; ok { + source = srctype + ":" + source + } + } + value := m.(prom2json.Metric).Value + fval, err := strconv.ParseFloat(value, 32) + if err != nil { + log.Errorf("Unexpected int value %s : %s", value, err) + continue + } + ival := int(fval) + + switch fam.Name { + case "cs_reader_hits_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + stats[source]["parsed"] = 0 + stats[source]["reads"] = 0 + stats[source]["unparsed"] = 0 + stats[source]["hits"] = 0 + } + stats[source]["reads"] += ival + case "cs_parser_hits_ok_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["parsed"] += ival + case "cs_parser_hits_ko_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["unparsed"] += ival + case "cs_node_hits_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["hits"] += ival + case "cs_node_hits_ok_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["parsed"] += ival + case "cs_node_hits_ko_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["unparsed"] += ival + default: + continue + } + } + } + return stats +} + +func GetScenarioMetric(url string, itemName string) map[string]int { + stats := make(map[string]int) + + stats["instantiation"] = 0 + stats["curr_count"] = 0 + stats["overflow"] = 0 + stats["pour"] = 0 + stats["underflow"] = 0 + + result := GetPrometheusMetric(url) + for idx, fam := range result { + if !strings.HasPrefix(fam.Name, "cs_") { + continue + } + log.Tracef("round %d", idx) + for _, m := range fam.Metrics { + metric, ok := m.(prom2json.Metric) + if !ok { + log.Debugf("failed to convert metric to prom2json.Metric") + continue + } + name, ok := metric.Labels["name"] + if !ok { + log.Debugf("no name in Metric %v", metric.Labels) + } + if name != itemName { + continue + } + value := m.(prom2json.Metric).Value + fval, err := strconv.ParseFloat(value, 32) + if err != nil { + log.Errorf("Unexpected int value %s : %s", value, err) + continue + } + ival := int(fval) + + switch fam.Name { + case "cs_bucket_created_total": + stats["instantiation"] += ival + case "cs_buckets": + stats["curr_count"] += ival + case "cs_bucket_overflowed_total": + stats["overflow"] += ival + case "cs_bucket_poured_total": + stats["pour"] += ival + case "cs_bucket_underflowed_total": + stats["underflow"] += ival + default: + continue + } + } + } + return stats +} + +func GetPrometheusMetric(url string) []*prom2json.Family { + mfChan := make(chan *dto.MetricFamily, 1024) + + // Start with the DefaultTransport for sane defaults. + transport := http.DefaultTransport.(*http.Transport).Clone() + // Conservatively disable HTTP keep-alives as this program will only + // ever need a single HTTP request. + transport.DisableKeepAlives = true + // Timeout early if the server doesn't even return the headers. + transport.ResponseHeaderTimeout = time.Minute + + go func() { + defer trace.CatchPanic("crowdsec/GetPrometheusMetric") + err := prom2json.FetchMetricFamilies(url, mfChan, transport) + if err != nil { + log.Fatalf("failed to fetch prometheus metrics : %v", err) + } + }() + + result := []*prom2json.Family{} + for mf := range mfChan { + result = append(result, prom2json.NewFamily(mf)) + } + log.Debugf("Finished reading prometheus output, %d entries", len(result)) + + return result +} + +type unit struct { + value int64 + symbol string +} + +var ranges = []unit{ + {value: 1e18, symbol: "E"}, + {value: 1e15, symbol: "P"}, + {value: 1e12, symbol: "T"}, + {value: 1e9, symbol: "G"}, + {value: 1e6, symbol: "M"}, + {value: 1e3, symbol: "k"}, + {value: 1, symbol: ""}, +} + +func formatNumber(num int) string { + goodUnit := unit{} + for _, u := range ranges { + if int64(num) >= u.value { + goodUnit = u + break + } + } + + if goodUnit.value == 1 { + return fmt.Sprintf("%d%s", num, goodUnit.symbol) + } + + res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100 + return fmt.Sprintf("%.2f%s", res, goodUnit.symbol) +} diff --git a/cmd/crowdsec-cli/item_suggest.go b/cmd/crowdsec-cli/item_suggest.go new file mode 100644 index 000000000..c52239fb4 --- /dev/null +++ b/cmd/crowdsec-cli/item_suggest.go @@ -0,0 +1,93 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/agext/levenshtein" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "slices" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +const MaxDistance = 7 + +func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) { + errMsg := "" + if score < MaxDistance { + errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem) + } else { + errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType) + } + if ignoreErr { + log.Error(errMsg) + } else { + log.Fatalf(errMsg) + } +} + +func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { + allItems := make([]string, 0) + nearestScore := 100 + nearestItem := &cwhub.Item{} + hubItems := cwhub.GetItemMap(itemType) + for _, item := range hubItems { + allItems = append(allItems, item.Name) + } + + for _, s := range allItems { + d := levenshtein.Distance(itemName, s, nil) + if d < nearestScore { + nearestScore = d + nearestItem = cwhub.GetItem(itemType, s) + } + } + return nearestItem, nearestScore +} + +func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if err := require.Hub(csConfig); err != nil { + return nil, cobra.ShellCompDirectiveDefault + } + + comp := make([]string, 0) + hubItems := cwhub.GetItemMap(itemType) + for _, item := range hubItems { + if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { + comp = append(comp, item.Name) + } + } + cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) + return comp, cobra.ShellCompDirectiveNoFileComp +} + +func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if err := require.Hub(csConfig); err != nil { + return nil, cobra.ShellCompDirectiveDefault + } + + items, err := cwhub.GetInstalledItemsAsString(itemType) + if err != nil { + cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) + return nil, cobra.ShellCompDirectiveDefault + } + + comp := make([]string, 0) + + if toComplete != "" { + for _, item := range items { + if strings.Contains(item, toComplete) { + comp = append(comp, item) + } + } + } else { + comp = items + } + + cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) + + return comp, cobra.ShellCompDirectiveNoFileComp +} diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go new file mode 100644 index 000000000..cebf5b723 --- /dev/null +++ b/cmd/crowdsec-cli/items.go @@ -0,0 +1,174 @@ +package main + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "io" + "slices" + "sort" + "strings" + + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + + +func selectItems(itemType string, args []string, installedOnly bool) ([]string, error) { + itemNames := cwhub.GetItemNames(itemType) + + notExist := []string{} + if len(args) > 0 { + installedOnly = false + for _, arg := range args { + if !slices.Contains(itemNames, arg) { + notExist = append(notExist, arg) + } + } + } + + if len(notExist) > 0 { + return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType) + } + + if len(args) > 0 { + itemNames = args + } + + if installedOnly { + installed := []string{} + for _, item := range itemNames { + if cwhub.GetItem(itemType, item).Installed { + installed = append(installed, item) + } + } + return installed, nil + } + return itemNames, nil +} + + +func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error { + var err error + items := make(map[string][]string) + for _, itemType := range itemTypes { + if items[itemType], err = selectItems(itemType, args, !all); err != nil { + return err + } + } + + if csConfig.Cscli.Output == "human" { + for _, itemType := range itemTypes { + listHubItemTable(out, "\n"+strings.ToUpper(itemType), itemType, items[itemType]) + } + } else if csConfig.Cscli.Output == "json" { + type itemHubStatus struct { + Name string `json:"name"` + LocalVersion string `json:"local_version"` + LocalPath string `json:"local_path"` + Description string `json:"description"` + UTF8Status string `json:"utf8_status"` + Status string `json:"status"` + } + + hubStatus := make(map[string][]itemHubStatus) + for _, itemType := range itemTypes { + // empty slice in case there are no items of this type + hubStatus[itemType] = make([]itemHubStatus, len(items[itemType])) + for i, itemName := range items[itemType] { + item := cwhub.GetItem(itemType, itemName) + status, emo := item.Status() + hubStatus[itemType][i] = itemHubStatus{ + Name: item.Name, + LocalVersion: item.LocalVersion, + LocalPath: item.LocalPath, + Description: item.Description, + Status: status, + UTF8Status: fmt.Sprintf("%v %s", emo, status), + } + } + h := hubStatus[itemType] + sort.Slice(h, func(i, j int) bool { return h[i].Name < h[j].Name }) + } + x, err := json.MarshalIndent(hubStatus, "", " ") + if err != nil { + log.Fatalf("failed to unmarshal") + } + out.Write(x) + } else if csConfig.Cscli.Output == "raw" { + csvwriter := csv.NewWriter(out) + if showHeader { + header := []string{"name", "status", "version", "description"} + if showType { + header = append(header, "type") + } + err := csvwriter.Write(header) + if err != nil { + log.Fatalf("failed to write header: %s", err) + } + + } + for _, itemType := range itemTypes { + for _, itemName := range items[itemType] { + item := cwhub.GetItem(itemType, itemName) + status, _ := item.Status() + if item.LocalVersion == "" { + item.LocalVersion = "n/a" + } + row := []string{ + item.Name, + status, + item.LocalVersion, + item.Description, + } + if showType { + row = append(row, itemType) + } + err := csvwriter.Write(row) + if err != nil { + log.Fatalf("failed to write raw output : %s", err) + } + } + } + csvwriter.Flush() + } + return nil +} + +func InspectItem(name string, itemType string, noMetrics bool) error { + hubItem := cwhub.GetItem(itemType, name) + if hubItem == nil { + return fmt.Errorf("can't find '%s' in %s", name, itemType) + } + + var ( + b []byte + err error + ) + + switch csConfig.Cscli.Output { + case "human", "raw": + b, err = yaml.Marshal(*hubItem) + if err != nil { + return fmt.Errorf("unable to marshal item: %s", err) + } + case "json": + b, err = json.MarshalIndent(*hubItem, "", " ") + if err != nil { + return fmt.Errorf("unable to marshal item: %s", err) + } + } + + fmt.Printf("%s", string(b)) + + if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { + return nil + } + + fmt.Printf("\nCurrent metrics: \n") + ShowMetrics(hubItem) + + return nil +} diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go index aee11abd0..49e301a6c 100644 --- a/cmd/crowdsec-cli/parsers.go +++ b/cmd/crowdsec-cli/parsers.go @@ -279,8 +279,9 @@ func runParsersList(cmd *cobra.Command, args []string) error { return err } - // XXX: will happily ignore missing parsers - ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all) + if err = ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all); err != nil { + return err + } return nil } diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go index e8926d7cd..7ff245a9f 100644 --- a/cmd/crowdsec-cli/postoverflows.go +++ b/cmd/crowdsec-cli/postoverflows.go @@ -280,8 +280,9 @@ func runPostOverflowsList(cmd *cobra.Command, args []string) error { return err } - // XXX: will happily ignore missing postoverflows - ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all) + if err = ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all); err != nil { + return err + } return nil } diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index 25044d5f9..863e50236 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -279,8 +279,9 @@ func runScenariosList(cmd *cobra.Command, args []string) error { return err } - // XXX: will happily ignore missing scenarios - ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all) + if err = ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all); err != nil { + return err + } return nil } diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index ed6d9a858..647738372 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -131,7 +131,9 @@ func collectOSInfo() ([]byte, error) { func collectHubItems(itemType string) []byte { out := bytes.NewBuffer(nil) log.Infof("Collecting %s list", itemType) - ListItems(out, []string{itemType}, []string{}, false, true, false) + if err := ListItems(out, []string{itemType}, []string{}, false, true, false); err != nil { + log.Warnf("could not collect %s list: %s", itemType, err) + } return out.Bytes() } diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 27c79f664..eb7fb51e0 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -1,36 +1,17 @@ package main import ( - "encoding/csv" - "encoding/json" "fmt" - "io" - "math" "net" - "net/http" - "slices" - "strconv" "strings" - "time" - "github.com/fatih/color" - dto "github.com/prometheus/client_model/go" - "github.com/prometheus/prom2json" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/agext/levenshtein" - "gopkg.in/yaml.v2" - "github.com/crowdsecurity/go-cs-lib/trace" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/types" ) -const MaxDistance = 7 - func printHelp(cmd *cobra.Command) { err := cmd.Help() if err != nil { @@ -38,189 +19,6 @@ func printHelp(cmd *cobra.Command) { } } -func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) { - errMsg := "" - if score < MaxDistance { - errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem) - } else { - errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType) - } - if ignoreErr { - log.Error(errMsg) - } else { - log.Fatalf(errMsg) - } -} - -func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { - allItems := make([]string, 0) - nearestScore := 100 - nearestItem := &cwhub.Item{} - hubItems := cwhub.GetHubStatusForItemType(itemType, "", true) - for _, item := range hubItems { - allItems = append(allItems, item.Name) - } - - for _, s := range allItems { - d := levenshtein.Distance(itemName, s, nil) - if d < nearestScore { - nearestScore = d - nearestItem = cwhub.GetItem(itemType, s) - } - } - return nearestItem, nearestScore -} - -func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { - return nil, cobra.ShellCompDirectiveDefault - } - - comp := make([]string, 0) - hubItems := cwhub.GetHubStatusForItemType(itemType, "", true) - for _, item := range hubItems { - if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { - comp = append(comp, item.Name) - } - } - cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) - return comp, cobra.ShellCompDirectiveNoFileComp -} - -func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { - return nil, cobra.ShellCompDirectiveDefault - } - - items, err := cwhub.GetInstalledItemsAsString(itemType) - if err != nil { - cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) - return nil, cobra.ShellCompDirectiveDefault - } - - comp := make([]string, 0) - - if toComplete != "" { - for _, item := range items { - if strings.Contains(item, toComplete) { - comp = append(comp, item) - } - } - } else { - comp = items - } - - cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) - - return comp, cobra.ShellCompDirectiveNoFileComp -} - -func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) { - var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus) - - for _, itemType := range itemTypes { - itemName := "" - if len(args) == 1 { - itemName = args[0] - } - hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all) - } - - if csConfig.Cscli.Output == "human" { - for _, itemType := range itemTypes { - var statuses []cwhub.ItemHubStatus - var ok bool - if statuses, ok = hubStatusByItemType[itemType]; !ok { - log.Errorf("unknown item type: %s", itemType) - continue - } - listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses) - } - } else if csConfig.Cscli.Output == "json" { - x, err := json.MarshalIndent(hubStatusByItemType, "", " ") - if err != nil { - log.Fatalf("failed to unmarshal") - } - out.Write(x) - } else if csConfig.Cscli.Output == "raw" { - csvwriter := csv.NewWriter(out) - if showHeader { - header := []string{"name", "status", "version", "description"} - if showType { - header = append(header, "type") - } - err := csvwriter.Write(header) - if err != nil { - log.Fatalf("failed to write header: %s", err) - } - - } - for _, itemType := range itemTypes { - var statuses []cwhub.ItemHubStatus - var ok bool - if statuses, ok = hubStatusByItemType[itemType]; !ok { - log.Errorf("unknown item type: %s", itemType) - continue - } - for _, status := range statuses { - if status.LocalVersion == "" { - status.LocalVersion = "n/a" - } - row := []string{ - status.Name, - status.Status, - status.LocalVersion, - status.Description, - } - if showType { - row = append(row, itemType) - } - err := csvwriter.Write(row) - if err != nil { - log.Fatalf("failed to write raw output : %s", err) - } - } - } - csvwriter.Flush() - } -} - -func InspectItem(name string, itemType string, noMetrics bool) error { - hubItem := cwhub.GetItem(itemType, name) - if hubItem == nil { - return fmt.Errorf("can't find '%s' in %s", name, itemType) - } - - var ( - b []byte - err error - ) - - switch csConfig.Cscli.Output { - case "human", "raw": - b, err = yaml.Marshal(*hubItem) - if err != nil { - return fmt.Errorf("unable to marshal item: %s", err) - } - case "json": - b, err = json.MarshalIndent(*hubItem, "", " ") - if err != nil { - return fmt.Errorf("unable to marshal item: %s", err) - } - } - - fmt.Printf("%s", string(b)) - - if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { - return nil - } - - fmt.Printf("\nCurrent metrics: \n") - ShowMetrics(hubItem) - - return nil -} - func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error { /*if a range is provided, change the scope*/ @@ -251,232 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value * return nil } -func ShowMetrics(hubItem *cwhub.Item) { - switch hubItem.Type { - case cwhub.PARSERS: - metrics := GetParserMetric(hubItem.Name) - parserMetricsTable(color.Output, hubItem.Name, metrics) - case cwhub.SCENARIOS: - metrics := GetScenarioMetric(hubItem.Name) - scenarioMetricsTable(color.Output, hubItem.Name, metrics) - case cwhub.COLLECTIONS: - for _, item := range hubItem.Parsers { - metrics := GetParserMetric(item) - parserMetricsTable(color.Output, item, metrics) - } - for _, item := range hubItem.Scenarios { - metrics := GetScenarioMetric(item) - scenarioMetricsTable(color.Output, item, metrics) - } - for _, item := range hubItem.Collections { - hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item) - if hubItem == nil { - log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name) - } - ShowMetrics(hubItem) - } - default: - log.Errorf("item of type '%s' is unknown", hubItem.Type) - } -} - -// GetParserMetric is a complete rip from prom2json -func GetParserMetric(itemName string) map[string]map[string]int { - stats := make(map[string]map[string]int) - - result := GetPrometheusMetric() - for idx, fam := range result { - if !strings.HasPrefix(fam.Name, "cs_") { - continue - } - log.Tracef("round %d", idx) - for _, m := range fam.Metrics { - metric, ok := m.(prom2json.Metric) - if !ok { - log.Debugf("failed to convert metric to prom2json.Metric") - continue - } - name, ok := metric.Labels["name"] - if !ok { - log.Debugf("no name in Metric %v", metric.Labels) - } - if name != itemName { - continue - } - source, ok := metric.Labels["source"] - if !ok { - log.Debugf("no source in Metric %v", metric.Labels) - } else { - if srctype, ok := metric.Labels["type"]; ok { - source = srctype + ":" + source - } - } - value := m.(prom2json.Metric).Value - fval, err := strconv.ParseFloat(value, 32) - if err != nil { - log.Errorf("Unexpected int value %s : %s", value, err) - continue - } - ival := int(fval) - - switch fam.Name { - case "cs_reader_hits_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - stats[source]["parsed"] = 0 - stats[source]["reads"] = 0 - stats[source]["unparsed"] = 0 - stats[source]["hits"] = 0 - } - stats[source]["reads"] += ival - case "cs_parser_hits_ok_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["parsed"] += ival - case "cs_parser_hits_ko_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["unparsed"] += ival - case "cs_node_hits_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["hits"] += ival - case "cs_node_hits_ok_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["parsed"] += ival - case "cs_node_hits_ko_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["unparsed"] += ival - default: - continue - } - } - } - return stats -} - -func GetScenarioMetric(itemName string) map[string]int { - stats := make(map[string]int) - - stats["instantiation"] = 0 - stats["curr_count"] = 0 - stats["overflow"] = 0 - stats["pour"] = 0 - stats["underflow"] = 0 - - result := GetPrometheusMetric() - for idx, fam := range result { - if !strings.HasPrefix(fam.Name, "cs_") { - continue - } - log.Tracef("round %d", idx) - for _, m := range fam.Metrics { - metric, ok := m.(prom2json.Metric) - if !ok { - log.Debugf("failed to convert metric to prom2json.Metric") - continue - } - name, ok := metric.Labels["name"] - if !ok { - log.Debugf("no name in Metric %v", metric.Labels) - } - if name != itemName { - continue - } - value := m.(prom2json.Metric).Value - fval, err := strconv.ParseFloat(value, 32) - if err != nil { - log.Errorf("Unexpected int value %s : %s", value, err) - continue - } - ival := int(fval) - - switch fam.Name { - case "cs_bucket_created_total": - stats["instantiation"] += ival - case "cs_buckets": - stats["curr_count"] += ival - case "cs_bucket_overflowed_total": - stats["overflow"] += ival - case "cs_bucket_poured_total": - stats["pour"] += ival - case "cs_bucket_underflowed_total": - stats["underflow"] += ival - default: - continue - } - } - } - return stats -} - -func GetPrometheusMetric() []*prom2json.Family { - mfChan := make(chan *dto.MetricFamily, 1024) - - // Start with the DefaultTransport for sane defaults. - transport := http.DefaultTransport.(*http.Transport).Clone() - // Conservatively disable HTTP keep-alives as this program will only - // ever need a single HTTP request. - transport.DisableKeepAlives = true - // Timeout early if the server doesn't even return the headers. - transport.ResponseHeaderTimeout = time.Minute - - go func() { - defer trace.CatchPanic("crowdsec/GetPrometheusMetric") - err := prom2json.FetchMetricFamilies(csConfig.Cscli.PrometheusUrl, mfChan, transport) - if err != nil { - log.Fatalf("failed to fetch prometheus metrics : %v", err) - } - }() - - result := []*prom2json.Family{} - for mf := range mfChan { - result = append(result, prom2json.NewFamily(mf)) - } - log.Debugf("Finished reading prometheus output, %d entries", len(result)) - - return result -} - -type unit struct { - value int64 - symbol string -} - -var ranges = []unit{ - {value: 1e18, symbol: "E"}, - {value: 1e15, symbol: "P"}, - {value: 1e12, symbol: "T"}, - {value: 1e9, symbol: "G"}, - {value: 1e6, symbol: "M"}, - {value: 1e3, symbol: "k"}, - {value: 1, symbol: ""}, -} - -func formatNumber(num int) string { - goodUnit := unit{} - for _, u := range ranges { - if int64(num) >= u.value { - goodUnit = u - break - } - } - - if goodUnit.value == 1 { - return fmt.Sprintf("%d%s", num, goodUnit.symbol) - } - - res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100 - return fmt.Sprintf("%.2f%s", res, goodUnit.symbol) -} - func getDBClient() (*database.Client, error) { var err error if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI { @@ -510,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string { } return slice - } diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index 16f42d72a..840250f9c 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -10,14 +10,16 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) { +func listHubItemTable(out io.Writer, title string, itemType string, itemNames []string) { t := newLightTable(out) t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) - for _, status := range statuses { - t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath) + for itemName := range itemNames { + item := cwhub.GetItem(itemType, itemNames[itemName]) + status, emo := item.Status() + t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.LocalVersion, item.LocalPath) } renderTableTitle(out, title) t.Render() diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 2fe45bcda..bf4d69d6d 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -1,10 +1,13 @@ +// Package cwhub is responsible for installing and upgrading the local hub files. +// +// This includes retrieving the index, the items to install (parsers, scenarios, data files...) +// and managing the dependencies and taints. package cwhub import ( "fmt" "os" "path/filepath" - "sort" "strings" "github.com/enescakir/emoji" @@ -36,18 +39,12 @@ var ( hubIdx map[string]map[string]Item ) +// ItemVersion is used to detect the version of a given item +// by comparing the hash of each version to the local file. +// If the item does not match any known version, it is considered tainted. type ItemVersion struct { - Digest string `json:"digest,omitempty"` // meow - Deprecated bool `json:"deprecated,omitempty"` -} - -type ItemHubStatus struct { - Name string `json:"name"` - LocalVersion string `json:"local_version"` - LocalPath string `json:"local_path"` - Description string `json:"description"` - UTF8Status string `json:"utf8_status"` - Status string `json:"status"` + Digest string `json:"digest,omitempty"` // meow + Deprecated bool `json:"deprecated,omitempty"` // XXX: do we keep this? } // Item can be: parser, scenario, collection.. @@ -84,7 +81,7 @@ type Item struct { Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` } -func (i *Item) status() (string, emoji.Emoji) { +func (i *Item) Status() (string, emoji.Emoji) { status := "disabled" ok := false @@ -124,19 +121,6 @@ func (i *Item) status() (string, emoji.Emoji) { return status, emo } -func (i *Item) hubStatus() ItemHubStatus { - status, emo := i.status() - - return ItemHubStatus{ - Name: i.Name, - LocalVersion: i.LocalVersion, - LocalPath: i.LocalPath, - Description: i.Description, - Status: status, - UTF8Status: fmt.Sprintf("%v %s", emo, status), - } -} - // versionStatus: semver requires 'v' prefix func (i *Item) versionStatus() int { return semver.Compare("v"+i.Version, "v"+i.LocalVersion) @@ -206,6 +190,23 @@ func GetItem(itemType string, itemName string) *Item { return nil } +// GetItemNames returns the list of item (full) names for a given type +// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx +// The names can be used to retrieve the item with GetItem() +func GetItemNames(itemType string) []string { + m := GetItemMap(itemType) + if m == nil { + return nil + } + + names := make([]string, 0, len(m)) + for k := range m { + names = append(names, k) + } + + return names +} + func AddItem(itemType string, item Item) error { for _, itype := range ItemTypes { if itype == itemType { @@ -257,32 +258,3 @@ func GetInstalledItemsAsString(itemType string) ([]string, error) { return retStr, nil } - -// Returns a slice of entries for packages: name, status, local_path, local_version, utf8_status (fancy) -func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus { - if _, ok := hubIdx[itemType]; !ok { - log.Errorf("type %s doesn't exist", itemType) - - return nil - } - - ret := make([]ItemHubStatus, 0) - - // remember, you do it for the user :) - for _, item := range hubIdx[itemType] { - if name != "" && name != item.Name { - // user has requested a specific name - continue - } - // Only enabled items ? - if !all && !item.Installed { - continue - } - // Check the item status - ret = append(ret, item.hubStatus()) - } - - sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) - - return ret -} diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 2fef828c2..9fafb3f1c 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -53,7 +53,7 @@ func TestItemStatus(t *testing.T) { item.Local = false item.Tainted = false - txt, _ := item.status() + txt, _ := item.Status() require.Equal(t, "enabled,update-available", txt) item.Installed = false @@ -61,7 +61,7 @@ func TestItemStatus(t *testing.T) { item.Local = true item.Tainted = false - txt, _ = item.status() + txt, _ = item.Status() require.Equal(t, "disabled,local", txt) } @@ -273,10 +273,8 @@ func TestInstallParser(t *testing.T) { for _, it := range hubIdx[PARSERS] { testInstallItem(cfg.Hub, t, it) it = hubIdx[PARSERS][it.Name] - _ = GetHubStatusForItemType(PARSERS, it.Name, false) testTaintItem(cfg.Hub, t, it) it = hubIdx[PARSERS][it.Name] - _ = GetHubStatusForItemType(PARSERS, it.Name, false) testUpdateItem(cfg.Hub, t, it) it = hubIdx[PARSERS][it.Name] testDisableItem(cfg.Hub, t, it) @@ -309,11 +307,6 @@ func TestInstallCollection(t *testing.T) { testUpdateItem(cfg.Hub, t, it) it = hubIdx[COLLECTIONS][it.Name] testDisableItem(cfg.Hub, t, it) - - it = hubIdx[COLLECTIONS][it.Name] - x := GetHubStatusForItemType(COLLECTIONS, it.Name, false) - log.Infof("%+v", x) - break } } diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats index 1840d14ec..77c3dc089 100644 --- a/test/bats/20_hub_parsers.bats +++ b/test/bats/20_hub_parsers.bats @@ -79,41 +79,42 @@ teardown() { assert_output "$expected" } - @test "cscli parsers list [parser]..." { + # non-existent + rune -1 cscli parsers install foo/bar + assert_stderr --partial "can't find 'foo/bar' in parsers" + + # not installed + rune -0 cscli parsers list crowdsecurity/whitelists + assert_output --regexp 'crowdsecurity/whitelists.*disabled' + + # install two items rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth - # list one item + # list an installed item rune -0 cscli parsers list crowdsecurity/whitelists - assert_output --partial "crowdsecurity/whitelists" + assert_output --regexp "crowdsecurity/whitelists.*enabled" refute_output --partial "crowdsecurity/windows-auth" - # list multiple items - rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth + # list multiple installed and non installed items + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs assert_output --partial "crowdsecurity/whitelists" assert_output --partial "crowdsecurity/windows-auth" + assert_output --partial "crowdsecurity/traefik-logs" rune -0 cscli parsers list crowdsecurity/whitelists -o json rune -0 jq '.parsers | length' <(output) assert_output "1" - rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth -o json + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o json rune -0 jq '.parsers | length' <(output) - assert_output "2" + assert_output "3" rune -0 cscli parsers list crowdsecurity/whitelists -o raw rune -0 grep -vc 'name,status,version,description' <(output) assert_output "1" - rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth -o raw + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o raw rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" -} - -@test "cscli parsers list [parser]... (not installed / not existing)" { - skip "not implemented yet" - # not installed - rune -1 cscli parsers list crowdsecurity/whitelists - # not existing - rune -1 cscli parsers list blahblah/blahblah + assert_output "3" } @test "cscli parsers install [parser]..." { diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats index ecec032d1..2eeb6146b 100644 --- a/test/bats/20_hub_scenarios.bats +++ b/test/bats/20_hub_scenarios.bats @@ -77,41 +77,42 @@ teardown() { assert_output "$expected" } - @test "cscli scenarios list [scenario]..." { + # non-existent + rune -1 cscli scenario install foo/bar + assert_stderr --partial "can't find 'foo/bar' in scenarios" + + # not installed + rune -0 cscli scenarios list crowdsecurity/ssh-bf + assert_output --regexp 'crowdsecurity/ssh-bf.*disabled' + + # install two items rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf - # list one item + # list an installed item rune -0 cscli scenarios list crowdsecurity/ssh-bf - assert_output --partial "crowdsecurity/ssh-bf" + assert_output --regexp "crowdsecurity/ssh-bf.*enabled" refute_output --partial "crowdsecurity/telnet-bf" - # list multiple items - rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf + # list multiple installed and non installed items + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf crowdsecurity/aws-bf assert_output --partial "crowdsecurity/ssh-bf" assert_output --partial "crowdsecurity/telnet-bf" + assert_output --partial "crowdsecurity/aws-bf" rune -0 cscli scenarios list crowdsecurity/ssh-bf -o json rune -0 jq '.scenarios | length' <(output) assert_output "1" - rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o json rune -0 jq '.scenarios | length' <(output) - assert_output "2" + assert_output "3" rune -0 cscli scenarios list crowdsecurity/ssh-bf -o raw rune -0 grep -vc 'name,status,version,description' <(output) assert_output "1" - rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o raw rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" -} - -@test "cscli scenarios list [scenario]... (not installed / not existing)" { - skip "not implemented yet" - # not installed - rune -1 cscli scenarios list crowdsecurity/ssh-bf - # not existing - rune -1 cscli scenarios list blahblah/blahblah + assert_output "3" } @test "cscli scenarios install [scenario]..." {