package main import ( "encoding/csv" "encoding/json" "fmt" "io" "math" "net" "net/http" "slices" "strconv" "strings" "time" "github.com/fatih/color" dto "github.com/prometheus/client_model/go" "github.com/prometheus/prom2json" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/agext/levenshtein" "gopkg.in/yaml.v2" "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/types" ) const MaxDistance = 7 func printHelp(cmd *cobra.Command) { err := cmd.Help() if err != nil { log.Fatalf("unable to print help(): %s", err) } } func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) { errMsg := "" if score < MaxDistance { errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem) } else { errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem) } if ignoreErr { log.Error(errMsg) } else { log.Fatalf(errMsg) } } func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { allItems := make([]string, 0) nearestScore := 100 nearestItem := &cwhub.Item{} hubItems := cwhub.GetHubStatusForItemType(itemType, "", true) for _, item := range hubItems { allItems = append(allItems, item.Name) } for _, s := range allItems { d := levenshtein.Distance(itemName, s, nil) if d < nearestScore { nearestScore = d nearestItem = cwhub.GetItem(itemType, s) } } return nearestItem, nearestScore } func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if err := require.Hub(csConfig); err != nil { return nil, cobra.ShellCompDirectiveDefault } comp := make([]string, 0) hubItems := cwhub.GetHubStatusForItemType(itemType, "", true) for _, item := range hubItems { if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { comp = append(comp, item.Name) } } cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) return comp, cobra.ShellCompDirectiveNoFileComp } func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if err := require.Hub(csConfig); err != nil { return nil, cobra.ShellCompDirectiveDefault } items, err := cwhub.GetInstalledItemsAsString(itemType) if err != nil { cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) return nil, cobra.ShellCompDirectiveDefault } comp := make([]string, 0) if toComplete != "" { for _, item := range items { if strings.Contains(item, toComplete) { comp = append(comp, item) } } } else { comp = items } cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) return comp, cobra.ShellCompDirectiveNoFileComp } func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) { var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus) for _, itemType := range itemTypes { itemName := "" if len(args) == 1 { itemName = args[0] } hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all) } if csConfig.Cscli.Output == "human" { for _, itemType := range itemTypes { var statuses []cwhub.ItemHubStatus var ok bool if statuses, ok = hubStatusByItemType[itemType]; !ok { log.Errorf("unknown item type: %s", itemType) continue } listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses) } } else if csConfig.Cscli.Output == "json" { x, err := json.MarshalIndent(hubStatusByItemType, "", " ") if err != nil { log.Fatalf("failed to unmarshal") } out.Write(x) } else if csConfig.Cscli.Output == "raw" { csvwriter := csv.NewWriter(out) if showHeader { header := []string{"name", "status", "version", "description"} if showType { header = append(header, "type") } err := csvwriter.Write(header) if err != nil { log.Fatalf("failed to write header: %s", err) } } for _, itemType := range itemTypes { var statuses []cwhub.ItemHubStatus var ok bool if statuses, ok = hubStatusByItemType[itemType]; !ok { log.Errorf("unknown item type: %s", itemType) continue } for _, status := range statuses { if status.LocalVersion == "" { status.LocalVersion = "n/a" } row := []string{ status.Name, status.Status, status.LocalVersion, status.Description, } if showType { row = append(row, itemType) } err := csvwriter.Write(row) if err != nil { log.Fatalf("failed to write raw output : %s", err) } } } csvwriter.Flush() } } func InspectItem(name string, objecitemType string) { hubItem := cwhub.GetItem(objecitemType, name) if hubItem == nil { log.Fatalf("unable to retrieve item.") } var b []byte var err error switch csConfig.Cscli.Output { case "human", "raw": b, err = yaml.Marshal(*hubItem) if err != nil { log.Fatalf("unable to marshal item : %s", err) } case "json": b, err = json.MarshalIndent(*hubItem, "", " ") if err != nil { log.Fatalf("unable to marshal item : %s", err) } } fmt.Printf("%s", string(b)) if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { return } if prometheusURL == "" { //This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus //But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case listenAddr := csConfig.Prometheus.ListenAddr if listenAddr == "" { listenAddr = "127.0.0.1" } listenPort := csConfig.Prometheus.ListenPort if listenPort == 0 { listenPort = 6060 } prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort) log.Debugf("No prometheus URL provided using: %s", prometheusURL) } fmt.Printf("\nCurrent metrics : \n") ShowMetrics(hubItem) } func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error { /*if a range is provided, change the scope*/ if *ipRange != "" { _, _, err := net.ParseCIDR(*ipRange) if err != nil { return fmt.Errorf("%s isn't a valid range", *ipRange) } } if *ip != "" { ipRepr := net.ParseIP(*ip) if ipRepr == nil { return fmt.Errorf("%s isn't a valid ip", *ip) } } //avoid confusion on scope (ip vs Ip and range vs Range) switch strings.ToLower(*scope) { case "ip": *scope = types.Ip case "range": *scope = types.Range case "country": *scope = types.Country case "as": *scope = types.AS } return nil } func ShowMetrics(hubItem *cwhub.Item) { switch hubItem.Type { case cwhub.PARSERS: metrics := GetParserMetric(prometheusURL, hubItem.Name) parserMetricsTable(color.Output, hubItem.Name, metrics) case cwhub.SCENARIOS: metrics := GetScenarioMetric(prometheusURL, hubItem.Name) scenarioMetricsTable(color.Output, hubItem.Name, metrics) case cwhub.COLLECTIONS: for _, item := range hubItem.Parsers { metrics := GetParserMetric(prometheusURL, item) parserMetricsTable(color.Output, item, metrics) } for _, item := range hubItem.Scenarios { metrics := GetScenarioMetric(prometheusURL, item) scenarioMetricsTable(color.Output, item, metrics) } for _, item := range hubItem.Collections { hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item) if hubItem == nil { log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name) } ShowMetrics(hubItem) } default: log.Errorf("item of type '%s' is unknown", hubItem.Type) } } // GetParserMetric is a complete rip from prom2json func GetParserMetric(url string, itemName string) map[string]map[string]int { stats := make(map[string]map[string]int) result := GetPrometheusMetric(url) for idx, fam := range result { if !strings.HasPrefix(fam.Name, "cs_") { continue } log.Tracef("round %d", idx) for _, m := range fam.Metrics { metric, ok := m.(prom2json.Metric) if !ok { log.Debugf("failed to convert metric to prom2json.Metric") continue } name, ok := metric.Labels["name"] if !ok { log.Debugf("no name in Metric %v", metric.Labels) } if name != itemName { continue } source, ok := metric.Labels["source"] if !ok { log.Debugf("no source in Metric %v", metric.Labels) } else { if srctype, ok := metric.Labels["type"]; ok { source = srctype + ":" + source } } value := m.(prom2json.Metric).Value fval, err := strconv.ParseFloat(value, 32) if err != nil { log.Errorf("Unexpected int value %s : %s", value, err) continue } ival := int(fval) switch fam.Name { case "cs_reader_hits_total": if _, ok := stats[source]; !ok { stats[source] = make(map[string]int) stats[source]["parsed"] = 0 stats[source]["reads"] = 0 stats[source]["unparsed"] = 0 stats[source]["hits"] = 0 } stats[source]["reads"] += ival case "cs_parser_hits_ok_total": if _, ok := stats[source]; !ok { stats[source] = make(map[string]int) } stats[source]["parsed"] += ival case "cs_parser_hits_ko_total": if _, ok := stats[source]; !ok { stats[source] = make(map[string]int) } stats[source]["unparsed"] += ival case "cs_node_hits_total": if _, ok := stats[source]; !ok { stats[source] = make(map[string]int) } stats[source]["hits"] += ival case "cs_node_hits_ok_total": if _, ok := stats[source]; !ok { stats[source] = make(map[string]int) } stats[source]["parsed"] += ival case "cs_node_hits_ko_total": if _, ok := stats[source]; !ok { stats[source] = make(map[string]int) } stats[source]["unparsed"] += ival default: continue } } } return stats } func GetScenarioMetric(url string, itemName string) map[string]int { stats := make(map[string]int) stats["instantiation"] = 0 stats["curr_count"] = 0 stats["overflow"] = 0 stats["pour"] = 0 stats["underflow"] = 0 result := GetPrometheusMetric(url) for idx, fam := range result { if !strings.HasPrefix(fam.Name, "cs_") { continue } log.Tracef("round %d", idx) for _, m := range fam.Metrics { metric, ok := m.(prom2json.Metric) if !ok { log.Debugf("failed to convert metric to prom2json.Metric") continue } name, ok := metric.Labels["name"] if !ok { log.Debugf("no name in Metric %v", metric.Labels) } if name != itemName { continue } value := m.(prom2json.Metric).Value fval, err := strconv.ParseFloat(value, 32) if err != nil { log.Errorf("Unexpected int value %s : %s", value, err) continue } ival := int(fval) switch fam.Name { case "cs_bucket_created_total": stats["instantiation"] += ival case "cs_buckets": stats["curr_count"] += ival case "cs_bucket_overflowed_total": stats["overflow"] += ival case "cs_bucket_poured_total": stats["pour"] += ival case "cs_bucket_underflowed_total": stats["underflow"] += ival default: continue } } } return stats } func GetPrometheusMetric(url string) []*prom2json.Family { mfChan := make(chan *dto.MetricFamily, 1024) // Start with the DefaultTransport for sane defaults. transport := http.DefaultTransport.(*http.Transport).Clone() // Conservatively disable HTTP keep-alives as this program will only // ever need a single HTTP request. transport.DisableKeepAlives = true // Timeout early if the server doesn't even return the headers. transport.ResponseHeaderTimeout = time.Minute go func() { defer trace.CatchPanic("crowdsec/GetPrometheusMetric") err := prom2json.FetchMetricFamilies(url, mfChan, transport) if err != nil { log.Fatalf("failed to fetch prometheus metrics : %v", err) } }() result := []*prom2json.Family{} for mf := range mfChan { result = append(result, prom2json.NewFamily(mf)) } log.Debugf("Finished reading prometheus output, %d entries", len(result)) return result } type unit struct { value int64 symbol string } var ranges = []unit{ {value: 1e18, symbol: "E"}, {value: 1e15, symbol: "P"}, {value: 1e12, symbol: "T"}, {value: 1e9, symbol: "G"}, {value: 1e6, symbol: "M"}, {value: 1e3, symbol: "k"}, {value: 1, symbol: ""}, } func formatNumber(num int) string { goodUnit := unit{} for _, u := range ranges { if int64(num) >= u.value { goodUnit = u break } } if goodUnit.value == 1 { return fmt.Sprintf("%d%s", num, goodUnit.symbol) } res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100 return fmt.Sprintf("%.2f%s", res, goodUnit.symbol) } func getDBClient() (*database.Client, error) { var err error if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI { return nil, err } ret, err := database.NewClient(csConfig.DbConfig) if err != nil { return nil, err } return ret, nil } func removeFromSlice(val string, slice []string) []string { var i int var value string valueFound := false // get the index for i, value = range slice { if value == val { valueFound = true break } } if valueFound { slice[i] = slice[len(slice)-1] slice[len(slice)-1] = "" slice = slice[:len(slice)-1] } return slice }