beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Refact cscli item listing, tests (#2547)

Browse source 
* hub diet; taint tests
* cmd/crowdsec-cli: split utils.go, moved cwhub.GetHubStatusForItemType()
* cscli: refactor hub list commands, fix edge cases
This commit is contained in:
mmetc 2023-10-16 14:33:36 +02:00 committed by Marco Mariani
parent f496bd1692
commit 325003bb69
15 changed files with 608 additions and 542 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmd/crowdsec-cli/collections.go
View file

@ -254,6 +254,11 @@ func runCollectionsInspect(cmd *cobra.Command, args []string) error {
return err
}
noMetrics, err := flags.GetBool("no-metrics")
if err != nil {
return err
}
for _, name := range args {
if err = InspectItem(name, cwhub.COLLECTIONS, noMetrics); err != nil {
return err
@ -292,8 +297,9 @@ func runCollectionsList(cmd *cobra.Command, args []string) error {
return err
}
// XXX: will happily ignore missing collections
ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
if err = ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all); err != nil {
return err
}
return nil
}

9
cmd/crowdsec-cli/hub.go
View file

@ -50,7 +50,7 @@ func runHubList(cmd *cobra.Command, args []string) error {
return err
}
if err := require.Hub(csConfig); err != nil {
if err = require.Hub(csConfig); err != nil {
return err
}
@ -62,9 +62,12 @@ func runHubList(cmd *cobra.Command, args []string) error {
cwhub.DisplaySummary()
ListItems(color.Output, []string{
err = ListItems(color.Output, []string{
cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
}, args, true, false, all)
}, nil, true, false, all)
if err != nil {
return err
}
return nil
}

245
cmd/crowdsec-cli/item_metrics.go Normal file
View file

@ -0,0 +1,245 @@
package main
import (
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func ShowMetrics(hubItem *cwhub.Item) {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(prometheusURL, hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, item := range hubItem.Parsers {
metrics := GetParserMetric(prometheusURL, item)
parserMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Scenarios {
metrics := GetScenarioMetric(prometheusURL, item)
scenarioMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Collections {
hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
if hubItem == nil {
log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
}
ShowMetrics(hubItem)
}
default:
log.Errorf("item of type '%s' is unknown", hubItem.Type)
}
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(url string, itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{value: 1e18, symbol: "E"},
{value: 1e15, symbol: "P"},
{value: 1e12, symbol: "T"},
{value: 1e9, symbol: "G"},
{value: 1e6, symbol: "M"},
{value: 1e3, symbol: "k"},
{value: 1, symbol: ""},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}

93
cmd/crowdsec-cli/item_suggest.go Normal file
View file

@ -0,0 +1,93 @@
package main
import (
"fmt"
"strings"
"github.com/agext/levenshtein"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
const MaxDistance = 7
func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
errMsg := ""
if score < MaxDistance {
errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem)
} else {
errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType)
}
if ignoreErr {
log.Error(errMsg)
} else {
log.Fatalf(errMsg)
}
}
func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
allItems := make([]string, 0)
nearestScore := 100
nearestItem := &cwhub.Item{}
hubItems := cwhub.GetItemMap(itemType)
for _, item := range hubItems {
allItems = append(allItems, item.Name)
}
for _, s := range allItems {
d := levenshtein.Distance(itemName, s, nil)
if d < nearestScore {
nearestScore = d
nearestItem = cwhub.GetItem(itemType, s)
}
}
return nearestItem, nearestScore
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
hubItems := cwhub.GetItemMap(itemType)
for _, item := range hubItems {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := cwhub.GetInstalledItemsAsString(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}

174
cmd/crowdsec-cli/items.go Normal file
View file

@ -0,0 +1,174 @@
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"slices"
"sort"
"strings"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func selectItems(itemType string, args []string, installedOnly bool) ([]string, error) {
itemNames := cwhub.GetItemNames(itemType)
notExist := []string{}
if len(args) > 0 {
installedOnly = false
for _, arg := range args {
if !slices.Contains(itemNames, arg) {
notExist = append(notExist, arg)
}
}
}
if len(notExist) > 0 {
return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
}
if len(args) > 0 {
itemNames = args
}
if installedOnly {
installed := []string{}
for _, item := range itemNames {
if cwhub.GetItem(itemType, item).Installed {
installed = append(installed, item)
}
}
return installed, nil
}
return itemNames, nil
}
func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error {
var err error
items := make(map[string][]string)
for _, itemType := range itemTypes {
if items[itemType], err = selectItems(itemType, args, !all); err != nil {
return err
}
}
if csConfig.Cscli.Output == "human" {
for _, itemType := range itemTypes {
listHubItemTable(out, "\n"+strings.ToUpper(itemType), itemType, items[itemType])
}
} else if csConfig.Cscli.Output == "json" {
type itemHubStatus struct {
Name string `json:"name"`
LocalVersion string `json:"local_version"`
LocalPath string `json:"local_path"`
Description string `json:"description"`
UTF8Status string `json:"utf8_status"`
Status string `json:"status"`
}
hubStatus := make(map[string][]itemHubStatus)
for _, itemType := range itemTypes {
// empty slice in case there are no items of this type
hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
for i, itemName := range items[itemType] {
item := cwhub.GetItem(itemType, itemName)
status, emo := item.Status()
hubStatus[itemType][i] = itemHubStatus{
Name: item.Name,
LocalVersion: item.LocalVersion,
LocalPath: item.LocalPath,
Description: item.Description,
Status: status,
UTF8Status: fmt.Sprintf("%v %s", emo, status),
}
}
h := hubStatus[itemType]
sort.Slice(h, func(i, j int) bool { return h[i].Name < h[j].Name })
}
x, err := json.MarshalIndent(hubStatus, "", " ")
if err != nil {
log.Fatalf("failed to unmarshal")
}
out.Write(x)
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
if showHeader {
header := []string{"name", "status", "version", "description"}
if showType {
header = append(header, "type")
}
err := csvwriter.Write(header)
if err != nil {
log.Fatalf("failed to write header: %s", err)
}
}
for _, itemType := range itemTypes {
for _, itemName := range items[itemType] {
item := cwhub.GetItem(itemType, itemName)
status, _ := item.Status()
if item.LocalVersion == "" {
item.LocalVersion = "n/a"
}
row := []string{
item.Name,
status,
item.LocalVersion,
item.Description,
}
if showType {
row = append(row, itemType)
}
err := csvwriter.Write(row)
if err != nil {
log.Fatalf("failed to write raw output : %s", err)
}
}
}
csvwriter.Flush()
}
return nil
}
func InspectItem(name string, itemType string, noMetrics bool) error {
hubItem := cwhub.GetItem(itemType, name)
if hubItem == nil {
return fmt.Errorf("can't find '%s' in %s", name, itemType)
}
var (
b []byte
err error
)
switch csConfig.Cscli.Output {
case "human", "raw":
b, err = yaml.Marshal(*hubItem)
if err != nil {
return fmt.Errorf("unable to marshal item: %s", err)
}
case "json":
b, err = json.MarshalIndent(*hubItem, "", " ")
if err != nil {
return fmt.Errorf("unable to marshal item: %s", err)
}
}
fmt.Printf("%s", string(b))
if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
return nil
}
fmt.Printf("\nCurrent metrics: \n")
ShowMetrics(hubItem)
return nil
}

5
cmd/crowdsec-cli/parsers.go
View file

@ -279,8 +279,9 @@ func runParsersList(cmd *cobra.Command, args []string) error {
return err
}
// XXX: will happily ignore missing parsers
ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
if err = ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all); err != nil {
return err
}
return nil
}

5
cmd/crowdsec-cli/postoverflows.go
View file

@ -280,8 +280,9 @@ func runPostOverflowsList(cmd *cobra.Command, args []string) error {
return err
}
// XXX: will happily ignore missing postoverflows
ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
if err = ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all); err != nil {
return err
}
return nil
}

5
cmd/crowdsec-cli/scenarios.go
View file

@ -279,8 +279,9 @@ func runScenariosList(cmd *cobra.Command, args []string) error {
return err
}
// XXX: will happily ignore missing scenarios
ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
if err = ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all); err != nil {
return err
}
return nil
}

4
cmd/crowdsec-cli/support.go
View file

@ -131,7 +131,9 @@ func collectOSInfo() ([]byte, error) {
func collectHubItems(itemType string) []byte {
out := bytes.NewBuffer(nil)
log.Infof("Collecting %s list", itemType)
ListItems(out, []string{itemType}, []string{}, false, true, false)
if err := ListItems(out, []string{itemType}, []string{}, false, true, false); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
return out.Bytes()
}

429
cmd/crowdsec-cli/utils.go
View file

@ -1,36 +1,17 @@
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"math"
"net"
"net/http"
"slices"
"strconv"
"strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/agext/levenshtein"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const MaxDistance = 7
func printHelp(cmd *cobra.Command) {
err := cmd.Help()
if err != nil {
@ -38,189 +19,6 @@ func printHelp(cmd *cobra.Command) {
}
}
func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
errMsg := ""
if score < MaxDistance {
errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem)
} else {
errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType)
}
if ignoreErr {
log.Error(errMsg)
} else {
log.Fatalf(errMsg)
}
}
func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
allItems := make([]string, 0)
nearestScore := 100
nearestItem := &cwhub.Item{}
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
allItems = append(allItems, item.Name)
}
for _, s := range allItems {
d := levenshtein.Distance(itemName, s, nil)
if d < nearestScore {
nearestScore = d
nearestItem = cwhub.GetItem(itemType, s)
}
}
return nearestItem, nearestScore
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := cwhub.GetInstalledItemsAsString(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
for _, itemType := range itemTypes {
itemName := ""
if len(args) == 1 {
itemName = args[0]
}
hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
}
if csConfig.Cscli.Output == "human" {
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
}
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
if err != nil {
log.Fatalf("failed to unmarshal")
}
out.Write(x)
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
if showHeader {
header := []string{"name", "status", "version", "description"}
if showType {
header = append(header, "type")
}
err := csvwriter.Write(header)
if err != nil {
log.Fatalf("failed to write header: %s", err)
}
}
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
for _, status := range statuses {
if status.LocalVersion == "" {
status.LocalVersion = "n/a"
}
row := []string{
status.Name,
status.Status,
status.LocalVersion,
status.Description,
}
if showType {
row = append(row, itemType)
}
err := csvwriter.Write(row)
if err != nil {
log.Fatalf("failed to write raw output : %s", err)
}
}
}
csvwriter.Flush()
}
}
func InspectItem(name string, itemType string, noMetrics bool) error {
hubItem := cwhub.GetItem(itemType, name)
if hubItem == nil {
return fmt.Errorf("can't find '%s' in %s", name, itemType)
}
var (
b []byte
err error
)
switch csConfig.Cscli.Output {
case "human", "raw":
b, err = yaml.Marshal(*hubItem)
if err != nil {
return fmt.Errorf("unable to marshal item: %s", err)
}
case "json":
b, err = json.MarshalIndent(*hubItem, "", " ")
if err != nil {
return fmt.Errorf("unable to marshal item: %s", err)
}
}
fmt.Printf("%s", string(b))
if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
return nil
}
fmt.Printf("\nCurrent metrics: \n")
ShowMetrics(hubItem)
return nil
}
func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
/*if a range is provided, change the scope*/
@ -251,232 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
return nil
}
func ShowMetrics(hubItem *cwhub.Item) {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, item := range hubItem.Parsers {
metrics := GetParserMetric(item)
parserMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Scenarios {
metrics := GetScenarioMetric(item)
scenarioMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Collections {
hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
if hubItem == nil {
log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
}
ShowMetrics(hubItem)
}
default:
log.Errorf("item of type '%s' is unknown", hubItem.Type)
}
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric()
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric()
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
func GetPrometheusMetric() []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(csConfig.Cscli.PrometheusUrl, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{value: 1e18, symbol: "E"},
{value: 1e15, symbol: "P"},
{value: 1e12, symbol: "T"},
{value: 1e9, symbol: "G"},
{value: 1e6, symbol: "M"},
{value: 1e3, symbol: "k"},
{value: 1, symbol: ""},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}
func getDBClient() (*database.Client, error) {
var err error
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
@ -510,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string {
}
return slice
}

8
cmd/crowdsec-cli/utils_table.go
View file

@ -10,14 +10,16 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
func listHubItemTable(out io.Writer, title string, itemType string, itemNames []string) {
t := newLightTable(out)
t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, status := range statuses {
t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
for itemName := range itemNames {
item := cwhub.GetItem(itemType, itemNames[itemName])
status, emo := item.Status()
t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.LocalVersion, item.LocalPath)
}
renderTableTitle(out, title)
t.Render()

82
pkg/cwhub/cwhub.go
View file

@ -1,10 +1,13 @@
// Package cwhub is responsible for installing and upgrading the local hub files.
//
// This includes retrieving the index, the items to install (parsers, scenarios, data files...)
// and managing the dependencies and taints.
package cwhub
import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"github.com/enescakir/emoji"
@ -36,18 +39,12 @@ var (
hubIdx map[string]map[string]Item
)
// ItemVersion is used to detect the version of a given item
// by comparing the hash of each version to the local file.
// If the item does not match any known version, it is considered tainted.
type ItemVersion struct {
Digest string `json:"digest,omitempty"` // meow
Deprecated bool `json:"deprecated,omitempty"`
}
type ItemHubStatus struct {
Name string `json:"name"`
LocalVersion string `json:"local_version"`
LocalPath string `json:"local_path"`
Description string `json:"description"`
UTF8Status string `json:"utf8_status"`
Status string `json:"status"`
Digest string `json:"digest,omitempty"` // meow
Deprecated bool `json:"deprecated,omitempty"` // XXX: do we keep this?
}
// Item can be: parser, scenario, collection..
@ -84,7 +81,7 @@ type Item struct {
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
}
func (i *Item) status() (string, emoji.Emoji) {
func (i *Item) Status() (string, emoji.Emoji) {
status := "disabled"
ok := false
@ -124,19 +121,6 @@ func (i *Item) status() (string, emoji.Emoji) {
return status, emo
}
func (i *Item) hubStatus() ItemHubStatus {
status, emo := i.status()
return ItemHubStatus{
Name: i.Name,
LocalVersion: i.LocalVersion,
LocalPath: i.LocalPath,
Description: i.Description,
Status: status,
UTF8Status: fmt.Sprintf("%v %s", emo, status),
}
}
// versionStatus: semver requires 'v' prefix
func (i *Item) versionStatus() int {
return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
@ -206,6 +190,23 @@ func GetItem(itemType string, itemName string) *Item {
return nil
}
// GetItemNames returns the list of item (full) names for a given type
// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx
// The names can be used to retrieve the item with GetItem()
func GetItemNames(itemType string) []string {
m := GetItemMap(itemType)
if m == nil {
return nil
}
names := make([]string, 0, len(m))
for k := range m {
names = append(names, k)
}
return names
}
func AddItem(itemType string, item Item) error {
for _, itype := range ItemTypes {
if itype == itemType {
@ -257,32 +258,3 @@ func GetInstalledItemsAsString(itemType string) ([]string, error) {
return retStr, nil
}
// Returns a slice of entries for packages: name, status, local_path, local_version, utf8_status (fancy)
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
if _, ok := hubIdx[itemType]; !ok {
log.Errorf("type %s doesn't exist", itemType)
return nil
}
ret := make([]ItemHubStatus, 0)
// remember, you do it for the user :)
for _, item := range hubIdx[itemType] {
if name != "" && name != item.Name {
// user has requested a specific name
continue
}
// Only enabled items ?
if !all && !item.Installed {
continue
}
// Check the item status
ret = append(ret, item.hubStatus())
}
sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
return ret
}

11
pkg/cwhub/cwhub_test.go
View file

@ -53,7 +53,7 @@ func TestItemStatus(t *testing.T) {
item.Local = false
item.Tainted = false
txt, _ := item.status()
txt, _ := item.Status()
require.Equal(t, "enabled,update-available", txt)
item.Installed = false
@ -61,7 +61,7 @@ func TestItemStatus(t *testing.T) {
item.Local = true
item.Tainted = false
txt, _ = item.status()
txt, _ = item.Status()
require.Equal(t, "disabled,local", txt)
}
@ -273,10 +273,8 @@ func TestInstallParser(t *testing.T) {
for _, it := range hubIdx[PARSERS] {
testInstallItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
_ = GetHubStatusForItemType(PARSERS, it.Name, false)
testTaintItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
_ = GetHubStatusForItemType(PARSERS, it.Name, false)
testUpdateItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
testDisableItem(cfg.Hub, t, it)
@ -309,11 +307,6 @@ func TestInstallCollection(t *testing.T) {
testUpdateItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
testDisableItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
log.Infof("%+v", x)
break
}
}

35
test/bats/20_hub_parsers.bats
View file

@ -79,41 +79,42 @@ teardown() {
assert_output "$expected"
}
@test "cscli parsers list [parser]..." {
# non-existent
rune -1 cscli parsers install foo/bar
assert_stderr --partial "can't find 'foo/bar' in parsers"
# not installed
rune -0 cscli parsers list crowdsecurity/whitelists
assert_output --regexp 'crowdsecurity/whitelists.*disabled'
# install two items
rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth
# list one item
# list an installed item
rune -0 cscli parsers list crowdsecurity/whitelists
assert_output --partial "crowdsecurity/whitelists"
assert_output --regexp "crowdsecurity/whitelists.*enabled"
refute_output --partial "crowdsecurity/windows-auth"
# list multiple items
rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth
# list multiple installed and non installed items
rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs
assert_output --partial "crowdsecurity/whitelists"
assert_output --partial "crowdsecurity/windows-auth"
assert_output --partial "crowdsecurity/traefik-logs"
rune -0 cscli parsers list crowdsecurity/whitelists -o json
rune -0 jq '.parsers | length' <(output)
assert_output "1"
rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth -o json
rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o json
rune -0 jq '.parsers | length' <(output)
assert_output "2"
assert_output "3"
rune -0 cscli parsers list crowdsecurity/whitelists -o raw
rune -0 grep -vc 'name,status,version,description' <(output)
assert_output "1"
rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth -o raw
rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o raw
rune -0 grep -vc 'name,status,version,description' <(output)
assert_output "2"
}
@test "cscli parsers list [parser]... (not installed / not existing)" {
skip "not implemented yet"
# not installed
rune -1 cscli parsers list crowdsecurity/whitelists
# not existing
rune -1 cscli parsers list blahblah/blahblah
assert_output "3"
}
@test "cscli parsers install [parser]..." {

35
test/bats/20_hub_scenarios.bats
View file

@ -77,41 +77,42 @@ teardown() {
assert_output "$expected"
}
@test "cscli scenarios list [scenario]..." {
# non-existent
rune -1 cscli scenario install foo/bar
assert_stderr --partial "can't find 'foo/bar' in scenarios"
# not installed
rune -0 cscli scenarios list crowdsecurity/ssh-bf
assert_output --regexp 'crowdsecurity/ssh-bf.*disabled'
# install two items
rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf
# list one item
# list an installed item
rune -0 cscli scenarios list crowdsecurity/ssh-bf
assert_output --partial "crowdsecurity/ssh-bf"
assert_output --regexp "crowdsecurity/ssh-bf.*enabled"
refute_output --partial "crowdsecurity/telnet-bf"
# list multiple items
rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf
# list multiple installed and non installed items
rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf crowdsecurity/aws-bf
assert_output --partial "crowdsecurity/ssh-bf"
assert_output --partial "crowdsecurity/telnet-bf"
assert_output --partial "crowdsecurity/aws-bf"
rune -0 cscli scenarios list crowdsecurity/ssh-bf -o json
rune -0 jq '.scenarios | length' <(output)
assert_output "1"
rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json
rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o json
rune -0 jq '.scenarios | length' <(output)
assert_output "2"
assert_output "3"
rune -0 cscli scenarios list crowdsecurity/ssh-bf -o raw
rune -0 grep -vc 'name,status,version,description' <(output)
assert_output "1"
rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw
rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o raw
rune -0 grep -vc 'name,status,version,description' <(output)
assert_output "2"
}
@test "cscli scenarios list [scenario]... (not installed / not existing)" {
skip "not implemented yet"
# not installed
rune -1 cscli scenarios list crowdsecurity/ssh-bf
# not existing
rune -1 cscli scenarios list blahblah/blahblah
assert_output "3"
}
@test "cscli scenarios install [scenario]..." {