crowdsec/cmd/crowdsec-cli/metrics.go

585 lines
16 KiB
Go
Raw Normal View History

2020-05-15 09:39:16 +00:00
package main
import (
"encoding/json"
"fmt"
"io"
2020-05-15 09:39:16 +00:00
"net/http"
"strconv"
"strings"
"time"
"github.com/fatih/color"
2020-05-15 09:39:16 +00:00
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
2020-05-15 09:39:16 +00:00
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
2024-02-06 09:07:05 +00:00
"github.com/crowdsecurity/go-cs-lib/maptools"
"github.com/crowdsecurity/go-cs-lib/trace"
)
2024-02-02 09:40:55 +00:00
type (
statAcquis map[string]map[string]int
statParser map[string]map[string]int
statBucket map[string]map[string]int
statWhitelist map[string]map[string]map[string]int
statLapi map[string]map[string]int
statLapiMachine map[string]map[string]map[string]int
statLapiBouncer map[string]map[string]map[string]int
2024-02-02 09:40:55 +00:00
statLapiDecision map[string]struct {
NonEmpty int
Empty int
}
statDecision map[string]map[string]map[string]int
2024-02-02 09:40:55 +00:00
statAppsecEngine map[string]map[string]int
statAppsecRule map[string]map[string]map[string]int
statAlert map[string]int
statStash map[string]struct {
2024-02-02 09:40:55 +00:00
Type string
Count int
}
)
2024-02-06 09:07:05 +00:00
type metricSection interface {
Table(io.Writer, bool, bool)
Description() (string, string)
2024-02-02 08:45:03 +00:00
}
2024-02-06 09:07:05 +00:00
type metricStore map[string]metricSection
func NewMetricStore() metricStore {
return metricStore{
"acquisition": statAcquis{},
"buckets": statBucket{},
"parsers": statParser{},
"lapi": statLapi{},
"lapi-machine": statLapiMachine{},
"lapi-bouncer": statLapiBouncer{},
2024-02-06 09:07:05 +00:00
"lapi-decisions": statLapiDecision{},
"decisions": statDecision{},
"alerts": statAlert{},
"stash": statStash{},
"appsec-engine": statAppsecEngine{},
"appsec-rule": statAppsecRule{},
"whitelists": statWhitelist{},
2024-02-02 08:45:03 +00:00
}
}
2024-02-06 09:07:05 +00:00
func (ms metricStore) Fetch(url string) error {
2020-05-15 09:39:16 +00:00
mfChan := make(chan *dto.MetricFamily, 1024)
errChan := make(chan error, 1)
2020-05-15 09:39:16 +00:00
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/ShowPrometheus")
2024-02-06 09:07:05 +00:00
2020-05-15 09:39:16 +00:00
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
2024-02-06 09:07:05 +00:00
errChan <- fmt.Errorf("failed to fetch metrics: %w", err)
return
2020-05-15 09:39:16 +00:00
}
errChan <- nil
2020-05-15 09:39:16 +00:00
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
if err := <-errChan; err != nil {
return err
}
2024-02-06 09:07:05 +00:00
log.Debugf("Finished reading metrics output, %d entries", len(result))
2020-05-15 09:39:16 +00:00
/*walk*/
2024-02-02 09:40:55 +00:00
2024-02-06 09:07:05 +00:00
mAcquis := ms["acquisition"].(statAcquis)
mParser := ms["parsers"].(statParser)
mBucket := ms["buckets"].(statBucket)
mLapi := ms["lapi"].(statLapi)
mLapiMachine := ms["lapi-machine"].(statLapiMachine)
mLapiBouncer := ms["lapi-bouncer"].(statLapiBouncer)
mLapiDecision := ms["lapi-decisions"].(statLapiDecision)
mDecision := ms["decisions"].(statDecision)
mAppsecEngine := ms["appsec-engine"].(statAppsecEngine)
mAppsecRule := ms["appsec-rule"].(statAppsecRule)
mAlert := ms["alerts"].(statAlert)
mStash := ms["stash"].(statStash)
mWhitelist := ms["whitelists"].(statWhitelist)
2020-05-15 09:39:16 +00:00
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
2020-05-15 09:39:16 +00:00
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
2020-05-15 09:39:16 +00:00
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
2020-05-15 09:39:16 +00:00
}
2020-05-15 09:39:16 +00:00
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v for %s", metric.Labels, fam.Name)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
2020-05-15 09:39:16 +00:00
}
2020-05-15 09:39:16 +00:00
value := m.(prom2json.Metric).Value
machine := metric.Labels["machine"]
bouncer := metric.Labels["bouncer"]
route := metric.Labels["route"]
method := metric.Labels["method"]
reason := metric.Labels["reason"]
origin := metric.Labels["origin"]
action := metric.Labels["action"]
mtype := metric.Labels["type"]
fval, err := strconv.ParseFloat(value, 32)
2020-05-15 09:39:16 +00:00
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
}
ival := int(fval)
2020-05-15 09:39:16 +00:00
switch fam.Name {
//
// buckets
//
case "cs_bucket_created_total":
2024-02-02 09:40:55 +00:00
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)
2020-05-15 09:39:16 +00:00
}
2024-02-02 09:40:55 +00:00
mBucket[name]["instantiation"] += ival
case "cs_buckets":
2024-02-02 09:40:55 +00:00
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)
}
2024-02-02 09:40:55 +00:00
mBucket[name]["curr_count"] += ival
case "cs_bucket_overflowed_total":
2024-02-02 09:40:55 +00:00
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)
2020-05-15 09:39:16 +00:00
}
2024-02-02 09:40:55 +00:00
mBucket[name]["overflow"] += ival
case "cs_bucket_poured_total":
2024-02-02 09:40:55 +00:00
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)
2020-05-15 09:39:16 +00:00
}
2024-02-02 09:40:55 +00:00
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)
2020-05-15 09:39:16 +00:00
}
2024-02-02 09:40:55 +00:00
mBucket[name]["pour"] += ival
mAcquis[source]["pour"] += ival
case "cs_bucket_underflowed_total":
2024-02-02 09:40:55 +00:00
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)
2020-05-15 09:39:16 +00:00
}
2024-02-02 09:40:55 +00:00
mBucket[name]["underflow"] += ival
//
// parsers
//
case "cs_parser_hits_total":
2024-02-02 09:40:55 +00:00
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)
2020-05-15 09:39:16 +00:00
}
2024-02-02 09:40:55 +00:00
mAcquis[source]["reads"] += ival
case "cs_parser_hits_ok_total":
2024-02-02 09:40:55 +00:00
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)
2020-05-15 09:39:16 +00:00
}
2024-02-02 09:40:55 +00:00
mAcquis[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
2024-02-02 09:40:55 +00:00
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)
2020-05-15 09:39:16 +00:00
}
2024-02-02 09:40:55 +00:00
mAcquis[source]["unparsed"] += ival
case "cs_node_hits_total":
2024-02-02 09:40:55 +00:00
if _, ok := mParser[name]; !ok {
mParser[name] = make(map[string]int)
2020-05-15 09:39:16 +00:00
}
2024-02-02 09:40:55 +00:00
mParser[name]["hits"] += ival
case "cs_node_hits_ok_total":
2024-02-02 09:40:55 +00:00
if _, ok := mParser[name]; !ok {
mParser[name] = make(map[string]int)
2020-05-15 09:39:16 +00:00
}
2024-02-02 09:40:55 +00:00
mParser[name]["parsed"] += ival
case "cs_node_hits_ko_total":
2024-02-02 09:40:55 +00:00
if _, ok := mParser[name]; !ok {
mParser[name] = make(map[string]int)
}
2024-02-02 09:40:55 +00:00
mParser[name]["unparsed"] += ival
//
// whitelists
//
case "cs_node_wl_hits_total":
if _, ok := mWhitelist[name]; !ok {
mWhitelist[name] = make(map[string]map[string]int)
}
if _, ok := mWhitelist[name][reason]; !ok {
mWhitelist[name][reason] = make(map[string]int)
}
mWhitelist[name][reason]["hits"] += ival
case "cs_node_wl_hits_ok_total":
if _, ok := mWhitelist[name]; !ok {
mWhitelist[name] = make(map[string]map[string]int)
}
if _, ok := mWhitelist[name][reason]; !ok {
mWhitelist[name][reason] = make(map[string]int)
}
mWhitelist[name][reason]["whitelisted"] += ival
// track as well whitelisted lines at acquis level
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)
}
mAcquis[source]["whitelisted"] += ival
//
// lapi
//
case "cs_lapi_route_requests_total":
2024-02-02 09:40:55 +00:00
if _, ok := mLapi[route]; !ok {
mLapi[route] = make(map[string]int)
}
2024-02-02 09:40:55 +00:00
mLapi[route][method] += ival
case "cs_lapi_machine_requests_total":
2024-02-02 09:40:55 +00:00
if _, ok := mLapiMachine[machine]; !ok {
mLapiMachine[machine] = make(map[string]map[string]int)
}
2024-02-02 09:40:55 +00:00
if _, ok := mLapiMachine[machine][route]; !ok {
mLapiMachine[machine][route] = make(map[string]int)
}
2024-02-02 09:40:55 +00:00
mLapiMachine[machine][route][method] += ival
case "cs_lapi_bouncer_requests_total":
2024-02-02 09:40:55 +00:00
if _, ok := mLapiBouncer[bouncer]; !ok {
mLapiBouncer[bouncer] = make(map[string]map[string]int)
}
2024-02-02 09:40:55 +00:00
if _, ok := mLapiBouncer[bouncer][route]; !ok {
mLapiBouncer[bouncer][route] = make(map[string]int)
}
2024-02-02 09:40:55 +00:00
mLapiBouncer[bouncer][route][method] += ival
case "cs_lapi_decisions_ko_total", "cs_lapi_decisions_ok_total":
2024-02-02 09:40:55 +00:00
if _, ok := mLapiDecision[bouncer]; !ok {
mLapiDecision[bouncer] = struct {
NonEmpty int
Empty int
}{}
}
2024-02-02 09:40:55 +00:00
x := mLapiDecision[bouncer]
if fam.Name == "cs_lapi_decisions_ko_total" {
x.Empty += ival
} else if fam.Name == "cs_lapi_decisions_ok_total" {
x.NonEmpty += ival
}
2024-02-02 09:40:55 +00:00
mLapiDecision[bouncer] = x
//
// decisions
//
case "cs_active_decisions":
2024-02-02 09:40:55 +00:00
if _, ok := mDecision[reason]; !ok {
mDecision[reason] = make(map[string]map[string]int)
}
2024-02-02 09:40:55 +00:00
if _, ok := mDecision[reason][origin]; !ok {
mDecision[reason][origin] = make(map[string]int)
}
2024-02-02 09:40:55 +00:00
mDecision[reason][origin][action] += ival
case "cs_alerts":
2024-02-02 09:40:55 +00:00
mAlert[reason] += ival
//
// stash
//
case "cs_cache_size":
2024-02-02 09:40:55 +00:00
mStash[name] = struct {
Type string
Count int
}{Type: mtype, Count: ival}
//
// appsec
//
case "cs_appsec_reqs_total":
2024-02-02 09:40:55 +00:00
if _, ok := mAppsecEngine[metric.Labels["appsec_engine"]]; !ok {
mAppsecEngine[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
}
2024-02-02 09:40:55 +00:00
mAppsecEngine[metric.Labels["appsec_engine"]]["processed"] = ival
case "cs_appsec_block_total":
2024-02-02 09:40:55 +00:00
if _, ok := mAppsecEngine[metric.Labels["appsec_engine"]]; !ok {
mAppsecEngine[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
}
2024-02-02 09:40:55 +00:00
mAppsecEngine[metric.Labels["appsec_engine"]]["blocked"] = ival
case "cs_appsec_rule_hits":
appsecEngine := metric.Labels["appsec_engine"]
ruleID := metric.Labels["rule_name"]
2024-02-02 09:40:55 +00:00
if _, ok := mAppsecRule[appsecEngine]; !ok {
mAppsecRule[appsecEngine] = make(map[string]map[string]int, 0)
}
2024-02-02 09:40:55 +00:00
if _, ok := mAppsecRule[appsecEngine][ruleID]; !ok {
mAppsecRule[appsecEngine][ruleID] = make(map[string]int, 0)
}
2024-02-02 09:40:55 +00:00
mAppsecRule[appsecEngine][ruleID]["triggered"] = ival
2020-05-15 09:39:16 +00:00
default:
log.Debugf("unknown: %+v", fam.Name)
2020-05-15 09:39:16 +00:00
continue
}
}
}
2024-02-06 09:07:05 +00:00
return nil
}
type cliMetrics struct {
cfg configGetter
}
func NewCLIMetrics(cfg configGetter) *cliMetrics {
2024-02-06 09:07:05 +00:00
return &cliMetrics{
cfg: cfg,
}
2024-02-06 09:07:05 +00:00
}
2022-08-18 09:54:01 +00:00
2024-02-06 09:07:05 +00:00
func (ms metricStore) Format(out io.Writer, sections []string, formatType string, noUnit bool) error {
// copy only the sections we want
want := map[string]metricSection{}
2024-02-06 09:07:05 +00:00
// if explicitly asking for sections, we want to show empty tables
showEmpty := len(sections) > 0
// if no sections are specified, we want all of them
if len(sections) == 0 {
for section := range ms {
sections = append(sections, section)
}
}
for _, section := range sections {
want[section] = ms[section]
}
switch formatType {
2024-02-06 09:07:05 +00:00
case "human":
for section := range want {
want[section].Table(out, noUnit, showEmpty)
}
case "json":
2024-02-06 09:07:05 +00:00
x, err := json.MarshalIndent(want, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
2020-05-15 09:39:16 +00:00
}
out.Write(x)
case "raw":
2024-02-06 09:07:05 +00:00
x, err := yaml.Marshal(want)
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
}
out.Write(x)
default:
return fmt.Errorf("unknown format type %s", formatType)
2020-05-15 09:39:16 +00:00
}
return nil
2020-05-15 09:39:16 +00:00
}
2024-02-06 09:07:05 +00:00
func (cli *cliMetrics) show(sections []string, url string, noUnit bool) error {
2024-02-02 08:45:03 +00:00
cfg := cli.cfg()
if url != "" {
2024-02-02 08:45:03 +00:00
cfg.Cscli.PrometheusUrl = url
}
2024-02-02 08:45:03 +00:00
if cfg.Prometheus == nil {
return fmt.Errorf("prometheus section missing, can't show metrics")
}
2024-02-02 08:45:03 +00:00
if !cfg.Prometheus.Enabled {
return fmt.Errorf("prometheus is not enabled, can't show metrics")
}
2024-02-06 09:07:05 +00:00
ms := NewMetricStore()
if err := ms.Fetch(cfg.Cscli.PrometheusUrl); err != nil {
return err
}
// any section that we don't have in the store is an error
for _, section := range sections {
if _, ok := ms[section]; !ok {
return fmt.Errorf("unknown metrics type: %s", section)
}
}
if err := ms.Format(color.Output, sections, cfg.Cscli.Output, noUnit); err != nil {
return err
}
return nil
}
2024-02-02 08:45:03 +00:00
func (cli *cliMetrics) NewCommand() *cobra.Command {
var (
url string
2024-02-02 08:45:03 +00:00
noUnit bool
)
cmd := &cobra.Command{
Use: "metrics",
Short: "Display crowdsec prometheus metrics.",
Long: `Fetch metrics from a Local API server and display them`,
Example: `# Show all Metrics, skip empty tables (same as "cecli metrics show")
2024-02-06 09:07:05 +00:00
cscli metrics
# Show only some metrics, connect to a different url
cscli metrics --url http://lapi.local:6060/metrics show acquisition parsers
# List available metric types
cscli metrics list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
2024-02-02 08:45:03 +00:00
RunE: func(cmd *cobra.Command, args []string) error {
2024-02-06 09:07:05 +00:00
return cli.show(nil, url, noUnit)
2024-02-02 08:45:03 +00:00
},
2020-05-15 09:39:16 +00:00
}
2024-02-02 08:45:03 +00:00
flags := cmd.Flags()
flags.StringVarP(&url, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
flags.BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
2020-05-15 09:39:16 +00:00
2024-02-06 09:07:05 +00:00
cmd.AddCommand(cli.newShowCmd())
cmd.AddCommand(cli.newListCmd())
return cmd
}
// expandAlias returns a list of sections. The input can be a list of sections or alias.
func (cli *cliMetrics) expandSectionGroups(args []string) []string {
ret := []string{}
for _, section := range args {
switch section {
case "engine":
ret = append(ret, "acquisition", "parsers", "buckets", "stash", "whitelists")
2024-02-06 09:07:05 +00:00
case "lapi":
ret = append(ret, "alerts", "decisions", "lapi", "lapi-bouncer", "lapi-decisions", "lapi-machine")
case "appsec":
ret = append(ret, "appsec-engine", "appsec-rule")
default:
ret = append(ret, section)
}
}
return ret
}
func (cli *cliMetrics) newShowCmd() *cobra.Command {
var (
url string
2024-02-06 09:07:05 +00:00
noUnit bool
)
cmd := &cobra.Command{
Use: "show [type]...",
Short: "Display all or part of the available metrics.",
Long: `Fetch metrics from a Local API server and display them, optionally filtering on specific types.`,
Example: `# Show all Metrics, skip empty tables
2024-02-06 09:07:05 +00:00
cscli metrics show
# Use an alias: "engine", "lapi" or "appsec" to show a group of metrics
cscli metrics show engine
# Show some specific metrics, show empty tables, connect to a different url
cscli metrics show acquisition parsers buckets stash --url http://lapi.local:6060/metrics
# Show metrics in json format
cscli metrics show acquisition parsers buckets stash -o json`,
// Positional args are optional
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
args = cli.expandSectionGroups(args)
return cli.show(args, url, noUnit)
},
}
flags := cmd.Flags()
flags.StringVarP(&url, "url", "u", "", "Metrics url (http://<ip>:<port>/metrics)")
flags.BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
return cmd
}
func (cli *cliMetrics) list() error {
type metricType struct {
Type string `json:"type" yaml:"type"`
Title string `json:"title" yaml:"title"`
Description string `json:"description" yaml:"description"`
2024-02-06 09:07:05 +00:00
}
var allMetrics []metricType
ms := NewMetricStore()
for _, section := range maptools.SortedKeys(ms) {
title, description := ms[section].Description()
allMetrics = append(allMetrics, metricType{
Type: section,
Title: title,
Description: description,
})
}
switch cli.cfg().Cscli.Output {
case "human":
t := newTable(color.Output)
t.SetRowLines(true)
t.SetHeaders("Type", "Title", "Description")
for _, metric := range allMetrics {
t.AddRow(metric.Type, metric.Title, metric.Description)
}
t.Render()
case "json":
x, err := json.MarshalIndent(allMetrics, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal metrics: %w", err)
}
fmt.Println(string(x))
case "raw":
x, err := yaml.Marshal(allMetrics)
if err != nil {
return fmt.Errorf("failed to unmarshal metrics: %w", err)
}
fmt.Println(string(x))
}
return nil
}
func (cli *cliMetrics) newListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "List available types of metrics.",
Long: `List available types of metrics.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
cli.list()
return nil
},
}
2024-02-02 08:45:03 +00:00
return cmd
2020-05-15 09:39:16 +00:00
}