beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

rename metrics and update metrics helps (#152)

Browse source 
* rename metrics and update metrics helps

* add meta info about crowdsec

Co-authored-by: erenJag <erenJag>
This commit is contained in:
erenJag 2020-07-29 15:03:15 +02:00 committed by GitHub
parent acc0960c17
commit 89c8d1a527
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 63 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
cmd/crowdsec-cli/metrics.go
View file

@ -103,22 +103,22 @@ func ShowPrometheus(url string) {
ival := int(fval)
switch fam.Name {
/*buckets*/
case "cs_bucket_create":
case "cs_bucket_created_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["instanciation"] += ival
case "cs_bucket_count":
case "cs_buckets":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["curr_count"] += ival
case "cs_bucket_overflow":
case "cs_bucket_overflowed_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["overflow"] += ival
case "cs_bucket_pour":
case "cs_bucket_poured_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
@ -127,38 +127,38 @@ func ShowPrometheus(url string) {
}
buckets_stats[name]["pour"] += ival
acquis_stats[source]["pour"] += ival
case "cs_bucket_underflow":
case "cs_bucket_underflowed_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["underflow"] += ival
/*acquis*/
case "cs_reader_hits":
case "cs_reader_hits_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["reads"] += ival
case "cs_parser_hits_ok":
case "cs_parser_hits_ok_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["parsed"] += ival
case "cs_parser_hits_ko":
case "cs_parser_hits_ko_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["unparsed"] += ival
case "cs_node_hits":
case "cs_node_hits_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}
parsers_stats[name]["hits"] += ival
case "cs_node_hits_ok":
case "cs_node_hits_ok_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}
parsers_stats[name]["parsed"] += ival
case "cs_node_hits_ko":
case "cs_node_hits_ko_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}

31
cmd/crowdsec/metrics.go
View file

@ -4,6 +4,7 @@ import (
"time"
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/jamiealquiza/tachymeter"
@ -32,37 +33,45 @@ var (
/*prometheus*/
var globalParserHits = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_parser_hits",
Help: "How many time an event entered the parser.",
Name: "cs_parser_hits_total",
Help: "Total events entered the parser.",
},
[]string{"source"},
)
var globalParserHitsOk = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_parser_hits_ok",
Help: "How many time an event was successfully parsed.",
Name: "cs_parser_hits_ok_total",
Help: "Total events were successfully parsed.",
},
[]string{"source"},
)
var globalParserHitsKo = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_parser_hits_ko",
Help: "How many time an event was unsuccessfully parsed.",
Name: "cs_parser_hits_ko_total",
Help: "Total events were unsuccessfully parsed.",
},
[]string{"source"},
)
var globalBucketPourKo = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "cs_bucket_pour_ko",
Help: "How many time an event was poured in no bucket.",
Name: "cs_bucket_pour_ko_total",
Help: "Total events were not poured in a bucket.",
},
)
var globalBucketPourOk = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "cs_bucket_pour_ok",
Help: "How many time an event was poured in at least one bucket.",
Name: "cs_bucket_pour_ok_total",
Help: "Total events were poured in at least one bucket.",
},
)
var globalCsInfo = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "cs_info",
Help: "Information about Crowdsec.",
ConstLabels: prometheus.Labels{"version": cwversion.VersionStr()},
},
)
@ -122,7 +131,7 @@ func registerPrometheus() {
log.Warningf("Loading prometheus collectors")
prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
parser.NodesHits, parser.NodesHitsOk, parser.NodesHitsKo,
acquisition.ReaderHits,
acquisition.ReaderHits, globalCsInfo,
leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsInstanciation, leaky.BucketsOverflow, leaky.BucketsCurrentCount)
http.Handle("/metrics", promhttp.Handler())
}

28
docs/observability/metrics.md
View file

@ -79,40 +79,40 @@ All the counters are "since {{crowdsec.name}} start".
### Scenarios
- `cs_bucket_create` : number of instantiation of each scenario
- `cs_bucket_overflow` : number of overflow of each scenario
- `cs_bucket_underflow` : number of underflow of each scenario (bucket was created but expired because of lack of events)
- `cs_bucket_pour` : number of event poured to each scenario with source as complementary key :
- `cs_bucket_created_total` : number of instantiation of each scenario
- `cs_bucket_overflowed_total` : number of overflow of each scenario
- `cs_bucket_underflowed_total` : number of underflow of each scenario (bucket was created but expired because of lack of events)
- `cs_bucket_poured_total` : number of event poured to each scenario with source as complementary key :
```
#2030 lines from `/var/log/nginx/access.log` were poured to `crowdsecurity/http-scan-uniques_404` scenario
cs_bucket_pour{name="crowdsecurity/http-scan-uniques_404",source="/var/log/nginx/access.log"} 2030
cs_bucket_poured_total{name="crowdsecurity/http-scan-uniques_404",source="/var/log/nginx/access.log"} 2030
```
### Parsers
- `cs_node_hits` : how many time an event from a specific source was processed by a parser node :
- `cs_node_hits_total` : how many time an event from a specific source was processed by a parser node :
```
# 235 lines from `auth.log` were processed by the `crowdsecurity/dateparse-enrich` parser
cs_node_hits{name="crowdsecurity/dateparse-enrich",source="/var/log/auth.log"} 235
cs_node_hits_total{name="crowdsecurity/dateparse-enrich",source="/var/log/auth.log"} 235
```
- `cs_node_hits_ko` : how many times an event from a specific was unsuccessfully parsed by a specific parser
- `cs_node_hits_ko_total` : how many times an event from a specific was unsuccessfully parsed by a specific parser
```
# 2112 lines from `error.log` failed to be parsed by `crowdsecurity/http-logs`
cs_node_hits_ko{name="crowdsecurity/http-logs",source="/var/log/nginx/error.log"} 2112
cs_node_hits_ko_total{name="crowdsecurity/http-logs",source="/var/log/nginx/error.log"} 2112
```
- `cs_node_hits_ok` : how many times an event from a specific source was successfully parsed by a specific parser
- `cs_node_hits_ok_total` : how many times an event from a specific source was successfully parsed by a specific parser
- `cs_parser_hits` : how many times an event from a source has hit the parser
- `cs_parser_hits_ok` : how many times an event from a source was successfully parsed
- `cs_parser_hits_ko` : how many times an event from a source was unsuccessfully parsed
- `cs_parser_hits_total` : how many times an event from a source has hit the parser
- `cs_parser_hits_ok_total` : how many times an event from a source was successfully parsed
- `cs_parser_hits_ko_total` : how many times an event from a source was unsuccessfully parsed
### Acquisition
- `cs_reader_hits` : how many events were read from a specific source
- `cs_reader_hits_total` : how many events were read from a specific source

4
pkg/acquisition/file_reader.go
View file

@ -61,8 +61,8 @@ const (
var ReaderHits = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_reader_hits",
Help: "How many lines where read.",
Name: "cs_reader_hits_total",
Help: "Total lines where read.",
},
[]string{"source"},
)

20
pkg/leakybucket/bucket.go
View file

@ -68,40 +68,40 @@ type Leaky struct {
var BucketsPour = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_bucket_pour",
Help: "How many time an event was poured in this bucket.",
Name: "cs_bucket_poured_total",
Help: "Total events were poured in bucket.",
},
[]string{"source", "name"},
)
var BucketsOverflow = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_bucket_overflow",
Help: "How many time this bucket overflowed.",
Name: "cs_bucket_overflowed_total",
Help: "Total buckets overflowed.",
},
[]string{"name"},
)
var BucketsUnderflow = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_bucket_underflow",
Help: "How many time this bucket has underflowed.",
Name: "cs_bucket_underflowed_total",
Help: "Total buckets underflowed.",
},
[]string{"name"},
)
var BucketsInstanciation = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_bucket_create",
Help: "How many time this bucket was instanciated.",
Name: "cs_bucket_created_total",
Help: "Total buckets were instanciated.",
},
[]string{"name"},
)
var BucketsCurrentCount = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "cs_bucket_count",
Help: "How many instances of this bucket exist.",
Name: "cs_buckets",
Help: "Number of buckets that currently exist.",
},
[]string{"name"},
)

12
pkg/parser/runtime.go
View file

@ -196,24 +196,24 @@ func ProcessStatics(statics []types.ExtraField, p *types.Event, clog *logrus.Ent
var NodesHits = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_node_hits",
Help: "How many time an event entered this node.",
Name: "cs_node_hits_total",
Help: "Total events entered node.",
},
[]string{"source", "name"},
)
var NodesHitsOk = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_node_hits_ok",
Help: "How many time an event successfuly exited this node.",
Name: "cs_node_hits_ok_total",
Help: "Total events successfuly exited node.",
},
[]string{"source", "name"},
)
var NodesHitsKo = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_node_hits_ko",
Help: "How many time an event unsuccessfuly exited this node.",
Name: "cs_node_hits_ko_total",
Help: "Total events unsuccessfuly exited node.",
},
[]string{"source", "name"},
)