add support for 'prometheus_mode' configuration directive that can be set to 'aggregation' to limit the cardinality of prometheus metrics (#192)

cmd/crowdsec/main.go

@@ -276,7 +276,7 @@ func main() {
 
 	// Enable profiling early
 	if cConfig.Prometheus {
-		registerPrometheus()
+		registerPrometheus(cConfig.PrometheusMode)
 		cConfig.Profiling = true
 	}
 	if cConfig.Profiling {

cmd/crowdsec/metrics.go

@@ -126,12 +126,22 @@ func runTachymeter(HTTPListen string) {
 	log.Fatal(http.ListenAndServe(HTTPListen, nil))
 }
 
-func registerPrometheus() {
+func registerPrometheus(mode string) {
 	/*Registering prometheus*/
-	log.Warningf("Loading prometheus collectors")
-	prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
-		parser.NodesHits, parser.NodesHitsOk, parser.NodesHitsKo,
-		acquisition.ReaderHits, globalCsInfo,
-		leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsInstanciation, leaky.BucketsOverflow, leaky.BucketsCurrentCount)
+	/*If in aggregated mode, do not register events associated to a source, keeps cardinality low*/
+	if mode == "aggregated" {
+		log.Infof("Loading aggregated prometheus collectors")
+		prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
+			acquisition.ReaderHits, globalCsInfo,
+			leaky.BucketsUnderflow, leaky.BucketsInstanciation, leaky.BucketsOverflow,
+			leaky.BucketsCurrentCount)
+	} else {
+		log.Infof("Loading prometheus collectors")
+		prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
+			parser.NodesHits, parser.NodesHitsOk, parser.NodesHitsKo,
+			acquisition.ReaderHits, globalCsInfo,
+			leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsInstanciation, leaky.BucketsOverflow, leaky.BucketsCurrentCount)
+
+	}
 	http.Handle("/metrics", promhttp.Handler())
 }

docs/guide/crowdsec/overview.md

@@ -73,6 +73,9 @@ To enable or disable {{crowdsec.Name}} daemon mode.
 #### `prometheus:`
 To enable or disable Prometheus metrics.
 
+### `prometheus_mode:`
+If `prometheus` is enabled, and is set to `aggregated`, will restrict prometheus metrics to global ones. All metrics containing a source as a label will be unregistered. Meant to keep cardinality low when relevant.
+
 #### `http_listen:`
 To configure the Prometheus service listening `address:port` or {{crowdsec.Name}} profiling
 

pkg/csconfig/config.go

@@ -41,6 +41,7 @@ type CrowdSec struct {
 	SimulationCfg     *SimulationConfig
 	Linter            bool
 	Prometheus        bool
+	PrometheusMode    string `yaml:"prometheus_mode"`
 	HTTPListen        string `yaml:"http_listen,omitempty"`
 	RestoreMode       string
 	DumpBuckets       bool