Dedicated whitelist metrics (#2813)
* add proper whitelist metrics : both its own table and an extension to acquis metrics to track discarded/whitelisted lines
This commit is contained in:
parent
4e724f6c0a
commit
3208a40ef3
|
@ -11,7 +11,7 @@ run:
|
||||||
linters-settings:
|
linters-settings:
|
||||||
cyclop:
|
cyclop:
|
||||||
# lower this after refactoring
|
# lower this after refactoring
|
||||||
max-complexity: 66
|
max-complexity: 70
|
||||||
|
|
||||||
gci:
|
gci:
|
||||||
sections:
|
sections:
|
||||||
|
@ -22,11 +22,11 @@ linters-settings:
|
||||||
|
|
||||||
gocognit:
|
gocognit:
|
||||||
# lower this after refactoring
|
# lower this after refactoring
|
||||||
min-complexity: 145
|
min-complexity: 150
|
||||||
|
|
||||||
gocyclo:
|
gocyclo:
|
||||||
# lower this after refactoring
|
# lower this after refactoring
|
||||||
min-complexity: 64
|
min-complexity: 70
|
||||||
|
|
||||||
funlen:
|
funlen:
|
||||||
# Checks the number of lines in a function.
|
# Checks the number of lines in a function.
|
||||||
|
|
|
@ -24,6 +24,7 @@ type (
|
||||||
statAcquis map[string]map[string]int
|
statAcquis map[string]map[string]int
|
||||||
statParser map[string]map[string]int
|
statParser map[string]map[string]int
|
||||||
statBucket map[string]map[string]int
|
statBucket map[string]map[string]int
|
||||||
|
statWhitelist map[string]map[string]map[string]int
|
||||||
statLapi map[string]map[string]int
|
statLapi map[string]map[string]int
|
||||||
statLapiMachine map[string]map[string]map[string]int
|
statLapiMachine map[string]map[string]map[string]int
|
||||||
statLapiBouncer map[string]map[string]map[string]int
|
statLapiBouncer map[string]map[string]map[string]int
|
||||||
|
@ -62,6 +63,7 @@ func NewMetricStore() metricStore {
|
||||||
"stash": statStash{},
|
"stash": statStash{},
|
||||||
"appsec-engine": statAppsecEngine{},
|
"appsec-engine": statAppsecEngine{},
|
||||||
"appsec-rule": statAppsecRule{},
|
"appsec-rule": statAppsecRule{},
|
||||||
|
"whitelists": statWhitelist{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -111,6 +113,7 @@ func (ms metricStore) Fetch(url string) error {
|
||||||
mAppsecRule := ms["appsec-rule"].(statAppsecRule)
|
mAppsecRule := ms["appsec-rule"].(statAppsecRule)
|
||||||
mAlert := ms["alerts"].(statAlert)
|
mAlert := ms["alerts"].(statAlert)
|
||||||
mStash := ms["stash"].(statStash)
|
mStash := ms["stash"].(statStash)
|
||||||
|
mWhitelist := ms["whitelists"].(statWhitelist)
|
||||||
|
|
||||||
for idx, fam := range result {
|
for idx, fam := range result {
|
||||||
if !strings.HasPrefix(fam.Name, "cs_") {
|
if !strings.HasPrefix(fam.Name, "cs_") {
|
||||||
|
@ -160,7 +163,9 @@ func (ms metricStore) Fetch(url string) error {
|
||||||
|
|
||||||
ival := int(fval)
|
ival := int(fval)
|
||||||
switch fam.Name {
|
switch fam.Name {
|
||||||
/*buckets*/
|
//
|
||||||
|
// buckets
|
||||||
|
//
|
||||||
case "cs_bucket_created_total":
|
case "cs_bucket_created_total":
|
||||||
if _, ok := mBucket[name]; !ok {
|
if _, ok := mBucket[name]; !ok {
|
||||||
mBucket[name] = make(map[string]int)
|
mBucket[name] = make(map[string]int)
|
||||||
|
@ -190,7 +195,9 @@ func (ms metricStore) Fetch(url string) error {
|
||||||
mBucket[name] = make(map[string]int)
|
mBucket[name] = make(map[string]int)
|
||||||
}
|
}
|
||||||
mBucket[name]["underflow"] += ival
|
mBucket[name]["underflow"] += ival
|
||||||
/*acquis*/
|
//
|
||||||
|
// parsers
|
||||||
|
//
|
||||||
case "cs_parser_hits_total":
|
case "cs_parser_hits_total":
|
||||||
if _, ok := mAcquis[source]; !ok {
|
if _, ok := mAcquis[source]; !ok {
|
||||||
mAcquis[source] = make(map[string]int)
|
mAcquis[source] = make(map[string]int)
|
||||||
|
@ -221,6 +228,33 @@ func (ms metricStore) Fetch(url string) error {
|
||||||
mParser[name] = make(map[string]int)
|
mParser[name] = make(map[string]int)
|
||||||
}
|
}
|
||||||
mParser[name]["unparsed"] += ival
|
mParser[name]["unparsed"] += ival
|
||||||
|
//
|
||||||
|
// whitelists
|
||||||
|
//
|
||||||
|
case "cs_node_wl_hits_total":
|
||||||
|
if _, ok := mWhitelist[name]; !ok {
|
||||||
|
mWhitelist[name] = make(map[string]map[string]int)
|
||||||
|
}
|
||||||
|
if _, ok := mWhitelist[name][reason]; !ok {
|
||||||
|
mWhitelist[name][reason] = make(map[string]int)
|
||||||
|
}
|
||||||
|
mWhitelist[name][reason]["hits"] += ival
|
||||||
|
case "cs_node_wl_hits_ok_total":
|
||||||
|
if _, ok := mWhitelist[name]; !ok {
|
||||||
|
mWhitelist[name] = make(map[string]map[string]int)
|
||||||
|
}
|
||||||
|
if _, ok := mWhitelist[name][reason]; !ok {
|
||||||
|
mWhitelist[name][reason] = make(map[string]int)
|
||||||
|
}
|
||||||
|
mWhitelist[name][reason]["whitelisted"] += ival
|
||||||
|
// track as well whitelisted lines at acquis level
|
||||||
|
if _, ok := mAcquis[source]; !ok {
|
||||||
|
mAcquis[source] = make(map[string]int)
|
||||||
|
}
|
||||||
|
mAcquis[source]["whitelisted"] += ival
|
||||||
|
//
|
||||||
|
// lapi
|
||||||
|
//
|
||||||
case "cs_lapi_route_requests_total":
|
case "cs_lapi_route_requests_total":
|
||||||
if _, ok := mLapi[route]; !ok {
|
if _, ok := mLapi[route]; !ok {
|
||||||
mLapi[route] = make(map[string]int)
|
mLapi[route] = make(map[string]int)
|
||||||
|
@ -256,6 +290,9 @@ func (ms metricStore) Fetch(url string) error {
|
||||||
x.NonEmpty += ival
|
x.NonEmpty += ival
|
||||||
}
|
}
|
||||||
mLapiDecision[bouncer] = x
|
mLapiDecision[bouncer] = x
|
||||||
|
//
|
||||||
|
// decisions
|
||||||
|
//
|
||||||
case "cs_active_decisions":
|
case "cs_active_decisions":
|
||||||
if _, ok := mDecision[reason]; !ok {
|
if _, ok := mDecision[reason]; !ok {
|
||||||
mDecision[reason] = make(map[string]map[string]int)
|
mDecision[reason] = make(map[string]map[string]int)
|
||||||
|
@ -265,15 +302,18 @@ func (ms metricStore) Fetch(url string) error {
|
||||||
}
|
}
|
||||||
mDecision[reason][origin][action] += ival
|
mDecision[reason][origin][action] += ival
|
||||||
case "cs_alerts":
|
case "cs_alerts":
|
||||||
/*if _, ok := mAlert[scenario]; !ok {
|
|
||||||
mAlert[scenario] = make(map[string]int)
|
|
||||||
}*/
|
|
||||||
mAlert[reason] += ival
|
mAlert[reason] += ival
|
||||||
|
//
|
||||||
|
// stash
|
||||||
|
//
|
||||||
case "cs_cache_size":
|
case "cs_cache_size":
|
||||||
mStash[name] = struct {
|
mStash[name] = struct {
|
||||||
Type string
|
Type string
|
||||||
Count int
|
Count int
|
||||||
}{Type: mtype, Count: ival}
|
}{Type: mtype, Count: ival}
|
||||||
|
//
|
||||||
|
// appsec
|
||||||
|
//
|
||||||
case "cs_appsec_reqs_total":
|
case "cs_appsec_reqs_total":
|
||||||
if _, ok := mAppsecEngine[metric.Labels["appsec_engine"]]; !ok {
|
if _, ok := mAppsecEngine[metric.Labels["appsec_engine"]]; !ok {
|
||||||
mAppsecEngine[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
|
mAppsecEngine[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
|
||||||
|
@ -431,7 +471,7 @@ func (cli *cliMetrics) expandSectionGroups(args []string) []string {
|
||||||
for _, section := range args {
|
for _, section := range args {
|
||||||
switch section {
|
switch section {
|
||||||
case "engine":
|
case "engine":
|
||||||
ret = append(ret, "acquisition", "parsers", "buckets", "stash")
|
ret = append(ret, "acquisition", "parsers", "buckets", "stash", "whitelists")
|
||||||
case "lapi":
|
case "lapi":
|
||||||
ret = append(ret, "alerts", "decisions", "lapi", "lapi-bouncer", "lapi-decisions", "lapi-machine")
|
ret = append(ret, "alerts", "decisions", "lapi", "lapi-bouncer", "lapi-decisions", "lapi-machine")
|
||||||
case "appsec":
|
case "appsec":
|
||||||
|
|
|
@ -45,6 +45,38 @@ func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]i
|
||||||
return numRows
|
return numRows
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func wlMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int, noUnit bool) (int, error) {
|
||||||
|
if t == nil {
|
||||||
|
return 0, fmt.Errorf("nil table")
|
||||||
|
}
|
||||||
|
|
||||||
|
numRows := 0
|
||||||
|
|
||||||
|
for _, name := range maptools.SortedKeys(stats) {
|
||||||
|
for _, reason := range maptools.SortedKeys(stats[name]) {
|
||||||
|
row := make([]string, 4)
|
||||||
|
row[0] = name
|
||||||
|
row[1] = reason
|
||||||
|
row[2] = "-"
|
||||||
|
row[3] = "-"
|
||||||
|
|
||||||
|
for _, action := range maptools.SortedKeys(stats[name][reason]) {
|
||||||
|
value := stats[name][reason][action]
|
||||||
|
if action == "whitelisted" {
|
||||||
|
row[3] = fmt.Sprintf("%d", value)
|
||||||
|
} else if action == "hits" {
|
||||||
|
row[2] = fmt.Sprintf("%d", value)
|
||||||
|
} else {
|
||||||
|
log.Debugf("unexpected counter '%s' for whitelists = %d", action, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.AddRow(row...)
|
||||||
|
numRows++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return numRows, nil
|
||||||
|
}
|
||||||
|
|
||||||
func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string, noUnit bool) (int, error) {
|
func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string, noUnit bool) (int, error) {
|
||||||
if t == nil {
|
if t == nil {
|
||||||
return 0, fmt.Errorf("nil table")
|
return 0, fmt.Errorf("nil table")
|
||||||
|
@ -95,7 +127,7 @@ func (s statBucket) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
log.Warningf("while collecting bucket stats: %s", err)
|
log.Warningf("while collecting bucket stats: %s", err)
|
||||||
} else if numRows > 0 || showEmpty {
|
} else if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n" + title + ":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -108,16 +140,16 @@ func (s statAcquis) Description() (string, string) {
|
||||||
func (s statAcquis) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statAcquis) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t := newTable(out)
|
t := newTable(out)
|
||||||
t.SetRowLines(false)
|
t.SetRowLines(false)
|
||||||
t.SetHeaders("Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket")
|
t.SetHeaders("Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket", "Lines whitelisted")
|
||||||
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
||||||
|
|
||||||
keys := []string{"reads", "parsed", "unparsed", "pour"}
|
keys := []string{"reads", "parsed", "unparsed", "pour", "whitelisted"}
|
||||||
|
|
||||||
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
|
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
|
||||||
log.Warningf("while collecting acquis stats: %s", err)
|
log.Warningf("while collecting acquis stats: %s", err)
|
||||||
} else if numRows > 0 || showEmpty {
|
} else if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n" + title + ":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -137,7 +169,7 @@ func (s statAppsecEngine) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
log.Warningf("while collecting appsec stats: %s", err)
|
log.Warningf("while collecting appsec stats: %s", err)
|
||||||
} else if numRows > 0 || showEmpty {
|
} else if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n" + title + ":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -156,7 +188,7 @@ func (s statAppsecRule) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
keys := []string{"triggered"}
|
keys := []string{"triggered"}
|
||||||
if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys, noUnit); err != nil {
|
if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys, noUnit); err != nil {
|
||||||
log.Warningf("while collecting appsec rules stats: %s", err)
|
log.Warningf("while collecting appsec rules stats: %s", err)
|
||||||
} else if numRows > 0 || showEmpty{
|
} else if numRows > 0 || showEmpty {
|
||||||
renderTableTitle(out, fmt.Sprintf("\nAppsec '%s' Rules Metrics:", appsecEngine))
|
renderTableTitle(out, fmt.Sprintf("\nAppsec '%s' Rules Metrics:", appsecEngine))
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
|
@ -164,6 +196,26 @@ func (s statAppsecRule) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s statWhitelist) Description() (string, string) {
|
||||||
|
return "Whitelist Metrics",
|
||||||
|
`Tracks the number of events processed and possibly whitelisted by each parser whitelist.`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s statWhitelist) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
t := newTable(out)
|
||||||
|
t.SetRowLines(false)
|
||||||
|
t.SetHeaders("Whitelist", "Reason", "Hits", "Whitelisted")
|
||||||
|
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
||||||
|
|
||||||
|
if numRows, err := wlMetricsToTable(t, s, noUnit); err != nil {
|
||||||
|
log.Warningf("while collecting parsers stats: %s", err)
|
||||||
|
} else if numRows > 0 || showEmpty {
|
||||||
|
title, _ := s.Description()
|
||||||
|
renderTableTitle(out, "\n"+title+":")
|
||||||
|
t.Render()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s statParser) Description() (string, string) {
|
func (s statParser) Description() (string, string) {
|
||||||
return "Parser Metrics",
|
return "Parser Metrics",
|
||||||
`Tracks the number of events processed by each parser and indicates success of failure. Zero parsed lines means the parer(s) failed. Non-zero unparsed lines are fine as crowdsec select relevant lines.`
|
`Tracks the number of events processed by each parser and indicates success of failure. Zero parsed lines means the parer(s) failed. Non-zero unparsed lines are fine as crowdsec select relevant lines.`
|
||||||
|
@ -181,7 +233,7 @@ func (s statParser) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
log.Warningf("while collecting parsers stats: %s", err)
|
log.Warningf("while collecting parsers stats: %s", err)
|
||||||
} else if numRows > 0 || showEmpty {
|
} else if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n" + title + ":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -213,7 +265,7 @@ func (s statStash) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
}
|
}
|
||||||
if numRows > 0 || showEmpty {
|
if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n" + title + ":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -254,7 +306,7 @@ func (s statLapi) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
|
||||||
if numRows > 0 || showEmpty {
|
if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n" + title + ":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -272,9 +324,9 @@ func (s statLapiMachine) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
|
||||||
numRows := lapiMetricsToTable(t, s)
|
numRows := lapiMetricsToTable(t, s)
|
||||||
|
|
||||||
if numRows > 0 || showEmpty{
|
if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n" + title + ":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -294,7 +346,7 @@ func (s statLapiBouncer) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
|
||||||
if numRows > 0 || showEmpty {
|
if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n" + title + ":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -320,9 +372,9 @@ func (s statLapiDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
numRows++
|
numRows++
|
||||||
}
|
}
|
||||||
|
|
||||||
if numRows > 0 || showEmpty{
|
if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n" + title + ":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -353,9 +405,9 @@ func (s statDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if numRows > 0 || showEmpty{
|
if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n" + title + ":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -380,9 +432,9 @@ func (s statAlert) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
numRows++
|
numRows++
|
||||||
}
|
}
|
||||||
|
|
||||||
if numRows > 0 || showEmpty{
|
if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n" + title + ":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -161,7 +161,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
|
||||||
leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow,
|
leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow,
|
||||||
v1.LapiRouteHits,
|
v1.LapiRouteHits,
|
||||||
leaky.BucketsCurrentCount,
|
leaky.BucketsCurrentCount,
|
||||||
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
|
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, parser.NodesWlHitsOk, parser.NodesWlHits,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
log.Infof("Loading prometheus collectors")
|
log.Infof("Loading prometheus collectors")
|
||||||
|
@ -170,7 +170,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
|
||||||
globalCsInfo, globalParsingHistogram, globalPourHistogram,
|
globalCsInfo, globalParsingHistogram, globalPourHistogram,
|
||||||
v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime,
|
v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime,
|
||||||
leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount,
|
leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount,
|
||||||
globalActiveDecisions, globalAlerts,
|
globalActiveDecisions, globalAlerts, parser.NodesWlHitsOk, parser.NodesWlHits,
|
||||||
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
|
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -168,9 +168,9 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
|
NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
|
||||||
}
|
}
|
||||||
exprErr := error(nil)
|
exprErr := error(nil)
|
||||||
isWhitelisted := n.CheckIPsWL(p.ParseIPSources())
|
isWhitelisted := n.CheckIPsWL(p)
|
||||||
if !isWhitelisted {
|
if !isWhitelisted {
|
||||||
isWhitelisted, exprErr = n.CheckExprWL(cachedExprEnv)
|
isWhitelisted, exprErr = n.CheckExprWL(cachedExprEnv, p)
|
||||||
}
|
}
|
||||||
if exprErr != nil {
|
if exprErr != nil {
|
||||||
// Previous code returned nil if there was an error, so we keep this behavior
|
// Previous code returned nil if there was an error, so we keep this behavior
|
||||||
|
|
|
@ -221,6 +221,24 @@ var NodesHitsKo = prometheus.NewCounterVec(
|
||||||
[]string{"source", "type", "name"},
|
[]string{"source", "type", "name"},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
//
|
||||||
|
|
||||||
|
var NodesWlHitsOk = prometheus.NewCounterVec(
|
||||||
|
prometheus.CounterOpts{
|
||||||
|
Name: "cs_node_wl_hits_ok_total",
|
||||||
|
Help: "Total events successfully whitelisted by node.",
|
||||||
|
},
|
||||||
|
[]string{"source", "type", "name", "reason"},
|
||||||
|
)
|
||||||
|
|
||||||
|
var NodesWlHits = prometheus.NewCounterVec(
|
||||||
|
prometheus.CounterOpts{
|
||||||
|
Name: "cs_node_wl_hits_total",
|
||||||
|
Help: "Total events processed by whitelist node.",
|
||||||
|
},
|
||||||
|
[]string{"source", "type", "name", "reason"},
|
||||||
|
)
|
||||||
|
|
||||||
func stageidx(stage string, stages []string) int {
|
func stageidx(stage string, stages []string) int {
|
||||||
for i, v := range stages {
|
for i, v := range stages {
|
||||||
if stage == v {
|
if stage == v {
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"github.com/antonmedv/expr/vm"
|
"github.com/antonmedv/expr/vm"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Whitelist struct {
|
type Whitelist struct {
|
||||||
|
@ -36,11 +37,13 @@ func (n *Node) ContainsIPLists() bool {
|
||||||
return len(n.Whitelist.B_Ips) > 0 || len(n.Whitelist.B_Cidrs) > 0
|
return len(n.Whitelist.B_Ips) > 0 || len(n.Whitelist.B_Cidrs) > 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Node) CheckIPsWL(srcs []net.IP) bool {
|
func (n *Node) CheckIPsWL(p *types.Event) bool {
|
||||||
|
srcs := p.ParseIPSources()
|
||||||
isWhitelisted := false
|
isWhitelisted := false
|
||||||
if !n.ContainsIPLists() {
|
if !n.ContainsIPLists() {
|
||||||
return isWhitelisted
|
return isWhitelisted
|
||||||
}
|
}
|
||||||
|
NodesWlHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason}).Inc()
|
||||||
for _, src := range srcs {
|
for _, src := range srcs {
|
||||||
if isWhitelisted {
|
if isWhitelisted {
|
||||||
break
|
break
|
||||||
|
@ -62,15 +65,19 @@ func (n *Node) CheckIPsWL(srcs []net.IP) bool {
|
||||||
n.Logger.Tracef("whitelist: %s not in [%s]", src, v)
|
n.Logger.Tracef("whitelist: %s not in [%s]", src, v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if isWhitelisted {
|
||||||
|
NodesWlHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason}).Inc()
|
||||||
|
}
|
||||||
return isWhitelisted
|
return isWhitelisted
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Node) CheckExprWL(cachedExprEnv map[string]interface{}) (bool, error) {
|
func (n *Node) CheckExprWL(cachedExprEnv map[string]interface{}, p *types.Event) (bool, error) {
|
||||||
isWhitelisted := false
|
isWhitelisted := false
|
||||||
|
|
||||||
if !n.ContainsExprLists() {
|
if !n.ContainsExprLists() {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
NodesWlHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason}).Inc()
|
||||||
/* run whitelist expression tests anyway */
|
/* run whitelist expression tests anyway */
|
||||||
for eidx, e := range n.Whitelist.B_Exprs {
|
for eidx, e := range n.Whitelist.B_Exprs {
|
||||||
//if we already know the event is whitelisted, skip the rest of the expressions
|
//if we already know the event is whitelisted, skip the rest of the expressions
|
||||||
|
@ -94,6 +101,9 @@ func (n *Node) CheckExprWL(cachedExprEnv map[string]interface{}) (bool, error) {
|
||||||
n.Logger.Errorf("unexpected type %t (%v) while running '%s'", output, output, n.Whitelist.Exprs[eidx])
|
n.Logger.Errorf("unexpected type %t (%v) while running '%s'", output, output, n.Whitelist.Exprs[eidx])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if isWhitelisted {
|
||||||
|
NodesWlHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason}).Inc()
|
||||||
|
}
|
||||||
return isWhitelisted, nil
|
return isWhitelisted, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -289,9 +289,9 @@ func TestWhitelistCheck(t *testing.T) {
|
||||||
var err error
|
var err error
|
||||||
node.Whitelist = tt.whitelist
|
node.Whitelist = tt.whitelist
|
||||||
node.CompileWLs()
|
node.CompileWLs()
|
||||||
isWhitelisted := node.CheckIPsWL(tt.event.ParseIPSources())
|
isWhitelisted := node.CheckIPsWL(tt.event)
|
||||||
if !isWhitelisted {
|
if !isWhitelisted {
|
||||||
isWhitelisted, err = node.CheckExprWL(map[string]interface{}{"evt": tt.event})
|
isWhitelisted, err = node.CheckExprWL(map[string]interface{}{"evt": tt.event}, tt.event)
|
||||||
}
|
}
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, tt.expected, isWhitelisted)
|
require.Equal(t, tt.expected, isWhitelisted)
|
||||||
|
|
Loading…
Reference in a new issue