Add conditional bucket (#1962)
This commit is contained in:
parent
822e441d3a
commit
a84e4b6b15
|
@ -63,6 +63,7 @@ type Leaky struct {
|
||||||
//Profiling when set to true enables profiling of bucket
|
//Profiling when set to true enables profiling of bucket
|
||||||
Profiling bool
|
Profiling bool
|
||||||
timedOverflow bool
|
timedOverflow bool
|
||||||
|
conditionalOverflow bool
|
||||||
logger *log.Entry
|
logger *log.Entry
|
||||||
scopeType types.ScopeType
|
scopeType types.ScopeType
|
||||||
hash string
|
hash string
|
||||||
|
@ -188,6 +189,10 @@ func FromFactory(bucketFactory BucketFactory) *Leaky {
|
||||||
l.timedOverflow = true
|
l.timedOverflow = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if l.BucketConfig.Type == "conditional" {
|
||||||
|
l.conditionalOverflow = true
|
||||||
|
l.Duration = l.BucketConfig.leakspeed
|
||||||
|
}
|
||||||
return l
|
return l
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -247,6 +252,14 @@ func LeakRoutine(leaky *Leaky) error {
|
||||||
BucketsPour.With(prometheus.Labels{"name": leaky.Name, "source": msg.Line.Src, "type": msg.Line.Module}).Inc()
|
BucketsPour.With(prometheus.Labels{"name": leaky.Name, "source": msg.Line.Src, "type": msg.Line.Module}).Inc()
|
||||||
|
|
||||||
leaky.Pour(leaky, *msg) // glue for now
|
leaky.Pour(leaky, *msg) // glue for now
|
||||||
|
|
||||||
|
for _, processor := range processors {
|
||||||
|
msg = processor.AfterBucketPour(leaky.BucketConfig)(*msg, leaky)
|
||||||
|
if msg == nil {
|
||||||
|
goto End
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
//Clear cache on behalf of pour
|
//Clear cache on behalf of pour
|
||||||
|
|
||||||
// if durationTicker isn't initialized, then we're pouring our first event
|
// if durationTicker isn't initialized, then we're pouring our first event
|
||||||
|
@ -337,7 +350,8 @@ func Pour(leaky *Leaky, msg types.Event) {
|
||||||
leaky.First_ts = time.Now().UTC()
|
leaky.First_ts = time.Now().UTC()
|
||||||
}
|
}
|
||||||
leaky.Last_ts = time.Now().UTC()
|
leaky.Last_ts = time.Now().UTC()
|
||||||
if leaky.Limiter.Allow() {
|
|
||||||
|
if leaky.Limiter.Allow() || leaky.conditionalOverflow {
|
||||||
leaky.Queue.Add(msg)
|
leaky.Queue.Add(msg)
|
||||||
} else {
|
} else {
|
||||||
leaky.Ovflw_ts = time.Now().UTC()
|
leaky.Ovflw_ts = time.Now().UTC()
|
||||||
|
|
61
pkg/leakybucket/conditional.go
Normal file
61
pkg/leakybucket/conditional.go
Normal file
|
@ -0,0 +1,61 @@
|
||||||
|
package leakybucket
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/antonmedv/expr"
|
||||||
|
"github.com/antonmedv/expr/vm"
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ConditionalOverflow struct {
|
||||||
|
ConditionalFilter string
|
||||||
|
ConditionalFilterRuntime *vm.Program
|
||||||
|
DumbProcessor
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewConditionalOverflow(g *BucketFactory) (*ConditionalOverflow, error) {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
c := ConditionalOverflow{}
|
||||||
|
c.ConditionalFilter = g.ConditionalOverflow
|
||||||
|
c.ConditionalFilterRuntime, err = expr.Compile(c.ConditionalFilter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{
|
||||||
|
"queue": &Queue{}, "leaky": &Leaky{}})))
|
||||||
|
if err != nil {
|
||||||
|
g.logger.Errorf("Unable to compile condition expression for conditional bucket : %s", err)
|
||||||
|
return nil, fmt.Errorf("unable to compile condition expression for conditional bucket : %v", err)
|
||||||
|
}
|
||||||
|
return &c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ConditionalOverflow) AfterBucketPour(b *BucketFactory) func(types.Event, *Leaky) *types.Event {
|
||||||
|
return func(msg types.Event, l *Leaky) *types.Event {
|
||||||
|
var condition, ok bool
|
||||||
|
if c.ConditionalFilterRuntime != nil {
|
||||||
|
l.logger.Debugf("Running condition expression : %s", c.ConditionalFilter)
|
||||||
|
ret, err := expr.Run(c.ConditionalFilterRuntime, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &msg, "queue": l.Queue, "leaky": l}))
|
||||||
|
if err != nil {
|
||||||
|
l.logger.Errorf("unable to run conditional filter : %s", err)
|
||||||
|
return &msg
|
||||||
|
}
|
||||||
|
|
||||||
|
l.logger.Debugf("Conditional bucket expression returned : %v", ret)
|
||||||
|
|
||||||
|
if condition, ok = ret.(bool); !ok {
|
||||||
|
l.logger.Warningf("overflow condition, unexpected non-bool return : %T", ret)
|
||||||
|
return &msg
|
||||||
|
}
|
||||||
|
|
||||||
|
if condition {
|
||||||
|
l.logger.Debugf("Conditional bucket overflow")
|
||||||
|
l.Ovflw_ts = time.Now().UTC()
|
||||||
|
l.Out <- l.Queue
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return &msg
|
||||||
|
}
|
||||||
|
}
|
|
@ -54,6 +54,7 @@ type BucketFactory struct {
|
||||||
CacheSize int `yaml:"cache_size"` //CacheSize, if > 0, limits the size of in-memory cache of the bucket
|
CacheSize int `yaml:"cache_size"` //CacheSize, if > 0, limits the size of in-memory cache of the bucket
|
||||||
Profiling bool `yaml:"profiling"` //Profiling, if true, will make the bucket record pours/overflows/etc.
|
Profiling bool `yaml:"profiling"` //Profiling, if true, will make the bucket record pours/overflows/etc.
|
||||||
OverflowFilter string `yaml:"overflow_filter"` //OverflowFilter if present, is a filter that must return true for the overflow to go through
|
OverflowFilter string `yaml:"overflow_filter"` //OverflowFilter if present, is a filter that must return true for the overflow to go through
|
||||||
|
ConditionalOverflow string `yaml:"condition"` //condition if present, is an expression that must return true for the bucket to overflow
|
||||||
ScopeType types.ScopeType `yaml:"scope,omitempty"` //to enforce a different remediation than blocking an IP. Will default this to IP
|
ScopeType types.ScopeType `yaml:"scope,omitempty"` //to enforce a different remediation than blocking an IP. Will default this to IP
|
||||||
BucketName string `yaml:"-"`
|
BucketName string `yaml:"-"`
|
||||||
Filename string `yaml:"-"`
|
Filename string `yaml:"-"`
|
||||||
|
@ -98,7 +99,7 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
|
||||||
}
|
}
|
||||||
} else if bucketFactory.Type == "counter" {
|
} else if bucketFactory.Type == "counter" {
|
||||||
if bucketFactory.Duration == "" {
|
if bucketFactory.Duration == "" {
|
||||||
return fmt.Errorf("duration ca't be empty for counter")
|
return fmt.Errorf("duration can't be empty for counter")
|
||||||
}
|
}
|
||||||
if bucketFactory.duration == 0 {
|
if bucketFactory.duration == 0 {
|
||||||
return fmt.Errorf("bad duration for counter bucket '%d'", bucketFactory.duration)
|
return fmt.Errorf("bad duration for counter bucket '%d'", bucketFactory.duration)
|
||||||
|
@ -110,6 +111,19 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
|
||||||
if bucketFactory.Capacity != 0 {
|
if bucketFactory.Capacity != 0 {
|
||||||
return fmt.Errorf("trigger bucket must have 0 capacity")
|
return fmt.Errorf("trigger bucket must have 0 capacity")
|
||||||
}
|
}
|
||||||
|
} else if bucketFactory.Type == "conditional" {
|
||||||
|
if bucketFactory.ConditionalOverflow == "" {
|
||||||
|
return fmt.Errorf("conditional bucket must have a condition")
|
||||||
|
}
|
||||||
|
if bucketFactory.Capacity != -1 {
|
||||||
|
bucketFactory.logger.Warnf("Using a value different than -1 as capacity for conditional bucket, this may lead to unexpected overflows")
|
||||||
|
}
|
||||||
|
if bucketFactory.LeakSpeed == "" {
|
||||||
|
return fmt.Errorf("leakspeed can't be empty for conditional bucket")
|
||||||
|
}
|
||||||
|
if bucketFactory.leakspeed == 0 {
|
||||||
|
return fmt.Errorf("bad leakspeed for conditional bucket '%s'", bucketFactory.LeakSpeed)
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
return fmt.Errorf("unknown bucket type '%s'", bucketFactory.Type)
|
return fmt.Errorf("unknown bucket type '%s'", bucketFactory.Type)
|
||||||
}
|
}
|
||||||
|
@ -304,6 +318,8 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error {
|
||||||
bucketFactory.processors = append(bucketFactory.processors, &Trigger{})
|
bucketFactory.processors = append(bucketFactory.processors, &Trigger{})
|
||||||
case "counter":
|
case "counter":
|
||||||
bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{})
|
bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{})
|
||||||
|
case "conditional":
|
||||||
|
bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{})
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("invalid type '%s' in %s : %v", bucketFactory.Type, bucketFactory.Filename, err)
|
return fmt.Errorf("invalid type '%s' in %s : %v", bucketFactory.Type, bucketFactory.Filename, err)
|
||||||
}
|
}
|
||||||
|
@ -338,6 +354,16 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error {
|
||||||
bucketFactory.processors = append(bucketFactory.processors, blackhole)
|
bucketFactory.processors = append(bucketFactory.processors, blackhole)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if bucketFactory.ConditionalOverflow != "" {
|
||||||
|
bucketFactory.logger.Tracef("Adding conditional overflow.")
|
||||||
|
condovflw, err := NewConditionalOverflow(bucketFactory)
|
||||||
|
if err != nil {
|
||||||
|
bucketFactory.logger.Errorf("Error creating conditional overflow : %s", err)
|
||||||
|
return fmt.Errorf("error creating conditional overflow : %s", err)
|
||||||
|
}
|
||||||
|
bucketFactory.processors = append(bucketFactory.processors, condovflw)
|
||||||
|
}
|
||||||
|
|
||||||
if len(bucketFactory.Data) > 0 {
|
if len(bucketFactory.Data) > 0 {
|
||||||
for _, data := range bucketFactory.Data {
|
for _, data := range bucketFactory.Data {
|
||||||
if data.DestPath == "" {
|
if data.DestPath == "" {
|
||||||
|
|
|
@ -6,6 +6,8 @@ type Processor interface {
|
||||||
OnBucketInit(Bucket *BucketFactory) error
|
OnBucketInit(Bucket *BucketFactory) error
|
||||||
OnBucketPour(Bucket *BucketFactory) func(types.Event, *Leaky) *types.Event
|
OnBucketPour(Bucket *BucketFactory) func(types.Event, *Leaky) *types.Event
|
||||||
OnBucketOverflow(Bucket *BucketFactory) func(*Leaky, types.RuntimeAlert, *Queue) (types.RuntimeAlert, *Queue)
|
OnBucketOverflow(Bucket *BucketFactory) func(*Leaky, types.RuntimeAlert, *Queue) (types.RuntimeAlert, *Queue)
|
||||||
|
|
||||||
|
AfterBucketPour(Bucket *BucketFactory) func(types.Event, *Leaky) *types.Event
|
||||||
}
|
}
|
||||||
|
|
||||||
type DumbProcessor struct {
|
type DumbProcessor struct {
|
||||||
|
@ -25,5 +27,10 @@ func (d *DumbProcessor) OnBucketOverflow(b *BucketFactory) func(*Leaky, types.Ru
|
||||||
return func(leaky *Leaky, alert types.RuntimeAlert, queue *Queue) (types.RuntimeAlert, *Queue) {
|
return func(leaky *Leaky, alert types.RuntimeAlert, queue *Queue) (types.RuntimeAlert, *Queue) {
|
||||||
return alert, queue
|
return alert, queue
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *DumbProcessor) AfterBucketPour(bucketFactory *BucketFactory) func(types.Event, *Leaky) *types.Event {
|
||||||
|
return func(msg types.Event, leaky *Leaky) *types.Event {
|
||||||
|
return &msg
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -64,6 +64,12 @@ func (u *CancelOnFilter) OnBucketOverflow(bucketFactory *BucketFactory) func(*Le
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (u *CancelOnFilter) AfterBucketPour(bucketFactory *BucketFactory) func(types.Event, *Leaky) *types.Event {
|
||||||
|
return func(msg types.Event, leaky *Leaky) *types.Event {
|
||||||
|
return &msg
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (u *CancelOnFilter) OnBucketInit(bucketFactory *BucketFactory) error {
|
func (u *CancelOnFilter) OnBucketInit(bucketFactory *BucketFactory) error {
|
||||||
var err error
|
var err error
|
||||||
var compiledExpr struct {
|
var compiledExpr struct {
|
||||||
|
|
11
pkg/leakybucket/tests/conditional-bucket/bucket.yaml
Normal file
11
pkg/leakybucket/tests/conditional-bucket/bucket.yaml
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
type: conditional
|
||||||
|
name: test/conditional
|
||||||
|
#debug: true
|
||||||
|
description: "conditional bucket"
|
||||||
|
filter: "evt.Meta.log_type == 'http_access-log'"
|
||||||
|
groupby: evt.Meta.source_ip
|
||||||
|
condition: any(queue.Queue, {.Meta.http_path == "/"}) and any(queue.Queue, {.Meta.http_path == "/foo"})
|
||||||
|
leakspeed: 1s
|
||||||
|
capacity: -1
|
||||||
|
labels:
|
||||||
|
type: overflow_1
|
1
pkg/leakybucket/tests/conditional-bucket/scenarios.yaml
Normal file
1
pkg/leakybucket/tests/conditional-bucket/scenarios.yaml
Normal file
|
@ -0,0 +1 @@
|
||||||
|
- filename: {{.TestDirectory}}/bucket.yaml
|
50
pkg/leakybucket/tests/conditional-bucket/test.json
Normal file
50
pkg/leakybucket/tests/conditional-bucket/test.json
Normal file
|
@ -0,0 +1,50 @@
|
||||||
|
{
|
||||||
|
"lines": [
|
||||||
|
{
|
||||||
|
"Line": {
|
||||||
|
"Labels": {
|
||||||
|
"type": "nginx"
|
||||||
|
},
|
||||||
|
"Raw": "don't care"
|
||||||
|
},
|
||||||
|
"MarshaledTime": "2020-01-01T10:00:00.000Z",
|
||||||
|
"Meta": {
|
||||||
|
"source_ip": "2a00:1450:4007:816::200e",
|
||||||
|
"log_type": "http_access-log",
|
||||||
|
"http_path": "/"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Line": {
|
||||||
|
"Labels": {
|
||||||
|
"type": "nginx"
|
||||||
|
},
|
||||||
|
"Raw": "don't care"
|
||||||
|
},
|
||||||
|
"MarshaledTime": "2020-01-01T10:00:00.000Z",
|
||||||
|
"Meta": {
|
||||||
|
"source_ip": "2a00:1450:4007:816::200e",
|
||||||
|
"log_type": "http_access-log",
|
||||||
|
"http_path": "/foo"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"Type" : 1,
|
||||||
|
"Alert": {
|
||||||
|
"sources" : {
|
||||||
|
"2a00:1450:4007:816::200e": {
|
||||||
|
"ip": "2a00:1450:4007:816::200e",
|
||||||
|
"scope": "Ip",
|
||||||
|
"value": "2a00:1450:4007:816::200e"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"Alert" : {
|
||||||
|
"scenario": "test/conditional",
|
||||||
|
"events_count": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
|
@ -53,6 +53,12 @@ func (u *Uniq) OnBucketOverflow(bucketFactory *BucketFactory) func(*Leaky, types
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (u *Uniq) AfterBucketPour(bucketFactory *BucketFactory) func(types.Event, *Leaky) *types.Event {
|
||||||
|
return func(msg types.Event, leaky *Leaky) *types.Event {
|
||||||
|
return &msg
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (u *Uniq) OnBucketInit(bucketFactory *BucketFactory) error {
|
func (u *Uniq) OnBucketInit(bucketFactory *BucketFactory) error {
|
||||||
var err error
|
var err error
|
||||||
var compiledExpr *vm.Program
|
var compiledExpr *vm.Program
|
||||||
|
|
Loading…
Reference in a new issue