Add conditional bucket (#1962)
This commit is contained in:
parent
822e441d3a
commit
a84e4b6b15
|
@ -63,6 +63,7 @@ type Leaky struct {
|
|||
//Profiling when set to true enables profiling of bucket
|
||||
Profiling bool
|
||||
timedOverflow bool
|
||||
conditionalOverflow bool
|
||||
logger *log.Entry
|
||||
scopeType types.ScopeType
|
||||
hash string
|
||||
|
@ -188,6 +189,10 @@ func FromFactory(bucketFactory BucketFactory) *Leaky {
|
|||
l.timedOverflow = true
|
||||
}
|
||||
|
||||
if l.BucketConfig.Type == "conditional" {
|
||||
l.conditionalOverflow = true
|
||||
l.Duration = l.BucketConfig.leakspeed
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
|
@ -247,6 +252,14 @@ func LeakRoutine(leaky *Leaky) error {
|
|||
BucketsPour.With(prometheus.Labels{"name": leaky.Name, "source": msg.Line.Src, "type": msg.Line.Module}).Inc()
|
||||
|
||||
leaky.Pour(leaky, *msg) // glue for now
|
||||
|
||||
for _, processor := range processors {
|
||||
msg = processor.AfterBucketPour(leaky.BucketConfig)(*msg, leaky)
|
||||
if msg == nil {
|
||||
goto End
|
||||
}
|
||||
}
|
||||
|
||||
//Clear cache on behalf of pour
|
||||
|
||||
// if durationTicker isn't initialized, then we're pouring our first event
|
||||
|
@ -337,7 +350,8 @@ func Pour(leaky *Leaky, msg types.Event) {
|
|||
leaky.First_ts = time.Now().UTC()
|
||||
}
|
||||
leaky.Last_ts = time.Now().UTC()
|
||||
if leaky.Limiter.Allow() {
|
||||
|
||||
if leaky.Limiter.Allow() || leaky.conditionalOverflow {
|
||||
leaky.Queue.Add(msg)
|
||||
} else {
|
||||
leaky.Ovflw_ts = time.Now().UTC()
|
||||
|
|
61
pkg/leakybucket/conditional.go
Normal file
61
pkg/leakybucket/conditional.go
Normal file
|
@ -0,0 +1,61 @@
|
|||
package leakybucket
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/antonmedv/expr"
|
||||
"github.com/antonmedv/expr/vm"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||
)
|
||||
|
||||
type ConditionalOverflow struct {
|
||||
ConditionalFilter string
|
||||
ConditionalFilterRuntime *vm.Program
|
||||
DumbProcessor
|
||||
}
|
||||
|
||||
func NewConditionalOverflow(g *BucketFactory) (*ConditionalOverflow, error) {
|
||||
var err error
|
||||
|
||||
c := ConditionalOverflow{}
|
||||
c.ConditionalFilter = g.ConditionalOverflow
|
||||
c.ConditionalFilterRuntime, err = expr.Compile(c.ConditionalFilter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{
|
||||
"queue": &Queue{}, "leaky": &Leaky{}})))
|
||||
if err != nil {
|
||||
g.logger.Errorf("Unable to compile condition expression for conditional bucket : %s", err)
|
||||
return nil, fmt.Errorf("unable to compile condition expression for conditional bucket : %v", err)
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
func (c *ConditionalOverflow) AfterBucketPour(b *BucketFactory) func(types.Event, *Leaky) *types.Event {
|
||||
return func(msg types.Event, l *Leaky) *types.Event {
|
||||
var condition, ok bool
|
||||
if c.ConditionalFilterRuntime != nil {
|
||||
l.logger.Debugf("Running condition expression : %s", c.ConditionalFilter)
|
||||
ret, err := expr.Run(c.ConditionalFilterRuntime, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &msg, "queue": l.Queue, "leaky": l}))
|
||||
if err != nil {
|
||||
l.logger.Errorf("unable to run conditional filter : %s", err)
|
||||
return &msg
|
||||
}
|
||||
|
||||
l.logger.Debugf("Conditional bucket expression returned : %v", ret)
|
||||
|
||||
if condition, ok = ret.(bool); !ok {
|
||||
l.logger.Warningf("overflow condition, unexpected non-bool return : %T", ret)
|
||||
return &msg
|
||||
}
|
||||
|
||||
if condition {
|
||||
l.logger.Debugf("Conditional bucket overflow")
|
||||
l.Ovflw_ts = time.Now().UTC()
|
||||
l.Out <- l.Queue
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return &msg
|
||||
}
|
||||
}
|
|
@ -54,6 +54,7 @@ type BucketFactory struct {
|
|||
CacheSize int `yaml:"cache_size"` //CacheSize, if > 0, limits the size of in-memory cache of the bucket
|
||||
Profiling bool `yaml:"profiling"` //Profiling, if true, will make the bucket record pours/overflows/etc.
|
||||
OverflowFilter string `yaml:"overflow_filter"` //OverflowFilter if present, is a filter that must return true for the overflow to go through
|
||||
ConditionalOverflow string `yaml:"condition"` //condition if present, is an expression that must return true for the bucket to overflow
|
||||
ScopeType types.ScopeType `yaml:"scope,omitempty"` //to enforce a different remediation than blocking an IP. Will default this to IP
|
||||
BucketName string `yaml:"-"`
|
||||
Filename string `yaml:"-"`
|
||||
|
@ -98,7 +99,7 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
|
|||
}
|
||||
} else if bucketFactory.Type == "counter" {
|
||||
if bucketFactory.Duration == "" {
|
||||
return fmt.Errorf("duration ca't be empty for counter")
|
||||
return fmt.Errorf("duration can't be empty for counter")
|
||||
}
|
||||
if bucketFactory.duration == 0 {
|
||||
return fmt.Errorf("bad duration for counter bucket '%d'", bucketFactory.duration)
|
||||
|
@ -110,6 +111,19 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
|
|||
if bucketFactory.Capacity != 0 {
|
||||
return fmt.Errorf("trigger bucket must have 0 capacity")
|
||||
}
|
||||
} else if bucketFactory.Type == "conditional" {
|
||||
if bucketFactory.ConditionalOverflow == "" {
|
||||
return fmt.Errorf("conditional bucket must have a condition")
|
||||
}
|
||||
if bucketFactory.Capacity != -1 {
|
||||
bucketFactory.logger.Warnf("Using a value different than -1 as capacity for conditional bucket, this may lead to unexpected overflows")
|
||||
}
|
||||
if bucketFactory.LeakSpeed == "" {
|
||||
return fmt.Errorf("leakspeed can't be empty for conditional bucket")
|
||||
}
|
||||
if bucketFactory.leakspeed == 0 {
|
||||
return fmt.Errorf("bad leakspeed for conditional bucket '%s'", bucketFactory.LeakSpeed)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("unknown bucket type '%s'", bucketFactory.Type)
|
||||
}
|
||||
|
@ -304,6 +318,8 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error {
|
|||
bucketFactory.processors = append(bucketFactory.processors, &Trigger{})
|
||||
case "counter":
|
||||
bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{})
|
||||
case "conditional":
|
||||
bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{})
|
||||
default:
|
||||
return fmt.Errorf("invalid type '%s' in %s : %v", bucketFactory.Type, bucketFactory.Filename, err)
|
||||
}
|
||||
|
@ -338,6 +354,16 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error {
|
|||
bucketFactory.processors = append(bucketFactory.processors, blackhole)
|
||||
}
|
||||
|
||||
if bucketFactory.ConditionalOverflow != "" {
|
||||
bucketFactory.logger.Tracef("Adding conditional overflow.")
|
||||
condovflw, err := NewConditionalOverflow(bucketFactory)
|
||||
if err != nil {
|
||||
bucketFactory.logger.Errorf("Error creating conditional overflow : %s", err)
|
||||
return fmt.Errorf("error creating conditional overflow : %s", err)
|
||||
}
|
||||
bucketFactory.processors = append(bucketFactory.processors, condovflw)
|
||||
}
|
||||
|
||||
if len(bucketFactory.Data) > 0 {
|
||||
for _, data := range bucketFactory.Data {
|
||||
if data.DestPath == "" {
|
||||
|
|
|
@ -6,6 +6,8 @@ type Processor interface {
|
|||
OnBucketInit(Bucket *BucketFactory) error
|
||||
OnBucketPour(Bucket *BucketFactory) func(types.Event, *Leaky) *types.Event
|
||||
OnBucketOverflow(Bucket *BucketFactory) func(*Leaky, types.RuntimeAlert, *Queue) (types.RuntimeAlert, *Queue)
|
||||
|
||||
AfterBucketPour(Bucket *BucketFactory) func(types.Event, *Leaky) *types.Event
|
||||
}
|
||||
|
||||
type DumbProcessor struct {
|
||||
|
@ -25,5 +27,10 @@ func (d *DumbProcessor) OnBucketOverflow(b *BucketFactory) func(*Leaky, types.Ru
|
|||
return func(leaky *Leaky, alert types.RuntimeAlert, queue *Queue) (types.RuntimeAlert, *Queue) {
|
||||
return alert, queue
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (d *DumbProcessor) AfterBucketPour(bucketFactory *BucketFactory) func(types.Event, *Leaky) *types.Event {
|
||||
return func(msg types.Event, leaky *Leaky) *types.Event {
|
||||
return &msg
|
||||
}
|
||||
}
|
||||
|
|
|
@ -64,6 +64,12 @@ func (u *CancelOnFilter) OnBucketOverflow(bucketFactory *BucketFactory) func(*Le
|
|||
}
|
||||
}
|
||||
|
||||
func (u *CancelOnFilter) AfterBucketPour(bucketFactory *BucketFactory) func(types.Event, *Leaky) *types.Event {
|
||||
return func(msg types.Event, leaky *Leaky) *types.Event {
|
||||
return &msg
|
||||
}
|
||||
}
|
||||
|
||||
func (u *CancelOnFilter) OnBucketInit(bucketFactory *BucketFactory) error {
|
||||
var err error
|
||||
var compiledExpr struct {
|
||||
|
|
11
pkg/leakybucket/tests/conditional-bucket/bucket.yaml
Normal file
11
pkg/leakybucket/tests/conditional-bucket/bucket.yaml
Normal file
|
@ -0,0 +1,11 @@
|
|||
type: conditional
|
||||
name: test/conditional
|
||||
#debug: true
|
||||
description: "conditional bucket"
|
||||
filter: "evt.Meta.log_type == 'http_access-log'"
|
||||
groupby: evt.Meta.source_ip
|
||||
condition: any(queue.Queue, {.Meta.http_path == "/"}) and any(queue.Queue, {.Meta.http_path == "/foo"})
|
||||
leakspeed: 1s
|
||||
capacity: -1
|
||||
labels:
|
||||
type: overflow_1
|
1
pkg/leakybucket/tests/conditional-bucket/scenarios.yaml
Normal file
1
pkg/leakybucket/tests/conditional-bucket/scenarios.yaml
Normal file
|
@ -0,0 +1 @@
|
|||
- filename: {{.TestDirectory}}/bucket.yaml
|
50
pkg/leakybucket/tests/conditional-bucket/test.json
Normal file
50
pkg/leakybucket/tests/conditional-bucket/test.json
Normal file
|
@ -0,0 +1,50 @@
|
|||
{
|
||||
"lines": [
|
||||
{
|
||||
"Line": {
|
||||
"Labels": {
|
||||
"type": "nginx"
|
||||
},
|
||||
"Raw": "don't care"
|
||||
},
|
||||
"MarshaledTime": "2020-01-01T10:00:00.000Z",
|
||||
"Meta": {
|
||||
"source_ip": "2a00:1450:4007:816::200e",
|
||||
"log_type": "http_access-log",
|
||||
"http_path": "/"
|
||||
}
|
||||
},
|
||||
{
|
||||
"Line": {
|
||||
"Labels": {
|
||||
"type": "nginx"
|
||||
},
|
||||
"Raw": "don't care"
|
||||
},
|
||||
"MarshaledTime": "2020-01-01T10:00:00.000Z",
|
||||
"Meta": {
|
||||
"source_ip": "2a00:1450:4007:816::200e",
|
||||
"log_type": "http_access-log",
|
||||
"http_path": "/foo"
|
||||
}
|
||||
}
|
||||
],
|
||||
"results": [
|
||||
{
|
||||
"Type" : 1,
|
||||
"Alert": {
|
||||
"sources" : {
|
||||
"2a00:1450:4007:816::200e": {
|
||||
"ip": "2a00:1450:4007:816::200e",
|
||||
"scope": "Ip",
|
||||
"value": "2a00:1450:4007:816::200e"
|
||||
}
|
||||
},
|
||||
"Alert" : {
|
||||
"scenario": "test/conditional",
|
||||
"events_count": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
|
@ -53,6 +53,12 @@ func (u *Uniq) OnBucketOverflow(bucketFactory *BucketFactory) func(*Leaky, types
|
|||
}
|
||||
}
|
||||
|
||||
func (u *Uniq) AfterBucketPour(bucketFactory *BucketFactory) func(types.Event, *Leaky) *types.Event {
|
||||
return func(msg types.Event, leaky *Leaky) *types.Event {
|
||||
return &msg
|
||||
}
|
||||
}
|
||||
|
||||
func (u *Uniq) OnBucketInit(bucketFactory *BucketFactory) error {
|
||||
var err error
|
||||
var compiledExpr *vm.Program
|
||||
|
|
Loading…
Reference in a new issue