do not create bucket if the overflow would be blackholed
This commit is contained in:
parent
9f72bbe725
commit
10c97e46af
|
@ -11,6 +11,7 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
|
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/leakybucket"
|
||||||
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
|
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/parser"
|
"github.com/crowdsecurity/crowdsec/pkg/parser"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
@ -47,6 +48,8 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
|
||||||
inputLineChan := make(chan types.Event)
|
inputLineChan := make(chan types.Event)
|
||||||
inputEventChan := make(chan types.Event)
|
inputEventChan := make(chan types.Event)
|
||||||
|
|
||||||
|
leaky.BlackholeTracking = &sync.Map{}
|
||||||
|
|
||||||
//start go-routines for parsing, buckets pour and outputs.
|
//start go-routines for parsing, buckets pour and outputs.
|
||||||
parserWg := &sync.WaitGroup{}
|
parserWg := &sync.WaitGroup{}
|
||||||
parsersTomb.Go(func() error {
|
parsersTomb.Go(func() error {
|
||||||
|
@ -67,6 +70,12 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
|
||||||
parserWg.Wait()
|
parserWg.Wait()
|
||||||
|
|
||||||
bucketWg := &sync.WaitGroup{}
|
bucketWg := &sync.WaitGroup{}
|
||||||
|
bucketsTomb.Go(func() error {
|
||||||
|
bucketWg.Add(1)
|
||||||
|
leakybucket.CleanupBlackhole(&bucketsTomb)
|
||||||
|
bucketWg.Done()
|
||||||
|
return nil
|
||||||
|
})
|
||||||
bucketsTomb.Go(func() error {
|
bucketsTomb.Go(func() error {
|
||||||
bucketWg.Add(1)
|
bucketWg.Add(1)
|
||||||
/*restore previous state as well if present*/
|
/*restore previous state as well if present*/
|
||||||
|
|
|
@ -48,6 +48,7 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
alertsToPush, err := dedupAlerts(alerts)
|
alertsToPush, err := dedupAlerts(alerts)
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "failed to transform alerts for api")
|
return errors.Wrap(err, "failed to transform alerts for api")
|
||||||
}
|
}
|
||||||
|
@ -55,6 +56,7 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "failed sending alert to LAPI")
|
return errors.Wrap(err, "failed sending alert to LAPI")
|
||||||
}
|
}
|
||||||
|
log.Tracef("sent %d alerts in %s", len(alertsToPush), time.Since(start))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -115,6 +115,7 @@ func (c *Controller) sendAlertToPluginChannel(alert *models.Alert, profileID uin
|
||||||
// CreateAlert : write received alerts in body to the database
|
// CreateAlert : write received alerts in body to the database
|
||||||
func (c *Controller) CreateAlert(gctx *gin.Context) {
|
func (c *Controller) CreateAlert(gctx *gin.Context) {
|
||||||
|
|
||||||
|
startCreateAlert := time.Now()
|
||||||
var input models.AddAlertsRequest
|
var input models.AddAlertsRequest
|
||||||
|
|
||||||
claims := jwt.ExtractClaims(gctx)
|
claims := jwt.ExtractClaims(gctx)
|
||||||
|
@ -130,6 +131,7 @@ func (c *Controller) CreateAlert(gctx *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
stopFlush := false
|
stopFlush := false
|
||||||
|
startAlertLoop := time.Now()
|
||||||
for _, alert := range input {
|
for _, alert := range input {
|
||||||
alert.MachineID = machineID
|
alert.MachineID = machineID
|
||||||
if len(alert.Decisions) != 0 {
|
if len(alert.Decisions) != 0 {
|
||||||
|
@ -175,12 +177,15 @@ func (c *Controller) CreateAlert(gctx *gin.Context) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
log.Tracef("alert loop in controller took %s", time.Since(startAlertLoop))
|
||||||
|
|
||||||
if stopFlush {
|
if stopFlush {
|
||||||
c.DBClient.CanFlush = false
|
c.DBClient.CanFlush = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
startDbCall := time.Now()
|
||||||
alerts, err := c.DBClient.CreateAlert(machineID, input)
|
alerts, err := c.DBClient.CreateAlert(machineID, input)
|
||||||
|
log.Tracef("CreateAlert : %s", time.Since(startDbCall))
|
||||||
c.DBClient.CanFlush = true
|
c.DBClient.CanFlush = true
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -197,6 +202,8 @@ func (c *Controller) CreateAlert(gctx *gin.Context) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Tracef("%d alerts created in %s", len(input), time.Since(startCreateAlert))
|
||||||
|
|
||||||
gctx.JSON(http.StatusCreated, alerts)
|
gctx.JSON(http.StatusCreated, alerts)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -306,6 +306,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([
|
||||||
c.Log.Debugf("writing %d items", len(alertList))
|
c.Log.Debugf("writing %d items", len(alertList))
|
||||||
bulk := make([]*ent.AlertCreate, 0, bulkSize)
|
bulk := make([]*ent.AlertCreate, 0, bulkSize)
|
||||||
alertDecisions := make([][]*ent.Decision, 0, bulkSize)
|
alertDecisions := make([][]*ent.Decision, 0, bulkSize)
|
||||||
|
alertLoopStart := time.Now()
|
||||||
for i, alertItem := range alertList {
|
for i, alertItem := range alertList {
|
||||||
var decisions []*ent.Decision
|
var decisions []*ent.Decision
|
||||||
var metas []*ent.Meta
|
var metas []*ent.Meta
|
||||||
|
@ -524,12 +525,16 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
c.Log.Tracef("alert loop took %s for %d elements", time.Since(alertLoopStart), len(alertList))
|
||||||
|
|
||||||
|
createBulkStart := time.Now()
|
||||||
alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX)
|
alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return []string{}, errors.Wrapf(BulkError, "leftovers creating alert : %s", err)
|
return []string{}, errors.Wrapf(BulkError, "leftovers creating alert : %s", err)
|
||||||
}
|
}
|
||||||
|
c.Log.Infof("createBulk took %s for %d elements", time.Since(createBulkStart), len(bulk))
|
||||||
|
|
||||||
|
decisionsUpdateTime := time.Now()
|
||||||
for alertIndex, a := range alerts {
|
for alertIndex, a := range alerts {
|
||||||
ret = append(ret, strconv.Itoa(a.ID))
|
ret = append(ret, strconv.Itoa(a.ID))
|
||||||
d := alertDecisions[alertIndex]
|
d := alertDecisions[alertIndex]
|
||||||
|
@ -542,6 +547,8 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
c.Log.Infof("decisions update took %s for %d elements", time.Since(decisionsUpdateTime), len(alerts))
|
||||||
|
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -888,7 +895,12 @@ func (c *Client) DeleteAlertWithFilter(filter map[string][]string) (int, error)
|
||||||
return 0, errors.Wrap(DeleteFail, "alert query failed")
|
return 0, errors.Wrap(DeleteFail, "alert query failed")
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, alertItem := range alertsToDelete {
|
c.Log.Debugf("Deleting %d alerts", len(alertsToDelete))
|
||||||
|
|
||||||
|
for p, alertItem := range alertsToDelete {
|
||||||
|
if p%100 == 0 {
|
||||||
|
c.Log.Debugf("Deleting alert %d", p)
|
||||||
|
}
|
||||||
err = c.DeleteAlertGraph(alertItem)
|
err = c.DeleteAlertGraph(alertItem)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.Log.Warningf("DeleteAlertWithFilter : %s", err)
|
c.Log.Warningf("DeleteAlertWithFilter : %s", err)
|
||||||
|
|
|
@ -5,6 +5,8 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
"gopkg.in/tomb.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
type HiddenKey struct {
|
type HiddenKey struct {
|
||||||
|
@ -31,38 +33,48 @@ func NewBlackhole(bucketFactory *BucketFactory) (*Blackhole, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func CleanupBlackhole(bucketsTomb *tomb.Tomb) error {
|
||||||
|
ticker := time.NewTicker(10 * time.Second)
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-bucketsTomb.Dying():
|
||||||
|
ticker.Stop()
|
||||||
|
BlackholeTracking.Range(func(key, value interface{}) bool {
|
||||||
|
BlackholeTracking.Delete(key)
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
case <-ticker.C:
|
||||||
|
BlackholeTracking.Range(func(key, value interface{}) bool {
|
||||||
|
expirationDate := value.(time.Time)
|
||||||
|
if expirationDate.Before(time.Now().UTC()) {
|
||||||
|
log.Debugf("Expiring blackhole for %s", key)
|
||||||
|
BlackholeTracking.Delete(key)
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (bl *Blackhole) OnBucketOverflow(bucketFactory *BucketFactory) func(*Leaky, types.RuntimeAlert, *Queue) (types.RuntimeAlert, *Queue) {
|
func (bl *Blackhole) OnBucketOverflow(bucketFactory *BucketFactory) func(*Leaky, types.RuntimeAlert, *Queue) (types.RuntimeAlert, *Queue) {
|
||||||
return func(leaky *Leaky, alert types.RuntimeAlert, queue *Queue) (types.RuntimeAlert, *Queue) {
|
return func(leaky *Leaky, alert types.RuntimeAlert, queue *Queue) (types.RuntimeAlert, *Queue) {
|
||||||
var blackholed bool = false
|
if expirationDate, ok := BlackholeTracking.Load(leaky.Mapkey); ok {
|
||||||
var tmp []HiddenKey
|
if expirationDate.(time.Time).After(time.Now().UTC()) {
|
||||||
// search if we are blackholed and refresh the slice
|
leaky.logger.Debugf("Blackhole already triggered for %s", leaky.Mapkey)
|
||||||
for _, element := range bl.hiddenKeys {
|
return types.RuntimeAlert{
|
||||||
|
Mapkey: leaky.Mapkey,
|
||||||
if element.key == leaky.Mapkey {
|
}, nil
|
||||||
if element.expiration.After(leaky.Ovflw_ts) {
|
|
||||||
leaky.logger.Debugf("Overflow discarded, still blackholed for %s", element.expiration.Sub(leaky.Ovflw_ts))
|
|
||||||
blackholed = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if element.expiration.After(leaky.Ovflw_ts) {
|
|
||||||
tmp = append(tmp, element)
|
|
||||||
} else {
|
} else {
|
||||||
leaky.logger.Debugf("%s left blackhole %s ago", element.key, leaky.Ovflw_ts.Sub(element.expiration))
|
leaky.logger.Debugf("Blackhole expired for %s", leaky.Mapkey)
|
||||||
|
BlackholeTracking.Delete(leaky.Mapkey)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
bl.hiddenKeys = tmp
|
|
||||||
|
|
||||||
if blackholed {
|
BlackholeTracking.Store(leaky.Mapkey, time.Now().UTC().Add(bl.duration))
|
||||||
leaky.logger.Tracef("Event is blackholed (%s)", leaky.First_ts)
|
|
||||||
return types.RuntimeAlert{
|
leaky.logger.Debugf("Blackhole triggered for %s", leaky.Mapkey)
|
||||||
Mapkey: leaky.Mapkey,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
bl.hiddenKeys = append(bl.hiddenKeys, HiddenKey{leaky.Mapkey, leaky.Ovflw_ts.Add(bl.duration)})
|
|
||||||
leaky.logger.Debugf("Adding overflow to blackhole (%s)", leaky.First_ts)
|
|
||||||
return alert, queue
|
return alert, queue
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -220,7 +220,6 @@ func LeakRoutine(leaky *Leaky) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
leaky.logger.Debugf("Leaky routine starting, lifetime : %s", leaky.Duration)
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
/*receiving an event*/
|
/*receiving an event*/
|
||||||
|
|
|
@ -38,6 +38,8 @@ func TestBucket(t *testing.T) {
|
||||||
log.Fatalf("exprhelpers init failed: %s", err)
|
log.Fatalf("exprhelpers init failed: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
BlackholeTracking = &sync.Map{}
|
||||||
|
|
||||||
if envSetting != "" {
|
if envSetting != "" {
|
||||||
if err := testOneBucket(t, envSetting, tomb); err != nil {
|
if err := testOneBucket(t, envSetting, tomb); err != nil {
|
||||||
t.Fatalf("Test '%s' failed : %s", envSetting, err)
|
t.Fatalf("Test '%s' failed : %s", envSetting, err)
|
||||||
|
@ -64,8 +66,8 @@ func TestBucket(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//during tests, we're likely to have only one scenario, and thus only one holder.
|
// during tests, we're likely to have only one scenario, and thus only one holder.
|
||||||
//we want to avoid the death of the tomb because all existing buckets have been destroyed.
|
// we want to avoid the death of the tomb because all existing buckets have been destroyed.
|
||||||
func watchTomb(tomb *tomb.Tomb) {
|
func watchTomb(tomb *tomb.Tomb) {
|
||||||
for {
|
for {
|
||||||
if tomb.Alive() == false {
|
if tomb.Alive() == false {
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"math"
|
"math"
|
||||||
"os"
|
"os"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
@ -23,9 +24,13 @@ var serialized map[string]Leaky
|
||||||
var BucketPourCache map[string][]types.Event
|
var BucketPourCache map[string][]types.Event
|
||||||
var BucketPourTrack bool
|
var BucketPourTrack bool
|
||||||
|
|
||||||
/*The leaky routines lifecycle are based on "real" time.
|
var BlackholeTracking *sync.Map
|
||||||
|
|
||||||
|
/*
|
||||||
|
The leaky routines lifecycle are based on "real" time.
|
||||||
But when we are running in time-machine mode, the reference time is in logs and not "real" time.
|
But when we are running in time-machine mode, the reference time is in logs and not "real" time.
|
||||||
Thus we need to garbage collect them to avoid a skyrocketing memory usage.*/
|
Thus we need to garbage collect them to avoid a skyrocketing memory usage.
|
||||||
|
*/
|
||||||
func GarbageCollectBuckets(deadline time.Time, buckets *Buckets) error {
|
func GarbageCollectBuckets(deadline time.Time, buckets *Buckets) error {
|
||||||
buckets.wgPour.Wait()
|
buckets.wgPour.Wait()
|
||||||
buckets.wgDumpState.Add(1)
|
buckets.wgDumpState.Add(1)
|
||||||
|
@ -340,7 +345,10 @@ func PourItemToHolders(parsed types.Event, holders []BucketFactory, buckets *Buc
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
buckey := GetKey(holders[idx], groupby)
|
buckey := GetKey(holders[idx], groupby)
|
||||||
|
if _, ok := BlackholeTracking.Load(buckey); ok {
|
||||||
|
holders[idx].logger.Tracef("Event is blackholed: %s", buckey)
|
||||||
|
continue
|
||||||
|
}
|
||||||
//we need to either find the existing bucket, or create a new one (if it's the first event to hit it for this partition key)
|
//we need to either find the existing bucket, or create a new one (if it's the first event to hit it for this partition key)
|
||||||
bucket, err := LoadOrStoreBucketFromHolder(buckey, buckets, holders[idx], parsed.ExpectMode)
|
bucket, err := LoadOrStoreBucketFromHolder(buckey, buckets, holders[idx], parsed.ExpectMode)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
Loading…
Reference in a new issue