beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

update

Browse source
This commit is contained in:
alteredCoder 2022-04-11 19:02:02 +02:00
parent f96cb2a70d
commit ab525fff6a
5 changed files with 114 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
cmd/crowdsec-cli/console.go
View file

@ -20,6 +20,7 @@ import (
"github.com/olekukonko/tablewriter" "github.com/olekukonko/tablewriter"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gopkg.in/yaml.v2"
) )
func NewConsoleCmd() *cobra.Command { func NewConsoleCmd() *cobra.Command {
@ -295,6 +296,21 @@ Disable given information push to the central API.`,
cmdLabelAdd.MarkFlagRequired("value") cmdLabelAdd.MarkFlagRequired("value")
cmdLabel.AddCommand(cmdLabelAdd) cmdLabel.AddCommand(cmdLabelAdd)
cmdLabelStatus := &cobra.Command{
Use: "status",
Short: "List label to send with alerts",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
dump, err := yaml.Marshal(csConfig.Crowdsec.LabelsToSend)
if err != nil {
log.Fatalf("unable to show labels status: %s", err)
}
fmt.Println(dump)
},
}
cmdLabel.AddCommand(cmdLabelStatus)
cmdConsole.AddCommand(cmdLabel) cmdConsole.AddCommand(cmdLabel)
return cmdConsole return cmdConsole

1
pkg/database/alerts.go
View file

@ -392,7 +392,6 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([
return []string{}, errors.Wrapf(BulkError, "creating alert events: %s", err) return []string{}, errors.Wrapf(BulkError, "creating alert events: %s", err)
} }
} }
if len(alertItem.Meta) > 0 { if len(alertItem.Meta) > 0 {
metaBulk := make([]*ent.MetaCreate, len(alertItem.Meta)) metaBulk := make([]*ent.MetaCreate, len(alertItem.Meta))
for i, metaItem := range alertItem.Meta { for i, metaItem := range alertItem.Meta {

3
pkg/leakybucket/bucket.go
View file

@ -7,6 +7,7 @@ import (
"time" "time"
//"log" //"log"
"github.com/antonmedv/expr/vm"
"github.com/crowdsecurity/crowdsec/pkg/time/rate" "github.com/crowdsecurity/crowdsec/pkg/time/rate"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/goombaio/namegenerator" "github.com/goombaio/namegenerator"
@ -71,6 +72,7 @@ type Leaky struct {
wgPour *sync.WaitGroup wgPour *sync.WaitGroup
wgDumpState *sync.WaitGroup wgDumpState *sync.WaitGroup
mutex *sync.Mutex //used only for TIMEMACHINE mode to allow garbage collection without races mutex *sync.Mutex //used only for TIMEMACHINE mode to allow garbage collection without races
LabelsToSend map[string][]*vm.Program
} }
var BucketsPour = prometheus.NewCounterVec( var BucketsPour = prometheus.NewCounterVec(
@ -179,6 +181,7 @@ func FromFactory(bucketFactory BucketFactory) *Leaky {
wgPour: bucketFactory.wgPour, wgPour: bucketFactory.wgPour,
wgDumpState: bucketFactory.wgDumpState, wgDumpState: bucketFactory.wgDumpState,
mutex: &sync.Mutex{}, mutex: &sync.Mutex{},
LabelsToSend: bucketFactory.LabelsToSendCompiled,
} }
if l.BucketConfig.Capacity > 0 && l.BucketConfig.leakspeed != time.Duration(0) { if l.BucketConfig.Capacity > 0 && l.BucketConfig.leakspeed != time.Duration(0) {
l.Duration = time.Duration(l.BucketConfig.Capacity+1) * l.BucketConfig.leakspeed l.Duration = time.Duration(l.BucketConfig.Capacity+1) * l.BucketConfig.leakspeed

95
pkg/leakybucket/manager_load.go
View file

@ -32,46 +32,48 @@ import (
// BucketFactory struct holds all fields for any bucket configuration. This is to have a // BucketFactory struct holds all fields for any bucket configuration. This is to have a
// generic struct for buckets. This can be seen as a bucket factory. // generic struct for buckets. This can be seen as a bucket factory.
type BucketFactory struct { type BucketFactory struct {
FormatVersion string `yaml:"format"` FormatVersion string `yaml:"format"`
Author string `yaml:"author"` Author string `yaml:"author"`
Description string `yaml:"description"` Description string `yaml:"description"`
References []string `yaml:"references"` References []string `yaml:"references"`
Type string `yaml:"type"` //Type can be : leaky, counter, trigger. It determines the main bucket characteristics Type string `yaml:"type"` //Type can be : leaky, counter, trigger. It determines the main bucket characteristics
Name string `yaml:"name"` //Name of the bucket, used later in log and user-messages. Should be unique Name string `yaml:"name"` //Name of the bucket, used later in log and user-messages. Should be unique
Capacity int `yaml:"capacity"` //Capacity is applicable to leaky buckets and determines the "burst" capacity Capacity int `yaml:"capacity"` //Capacity is applicable to leaky buckets and determines the "burst" capacity
LeakSpeed string `yaml:"leakspeed"` //Leakspeed is a float representing how many events per second leak out of the bucket LeakSpeed string `yaml:"leakspeed"` //Leakspeed is a float representing how many events per second leak out of the bucket
Duration string `yaml:"duration"` //Duration allows 'counter' buckets to have a fixed life-time Duration string `yaml:"duration"` //Duration allows 'counter' buckets to have a fixed life-time
Filter string `yaml:"filter"` //Filter is an expr that determines if an event is elligible for said bucket. Filter is evaluated against the Event struct Filter string `yaml:"filter"` //Filter is an expr that determines if an event is elligible for said bucket. Filter is evaluated against the Event struct
GroupBy string `yaml:"groupby,omitempty"` //groupy is an expr that allows to determine the partitions of the bucket. A common example is the source_ip GroupBy string `yaml:"groupby,omitempty"` //groupy is an expr that allows to determine the partitions of the bucket. A common example is the source_ip
Distinct string `yaml:"distinct"` //Distinct, when present, adds a `Pour()` processor that will only pour uniq items (based on distinct expr result) Distinct string `yaml:"distinct"` //Distinct, when present, adds a `Pour()` processor that will only pour uniq items (based on distinct expr result)
Debug bool `yaml:"debug"` //Debug, when set to true, will enable debugging for _this_ scenario specifically Debug bool `yaml:"debug"` //Debug, when set to true, will enable debugging for _this_ scenario specifically
Labels map[string]string `yaml:"labels"` //Labels is K:V list aiming at providing context the overflow Labels map[string]string `yaml:"labels"` //Labels is K:V list aiming at providing context the overflow
Blackhole string `yaml:"blackhole,omitempty"` //Blackhole is a duration that, if present, will prevent same bucket partition to overflow more often than $duration Blackhole string `yaml:"blackhole,omitempty"` //Blackhole is a duration that, if present, will prevent same bucket partition to overflow more often than $duration
logger *log.Entry `yaml:"-"` //logger is bucket-specific logger (used by Debug as well) logger *log.Entry `yaml:"-"` //logger is bucket-specific logger (used by Debug as well)
Reprocess bool `yaml:"reprocess"` //Reprocess, if true, will for the bucket to be re-injected into processing chain Reprocess bool `yaml:"reprocess"` //Reprocess, if true, will for the bucket to be re-injected into processing chain
CacheSize int `yaml:"cache_size"` //CacheSize, if > 0, limits the size of in-memory cache of the bucket CacheSize int `yaml:"cache_size"` //CacheSize, if > 0, limits the size of in-memory cache of the bucket
Profiling bool `yaml:"profiling"` //Profiling, if true, will make the bucket record pours/overflows/etc. Profiling bool `yaml:"profiling"` //Profiling, if true, will make the bucket record pours/overflows/etc.
OverflowFilter string `yaml:"overflow_filter"` //OverflowFilter if present, is a filter that must return true for the overflow to go through OverflowFilter string `yaml:"overflow_filter"` //OverflowFilter if present, is a filter that must return true for the overflow to go through
ScopeType types.ScopeType `yaml:"scope,omitempty"` //to enforce a different remediation than blocking an IP. Will default this to IP ScopeType types.ScopeType `yaml:"scope,omitempty"` //to enforce a different remediation than blocking an IP. Will default this to IP
BucketName string `yaml:"-"` BucketName string `yaml:"-"`
Filename string `yaml:"-"` Filename string `yaml:"-"`
RunTimeFilter *vm.Program `json:"-"` RunTimeFilter *vm.Program `json:"-"`
ExprDebugger *exprhelpers.ExprDebugger `yaml:"-" json:"-"` // used to debug expression by printing the content of each variable of the expression ExprDebugger *exprhelpers.ExprDebugger `yaml:"-" json:"-"` // used to debug expression by printing the content of each variable of the expression
RunTimeGroupBy *vm.Program `json:"-"` RunTimeGroupBy *vm.Program `json:"-"`
Data []*types.DataSource `yaml:"data,omitempty"` Data []*types.DataSource `yaml:"data,omitempty"`
DataDir string `yaml:"-"` DataDir string `yaml:"-"`
CancelOnFilter string `yaml:"cancel_on,omitempty"` //a filter that, if matched, kills the bucket CancelOnFilter string `yaml:"cancel_on,omitempty"` //a filter that, if matched, kills the bucket
leakspeed time.Duration //internal representation of `Leakspeed` leakspeed time.Duration //internal representation of `Leakspeed`
duration time.Duration //internal representation of `Duration` duration time.Duration //internal representation of `Duration`
ret chan types.Event //the bucket-specific output chan for overflows ret chan types.Event //the bucket-specific output chan for overflows
processors []Processor //processors is the list of hooks for pour/overflow/create (cf. uniq, blackhole etc.) processors []Processor //processors is the list of hooks for pour/overflow/create (cf. uniq, blackhole etc.)
output bool //?? output bool //??
ScenarioVersion string `yaml:"version,omitempty"` ScenarioVersion string `yaml:"version,omitempty"`
hash string `yaml:"-"` hash string `yaml:"-"`
Simulated bool `yaml:"simulated"` //Set to true if the scenario instanciating the bucket was in the exclusion list Simulated bool `yaml:"simulated"` //Set to true if the scenario instanciating the bucket was in the exclusion list
tomb *tomb.Tomb `yaml:"-"` tomb *tomb.Tomb `yaml:"-"`
wgPour *sync.WaitGroup `yaml:"-"` wgPour *sync.WaitGroup `yaml:"-"`
wgDumpState *sync.WaitGroup `yaml:"-"` wgDumpState *sync.WaitGroup `yaml:"-"`
LabelsToSend map[string][]string `yaml:"-"`
LabelsToSendCompiled map[string][]*vm.Program `yaml:"-"`
} }
func ValidateFactory(bucketFactory *BucketFactory) error { func ValidateFactory(bucketFactory *BucketFactory) error {
@ -216,6 +218,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
bucketFactory.wgDumpState = buckets.wgDumpState bucketFactory.wgDumpState = buckets.wgDumpState
bucketFactory.wgPour = buckets.wgPour bucketFactory.wgPour = buckets.wgPour
bucketFactory.LabelsToSend = cscfg.LabelsToSend
err = LoadBucket(&bucketFactory, tomb) err = LoadBucket(&bucketFactory, tomb)
if err != nil { if err != nil {
log.Errorf("Failed to load bucket %s : %v", bucketFactory.Name, err) log.Errorf("Failed to load bucket %s : %v", bucketFactory.Name, err)
@ -348,6 +351,18 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error {
return fmt.Errorf("invalid bucket from %s : %v", bucketFactory.Filename, err) return fmt.Errorf("invalid bucket from %s : %v", bucketFactory.Filename, err)
} }
bucketFactory.tomb = tomb bucketFactory.tomb = tomb
bucketFactory.LabelsToSendCompiled = make(map[string][]*vm.Program)
for key, values := range bucketFactory.LabelsToSend {
bucketFactory.LabelsToSendCompiled[key] = make([]*vm.Program, 0)
for _, value := range values {
valueCompiled, err := expr.Compile(value, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
if err != nil {
return fmt.Errorf("compilation of '%s' failed: %v", value, err)
}
bucketFactory.LabelsToSendCompiled[key] = append(bucketFactory.LabelsToSendCompiled[key], valueCompiled)
}
}
return nil return nil
} }

41
pkg/leakybucket/overflows.go
View file

@ -1,6 +1,7 @@
package leakybucket package leakybucket
import ( import (
"encoding/json"
"fmt" "fmt"
"net" "net"
"sort" "sort"
@ -14,6 +15,7 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/antonmedv/expr" "github.com/antonmedv/expr"
"github.com/antonmedv/expr/vm"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
) )
@ -232,7 +234,42 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st
return sources, source_type, nil return sources, source_type, nil
} }
func EventToLabel(Queue) { func EventToLabel(labels map[string][]*vm.Program, queue *Queue) models.Meta {
meta := make([]*models.MetaItems0, 0)
for _, evt := range queue.Queue {
for key, values := range labels {
tmpMeta := models.MetaItems0{}
tmpMeta.Key = key
tmpValue := make([]string, 0)
for _, value := range values {
var val string
output, err := expr.Run(value, exprhelpers.GetExprEnv(map[string]interface{}{"evt": evt}))
if err != nil {
log.Warningf("failed to get value of '%v': %v", value, err)
continue
}
switch out := output.(type) {
case string:
val = out
case int:
val = strconv.Itoa(out)
default:
log.Warningf("unexpected return type for label to send : %T", output)
continue
}
tmpValue = append(tmpValue, val)
}
valueBytes, err := json.Marshal(tmpValue)
if err != nil {
log.Warningf("unable to marshall label values to send: %s", err)
}
tmpMeta.Value = string(valueBytes)
meta = append(meta, &tmpMeta)
}
}
ret := models.Meta(meta)
return ret
} }
@ -296,6 +333,7 @@ func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) {
*apiAlert.Message = fmt.Sprintf("%s %s performed '%s' (%d events over %s) at %s", source_scope, sourceStr, leaky.Name, leaky.Total_count, leaky.Ovflw_ts.Sub(leaky.First_ts), leaky.Last_ts) *apiAlert.Message = fmt.Sprintf("%s %s performed '%s' (%d events over %s) at %s", source_scope, sourceStr, leaky.Name, leaky.Total_count, leaky.Ovflw_ts.Sub(leaky.First_ts), leaky.Last_ts)
//Get the events from Leaky/Queue //Get the events from Leaky/Queue
apiAlert.Events = EventsFromQueue(queue) apiAlert.Events = EventsFromQueue(queue)
apiAlert.Meta = EventToLabel(leaky.LabelsToSend, leaky.Queue)
//Loop over the Sources and generate appropriate number of ApiAlerts //Loop over the Sources and generate appropriate number of ApiAlerts
for _, srcValue := range sources { for _, srcValue := range sources {
@ -321,5 +359,6 @@ func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) {
if leaky.Reprocess { if leaky.Reprocess {
runtimeAlert.Reprocess = true runtimeAlert.Reprocess = true
} }
return runtimeAlert, nil return runtimeAlert, nil
} }