diff --git a/go.mod b/go.mod index a59dfd394..ae7c18a93 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ require ( github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/c-robinson/iplib v1.0.3 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/containerd/containerd v1.4.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 53008fb6c..8c103a3a0 100644 --- a/go.sum +++ b/go.sum @@ -90,6 +90,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU= +github.com/c-robinson/iplib v1.0.3/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/pkg/exprhelpers/exprlib.go b/pkg/exprhelpers/exprlib.go index 00c268cc9..c87b0d483 100644 --- a/pkg/exprhelpers/exprlib.go +++ b/pkg/exprhelpers/exprlib.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "github.com/c-robinson/iplib" + "github.com/davecgh/go-spew/spew" log "github.com/sirupsen/logrus" ) @@ -53,6 +55,7 @@ func GetExprEnv(ctx map[string]interface{}) map[string]interface{} { "QueryUnescape": QueryUnescape, "PathEscape": PathEscape, "QueryEscape": QueryEscape, + "IpToRange": IpToRange, } for k, v := range ctx { ExprLib[k] = v @@ -175,6 +178,27 @@ func IpInRange(ip string, ipRange string) bool { return false } +func IpToRange(ip string, cidr string) string { + cidr = strings.TrimPrefix(cidr, "/") + mask, err := strconv.Atoi(cidr) + if err != nil { + log.Errorf("bad cidr '%s': %s", cidr, err) + return "" + } + + ipAddr := net.ParseIP(ip) + if ipAddr == nil { + log.Errorf("can't parse IP address '%s'", ip) + return "" + } + ipRange := iplib.NewNet(ipAddr, mask) + if ipRange.IP() == nil { + log.Errorf("can't get cidr '%s' of '%s'", cidr, ip) + return "" + } + return ipRange.String() +} + func TimeNow() string { return time.Now().UTC().Format(time.RFC3339) } diff --git a/pkg/exprhelpers/exprlib_test.go b/pkg/exprhelpers/exprlib_test.go index 7fc2436f5..2f59e3e4e 100644 --- a/pkg/exprhelpers/exprlib_test.go +++ b/pkg/exprhelpers/exprlib_test.go @@ -342,6 +342,82 @@ func TestIpInRange(t *testing.T) { } +func TestIpToRange(t *testing.T) { + tests := []struct { + name string + env map[string]interface{} + code string + result string + err string + }{ + { + name: "IpToRange() test: IPv4", + env: map[string]interface{}{ + "ip": "192.168.1.1", + "netmask": "16", + "IpToRange": IpToRange, + }, + code: "IpToRange(ip, netmask)", + result: "192.168.0.0/16", + err: "", + }, + { + name: "IpToRange() test: IPv6", + env: map[string]interface{}{ + "ip": "2001:db8::1", + "netmask": "/64", + "IpToRange": IpToRange, + }, + code: "IpToRange(ip, netmask)", + result: "2001:db8::/64", + err: "", + }, + { + name: "IpToRange() test: malformed netmask", + env: map[string]interface{}{ + "ip": "192.168.0.1", + "netmask": "test", + "IpToRange": IpToRange, + }, + code: "IpToRange(ip, netmask)", + result: "", + err: "", + }, + { + name: "IpToRange() test: malformed IP", + env: map[string]interface{}{ + "ip": "a.b.c.d", + "netmask": "24", + "IpToRange": IpToRange, + }, + code: "IpToRange(ip, netmask)", + result: "", + err: "", + }, + { + name: "IpToRange() test: too high netmask", + env: map[string]interface{}{ + "ip": "192.168.1.1", + "netmask": "35", + "IpToRange": IpToRange, + }, + code: "IpToRange(ip, netmask)", + result: "", + err: "", + }, + } + + for _, test := range tests { + program, err := expr.Compile(test.code, expr.Env(test.env)) + require.NoError(t, err) + output, err := expr.Run(program, test.env) + require.NoError(t, err) + require.Equal(t, test.result, output) + log.Printf("test '%s' : OK", test.name) + } + +} + func TestAtof(t *testing.T) { testFloat := "1.5" expectedFloat := 1.5 diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index 0c9e5086f..7f595bd08 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -114,16 +114,29 @@ func ValidateFactory(bucketFactory *BucketFactory) error { bucketFactory.ScopeType.Scope = types.Ip case types.Ip: case types.Range: + var ( + runTimeFilter *vm.Program + err error + ) + if bucketFactory.ScopeType.Filter != "" { + if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))); err != nil { + return fmt.Errorf("Error compiling the scope filter: %s", err) + } + bucketFactory.ScopeType.RunTimeFilter = runTimeFilter + } + default: //Compile the scope filter var ( runTimeFilter *vm.Program err error ) - if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))); err != nil { - return fmt.Errorf("Error compiling the scope filter: %s", err) + if bucketFactory.ScopeType.Filter != "" { + if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))); err != nil { + return fmt.Errorf("Error compiling the scope filter: %s", err) + } + bucketFactory.ScopeType.RunTimeFilter = runTimeFilter } - bucketFactory.ScopeType.RunTimeFilter = runTimeFilter } return nil } diff --git a/pkg/leakybucket/overflows.go b/pkg/leakybucket/overflows.go index 658427ac9..8616a4391 100644 --- a/pkg/leakybucket/overflows.go +++ b/pkg/leakybucket/overflows.go @@ -20,7 +20,6 @@ import ( //SourceFromEvent extracts and formats a valid models.Source object from an Event func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, error) { srcs := make(map[string]models.Source) - /*if it's already an overflow, we have properly formatted sources. we can just twitch them to reflect the requested scope*/ if evt.Type == types.OVFLW { @@ -37,18 +36,32 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e if leaky.scopeType.Scope == types.Range { /*the original bucket was target IPs, check that we do have range*/ if *v.Scope == types.Ip { + src := models.Source{} + src.AsName = v.AsName + src.AsNumber = v.AsNumber + src.Cn = v.Cn + src.Latitude = v.Latitude + src.Longitude = v.Longitude + src.Range = v.Range + src.Value = new(string) + src.Scope = new(string) + *src.Scope = leaky.scopeType.Scope + *src.Value = "" if v.Range != "" { - src := models.Source{} - src.AsName = v.AsName - src.AsNumber = v.AsNumber - src.Cn = v.Cn - src.Latitude = v.Latitude - src.Longitude = v.Longitude - src.Range = v.Range - src.Value = new(string) - src.Scope = new(string) *src.Value = v.Range - *src.Scope = leaky.scopeType.Scope + } + if leaky.scopeType.RunTimeFilter != nil { + retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &evt})) + if err != nil { + return srcs, errors.Wrapf(err, "while running scope filter") + } + value, ok := retValue.(string) + if !ok { + value = "" + } + src.Value = &value + } + if *src.Value != "" { srcs[*src.Value] = src } else { log.Warningf("bucket %s requires scope Range, but none was provided. It seems that the %s wasn't enriched to include its range.", leaky.Name, *v.Value) @@ -112,6 +125,18 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e src.Value = &src.IP } else if leaky.scopeType.Scope == types.Range { src.Value = &src.Range + if leaky.scopeType.RunTimeFilter != nil { + retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &evt})) + if err != nil { + return srcs, errors.Wrapf(err, "while running scope filter") + } + + value, ok := retValue.(string) + if !ok { + value = "" + } + src.Value = &value + } } srcs[*src.Value] = src default: @@ -209,7 +234,6 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st //NewAlert will generate a RuntimeAlert and its APIAlert(s) from a bucket that overflowed func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) { - var runtimeAlert types.RuntimeAlert leaky.logger.Tracef("Overflow (start: %s, end: %s)", leaky.First_ts, leaky.Ovflw_ts) diff --git a/pkg/leakybucket/tests/leaky-scope-range-expression/bucket.yaml b/pkg/leakybucket/tests/leaky-scope-range-expression/bucket.yaml new file mode 100644 index 000000000..1c0c4a1bd --- /dev/null +++ b/pkg/leakybucket/tests/leaky-scope-range-expression/bucket.yaml @@ -0,0 +1,14 @@ +type: leaky +debug: true +name: test/leaky-scope-range-expression +description: "Leaky with scope range-expression" +filter: "evt.Line.Labels.type =='testlog'" +leakspeed: "10s" +capacity: 1 +groupby: evt.Meta.source_ip +labels: + type: overflow_1 +scope: + type: Range + expression: IpToRange(evt.Meta.source_ip, "/16") + diff --git a/pkg/leakybucket/tests/leaky-scope-range-expression/scenarios.yaml b/pkg/leakybucket/tests/leaky-scope-range-expression/scenarios.yaml new file mode 100644 index 000000000..05e1557cf --- /dev/null +++ b/pkg/leakybucket/tests/leaky-scope-range-expression/scenarios.yaml @@ -0,0 +1 @@ + - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/leaky-scope-range-expression/test.json b/pkg/leakybucket/tests/leaky-scope-range-expression/test.json new file mode 100644 index 000000000..38bc7ffc6 --- /dev/null +++ b/pkg/leakybucket/tests/leaky-scope-range-expression/test.json @@ -0,0 +1,47 @@ +{ + "lines": [ + { + "Line": { + "Labels": { + "type": "testlog" + }, + "Raw": "xxheader VALUE1 trailing stuff" + }, + "MarshaledTime": "2020-01-01T10:00:00+00:00", + "Meta": { + "source_ip": "192.168.1.1" + } + }, + { + "Line": { + "Labels": { + "type": "testlog" + }, + "Raw": "xxheader VALUE2 trailing stuff" + }, + "MarshaledTime": "2020-01-01T10:00:05+00:00", + "Meta": { + "source_ip": "192.168.1.1" + } + } + ], + "results": [ + { + "Alert": { + "sources": { + "192.168.0.0/16": { + "scope": "Range", + "value": "192.168.0.0/16", + "ip": "192.168.1.1" + } + }, + "Alert" : { + "scenario": "test/leaky-scope-range-expression", + "events_count": 2 + } + } + } + ] + } + + \ No newline at end of file