add IpToRange helpers and allows to have an expression with scope Range (#1260)
* add IpToRange helpers and allows to have an expression with scope Range
This commit is contained in:
parent
40ab8fa738
commit
5a0843852a
1
go.mod
1
go.mod
|
@ -70,6 +70,7 @@ require (
|
||||||
github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 // indirect
|
github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 // indirect
|
||||||
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
|
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
|
github.com/c-robinson/iplib v1.0.3 // indirect
|
||||||
github.com/cespare/xxhash/v2 v2.1.1 // indirect
|
github.com/cespare/xxhash/v2 v2.1.1 // indirect
|
||||||
github.com/containerd/containerd v1.4.3 // indirect
|
github.com/containerd/containerd v1.4.3 // indirect
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
|
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
|
||||||
|
|
2
go.sum
2
go.sum
|
@ -90,6 +90,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
|
||||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||||
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
|
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
|
||||||
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
|
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
|
||||||
|
github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU=
|
||||||
|
github.com/c-robinson/iplib v1.0.3/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo=
|
||||||
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
|
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
|
||||||
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||||
|
|
|
@ -12,6 +12,8 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/c-robinson/iplib"
|
||||||
|
|
||||||
"github.com/davecgh/go-spew/spew"
|
"github.com/davecgh/go-spew/spew"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
@ -53,6 +55,7 @@ func GetExprEnv(ctx map[string]interface{}) map[string]interface{} {
|
||||||
"QueryUnescape": QueryUnescape,
|
"QueryUnescape": QueryUnescape,
|
||||||
"PathEscape": PathEscape,
|
"PathEscape": PathEscape,
|
||||||
"QueryEscape": QueryEscape,
|
"QueryEscape": QueryEscape,
|
||||||
|
"IpToRange": IpToRange,
|
||||||
}
|
}
|
||||||
for k, v := range ctx {
|
for k, v := range ctx {
|
||||||
ExprLib[k] = v
|
ExprLib[k] = v
|
||||||
|
@ -175,6 +178,27 @@ func IpInRange(ip string, ipRange string) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func IpToRange(ip string, cidr string) string {
|
||||||
|
cidr = strings.TrimPrefix(cidr, "/")
|
||||||
|
mask, err := strconv.Atoi(cidr)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("bad cidr '%s': %s", cidr, err)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
ipAddr := net.ParseIP(ip)
|
||||||
|
if ipAddr == nil {
|
||||||
|
log.Errorf("can't parse IP address '%s'", ip)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
ipRange := iplib.NewNet(ipAddr, mask)
|
||||||
|
if ipRange.IP() == nil {
|
||||||
|
log.Errorf("can't get cidr '%s' of '%s'", cidr, ip)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return ipRange.String()
|
||||||
|
}
|
||||||
|
|
||||||
func TimeNow() string {
|
func TimeNow() string {
|
||||||
return time.Now().UTC().Format(time.RFC3339)
|
return time.Now().UTC().Format(time.RFC3339)
|
||||||
}
|
}
|
||||||
|
|
|
@ -342,6 +342,82 @@ func TestIpInRange(t *testing.T) {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIpToRange(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
env map[string]interface{}
|
||||||
|
code string
|
||||||
|
result string
|
||||||
|
err string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "IpToRange() test: IPv4",
|
||||||
|
env: map[string]interface{}{
|
||||||
|
"ip": "192.168.1.1",
|
||||||
|
"netmask": "16",
|
||||||
|
"IpToRange": IpToRange,
|
||||||
|
},
|
||||||
|
code: "IpToRange(ip, netmask)",
|
||||||
|
result: "192.168.0.0/16",
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IpToRange() test: IPv6",
|
||||||
|
env: map[string]interface{}{
|
||||||
|
"ip": "2001:db8::1",
|
||||||
|
"netmask": "/64",
|
||||||
|
"IpToRange": IpToRange,
|
||||||
|
},
|
||||||
|
code: "IpToRange(ip, netmask)",
|
||||||
|
result: "2001:db8::/64",
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IpToRange() test: malformed netmask",
|
||||||
|
env: map[string]interface{}{
|
||||||
|
"ip": "192.168.0.1",
|
||||||
|
"netmask": "test",
|
||||||
|
"IpToRange": IpToRange,
|
||||||
|
},
|
||||||
|
code: "IpToRange(ip, netmask)",
|
||||||
|
result: "",
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IpToRange() test: malformed IP",
|
||||||
|
env: map[string]interface{}{
|
||||||
|
"ip": "a.b.c.d",
|
||||||
|
"netmask": "24",
|
||||||
|
"IpToRange": IpToRange,
|
||||||
|
},
|
||||||
|
code: "IpToRange(ip, netmask)",
|
||||||
|
result: "",
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IpToRange() test: too high netmask",
|
||||||
|
env: map[string]interface{}{
|
||||||
|
"ip": "192.168.1.1",
|
||||||
|
"netmask": "35",
|
||||||
|
"IpToRange": IpToRange,
|
||||||
|
},
|
||||||
|
code: "IpToRange(ip, netmask)",
|
||||||
|
result: "",
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
program, err := expr.Compile(test.code, expr.Env(test.env))
|
||||||
|
require.NoError(t, err)
|
||||||
|
output, err := expr.Run(program, test.env)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, test.result, output)
|
||||||
|
log.Printf("test '%s' : OK", test.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
func TestAtof(t *testing.T) {
|
func TestAtof(t *testing.T) {
|
||||||
testFloat := "1.5"
|
testFloat := "1.5"
|
||||||
expectedFloat := 1.5
|
expectedFloat := 1.5
|
||||||
|
|
|
@ -114,16 +114,29 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
|
||||||
bucketFactory.ScopeType.Scope = types.Ip
|
bucketFactory.ScopeType.Scope = types.Ip
|
||||||
case types.Ip:
|
case types.Ip:
|
||||||
case types.Range:
|
case types.Range:
|
||||||
|
var (
|
||||||
|
runTimeFilter *vm.Program
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
if bucketFactory.ScopeType.Filter != "" {
|
||||||
|
if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))); err != nil {
|
||||||
|
return fmt.Errorf("Error compiling the scope filter: %s", err)
|
||||||
|
}
|
||||||
|
bucketFactory.ScopeType.RunTimeFilter = runTimeFilter
|
||||||
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
//Compile the scope filter
|
//Compile the scope filter
|
||||||
var (
|
var (
|
||||||
runTimeFilter *vm.Program
|
runTimeFilter *vm.Program
|
||||||
err error
|
err error
|
||||||
)
|
)
|
||||||
if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))); err != nil {
|
if bucketFactory.ScopeType.Filter != "" {
|
||||||
return fmt.Errorf("Error compiling the scope filter: %s", err)
|
if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))); err != nil {
|
||||||
|
return fmt.Errorf("Error compiling the scope filter: %s", err)
|
||||||
|
}
|
||||||
|
bucketFactory.ScopeType.RunTimeFilter = runTimeFilter
|
||||||
}
|
}
|
||||||
bucketFactory.ScopeType.RunTimeFilter = runTimeFilter
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,7 +20,6 @@ import (
|
||||||
//SourceFromEvent extracts and formats a valid models.Source object from an Event
|
//SourceFromEvent extracts and formats a valid models.Source object from an Event
|
||||||
func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, error) {
|
func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, error) {
|
||||||
srcs := make(map[string]models.Source)
|
srcs := make(map[string]models.Source)
|
||||||
|
|
||||||
/*if it's already an overflow, we have properly formatted sources.
|
/*if it's already an overflow, we have properly formatted sources.
|
||||||
we can just twitch them to reflect the requested scope*/
|
we can just twitch them to reflect the requested scope*/
|
||||||
if evt.Type == types.OVFLW {
|
if evt.Type == types.OVFLW {
|
||||||
|
@ -37,18 +36,32 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e
|
||||||
if leaky.scopeType.Scope == types.Range {
|
if leaky.scopeType.Scope == types.Range {
|
||||||
/*the original bucket was target IPs, check that we do have range*/
|
/*the original bucket was target IPs, check that we do have range*/
|
||||||
if *v.Scope == types.Ip {
|
if *v.Scope == types.Ip {
|
||||||
|
src := models.Source{}
|
||||||
|
src.AsName = v.AsName
|
||||||
|
src.AsNumber = v.AsNumber
|
||||||
|
src.Cn = v.Cn
|
||||||
|
src.Latitude = v.Latitude
|
||||||
|
src.Longitude = v.Longitude
|
||||||
|
src.Range = v.Range
|
||||||
|
src.Value = new(string)
|
||||||
|
src.Scope = new(string)
|
||||||
|
*src.Scope = leaky.scopeType.Scope
|
||||||
|
*src.Value = ""
|
||||||
if v.Range != "" {
|
if v.Range != "" {
|
||||||
src := models.Source{}
|
|
||||||
src.AsName = v.AsName
|
|
||||||
src.AsNumber = v.AsNumber
|
|
||||||
src.Cn = v.Cn
|
|
||||||
src.Latitude = v.Latitude
|
|
||||||
src.Longitude = v.Longitude
|
|
||||||
src.Range = v.Range
|
|
||||||
src.Value = new(string)
|
|
||||||
src.Scope = new(string)
|
|
||||||
*src.Value = v.Range
|
*src.Value = v.Range
|
||||||
*src.Scope = leaky.scopeType.Scope
|
}
|
||||||
|
if leaky.scopeType.RunTimeFilter != nil {
|
||||||
|
retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &evt}))
|
||||||
|
if err != nil {
|
||||||
|
return srcs, errors.Wrapf(err, "while running scope filter")
|
||||||
|
}
|
||||||
|
value, ok := retValue.(string)
|
||||||
|
if !ok {
|
||||||
|
value = ""
|
||||||
|
}
|
||||||
|
src.Value = &value
|
||||||
|
}
|
||||||
|
if *src.Value != "" {
|
||||||
srcs[*src.Value] = src
|
srcs[*src.Value] = src
|
||||||
} else {
|
} else {
|
||||||
log.Warningf("bucket %s requires scope Range, but none was provided. It seems that the %s wasn't enriched to include its range.", leaky.Name, *v.Value)
|
log.Warningf("bucket %s requires scope Range, but none was provided. It seems that the %s wasn't enriched to include its range.", leaky.Name, *v.Value)
|
||||||
|
@ -112,6 +125,18 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e
|
||||||
src.Value = &src.IP
|
src.Value = &src.IP
|
||||||
} else if leaky.scopeType.Scope == types.Range {
|
} else if leaky.scopeType.Scope == types.Range {
|
||||||
src.Value = &src.Range
|
src.Value = &src.Range
|
||||||
|
if leaky.scopeType.RunTimeFilter != nil {
|
||||||
|
retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &evt}))
|
||||||
|
if err != nil {
|
||||||
|
return srcs, errors.Wrapf(err, "while running scope filter")
|
||||||
|
}
|
||||||
|
|
||||||
|
value, ok := retValue.(string)
|
||||||
|
if !ok {
|
||||||
|
value = ""
|
||||||
|
}
|
||||||
|
src.Value = &value
|
||||||
|
}
|
||||||
}
|
}
|
||||||
srcs[*src.Value] = src
|
srcs[*src.Value] = src
|
||||||
default:
|
default:
|
||||||
|
@ -209,7 +234,6 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st
|
||||||
|
|
||||||
//NewAlert will generate a RuntimeAlert and its APIAlert(s) from a bucket that overflowed
|
//NewAlert will generate a RuntimeAlert and its APIAlert(s) from a bucket that overflowed
|
||||||
func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) {
|
func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) {
|
||||||
|
|
||||||
var runtimeAlert types.RuntimeAlert
|
var runtimeAlert types.RuntimeAlert
|
||||||
|
|
||||||
leaky.logger.Tracef("Overflow (start: %s, end: %s)", leaky.First_ts, leaky.Ovflw_ts)
|
leaky.logger.Tracef("Overflow (start: %s, end: %s)", leaky.First_ts, leaky.Ovflw_ts)
|
||||||
|
|
|
@ -0,0 +1,14 @@
|
||||||
|
type: leaky
|
||||||
|
debug: true
|
||||||
|
name: test/leaky-scope-range-expression
|
||||||
|
description: "Leaky with scope range-expression"
|
||||||
|
filter: "evt.Line.Labels.type =='testlog'"
|
||||||
|
leakspeed: "10s"
|
||||||
|
capacity: 1
|
||||||
|
groupby: evt.Meta.source_ip
|
||||||
|
labels:
|
||||||
|
type: overflow_1
|
||||||
|
scope:
|
||||||
|
type: Range
|
||||||
|
expression: IpToRange(evt.Meta.source_ip, "/16")
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
- filename: {{.TestDirectory}}/bucket.yaml
|
47
pkg/leakybucket/tests/leaky-scope-range-expression/test.json
Normal file
47
pkg/leakybucket/tests/leaky-scope-range-expression/test.json
Normal file
|
@ -0,0 +1,47 @@
|
||||||
|
{
|
||||||
|
"lines": [
|
||||||
|
{
|
||||||
|
"Line": {
|
||||||
|
"Labels": {
|
||||||
|
"type": "testlog"
|
||||||
|
},
|
||||||
|
"Raw": "xxheader VALUE1 trailing stuff"
|
||||||
|
},
|
||||||
|
"MarshaledTime": "2020-01-01T10:00:00+00:00",
|
||||||
|
"Meta": {
|
||||||
|
"source_ip": "192.168.1.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Line": {
|
||||||
|
"Labels": {
|
||||||
|
"type": "testlog"
|
||||||
|
},
|
||||||
|
"Raw": "xxheader VALUE2 trailing stuff"
|
||||||
|
},
|
||||||
|
"MarshaledTime": "2020-01-01T10:00:05+00:00",
|
||||||
|
"Meta": {
|
||||||
|
"source_ip": "192.168.1.1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"Alert": {
|
||||||
|
"sources": {
|
||||||
|
"192.168.0.0/16": {
|
||||||
|
"scope": "Range",
|
||||||
|
"value": "192.168.0.0/16",
|
||||||
|
"ip": "192.168.1.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"Alert" : {
|
||||||
|
"scenario": "test/leaky-scope-range-expression",
|
||||||
|
"events_count": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue