Merge branch 'crowdsecurity:master' into master
This commit is contained in:
commit
755f2103be
4
.github/workflows/ci_functests-install.yml
vendored
4
.github/workflows/ci_functests-install.yml
vendored
|
@ -21,10 +21,10 @@ jobs:
|
||||||
name: Install generated release and perform functional tests
|
name: Install generated release and perform functional tests
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Set up Go 1.16
|
- name: Set up Go 1.17
|
||||||
uses: actions/setup-go@v1
|
uses: actions/setup-go@v1
|
||||||
with:
|
with:
|
||||||
go-version: 1.16
|
go-version: 1.17
|
||||||
id: go
|
id: go
|
||||||
- name: Check out code into the Go module directory
|
- name: Check out code into the Go module directory
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
|
6
.github/workflows/ci_go-test.yml
vendored
6
.github/workflows/ci_go-test.yml
vendored
|
@ -52,10 +52,10 @@ jobs:
|
||||||
--health-timeout=5s
|
--health-timeout=5s
|
||||||
--health-retries=3
|
--health-retries=3
|
||||||
steps:
|
steps:
|
||||||
- name: Set up Go 1.16
|
- name: Set up Go 1.17
|
||||||
uses: actions/setup-go@v1
|
uses: actions/setup-go@v1
|
||||||
with:
|
with:
|
||||||
go-version: 1.16
|
go-version: 1.17
|
||||||
id: go
|
id: go
|
||||||
- name: Check out code into the Go module directory
|
- name: Check out code into the Go module directory
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
@ -73,4 +73,4 @@ jobs:
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
with:
|
with:
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
path-to-lcov: coverage.txt
|
path-to-lcov: coverage.txt
|
||||||
|
|
6
.github/workflows/ci_hubtest.yml
vendored
6
.github/workflows/ci_hubtest.yml
vendored
|
@ -13,10 +13,10 @@ jobs:
|
||||||
name: Hub tests
|
name: Hub tests
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Set up Go 1.16
|
- name: Set up Go 1.17
|
||||||
uses: actions/setup-go@v1
|
uses: actions/setup-go@v1
|
||||||
with:
|
with:
|
||||||
go-version: 1.16
|
go-version: 1.17
|
||||||
id: go
|
id: go
|
||||||
- name: Check out code into the Go module directory
|
- name: Check out code into the Go module directory
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
@ -68,4 +68,4 @@ jobs:
|
||||||
filename: crowdsec_scenarios_badge.json
|
filename: crowdsec_scenarios_badge.json
|
||||||
label: Hub Scenarios
|
label: Hub Scenarios
|
||||||
message: ${{ env.SCENARIOS_COV }}
|
message: ${{ env.SCENARIOS_COV }}
|
||||||
color: ${{ env.SCENARIO_BADGE_COLOR }}
|
color: ${{ env.SCENARIO_BADGE_COLOR }}
|
||||||
|
|
|
@ -10,10 +10,10 @@ jobs:
|
||||||
name: Build and upload binary package
|
name: Build and upload binary package
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Set up Go 1.16
|
- name: Set up Go 1.17
|
||||||
uses: actions/setup-go@v1
|
uses: actions/setup-go@v1
|
||||||
with:
|
with:
|
||||||
go-version: 1.16
|
go-version: 1.17
|
||||||
id: go
|
id: go
|
||||||
- name: Check out code into the Go module directory
|
- name: Check out code into the Go module directory
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
@ -29,10 +29,10 @@ jobs:
|
||||||
name: Build and upload binary package
|
name: Build and upload binary package
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Set up Go 1.16
|
- name: Set up Go 1.17
|
||||||
uses: actions/setup-go@v1
|
uses: actions/setup-go@v1
|
||||||
with:
|
with:
|
||||||
go-version: 1.16
|
go-version: 1.17
|
||||||
id: go
|
id: go
|
||||||
- name: Check out code into the Go module directory
|
- name: Check out code into the Go module directory
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
ARG GOVERSION=1.16
|
ARG GOVERSION=1.17
|
||||||
|
|
||||||
FROM golang:${GOVERSION}-alpine AS build
|
FROM golang:${GOVERSION}-alpine AS build
|
||||||
|
|
||||||
|
|
2
Makefile
2
Makefile
|
@ -35,7 +35,7 @@ GOARCH ?= $(shell go env GOARCH)
|
||||||
GO_MAJOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1)
|
GO_MAJOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1)
|
||||||
GO_MINOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f2)
|
GO_MINOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f2)
|
||||||
MINIMUM_SUPPORTED_GO_MAJOR_VERSION = 1
|
MINIMUM_SUPPORTED_GO_MAJOR_VERSION = 1
|
||||||
MINIMUM_SUPPORTED_GO_MINOR_VERSION = 13
|
MINIMUM_SUPPORTED_GO_MINOR_VERSION = 17
|
||||||
GO_VERSION_VALIDATION_ERR_MSG = Golang version ($(BUILD_GOVERSION)) is not supported, please use least $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION).$(MINIMUM_SUPPORTED_GO_MINOR_VERSION)
|
GO_VERSION_VALIDATION_ERR_MSG = Golang version ($(BUILD_GOVERSION)) is not supported, please use least $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION).$(MINIMUM_SUPPORTED_GO_MINOR_VERSION)
|
||||||
#Current versioning information from env
|
#Current versioning information from env
|
||||||
BUILD_VERSION?="$(shell git describe --tags `git rev-list --tags --max-count=1`)"
|
BUILD_VERSION?="$(shell git describe --tags `git rev-list --tags --max-count=1`)"
|
||||||
|
|
|
@ -82,7 +82,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
|
||||||
log.Infof("Loading aggregated prometheus collectors")
|
log.Infof("Loading aggregated prometheus collectors")
|
||||||
prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
|
prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
|
||||||
globalCsInfo,
|
globalCsInfo,
|
||||||
leaky.BucketsUnderflow, leaky.BucketsInstanciation, leaky.BucketsOverflow,
|
leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstanciation, leaky.BucketsOverflow,
|
||||||
v1.LapiRouteHits,
|
v1.LapiRouteHits,
|
||||||
leaky.BucketsCurrentCount)
|
leaky.BucketsCurrentCount)
|
||||||
} else {
|
} else {
|
||||||
|
@ -91,7 +91,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
|
||||||
parser.NodesHits, parser.NodesHitsOk, parser.NodesHitsKo,
|
parser.NodesHits, parser.NodesHitsOk, parser.NodesHitsKo,
|
||||||
globalCsInfo,
|
globalCsInfo,
|
||||||
v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions,
|
v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions,
|
||||||
leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsInstanciation, leaky.BucketsOverflow, leaky.BucketsCurrentCount)
|
leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstanciation, leaky.BucketsOverflow, leaky.BucketsCurrentCount)
|
||||||
|
|
||||||
}
|
}
|
||||||
http.Handle("/metrics", promhttp.Handler())
|
http.Handle("/metrics", promhttp.Handler())
|
||||||
|
|
105
go.mod
105
go.mod
|
@ -1,27 +1,22 @@
|
||||||
module github.com/crowdsecurity/crowdsec
|
module github.com/crowdsecurity/crowdsec
|
||||||
|
|
||||||
go 1.13
|
go 1.17
|
||||||
|
|
||||||
require (
|
require (
|
||||||
entgo.io/ent v0.9.1
|
entgo.io/ent v0.9.1
|
||||||
github.com/AlecAivazis/survey/v2 v2.2.7
|
github.com/AlecAivazis/survey/v2 v2.2.7
|
||||||
github.com/Masterminds/goutils v1.1.1 // indirect
|
|
||||||
github.com/Masterminds/semver v1.5.0 // indirect
|
|
||||||
github.com/Masterminds/sprig v2.22.0+incompatible
|
github.com/Masterminds/sprig v2.22.0+incompatible
|
||||||
github.com/Microsoft/go-winio v0.4.16 // indirect
|
|
||||||
github.com/ahmetb/dlog v0.0.0-20170105205344-4fb5f8204f26
|
github.com/ahmetb/dlog v0.0.0-20170105205344-4fb5f8204f26
|
||||||
github.com/alexliesenfeld/health v0.5.1
|
github.com/alexliesenfeld/health v0.5.1
|
||||||
github.com/antonmedv/expr v1.8.9
|
github.com/antonmedv/expr v1.8.9
|
||||||
github.com/appleboy/gin-jwt/v2 v2.6.4
|
github.com/appleboy/gin-jwt/v2 v2.6.4
|
||||||
github.com/aws/aws-sdk-go v1.38.34
|
github.com/aws/aws-sdk-go v1.38.34
|
||||||
github.com/buger/jsonparser v1.1.1
|
github.com/buger/jsonparser v1.1.1
|
||||||
github.com/containerd/containerd v1.4.3 // indirect
|
|
||||||
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
|
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
|
||||||
github.com/crowdsecurity/grokky v0.0.0-20210908095311-0b3373925934
|
github.com/crowdsecurity/grokky v0.0.0-20210908095311-0b3373925934
|
||||||
github.com/davecgh/go-spew v1.1.1
|
github.com/davecgh/go-spew v1.1.1
|
||||||
github.com/denisbrodbeck/machineid v1.0.1
|
github.com/denisbrodbeck/machineid v1.0.1
|
||||||
github.com/dghubble/sling v1.3.0
|
github.com/dghubble/sling v1.3.0
|
||||||
github.com/docker/distribution v2.7.1+incompatible // indirect
|
|
||||||
github.com/docker/docker v20.10.2+incompatible
|
github.com/docker/docker v20.10.2+incompatible
|
||||||
github.com/docker/go-connections v0.4.0
|
github.com/docker/go-connections v0.4.0
|
||||||
github.com/enescakir/emoji v1.0.0
|
github.com/enescakir/emoji v1.0.0
|
||||||
|
@ -33,31 +28,19 @@ require (
|
||||||
github.com/go-openapi/strfmt v0.19.11
|
github.com/go-openapi/strfmt v0.19.11
|
||||||
github.com/go-openapi/swag v0.19.12
|
github.com/go-openapi/swag v0.19.12
|
||||||
github.com/go-openapi/validate v0.20.0
|
github.com/go-openapi/validate v0.20.0
|
||||||
github.com/go-playground/validator/v10 v10.4.1 // indirect
|
|
||||||
github.com/go-sql-driver/mysql v1.5.1-0.20200311113236-681ffa848bae
|
github.com/go-sql-driver/mysql v1.5.1-0.20200311113236-681ffa848bae
|
||||||
github.com/gogo/protobuf v1.3.2 // indirect
|
|
||||||
github.com/google/go-querystring v1.0.0
|
github.com/google/go-querystring v1.0.0
|
||||||
github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e
|
github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e
|
||||||
github.com/hashicorp/go-hclog v0.14.1
|
github.com/hashicorp/go-hclog v0.14.1
|
||||||
github.com/hashicorp/go-plugin v1.4.2
|
github.com/hashicorp/go-plugin v1.4.2
|
||||||
github.com/hashicorp/go-version v1.2.1
|
github.com/hashicorp/go-version v1.2.1
|
||||||
github.com/huandu/xstrings v1.3.2 // indirect
|
|
||||||
github.com/imdario/mergo v0.3.12 // indirect
|
|
||||||
github.com/influxdata/go-syslog/v3 v3.0.0
|
github.com/influxdata/go-syslog/v3 v3.0.0
|
||||||
github.com/jszwec/csvutil v1.5.1 // indirect
|
github.com/jszwec/csvutil v1.5.1
|
||||||
github.com/leodido/go-urn v1.2.1 // indirect
|
|
||||||
github.com/lib/pq v1.10.2
|
github.com/lib/pq v1.10.2
|
||||||
github.com/mattn/go-runewidth v0.0.10 // indirect
|
|
||||||
github.com/mattn/go-sqlite3 v1.14.8
|
github.com/mattn/go-sqlite3 v1.14.8
|
||||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
|
|
||||||
github.com/mitchellh/copystructure v1.2.0 // indirect
|
|
||||||
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
|
|
||||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
|
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
|
||||||
github.com/morikuni/aec v1.0.0 // indirect
|
|
||||||
github.com/nxadm/tail v1.4.6
|
github.com/nxadm/tail v1.4.6
|
||||||
github.com/olekukonko/tablewriter v0.0.5
|
github.com/olekukonko/tablewriter v0.0.5
|
||||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
|
||||||
github.com/opencontainers/image-spec v1.0.1 // indirect
|
|
||||||
github.com/oschwald/geoip2-golang v1.4.0
|
github.com/oschwald/geoip2-golang v1.4.0
|
||||||
github.com/oschwald/maxminddb-golang v1.8.0
|
github.com/oschwald/maxminddb-golang v1.8.0
|
||||||
github.com/pkg/errors v0.9.1
|
github.com/pkg/errors v0.9.1
|
||||||
|
@ -65,19 +48,12 @@ require (
|
||||||
github.com/prometheus/client_model v0.2.0
|
github.com/prometheus/client_model v0.2.0
|
||||||
github.com/prometheus/prom2json v1.3.0
|
github.com/prometheus/prom2json v1.3.0
|
||||||
github.com/r3labs/diff/v2 v2.14.1
|
github.com/r3labs/diff/v2 v2.14.1
|
||||||
github.com/rivo/uniseg v0.2.0 // indirect
|
|
||||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
|
||||||
github.com/sirupsen/logrus v1.8.1
|
github.com/sirupsen/logrus v1.8.1
|
||||||
github.com/spf13/cobra v1.1.3
|
github.com/spf13/cobra v1.1.3
|
||||||
github.com/stretchr/testify v1.7.0
|
github.com/stretchr/testify v1.7.0
|
||||||
github.com/ugorji/go v1.2.3 // indirect
|
|
||||||
github.com/vjeantet/grok v1.0.1 // indirect
|
|
||||||
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
|
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
|
||||||
golang.org/x/mod v0.4.2
|
golang.org/x/mod v0.4.2
|
||||||
golang.org/x/sys v0.0.0-20210921065528-437939a70204
|
golang.org/x/sys v0.0.0-20210921065528-437939a70204
|
||||||
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
|
|
||||||
golang.org/x/text v0.3.5 // indirect
|
|
||||||
google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f // indirect
|
|
||||||
google.golang.org/grpc v1.35.0
|
google.golang.org/grpc v1.35.0
|
||||||
google.golang.org/protobuf v1.25.0
|
google.golang.org/protobuf v1.25.0
|
||||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0
|
gopkg.in/natefinch/lumberjack.v2 v2.0.0
|
||||||
|
@ -86,4 +62,81 @@ require (
|
||||||
gotest.tools/v3 v3.0.3
|
gotest.tools/v3 v3.0.3
|
||||||
)
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/Masterminds/goutils v1.1.1 // indirect
|
||||||
|
github.com/Masterminds/semver v1.5.0 // indirect
|
||||||
|
github.com/Microsoft/go-winio v0.4.16 // indirect
|
||||||
|
github.com/PuerkitoBio/purell v1.1.1 // indirect
|
||||||
|
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
|
||||||
|
github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 // indirect
|
||||||
|
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
|
||||||
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
|
github.com/cespare/xxhash/v2 v2.1.1 // indirect
|
||||||
|
github.com/containerd/containerd v1.4.3 // indirect
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
|
||||||
|
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
|
||||||
|
github.com/docker/distribution v2.7.1+incompatible // indirect
|
||||||
|
github.com/docker/go-units v0.4.0 // indirect
|
||||||
|
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||||
|
github.com/go-openapi/analysis v0.19.16 // indirect
|
||||||
|
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||||
|
github.com/go-openapi/jsonreference v0.19.5 // indirect
|
||||||
|
github.com/go-openapi/loads v0.20.0 // indirect
|
||||||
|
github.com/go-openapi/runtime v0.19.24 // indirect
|
||||||
|
github.com/go-openapi/spec v0.20.0 // indirect
|
||||||
|
github.com/go-playground/locales v0.13.0 // indirect
|
||||||
|
github.com/go-playground/universal-translator v0.17.0 // indirect
|
||||||
|
github.com/go-playground/validator/v10 v10.4.1 // indirect
|
||||||
|
github.com/go-stack/stack v1.8.0 // indirect
|
||||||
|
github.com/gogo/protobuf v1.3.2 // indirect
|
||||||
|
github.com/golang/protobuf v1.4.3 // indirect
|
||||||
|
github.com/google/go-cmp v0.5.4 // indirect
|
||||||
|
github.com/google/uuid v1.3.0 // indirect
|
||||||
|
github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
|
||||||
|
github.com/huandu/xstrings v1.3.2 // indirect
|
||||||
|
github.com/imdario/mergo v0.3.12 // indirect
|
||||||
|
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||||
|
github.com/josharian/intern v1.0.0 // indirect
|
||||||
|
github.com/json-iterator/go v1.1.10 // indirect
|
||||||
|
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||||
|
github.com/leodido/go-urn v1.2.1 // indirect
|
||||||
|
github.com/mailru/easyjson v0.7.6 // indirect
|
||||||
|
github.com/mattn/go-colorable v0.1.9 // indirect
|
||||||
|
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||||
|
github.com/mattn/go-runewidth v0.0.10 // indirect
|
||||||
|
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||||
|
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
|
||||||
|
github.com/mitchellh/copystructure v1.2.0 // indirect
|
||||||
|
github.com/mitchellh/go-testing-interface v1.0.0 // indirect
|
||||||
|
github.com/mitchellh/mapstructure v1.4.1 // indirect
|
||||||
|
github.com/mitchellh/reflectwalk v1.0.2 // indirect
|
||||||
|
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||||
|
github.com/modern-go/reflect2 v1.0.1 // indirect
|
||||||
|
github.com/morikuni/aec v1.0.0 // indirect
|
||||||
|
github.com/oklog/run v1.0.0 // indirect
|
||||||
|
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||||
|
github.com/opencontainers/image-spec v1.0.1 // indirect
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
|
github.com/prometheus/common v0.18.0 // indirect
|
||||||
|
github.com/prometheus/procfs v0.6.0 // indirect
|
||||||
|
github.com/rivo/uniseg v0.2.0 // indirect
|
||||||
|
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||||
|
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||||
|
github.com/spf13/pflag v1.0.5 // indirect
|
||||||
|
github.com/ugorji/go/codec v1.2.3 // indirect
|
||||||
|
github.com/vjeantet/grok v1.0.1 // indirect
|
||||||
|
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
|
||||||
|
go.mongodb.org/mongo-driver v1.4.4 // indirect
|
||||||
|
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
|
||||||
|
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||||
|
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
|
||||||
|
golang.org/x/text v0.3.5 // indirect
|
||||||
|
google.golang.org/appengine v1.6.6 // indirect
|
||||||
|
google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f // indirect
|
||||||
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||||
|
)
|
||||||
|
|
||||||
replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0
|
replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0
|
||||||
|
|
7
go.sum
7
go.sum
|
@ -45,6 +45,8 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx
|
||||||
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
|
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
|
||||||
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
|
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
|
||||||
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
|
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
|
||||||
|
github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:pzStYMLAXM7CNQjS/Wn+zK9MUxDhSUNfVvnHsyQyjs0=
|
||||||
|
github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:ilK+u7u1HoqaDk0mjhh27QJB7PyWMreGffEvOCoEKiY=
|
||||||
github.com/ahmetb/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:3YVZUqkoev4mL+aCwVOSWV4M7pN+NURHL38Z2zq5JKA=
|
github.com/ahmetb/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:3YVZUqkoev4mL+aCwVOSWV4M7pN+NURHL38Z2zq5JKA=
|
||||||
github.com/ahmetb/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:ymXt5bw5uSNu4jveerFxE0vNYxF8ncqbptntMaFMg3k=
|
github.com/ahmetb/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:ymXt5bw5uSNu4jveerFxE0vNYxF8ncqbptntMaFMg3k=
|
||||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||||
|
@ -117,7 +119,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||||
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
|
|
||||||
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/crowdsecurity/grokky v0.0.0-20210908095311-0b3373925934 h1:3kpyoNXTVgfqvpnpy5S8n8hNicgStCK7CiUc4IlYs2s=
|
github.com/crowdsecurity/grokky v0.0.0-20210908095311-0b3373925934 h1:3kpyoNXTVgfqvpnpy5S8n8hNicgStCK7CiUc4IlYs2s=
|
||||||
github.com/crowdsecurity/grokky v0.0.0-20210908095311-0b3373925934/go.mod h1:fx5UYUYAFIrOUNAkFCUOM2wJcsp9EWSQE9R0/9kaFJg=
|
github.com/crowdsecurity/grokky v0.0.0-20210908095311-0b3373925934/go.mod h1:fx5UYUYAFIrOUNAkFCUOM2wJcsp9EWSQE9R0/9kaFJg=
|
||||||
|
@ -154,7 +155,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
|
||||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||||
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
|
|
||||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||||
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
|
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
|
||||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||||
|
@ -486,15 +486,12 @@ github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kN
|
||||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||||
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
|
|
||||||
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
|
||||||
github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U=
|
github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U=
|
||||||
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||||
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
|
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
|
||||||
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
|
|
||||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||||
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
|
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
|
||||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||||
|
|
|
@ -296,7 +296,6 @@ func chunkDecisions(decisions []*ent.Decision, chunkSize int) [][]*ent.Decision
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([]string, error) {
|
func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([]string, error) {
|
||||||
|
|
||||||
ret := []string{}
|
ret := []string{}
|
||||||
bulkSize := 20
|
bulkSize := 20
|
||||||
|
|
||||||
|
@ -504,15 +503,14 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return []string{}, errors.Wrapf(BulkError, "bulk creating alert : %s", err)
|
return []string{}, errors.Wrapf(BulkError, "bulk creating alert : %s", err)
|
||||||
}
|
}
|
||||||
for _, a := range alerts {
|
for alertIndex, a := range alerts {
|
||||||
ret = append(ret, strconv.Itoa(a.ID))
|
ret = append(ret, strconv.Itoa(a.ID))
|
||||||
for _, d := range alertDecisions {
|
d := alertDecisions[alertIndex]
|
||||||
decisionsChunk := chunkDecisions(d, bulkSize)
|
decisionsChunk := chunkDecisions(d, bulkSize)
|
||||||
for _, d2 := range decisionsChunk {
|
for _, d2 := range decisionsChunk {
|
||||||
_, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX)
|
_, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return []string{}, fmt.Errorf("error while updating decisions: %s", err.Error())
|
return []string{}, fmt.Errorf("error while updating decisions: %s", err.Error())
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -531,15 +529,14 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([
|
||||||
return []string{}, errors.Wrapf(BulkError, "leftovers creating alert : %s", err)
|
return []string{}, errors.Wrapf(BulkError, "leftovers creating alert : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, a := range alerts {
|
for alertIndex, a := range alerts {
|
||||||
ret = append(ret, strconv.Itoa(a.ID))
|
ret = append(ret, strconv.Itoa(a.ID))
|
||||||
for _, d := range alertDecisions {
|
d := alertDecisions[alertIndex]
|
||||||
decisionsChunk := chunkDecisions(d, bulkSize)
|
decisionsChunk := chunkDecisions(d, bulkSize)
|
||||||
for _, d2 := range decisionsChunk {
|
for _, d2 := range decisionsChunk {
|
||||||
_, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX)
|
_, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return []string{}, fmt.Errorf("error while updating decisions: %s", err.Error())
|
return []string{}, fmt.Errorf("error while updating decisions: %s", err.Error())
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -48,6 +48,7 @@ type Leaky struct {
|
||||||
Mapkey string
|
Mapkey string
|
||||||
// chan for signaling
|
// chan for signaling
|
||||||
Signal chan bool `json:"-"`
|
Signal chan bool `json:"-"`
|
||||||
|
Suicide chan bool `json:"-"`
|
||||||
Reprocess bool
|
Reprocess bool
|
||||||
Simulated bool
|
Simulated bool
|
||||||
Uuid string
|
Uuid string
|
||||||
|
@ -88,6 +89,14 @@ var BucketsOverflow = prometheus.NewCounterVec(
|
||||||
[]string{"name"},
|
[]string{"name"},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var BucketsCanceled = prometheus.NewCounterVec(
|
||||||
|
prometheus.CounterOpts{
|
||||||
|
Name: "cs_bucket_canceled_total",
|
||||||
|
Help: "Total buckets canceled.",
|
||||||
|
},
|
||||||
|
[]string{"name"},
|
||||||
|
)
|
||||||
|
|
||||||
var BucketsUnderflow = prometheus.NewCounterVec(
|
var BucketsUnderflow = prometheus.NewCounterVec(
|
||||||
prometheus.CounterOpts{
|
prometheus.CounterOpts{
|
||||||
Name: "cs_bucket_underflowed_total",
|
Name: "cs_bucket_underflowed_total",
|
||||||
|
@ -153,6 +162,7 @@ func FromFactory(bucketFactory BucketFactory) *Leaky {
|
||||||
Queue: NewQueue(Qsize),
|
Queue: NewQueue(Qsize),
|
||||||
CacheSize: bucketFactory.CacheSize,
|
CacheSize: bucketFactory.CacheSize,
|
||||||
Out: make(chan *Queue, 1),
|
Out: make(chan *Queue, 1),
|
||||||
|
Suicide: make(chan bool, 1),
|
||||||
AllOut: bucketFactory.ret,
|
AllOut: bucketFactory.ret,
|
||||||
Capacity: bucketFactory.Capacity,
|
Capacity: bucketFactory.Capacity,
|
||||||
Leakspeed: bucketFactory.leakspeed,
|
Leakspeed: bucketFactory.leakspeed,
|
||||||
|
@ -237,7 +247,15 @@ func LeakRoutine(leaky *Leaky) error {
|
||||||
case ofw := <-leaky.Out:
|
case ofw := <-leaky.Out:
|
||||||
leaky.overflow(ofw)
|
leaky.overflow(ofw)
|
||||||
return nil
|
return nil
|
||||||
/*we underflow or reach bucket deadline (timers)*/
|
/*suiciiiide*/
|
||||||
|
case <-leaky.Suicide:
|
||||||
|
close(leaky.Signal)
|
||||||
|
BucketsCanceled.With(prometheus.Labels{"name": leaky.Name}).Inc()
|
||||||
|
leaky.logger.Debugf("Suicide triggered")
|
||||||
|
leaky.AllOut <- types.Event{Type: types.OVFLW, Overflow: types.RuntimeAlert{Mapkey: leaky.Mapkey}}
|
||||||
|
leaky.logger.Tracef("Returning from leaky routine.")
|
||||||
|
return nil
|
||||||
|
/*we underflow or reach bucket deadline (timers)*/
|
||||||
case <-durationTicker:
|
case <-durationTicker:
|
||||||
var (
|
var (
|
||||||
alert types.RuntimeAlert
|
alert types.RuntimeAlert
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -42,6 +43,7 @@ func TestBucket(t *testing.T) {
|
||||||
t.Fatalf("Test '%s' failed : %s", envSetting, err)
|
t.Fatalf("Test '%s' failed : %s", envSetting, err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
wg := new(sync.WaitGroup)
|
||||||
fds, err := ioutil.ReadDir("./tests/")
|
fds, err := ioutil.ReadDir("./tests/")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Unable to read test directory : %s", err)
|
t.Fatalf("Unable to read test directory : %s", err)
|
||||||
|
@ -50,12 +52,27 @@ func TestBucket(t *testing.T) {
|
||||||
fname := "./tests/" + fd.Name()
|
fname := "./tests/" + fd.Name()
|
||||||
log.Infof("Running test on %s", fname)
|
log.Infof("Running test on %s", fname)
|
||||||
tomb.Go(func() error {
|
tomb.Go(func() error {
|
||||||
|
wg.Add(1)
|
||||||
|
defer wg.Done()
|
||||||
if err := testOneBucket(t, fname, tomb); err != nil {
|
if err := testOneBucket(t, fname, tomb); err != nil {
|
||||||
t.Fatalf("Test '%s' failed : %s", fname, err)
|
t.Fatalf("Test '%s' failed : %s", fname, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//during tests, we're likely to have only one scenario, and thus only one holder.
|
||||||
|
//we want to avoid the death of the tomb because all existing buckets have been destroyed.
|
||||||
|
func watchTomb(tomb *tomb.Tomb) {
|
||||||
|
for {
|
||||||
|
if tomb.Alive() == false {
|
||||||
|
log.Warningf("Tomb is dead")
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -102,6 +119,10 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed loading bucket : %s", err)
|
t.Fatalf("failed loading bucket : %s", err)
|
||||||
}
|
}
|
||||||
|
tomb.Go(func() error {
|
||||||
|
watchTomb(tomb)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
if !testFile(t, dir+"/test.json", dir+"/in-buckets_state.json", holders, response, buckets) {
|
if !testFile(t, dir+"/test.json", dir+"/in-buckets_state.json", holders, response, buckets) {
|
||||||
return fmt.Errorf("tests from %s failed", dir)
|
return fmt.Errorf("tests from %s failed", dir)
|
||||||
}
|
}
|
||||||
|
|
|
@ -60,6 +60,7 @@ type BucketFactory struct {
|
||||||
RunTimeGroupBy *vm.Program `json:"-"`
|
RunTimeGroupBy *vm.Program `json:"-"`
|
||||||
Data []*types.DataSource `yaml:"data,omitempty"`
|
Data []*types.DataSource `yaml:"data,omitempty"`
|
||||||
DataDir string `yaml:"-"`
|
DataDir string `yaml:"-"`
|
||||||
|
CancelOnFilter string `yaml:"cancel_on,omitempty"` //a filter that, if matched, kills the bucket
|
||||||
leakspeed time.Duration //internal representation of `Leakspeed`
|
leakspeed time.Duration //internal representation of `Leakspeed`
|
||||||
duration time.Duration //internal representation of `Duration`
|
duration time.Duration //internal representation of `Duration`
|
||||||
ret chan types.Event //the bucket-specific output chan for overflows
|
ret chan types.Event //the bucket-specific output chan for overflows
|
||||||
|
@ -292,6 +293,11 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error {
|
||||||
bucketFactory.processors = append(bucketFactory.processors, &Uniq{})
|
bucketFactory.processors = append(bucketFactory.processors, &Uniq{})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if bucketFactory.CancelOnFilter != "" {
|
||||||
|
bucketFactory.logger.Tracef("Adding a cancel_on filter on %s.", bucketFactory.Name)
|
||||||
|
bucketFactory.processors = append(bucketFactory.processors, &CancelOnFilter{})
|
||||||
|
}
|
||||||
|
|
||||||
if bucketFactory.OverflowFilter != "" {
|
if bucketFactory.OverflowFilter != "" {
|
||||||
bucketFactory.logger.Tracef("Adding an overflow filter")
|
bucketFactory.logger.Tracef("Adding an overflow filter")
|
||||||
filovflw, err := NewOverflowFilter(bucketFactory)
|
filovflw, err := NewOverflowFilter(bucketFactory)
|
||||||
|
|
76
pkg/leakybucket/reset_filter.go
Normal file
76
pkg/leakybucket/reset_filter.go
Normal file
|
@ -0,0 +1,76 @@
|
||||||
|
package leakybucket
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/antonmedv/expr"
|
||||||
|
"github.com/antonmedv/expr/vm"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ResetFilter allows to kill the bucket (without overflowing), if a particular condition is met.
|
||||||
|
// An example would be a scenario to detect aggressive crawlers that *do not* fetch any static ressources :
|
||||||
|
// type : leaky
|
||||||
|
// filter: filter: "evt.Meta.log_type == 'http_access-log'
|
||||||
|
// reset_filter: evt.Parsed.request endswith '.css'
|
||||||
|
// ....
|
||||||
|
// Thus, if the bucket receives a request that matches fetching a static ressource (here css), it cancels itself
|
||||||
|
|
||||||
|
type CancelOnFilter struct {
|
||||||
|
CancelOnFilter *vm.Program
|
||||||
|
CancelOnFilterDebug *exprhelpers.ExprDebugger
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *CancelOnFilter) OnBucketPour(bucketFactory *BucketFactory) func(types.Event, *Leaky) *types.Event {
|
||||||
|
return func(msg types.Event, leaky *Leaky) *types.Event {
|
||||||
|
var condition, ok bool
|
||||||
|
if u.CancelOnFilter != nil {
|
||||||
|
leaky.logger.Tracef("running cancel_on filter")
|
||||||
|
output, err := expr.Run(u.CancelOnFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &msg}))
|
||||||
|
if err != nil {
|
||||||
|
leaky.logger.Warningf("cancel_on error : %s", err)
|
||||||
|
return &msg
|
||||||
|
}
|
||||||
|
//only run debugger expression if condition is false
|
||||||
|
if u.CancelOnFilterDebug != nil {
|
||||||
|
u.CancelOnFilterDebug.Run(leaky.logger, condition, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &msg}))
|
||||||
|
}
|
||||||
|
if condition, ok = output.(bool); !ok {
|
||||||
|
leaky.logger.Warningf("cancel_on, unexpected non-bool return : %T", output)
|
||||||
|
return &msg
|
||||||
|
}
|
||||||
|
if condition {
|
||||||
|
leaky.logger.Debugf("reset_filter matched, kill bucket")
|
||||||
|
leaky.Suicide <- true
|
||||||
|
return nil //counter intuitively, we need to keep the message so that it doesn't trigger an endless loop
|
||||||
|
} else {
|
||||||
|
leaky.logger.Debugf("reset_filter didn't match")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &msg
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *CancelOnFilter) OnBucketOverflow(bucketFactory *BucketFactory) func(*Leaky, types.RuntimeAlert, *Queue) (types.RuntimeAlert, *Queue) {
|
||||||
|
return func(leaky *Leaky, alert types.RuntimeAlert, queue *Queue) (types.RuntimeAlert, *Queue) {
|
||||||
|
return alert, queue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *CancelOnFilter) OnBucketInit(bucketFactory *BucketFactory) error {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
u.CancelOnFilter, err = expr.Compile(bucketFactory.CancelOnFilter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
|
||||||
|
if err != nil {
|
||||||
|
bucketFactory.logger.Errorf("reset_filter compile error : %s", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if bucketFactory.Debug {
|
||||||
|
u.CancelOnFilterDebug, err = exprhelpers.NewDebugger(bucketFactory.CancelOnFilter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
|
||||||
|
if err != nil {
|
||||||
|
bucketFactory.logger.Errorf("reset_filter debug error : %s", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
13
pkg/leakybucket/tests/simple-leaky-cancel_on/bucket.yaml
Normal file
13
pkg/leakybucket/tests/simple-leaky-cancel_on/bucket.yaml
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
type: leaky
|
||||||
|
debug: true
|
||||||
|
name: test/simple-leaky-cancel
|
||||||
|
description: "Simple leaky"
|
||||||
|
filter: "evt.Line.Labels.type =='testlog'"
|
||||||
|
cancel_on: evt.Parsed.random_value == '42'
|
||||||
|
leakspeed: "10s"
|
||||||
|
blackhole: 1m
|
||||||
|
capacity: 1
|
||||||
|
groupby: evt.Meta.source_ip
|
||||||
|
labels:
|
||||||
|
type: overflow_1
|
||||||
|
|
|
@ -0,0 +1,2 @@
|
||||||
|
- filename: {{.TestDirectory}}/bucket.yaml
|
||||||
|
|
117
pkg/leakybucket/tests/simple-leaky-cancel_on/test.json
Normal file
117
pkg/leakybucket/tests/simple-leaky-cancel_on/test.json
Normal file
|
@ -0,0 +1,117 @@
|
||||||
|
{
|
||||||
|
"lines": [
|
||||||
|
{
|
||||||
|
"Line": {
|
||||||
|
"Labels": {
|
||||||
|
"type": "testlog"
|
||||||
|
},
|
||||||
|
"Raw": "xxheader VALUE1 trailing stuff"
|
||||||
|
},
|
||||||
|
"MarshaledTime": "2020-01-01T10:00:00+00:00",
|
||||||
|
"Meta": {
|
||||||
|
"source_ip": "1.2.3.4"
|
||||||
|
},
|
||||||
|
"Parsed": {
|
||||||
|
"random_value" : "41"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Line": {
|
||||||
|
"Labels": {
|
||||||
|
"type": "testlog"
|
||||||
|
},
|
||||||
|
"Raw": "xxheader VALUE2 trailing stuff"
|
||||||
|
},
|
||||||
|
"MarshaledTime": "2020-01-01T10:00:05+00:00",
|
||||||
|
"Meta": {
|
||||||
|
"source_ip": "1.2.3.4"
|
||||||
|
},
|
||||||
|
"Parsed": {
|
||||||
|
"random_value" : "42"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Line": {
|
||||||
|
"Labels": {
|
||||||
|
"type": "testlog"
|
||||||
|
},
|
||||||
|
"Raw": "xxheader VALUE1 trailing stuff"
|
||||||
|
},
|
||||||
|
"MarshaledTime": "2020-01-01T10:00:00+00:00",
|
||||||
|
"Meta": {
|
||||||
|
"source_ip": "1.2.3.4"
|
||||||
|
},
|
||||||
|
"Parsed": {
|
||||||
|
"random_value" : "41"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Line": {
|
||||||
|
"Labels": {
|
||||||
|
"type": "testlog"
|
||||||
|
},
|
||||||
|
"Raw": "xxheader VALUE1 trailing stuff"
|
||||||
|
},
|
||||||
|
"MarshaledTime": "2020-01-01T10:00:00+00:00",
|
||||||
|
"Meta": {
|
||||||
|
"source_ip": "2.2.3.4"
|
||||||
|
},
|
||||||
|
"Parsed": {
|
||||||
|
"random_value" : "41"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Line": {
|
||||||
|
"Labels": {
|
||||||
|
"type": "testlog"
|
||||||
|
},
|
||||||
|
"Raw": "xxheader VALUE1 trailing stuff"
|
||||||
|
},
|
||||||
|
"MarshaledTime": "2020-01-01T10:00:00+00:00",
|
||||||
|
"Meta": {
|
||||||
|
"source_ip": "2.2.3.4"
|
||||||
|
},
|
||||||
|
"Parsed": {
|
||||||
|
"random_value" : "41"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Line": {
|
||||||
|
"Labels": {
|
||||||
|
"type": "testlog"
|
||||||
|
},
|
||||||
|
"Raw": "xxheader VALUE1 trailing stuff"
|
||||||
|
},
|
||||||
|
"MarshaledTime": "2020-01-01T10:00:00+00:00",
|
||||||
|
"Meta": {
|
||||||
|
"source_ip": "2.2.3.4"
|
||||||
|
},
|
||||||
|
"Parsed": {
|
||||||
|
"random_value" : "41"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"Alert": {
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Alert": {
|
||||||
|
"sources": {
|
||||||
|
"2.2.3.4": {
|
||||||
|
"scope": "Ip",
|
||||||
|
"value": "2.2.3.4",
|
||||||
|
"ip": "2.2.3.4"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"Alert" : {
|
||||||
|
"scenario": "test/simple-leaky-cancel",
|
||||||
|
"events_count": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
|
@ -44,7 +44,7 @@ type Config struct {
|
||||||
var (
|
var (
|
||||||
metabaseDefaultUser = "crowdsec@crowdsec.net"
|
metabaseDefaultUser = "crowdsec@crowdsec.net"
|
||||||
metabaseDefaultPassword = "!!Cr0wdS3c_M3t4b4s3??"
|
metabaseDefaultPassword = "!!Cr0wdS3c_M3t4b4s3??"
|
||||||
metabaseImage = "metabase/metabase:v0.41.4"
|
metabaseImage = "metabase/metabase:v0.41.5"
|
||||||
containerSharedFolder = "/metabase-data"
|
containerSharedFolder = "/metabase-data"
|
||||||
metabaseSQLiteDBURL = "https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/metabase_sqlite.zip"
|
metabaseSQLiteDBURL = "https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/metabase_sqlite.zip"
|
||||||
)
|
)
|
||||||
|
|
|
@ -7,6 +7,6 @@ CURRENT_DIR=$(pwd)
|
||||||
|
|
||||||
git clone https://github.com/crowdsecurity/hub.git
|
git clone https://github.com/crowdsecurity/hub.git
|
||||||
cd hub/
|
cd hub/
|
||||||
${CSCLI} hubtest run --all
|
${CSCLI} hubtest run --all --clean
|
||||||
|
|
||||||
cd "${CURRENT_DIR}"
|
cd "${CURRENT_DIR}"
|
||||||
|
|
|
@ -164,7 +164,7 @@ log_input_tags[linux]="type: syslog"
|
||||||
|
|
||||||
declare -A log_locations
|
declare -A log_locations
|
||||||
log_locations[apache2]='/var/log/apache2/*.log,/var/log/*httpd*.log,/var/log/httpd/*log'
|
log_locations[apache2]='/var/log/apache2/*.log,/var/log/*httpd*.log,/var/log/httpd/*log'
|
||||||
log_locations[nginx]='/var/log/nginx/*.log'
|
log_locations[nginx]='/var/log/nginx/*.log,/usr/local/openresty/nginx/logs/*.log'
|
||||||
log_locations[sshd]='/var/log/auth.log,/var/log/sshd.log,/var/log/secure'
|
log_locations[sshd]='/var/log/auth.log,/var/log/sshd.log,/var/log/secure'
|
||||||
log_locations[rsyslog]='/var/log/syslog'
|
log_locations[rsyslog]='/var/log/syslog'
|
||||||
log_locations[telnet]='/var/log/telnetd*.log'
|
log_locations[telnet]='/var/log/telnetd*.log'
|
||||||
|
|
Loading…
Reference in a new issue