beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Refactor Acquisition Interface (#773)

Browse source 
* Add new acquisition interface + new modules (cloudwatch, syslog)

Co-authored-by: Sebastien Blot <sebastien@crowdsec.net>
This commit is contained in:
Thibault "bui" Koechlin 2021-06-11 09:53:53 +02:00 committed by GitHub
parent 71c1d9431f
commit ce6a61df1c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
47 changed files with 4468 additions and 1378 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
.github/workflows/ci_go-test.yml vendored
View file

@ -1,5 +1,16 @@
name: tests
#those env variables are for localstack, so we can emulate aws services
env:
AWS_HOST: localstack
SERVICES: cloudwatch,logs
#those are to mimic aws config
AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
AWS_REGION: us-east-1
#and to override our endpoint in aws sdk
AWS_ENDPOINT_FORCE: http://localhost:4566
on:
push:
branches: [ master ]
@ -19,6 +30,27 @@ jobs:
build:
name: Build
runs-on: ubuntu-latest
services:
localstack:
image: localstack/localstack:0.12.11
ports:
- 4566:4566 # Localstack exposes all services on same port
env:
SERVICES: ${{ env.SERVICES }}
DEBUG: ""
DATA_DIR: ""
LAMBDA_EXECUTOR: ""
KINESIS_ERROR_PROBABILITY: ""
DOCKER_HOST: unix:///var/run/docker.sock
HOST_TMP_FOLDER: "/tmp"
HOSTNAME_EXTERNAL: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
# e.g sqs url will get localhost if we don't set this env to map our service
options: >-
--name=localstack
--health-cmd="curl -sS 127.0.0.1:4566 || exit 1"
--health-interval=10s
--health-timeout=5s
--health-retries=3
steps:
- name: Set up Go 1.13
uses: actions/setup-go@v1
@ -40,4 +72,4 @@ jobs:
uses: coverallsapp/github-action@master
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
path-to-lcov: coverage.txt
path-to-lcov: coverage.txt

9
cmd/crowdsec-cli/metrics.go
View file

@ -134,8 +134,13 @@ func ShowPrometheus(url string) {
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
log.Debugf("no source in Metric %v for %s", metric.Labels, fam.Name)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
machine := metric.Labels["machine"]
bouncer := metric.Labels["bouncer"]
@ -180,7 +185,7 @@ func ShowPrometheus(url string) {
}
buckets_stats[name]["underflow"] += ival
/*acquis*/
case "cs_reader_hits_total":
case "cs_parser_hits_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}

4
cmd/crowdsec-cli/utils.go
View file

@ -334,6 +334,10 @@ func GetParserMetric(url string, itemName string) map[string]map[string]int {
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)

12
cmd/crowdsec/crowdsec.go
View file

@ -11,6 +11,7 @@ import (
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
@ -114,6 +115,17 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
return err
}
if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
aggregated := false
if cConfig.Prometheus.Level == "aggregated" {
aggregated = true
}
if err := acquisition.GetMetrics(dataSources, aggregated); err != nil {
return errors.Wrap(err, "while fetching prometheus metrics for datasources.")
}
}
return nil
}

64
cmd/crowdsec/main.go
View file

@ -4,7 +4,6 @@ import (
"flag"
"fmt"
"os"
"strings"
_ "net/http/pprof"
"time"
@ -18,6 +17,7 @@ import (
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
@ -46,18 +46,16 @@ var (
)
type Flags struct {
ConfigFile string
TraceLevel bool
DebugLevel bool
InfoLevel bool
PrintVersion bool
SingleFilePath string
SingleJournalctlFilter string
SingleFileType string
SingleFileJsonOutput string
TestMode bool
DisableAgent bool
DisableAPI bool
ConfigFile string
TraceLevel bool
DebugLevel bool
InfoLevel bool
PrintVersion bool
SingleFileType string
OneShotDSN string
TestMode bool
DisableAgent bool
DisableAPI bool
}
type parsers struct {
@ -140,30 +138,19 @@ func LoadBuckets(cConfig *csconfig.Config) error {
func LoadAcquisition(cConfig *csconfig.Config) error {
var err error
if flags.SingleFilePath != "" || flags.SingleJournalctlFilter != "" {
tmpCfg := acquisition.DataSourceCfg{}
tmpCfg.Mode = acquisition.CAT_MODE
tmpCfg.Labels = map[string]string{"type": flags.SingleFileType}
if flags.SingleFilePath != "" {
tmpCfg.Filename = flags.SingleFilePath
} else if flags.SingleJournalctlFilter != "" {
tmpCfg.JournalctlFilters = strings.Split(flags.SingleJournalctlFilter, " ")
if flags.SingleFileType != "" || flags.OneShotDSN != "" {
if flags.OneShotDSN == "" || flags.SingleFileType == "" {
return fmt.Errorf("-type requires a -dsn argument")
}
datasrc, err := acquisition.DataSourceConfigure(tmpCfg)
dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.SingleFileType)
if err != nil {
return fmt.Errorf("while configuring specified file datasource : %s", err)
return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN)
}
if dataSources == nil {
dataSources = make([]acquisition.DataSource, 0)
}
dataSources = append(dataSources, datasrc)
} else {
dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec)
if err != nil {
log.Fatalf("While loading acquisition configuration : %s", err)
return errors.Wrap(err, "while loading acquisition configuration")
}
}
@ -177,8 +164,7 @@ func (f *Flags) Parse() {
flag.BoolVar(&f.DebugLevel, "debug", false, "print debug-level on stdout")
flag.BoolVar(&f.InfoLevel, "info", false, "print info-level on stdout")
flag.BoolVar(&f.PrintVersion, "version", false, "display version")
flag.StringVar(&f.SingleFilePath, "file", "", "Process a single file in time-machine")
flag.StringVar(&f.SingleJournalctlFilter, "jfilter", "", "Process a single journalctl output in time-machine")
flag.StringVar(&f.OneShotDSN, "dsn", "", "Process a single data source in time-machine")
flag.StringVar(&f.SingleFileType, "type", "", "Labels.type for file in time-machine")
flag.BoolVar(&f.TestMode, "t", false, "only test configs")
flag.BoolVar(&f.DisableAgent, "no-cs", false, "disable crowdsec agent")
@ -210,18 +196,6 @@ func LoadConfig(cConfig *csconfig.Config) error {
log.Fatalf("You must run at least the API Server or crowdsec")
}
if flags.SingleFilePath != "" {
if flags.SingleFileType == "" {
return fmt.Errorf("-file requires -type")
}
}
if flags.SingleJournalctlFilter != "" {
if flags.SingleFileType == "" {
return fmt.Errorf("-jfilter requires -type")
}
}
if flags.DebugLevel {
logLevel := log.DebugLevel
cConfig.Common.LogLevel = &logLevel
@ -239,7 +213,7 @@ func LoadConfig(cConfig *csconfig.Config) error {
cConfig.Crowdsec.LintOnly = true
}
if flags.SingleFilePath != "" || flags.SingleJournalctlFilter != "" {
if flags.SingleFileType != "" && flags.OneShotDSN != "" {
cConfig.API.Server.OnlineClient = nil
/*if the api is disabled as well, just read file and exit, don't daemonize*/
if flags.DisableAPI {

11
cmd/crowdsec/metrics.go
View file

@ -3,7 +3,6 @@ package main
import (
"fmt"
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
@ -24,21 +23,21 @@ var globalParserHits = prometheus.NewCounterVec(
Name: "cs_parser_hits_total",
Help: "Total events entered the parser.",
},
[]string{"source"},
[]string{"source", "type"},
)
var globalParserHitsOk = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_parser_hits_ok_total",
Help: "Total events were successfully parsed.",
},
[]string{"source"},
[]string{"source", "type"},
)
var globalParserHitsKo = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_parser_hits_ko_total",
Help: "Total events were unsuccessfully parsed.",
},
[]string{"source"},
[]string{"source", "type"},
)
var globalBucketPourKo = prometheus.NewCounter(
@ -82,7 +81,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
if config.Level == "aggregated" {
log.Infof("Loading aggregated prometheus collectors")
prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
acquisition.ReaderHits, globalCsInfo,
globalCsInfo,
leaky.BucketsUnderflow, leaky.BucketsInstanciation, leaky.BucketsOverflow,
v1.LapiRouteHits,
leaky.BucketsCurrentCount)
@ -90,7 +89,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
log.Infof("Loading prometheus collectors")
prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
parser.NodesHits, parser.NodesHitsOk, parser.NodesHitsKo,
acquisition.ReaderHits, globalCsInfo,
globalCsInfo,
v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions,
leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsInstanciation, leaky.BucketsOverflow, leaky.BucketsCurrentCount)

10
cmd/crowdsec/parse.go
View file

@ -22,7 +22,11 @@ LOOP:
if !event.Process {
continue
}
globalParserHits.With(prometheus.Labels{"source": event.Line.Src}).Inc()
if event.Line.Module == "" {
log.Errorf("empty event.Line.Module field, the acquisition module must set it ! : %+v", event.Line)
continue
}
globalParserHits.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc()
/* parse the log using magic */
parsed, error := parser.Parse(parserCTX, event, nodes)
@ -31,11 +35,11 @@ LOOP:
return errors.New("parsing failed :/")
}
if !parsed.Process {
globalParserHitsKo.With(prometheus.Labels{"source": event.Line.Src}).Inc()
globalParserHitsKo.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc()
log.Debugf("Discarding line %+v", parsed)
continue
}
globalParserHitsOk.With(prometheus.Labels{"source": event.Line.Src}).Inc()
globalParserHitsOk.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc()
if parsed.Whitelisted {
log.Debugf("event whitelisted, discard")
continue

14
go.mod
View file

@ -8,6 +8,7 @@ require (
github.com/Microsoft/go-winio v0.4.16 // indirect
github.com/antonmedv/expr v1.8.9
github.com/appleboy/gin-jwt/v2 v2.6.4
github.com/aws/aws-sdk-go v1.38.34
github.com/buger/jsonparser v1.1.1
github.com/containerd/containerd v1.4.3 // indirect
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
@ -18,6 +19,7 @@ require (
github.com/docker/docker v20.10.2+incompatible
github.com/docker/go-connections v0.4.0
github.com/enescakir/emoji v1.0.0
github.com/fsnotify/fsnotify v1.4.9
github.com/gin-gonic/gin v1.6.3
github.com/go-co-op/gocron v0.5.1
github.com/go-openapi/errors v0.19.9
@ -30,6 +32,7 @@ require (
github.com/google/go-querystring v1.0.0
github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e
github.com/hashicorp/go-version v1.2.1
github.com/influxdata/go-syslog/v3 v3.0.0 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lib/pq v1.10.0
github.com/logrusorgru/grokky v0.0.0-20180829062225-47edf017d42c
@ -47,21 +50,20 @@ require (
github.com/oschwald/geoip2-golang v1.4.0
github.com/oschwald/maxminddb-golang v1.8.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.9.0
github.com/prometheus/client_golang v1.10.0
github.com/prometheus/client_model v0.2.0
github.com/prometheus/procfs v0.3.0 // indirect
github.com/prometheus/prom2json v1.3.0
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sirupsen/logrus v1.7.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.1.3
github.com/stretchr/testify v1.7.0
github.com/ugorji/go v1.2.3 // indirect
github.com/vjeantet/grok v1.0.1 // indirect
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
golang.org/x/mod v0.4.1
golang.org/x/net v0.0.0-20201224014010-6772e930b67b
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
golang.org/x/net v0.0.0-20201224014010-6772e930b67b // indirect
golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 // indirect
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
golang.org/x/text v0.3.5 // indirect
google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f // indirect
@ -69,7 +71,7 @@ require (
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
gopkg.in/yaml.v2 v2.4.0
gotest.tools/v3 v3.0.3 // indirect
gotest.tools/v3 v3.0.3
)
replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0

109
go.sum
View file

@ -40,11 +40,9 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/antonmedv/expr v1.8.9 h1:O9stiHmHHww9b4ozhPx7T6BK7fXfOCHJ8ybxf0833zw=
@ -62,13 +60,15 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg=
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk=
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/aws/aws-sdk-go v1.38.34 h1:JSAyS6hSDLbRmCAz9VAkwDf5oh/olt9mBTrVBWGJcU8=
github.com/aws/aws-sdk-go v1.38.34/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@ -98,7 +98,6 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@ -147,7 +146,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@ -160,7 +158,6 @@ github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/go-bindata/go-bindata v1.0.1-0.20190711162640-ee3c2418e368 h1:WNHfSP1q2vuAa9vF54RrhCl4nqxCjVcXhlbsRXbGOSY=
github.com/go-bindata/go-bindata v1.0.1-0.20190711162640-ee3c2418e368/go.mod h1:7xCgX1lzlrXPHkfvn3EhumqHkmSlzt8at9q7v0ax19c=
github.com/go-co-op/gocron v0.5.1 h1:Cni1V7mt184+HnYTDYe6MH7siofCvf94PrGyIDI1v1U=
github.com/go-co-op/gocron v0.5.1/go.mod h1:6Btk4lVj3bnFAgbVfr76W8impTyhYrEi1pV5Pt4Tp/M=
@ -177,7 +174,6 @@ github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpR
github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU=
github.com/go-openapi/analysis v0.19.10 h1:5BHISBAXOc/aJK25irLZnx2D3s6WyYaY9D4gmuz9fdE=
github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ=
github.com/go-openapi/analysis v0.19.16 h1:Ub9e++M8sDwtHD+S587TYi+6ANBG1NRYGZDihqk0SaY=
github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk=
@ -186,25 +182,20 @@ github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQH
github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
github.com/go-openapi/errors v0.19.7 h1:Lcq+o0mSwCLKACMxZhreVHigB9ebghJ/lrmeaqASbjo=
github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
github.com/go-openapi/errors v0.19.8 h1:doM+tQdZbUm9gydV9yR+iQNmztbjj7I3sW4sIcAwIzc=
github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
github.com/go-openapi/errors v0.19.9 h1:9SnKdGhiPZHF3ttwFMiCBEb8jQ4IDdrK+5+a0oTygA4=
github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4=
github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4=
github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
@ -213,9 +204,7 @@ github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf
github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI=
github.com/go-openapi/loads v0.19.5 h1:jZVYWawIQiA1NBnHla28ktg6hrcfTHsCE+3QLVRBIls=
github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY=
github.com/go-openapi/loads v0.19.6 h1:6IAtnx22MNSjPocZZ2sV7EjgF6wW5rDC9r6ZkNxjiN8=
github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
github.com/go-openapi/loads v0.20.0 h1:Pymw1O8zDmWeNv4kVsHd0W3cvgdp8juRa4U/U/8D/Pk=
@ -224,7 +213,6 @@ github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6
github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo=
github.com/go-openapi/runtime v0.19.16 h1:tQMAY5s5BfmmCC31+ufDCsGrr8iO1A8UIdYfDo5ADvs=
github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98=
github.com/go-openapi/runtime v0.19.24 h1:TqagMVlRAOTwllE/7hNKx6rQ10O6T8ZzeJdMjSTKaD4=
github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk=
@ -233,7 +221,6 @@ github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsd
github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
github.com/go-openapi/spec v0.19.8 h1:qAdZLh1r6QF/hI/gTq+TJTvsQUodZsM7KLqkAJdiJNg=
github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU=
github.com/go-openapi/spec v0.20.0 h1:HGLc8AJ7ynOxwv0Lq4TsnwLsWMawHAYiJIFzbcML86I=
@ -244,7 +231,6 @@ github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+Z
github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
github.com/go-openapi/strfmt v0.19.5 h1:0utjKrw+BAh8s57XE9Xz8DUBsVvPmRUB6styvl9wWIM=
github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
github.com/go-openapi/strfmt v0.19.11 h1:0+YvbNh05rmBkgztd6zHp4OCFn7Mtu30bn46NQo2ZRw=
github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
@ -253,7 +239,6 @@ github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
github.com/go-openapi/swag v0.19.9 h1:1IxuqvBUU3S2Bi4YC7tlP9SJF1gVpCvqN0T2Qof4azE=
github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
github.com/go-openapi/swag v0.19.12 h1:Bc0bnY2c3AoF7Gc+IMIAQQsD8fLHjHpc19wXvYuayQI=
github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M=
@ -261,7 +246,6 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+
github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo=
github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8=
github.com/go-openapi/validate v0.19.12 h1:mPLM/bfbd00PGOCJlU0yJL7IulkZ+q9VjPv7U11RMQQ=
github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4=
github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI=
github.com/go-openapi/validate v0.20.0 h1:pzutNCCBZGZlE+u8HD3JZyWdc/TVbtVwlWUp8/vgUKk=
@ -272,7 +256,6 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
@ -309,11 +292,9 @@ github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY9
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@ -324,9 +305,7 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
@ -346,10 +325,10 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@ -392,7 +371,6 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
@ -410,9 +388,12 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/go-syslog/v3 v3.0.0 h1:jichmjSZlYK0VMmlz+k4WeOQd7z745YLsvGMqwtYt4I=
github.com/influxdata/go-syslog/v3 v3.0.0/go.mod h1:tulsOp+CecTAYC27u9miMgq21GqXRW6VdKbOG+QSP4Q=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
@ -440,7 +421,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@ -452,10 +432,10 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg=
github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
@ -470,14 +450,12 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8=
github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
@ -488,9 +466,7 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
@ -499,7 +475,6 @@ github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJK
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@ -512,10 +487,8 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg=
github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.0 h1:7ks8ZkOP5/ujthUsT07rNv+nkLXCQWKNHuwzOAesEks=
github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@ -569,7 +542,6 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh
github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9PvrpOfz+Ug=
github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng=
github.com/oschwald/maxminddb-golang v1.6.0 h1:KAJSjdHQ8Kv45nFIbtoLGrGWqHFajOIm7skTyz/+Dls=
github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w=
github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk=
github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
@ -583,7 +555,6 @@ github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -598,8 +569,8 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU=
github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU=
github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg=
github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@ -613,19 +584,16 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM=
github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y=
github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4=
github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.3.0 h1:Uehi/mxLK0eiUc0H0++5tpMGTexB8wZ598MIgU8VpDM=
github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y=
github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
@ -638,7 +606,6 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@ -647,15 +614,14 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0
github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@ -668,7 +634,6 @@ github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@ -684,9 +649,7 @@ github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@ -699,11 +662,9 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go v1.2.3 h1:WbFSXLxDFKVN69Sk8t+XHGzVCD7R8UoAATR8NqZgTbk=
github.com/ugorji/go v1.2.3/go.mod h1:5l8GZ8hZvmL4uMdy+mhCO1LjswGRYco9Q3HfuisB21A=
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ugorji/go/codec v1.2.3 h1:/mVYEV+Jo3IZKeA5gBngN0AvNnQltEDkR+eQikkWQu0=
github.com/ugorji/go/codec v1.2.3/go.mod h1:5FxzDJIgeiWJZslYHPj+LS1dq1ZBQVelZFnjsFGI/Uc=
@ -723,9 +684,7 @@ go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mI
go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
go.mongodb.org/mongo-driver v1.3.4 h1:zs/dKNwX0gYUtzwrN9lLiR15hCO0nDwQj5xXx+vjCdE=
go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
go.mongodb.org/mongo-driver v1.4.3 h1:moga+uhicpVshTyaqY9L23E6QqwcHRUv1sqyOsoyOO8=
go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
go.mongodb.org/mongo-driver v1.4.4 h1:bsPHfODES+/yx2PCWzUYMH8xj6PVniPI8DQrsJuSXSs=
go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
@ -755,7 +714,6 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
@ -769,7 +727,6 @@ golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+o
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@ -780,7 +737,6 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@ -801,19 +757,15 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw=
@ -829,6 +781,7 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -862,30 +815,28 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4 h1:sfkvUWPNGwSV+8/fNqctR5lS2AqCSqYwXdrjCxp/dXo=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 h1:cdsMqa2nXzqlgs183pHxtvoVwU7CyzaCTAUOg94af4c=
golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@ -901,7 +852,6 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@ -929,11 +879,9 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -948,7 +896,6 @@ google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@ -958,9 +905,7 @@ google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dT
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f h1:izedQ6yVIc5mZsRuXzmSreCOlzI0lCU1HpG8yEdMiKw=
google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@ -968,16 +913,13 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
@ -988,13 +930,11 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@ -1019,14 +959,11 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo=
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

135
pkg/acquisition/README.md Normal file
View file

@ -0,0 +1,135 @@
# pkg/acquisition
## What is it
`pkg/acquisition` is in charge of reading data sources and feeding events to the parser(s).
Most data sources can either be used :
- in [one-shot](https://doc.crowdsec.net/v1.X/docs/user_guide/forensic_mode/#forensic-mode) mode : data source (ie. file) is read at once
- in streaming mode : data source is constantly monitored, and events are fed to the parsers in real time
## Scope
This documentation aims at providing guidelines for implementation of new data sources.
# Writting modules
Each module must implement the `DataSource` interface.
```golang
type DataSource interface {
GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module
Configure([]byte, *log.Entry) error // Configure the datasource
ConfigureByDSN(string, string, *log.Entry) error // Configure the datasource
GetMode() string // Get the mode (TAIL, CAT or SERVER)
GetName() string
OneShotAcquisition(chan types.Event, *tomb.Tomb) error // Start one shot acquisition(eg, cat a file)
StreamingAcquisition(chan types.Event, *tomb.Tomb) error // Start live acquisition (eg, tail a file)
CanRun() error // Whether the datasource can run or not (eg, journalctl on BSD is a non-sense)
Dump() interface{}
}
```
Ground rules :
- All modules must respect the `tomb.Tomb`
- `StreamingAcquisition` starts dedicated routines (via the `tomb.Tomb`) and returns, while `OneShotAcquisition` returns when datasource is consumed
- `ConfigureByDSN` allows to configure datasource via cli for command-line invokation. Liberties can be taken with dsn format
- Each datasource will be given a logger at configuration time, that is configured according to `DataSourceCommonCfg`. It is advised to customize it via [`.WithFields`](https://pkg.go.dev/github.com/sirupsen/logrus#WithFields) to take advantage of structured logging.
- You must set `Module` in the `types.Event.Line` that you're pushing the chan
Note about configuration format :
- Each data source can have their custom configuration.
- All datasource share a "common" configuration section (`DataSourceCommonCfg`). To achieve this, you might want to inlines `DataSourceCommonCfg` in your datasource-specific configuration structure.
## Interface methods
### GetMetrics
Each data source can and should return custom prometheus metrics.
This is called for each data source that has at least one configured instance.
Using `cs_yoursource_` is advised, along with [prometheus naming good practices](https://prometheus.io/docs/practices/naming/) for reference.
### Configure
Configure is fed with the raw yaml configuration for your data source.
This is meant to allow freedom for each data source's configurations.
## ConfigureByDSN
When used in one-shot mode, your datasource is going to be configured via cli arguments.
The first argument is the `dsn`, the second on is the `label->type` to set on the logs.
Datasource implementations are allowed a lot of freedom around the [`dsn`](https://en.wikipedia.org/wiki/Data_source_name) specifications, but are expected :
- to allow `log_level` configuration via dsn (ie. `mod://source;log_level=trace`)
## GetMode
Returns the mode `TAIL_MODE` or `CAT_MODE` of the current instance.
## OneShotAcquisition
Start a one-shot (or `CAT_MODE`, commonly used for forensic) acquisition that is expected to return once the datasource has been consumed.
## StreamingAcquisition
Start a streaming (or `TAIL_MODE`, commonly used when crowdsec runs as a daemon) acquisition. Starts appropriate go-routines via the `tomb.Tomb` and returns.
## CanRun
Can be used to prevent specific data source to run on specific platforms (ie. journalctl on BSD)
## Dump
Simply return a pointer to the actual object, kept for future.
## GetName
Returns the name, as expected in the `source` argument of the configuration.
# BoilerPlate code
Taking a look at `acquisition_test.go` is advised for up-to-date boilerplate code.
```golang
type MockSource struct {
configuration.DataSourceCommonCfg `yaml:",inline"`
Toto string `yaml:"toto"`
logger *log.Entry
}
func (f *MockSource) Configure(cfg []byte, logger *log.Entry) error {
if err := yaml.UnmarshalStrict(cfg, &f); err != nil {
return errors.Wrap(err, "while unmarshaling to reader specific config")
}
if f.Mode == "" {
f.Mode = configuration.CAT_MODE
}
if f.Mode != configuration.CAT_MODE && f.Mode != configuration.TAIL_MODE {
return fmt.Errorf("mode %s is not supported", f.Mode)
}
if f.Toto == "" {
return fmt.Errorf("expect non-empty toto")
}
f.logger = logger.WithField("toto", f.Toto)
return nil
}
func (f *MockSource) GetMode() string { return f.Mode }
func (f *MockSource) OneShotAcquisition(chan types.Event, *tomb.Tomb) error { return nil }
func (f *MockSource) StreamingAcquisition(chan types.Event, *tomb.Tomb) error { return nil }
func (f *MockSource) CanRun() error { return nil }
func (f *MockSource) GetMetrics() []prometheus.Collector { return nil }
func (f *MockSource) Dump() interface{} { return f }
func (f *MockSource) GetName() string { return "mock" }
func (f *MockSource) ConfigureByDSN(string, string, *log.Entry) error {
return fmt.Errorf("not supported")
}
```

280
pkg/acquisition/acquisition.go
View file

@ -4,6 +4,13 @@ import (
"fmt"
"io"
"os"
"strings"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch"
fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file"
journalctlacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/journalctl"
syslogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -15,102 +22,137 @@ import (
tomb "gopkg.in/tomb.v2"
)
var ReaderHits = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_reader_hits_total",
Help: "Total lines where read.",
},
[]string{"source"},
)
/*
current limits :
- The acquisition is not yet modular (cf. traefik/yaegi), but we start with an interface to pave the road for it.
- The configuration item unmarshaled (DataSourceCfg) isn't generic neither yet.
- This changes should be made when we're ready to have acquisition managed by the hub & cscli
once this change is done, we might go for the following configuration format instead :
```yaml
---
type: nginx
source: journald
filter: "PROG=nginx"
---
type: nginx
source: files
filenames:
- "/var/log/nginx/*.log"
```
*/
/* Approach
We support acquisition in two modes :
- tail mode : we're following a stream of info (tail -f $src). this is used when monitoring live logs
- cat mode : we're reading a file/source one-shot (cat $src), and scenarios will match the timestamp extracted from logs.
One DataSourceCfg can lead to multiple goroutines, hence the Tombs passing around to allow proper tracking.
tail mode shouldn't return except on errors or when externally killed via tombs.
cat mode will return once source has been exhausted.
TBD in current iteration :
- how to deal with "file was not present at startup but might appear later" ?
*/
var TAIL_MODE = "tail"
var CAT_MODE = "cat"
type DataSourceCfg struct {
Mode string `yaml:"mode,omitempty"` //tail|cat|...
Filename string `yaml:"filename,omitempty"`
Filenames []string `yaml:"filenames,omitempty"`
JournalctlFilters []string `yaml:"journalctl_filter,omitempty"`
Labels map[string]string `yaml:"labels,omitempty"`
Profiling bool `yaml:"profiling,omitempty"`
}
// The interface each datasource must implement
type DataSource interface {
Configure(DataSourceCfg) error
/*the readers must watch the tomb (especially in tail mode) to know when to shutdown.
tomb is as well used to trigger general shutdown when a datasource errors */
StartReading(chan types.Event, *tomb.Tomb) error
Mode() string //return CAT_MODE or TAIL_MODE
//Not sure it makes sense to make those funcs part of the interface.
//While 'cat' and 'tail' are the only two modes we see now, other modes might appear
//StartTail(chan types.Event, *tomb.Tomb) error
//StartCat(chan types.Event, *tomb.Tomb) error
GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module
GetAggregMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module (aggregated mode, limits cardinality)
Configure([]byte, *log.Entry) error // Configure the datasource
ConfigureByDSN(string, string, *log.Entry) error // Configure the datasource
GetMode() string // Get the mode (TAIL, CAT or SERVER)
GetName() string // Get the name of the module
OneShotAcquisition(chan types.Event, *tomb.Tomb) error // Start one shot acquisition(eg, cat a file)
StreamingAcquisition(chan types.Event, *tomb.Tomb) error // Start live acquisition (eg, tail a file)
CanRun() error // Whether the datasource can run or not (eg, journalctl on BSD is a non-sense)
Dump() interface{}
}
func DataSourceConfigure(config DataSourceCfg) (DataSource, error) {
if config.Mode == "" { /*default mode is tail*/
config.Mode = TAIL_MODE
}
if len(config.Filename) > 0 || len(config.Filenames) > 0 { /*it's file acquisition*/
fileSrc := new(FileSource)
if err := fileSrc.Configure(config); err != nil {
return nil, errors.Wrap(err, "configuring file datasource")
}
return fileSrc, nil
} else if len(config.JournalctlFilters) > 0 { /*it's journald acquisition*/
journaldSrc := new(JournaldSource)
if err := journaldSrc.Configure(config); err != nil {
return nil, errors.Wrap(err, "configuring journald datasource")
}
return journaldSrc, nil
} else {
return nil, fmt.Errorf("empty filename(s) and journalctl filter, malformed datasource")
}
var AcquisitionSources = []struct {
name string
iface func() DataSource
}{
{
name: "file",
iface: func() DataSource { return &fileacquisition.FileSource{} },
},
{
name: "journalctl",
iface: func() DataSource { return &journalctlacquisition.JournalCtlSource{} },
},
{
name: "cloudwatch",
iface: func() DataSource { return &cloudwatchacquisition.CloudwatchSource{} },
},
{
name: "syslog",
iface: func() DataSource { return &syslogacquisition.SyslogSource{} },
},
}
func GetDataSourceIface(dataSourceType string) DataSource {
for _, source := range AcquisitionSources {
if source.name == dataSourceType {
return source.iface()
}
}
return nil
}
func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
//we dump it back to []byte, because we want to decode the yaml blob twice :
//once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
yamlConfig, err := yaml.Marshal(commonConfig)
if err != nil {
return nil, errors.Wrap(err, "unable to marshal back interface")
}
if dataSrc := GetDataSourceIface(commonConfig.Source); dataSrc != nil {
/* this logger will then be used by the datasource at runtime */
clog := log.New()
if err := types.ConfigureLogger(clog); err != nil {
return nil, errors.Wrap(err, "while configuring datasource logger")
}
if commonConfig.LogLevel != nil {
clog.SetLevel(*commonConfig.LogLevel)
}
customLog := log.Fields{
"type": commonConfig.Source,
}
if commonConfig.Name != "" {
customLog["name"] = commonConfig.Name
}
subLogger := clog.WithFields(customLog)
/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
if err := dataSrc.CanRun(); err != nil {
return nil, errors.Wrapf(err, "datasource %s cannot be run", commonConfig.Source)
}
/* configure the actual datasource */
if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
return nil, errors.Wrapf(err, "failed to configure datasource %s", commonConfig.Source)
}
return &dataSrc, nil
}
return nil, fmt.Errorf("cannot find source %s", commonConfig.Source)
}
//detectBackwardCompatAcquis : try to magically detect the type for backward compat (type was not mandatory then)
func detectBackwardCompatAcquis(sub configuration.DataSourceCommonCfg) string {
if _, ok := sub.Config["filename"]; ok {
return "file"
}
if _, ok := sub.Config["filenames"]; ok {
return "file"
}
if _, ok := sub.Config["journalctl_filter"]; ok {
return "journalctl"
}
return ""
}
func LoadAcquisitionFromDSN(dsn string, label string) ([]DataSource, error) {
var sources []DataSource
frags := strings.Split(dsn, ":")
if len(frags) == 1 {
return nil, fmt.Errorf("%s isn't valid dsn (no protocol)", dsn)
}
dataSrc := GetDataSourceIface(frags[0])
if dataSrc == nil {
return nil, fmt.Errorf("no acquisition for protocol %s://", frags[0])
}
/* this logger will then be used by the datasource at runtime */
clog := log.New()
if err := types.ConfigureLogger(clog); err != nil {
return nil, errors.Wrap(err, "while configuring datasource logger")
}
subLogger := clog.WithFields(log.Fields{
"type": dsn,
})
err := dataSrc.ConfigureByDSN(dsn, label, subLogger)
if err != nil {
return nil, errors.Wrapf(err, "while configuration datasource for %s", dsn)
}
sources = append(sources, dataSrc)
return sources, nil
}
// LoadAcquisitionFromFile unmarshals the configuration item and checks its availability
func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, error) {
var sources []DataSource
var acquisSources = config.AcquisitionFiles
for _, acquisFile := range acquisSources {
for _, acquisFile := range config.AcquisitionFiles {
log.Infof("loading acquisition file : %s", acquisFile)
yamlFile, err := os.Open(acquisFile)
if err != nil {
@ -119,36 +161,82 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
dec := yaml.NewDecoder(yamlFile)
dec.SetStrict(true)
for {
sub := DataSourceCfg{}
var sub configuration.DataSourceCommonCfg
err = dec.Decode(&sub)
if err != nil {
if err == io.EOF {
log.Tracef("End of yaml file")
break
}
return nil, errors.Wrap(err, fmt.Sprintf("failed to yaml decode %s", acquisFile))
return nil, errors.Wrapf(err, "failed to yaml decode %s", acquisFile)
}
//for backward compat ('type' was not mandatory, detect it)
if guessType := detectBackwardCompatAcquis(sub); guessType != "" {
sub.Source = guessType
}
//it's an empty item, skip it
if len(sub.Labels) == 0 {
if sub.Source == "" {
log.Debugf("skipping empty item in %s", acquisFile)
continue
}
return nil, fmt.Errorf("missing labels in %s", acquisFile)
}
if sub.Source == "" {
return nil, fmt.Errorf("data source type is empty ('source') in %s", acquisFile)
}
if GetDataSourceIface(sub.Source) == nil {
return nil, fmt.Errorf("unknown data source %s in %s", sub.Source, acquisFile)
}
src, err := DataSourceConfigure(sub)
if err != nil {
log.Warningf("while configuring datasource : %s", err)
continue
return nil, errors.Wrapf(err, "while configuring datasource of type %s from %s", sub.Source, acquisFile)
}
sources = append(sources, src)
sources = append(sources, *src)
}
}
return sources, nil
}
func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error {
func GetMetrics(sources []DataSource, aggregated bool) error {
var metrics []prometheus.Collector
for i := 0; i < len(sources); i++ {
if aggregated {
metrics = sources[i].GetMetrics()
} else {
metrics = sources[i].GetAggregMetrics()
}
for _, metric := range metrics {
if err := prometheus.Register(metric); err != nil {
if _, ok := err.(prometheus.AlreadyRegisteredError); ok {
//ignore the error
} else {
return errors.Wrapf(err, "could not register metrics for datasource %s", sources[i].GetName())
}
}
}
}
return nil
}
func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error {
for i := 0; i < len(sources); i++ {
subsrc := sources[i] //ensure its a copy
log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc)
AcquisTomb.Go(func() error {
defer types.CatchPanic("crowdsec/acquis")
if err := subsrc.StartReading(output, AcquisTomb); err != nil {
return err
var err error
if subsrc.GetMode() == configuration.TAIL_MODE {
err = subsrc.StreamingAcquisition(output, AcquisTomb)
} else {
err = subsrc.OneShotAcquisition(output, AcquisTomb)
}
if err != nil {
//if one of the acqusition returns an error, we kill the others to properly shutdown
AcquisTomb.Kill(err)
}
return nil
})

620
pkg/acquisition/acquisition_test.go
View file

@ -2,137 +2,571 @@ package acquisition
import (
"fmt"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
tomb "gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"gotest.tools/v3/assert"
)
func TestConfigLoading(t *testing.T) {
//bad filename
cfg := csconfig.CrowdsecServiceCfg{
AcquisitionFiles: []string{"./tests/xxx.yaml"},
type MockSource struct {
configuration.DataSourceCommonCfg `yaml:",inline"`
Toto string `yaml:"toto"`
logger *log.Entry
}
func (f *MockSource) Configure(cfg []byte, logger *log.Entry) error {
f.logger = logger
if err := yaml.UnmarshalStrict(cfg, &f); err != nil {
return errors.Wrap(err, "while unmarshaling to reader specific config")
}
_, err := LoadAcquisitionFromFile(&cfg)
assert.Contains(t, fmt.Sprintf("%s", err), "can't open ./tests/xxx.yaml: open ./tests/xxx.yaml: no such file or directory")
//bad config file
cfg = csconfig.CrowdsecServiceCfg{
AcquisitionFiles: []string{"./tests/test.log"},
if f.Mode == "" {
f.Mode = configuration.CAT_MODE
}
_, err = LoadAcquisitionFromFile(&cfg)
assert.Contains(t, fmt.Sprintf("%s", err), "failed to yaml decode ./tests/test.log: yaml: unmarshal errors")
//correct config file
cfg = csconfig.CrowdsecServiceCfg{
AcquisitionFiles: []string{"./tests/acquis_test.yaml"},
if f.Mode != configuration.CAT_MODE && f.Mode != configuration.TAIL_MODE {
return fmt.Errorf("mode %s is not supported", f.Mode)
}
srcs, err := LoadAcquisitionFromFile(&cfg)
if err != nil {
t.Fatalf("unexpected error : %s", err)
if f.Toto == "" {
return fmt.Errorf("expect non-empty toto")
}
return nil
}
func (f *MockSource) GetMode() string { return f.Mode }
func (f *MockSource) OneShotAcquisition(chan types.Event, *tomb.Tomb) error { return nil }
func (f *MockSource) StreamingAcquisition(chan types.Event, *tomb.Tomb) error { return nil }
func (f *MockSource) CanRun() error { return nil }
func (f *MockSource) GetMetrics() []prometheus.Collector { return nil }
func (f *MockSource) GetAggregMetrics() []prometheus.Collector { return nil }
func (f *MockSource) Dump() interface{} { return f }
func (f *MockSource) GetName() string { return "mock" }
func (f *MockSource) ConfigureByDSN(string, string, *log.Entry) error {
return fmt.Errorf("not supported")
}
//copy the mocksource, but this one can't run
type MockSourceCantRun struct {
MockSource
}
func (f *MockSourceCantRun) CanRun() error { return fmt.Errorf("can't run bro") }
func (f *MockSourceCantRun) GetName() string { return "mock_cant_run" }
//appendMockSource is only used to add mock source for tests
func appendMockSource() {
if GetDataSourceIface("mock") == nil {
mock := struct {
name string
iface func() DataSource
}{
name: "mock",
iface: func() DataSource { return &MockSource{} },
}
AcquisitionSources = append(AcquisitionSources, mock)
}
if GetDataSourceIface("mock_cant_run") == nil {
mock := struct {
name string
iface func() DataSource
}{
name: "mock_cant_run",
iface: func() DataSource { return &MockSourceCantRun{} },
}
AcquisitionSources = append(AcquisitionSources, mock)
}
assert.Equal(t, len(srcs), 1)
}
func TestDataSourceConfigure(t *testing.T) {
appendMockSource()
tests := []struct {
cfg DataSourceCfg
//tombState
config_error string
read_error string
tomb_error string
lines int
TestName string
RawBytes []byte
ExpectedError string
}{
{ //missing filename(s)
cfg: DataSourceCfg{
Mode: CAT_MODE,
},
config_error: "empty filename(s) and journalctl filter, malformed datasource",
{
TestName: "basic_valid_config",
RawBytes: []byte(`
mode: cat
labels:
test: foobar
log_level: info
source: mock
toto: test_value1
`),
},
{ //missing filename(s)
cfg: DataSourceCfg{
Mode: TAIL_MODE,
},
config_error: "empty filename(s) and journalctl filter, malformed datasource",
{
TestName: "basic_debug_config",
RawBytes: []byte(`
mode: cat
labels:
test: foobar
log_level: debug
source: mock
toto: test_value1
`),
},
{ //bad mode(s)
cfg: DataSourceCfg{
Filename: "./tests/test.log",
Mode: "ratata",
},
config_error: "configuring file datasource: unknown mode ratata for file acquisition",
{
TestName: "basic_tailmode_config",
RawBytes: []byte(`
mode: tail
labels:
test: foobar
log_level: debug
source: mock
toto: test_value1
`),
},
{ //ok test
cfg: DataSourceCfg{
Mode: CAT_MODE,
Filename: "./tests/test.log",
},
{
TestName: "bad_mode_config",
RawBytes: []byte(`
mode: ratata
labels:
test: foobar
log_level: debug
source: mock
toto: test_value1
`),
ExpectedError: "failed to configure datasource mock: mode ratata is not supported",
},
{ //missing mode, default to CAT_MODE
cfg: DataSourceCfg{
Filename: "./tests/test.log",
},
{
TestName: "bad_type_config",
RawBytes: []byte(`
mode: cat
labels:
test: foobar
log_level: debug
source: tutu
`),
ExpectedError: "cannot find source tutu",
},
{ //ok test for journalctl
cfg: DataSourceCfg{
Mode: CAT_MODE,
JournalctlFilters: []string{"-test.run=TestSimJournalctlCatOneLine", "--"},
},
{
TestName: "mismatch_config",
RawBytes: []byte(`
mode: cat
labels:
test: foobar
log_level: debug
source: mock
wowo: ajsajasjas
`),
ExpectedError: "field wowo not found in type acquisition.MockSource",
},
{
TestName: "cant_run_error",
RawBytes: []byte(`
mode: cat
labels:
test: foobar
log_level: debug
source: mock_cant_run
wowo: ajsajasjas
`),
ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
},
}
for tidx, test := range tests {
srcs, err := DataSourceConfigure(test.cfg)
if test.config_error != "" {
assert.Contains(t, fmt.Sprintf("%s", err), test.config_error)
log.Infof("expected config error ok : %s", test.config_error)
continue
for _, test := range tests {
common := configuration.DataSourceCommonCfg{}
yaml.Unmarshal(test.RawBytes, &common)
ds, err := DataSourceConfigure(common)
if test.ExpectedError != "" {
if err == nil {
t.Fatalf("expected error %s, got none", test.ExpectedError)
}
if !strings.Contains(err.Error(), test.ExpectedError) {
t.Fatalf("%s : expected error '%s' in '%s'", test.TestName, test.ExpectedError, err.Error())
} else {
continue
}
} else {
if err != nil {
t.Fatalf("%d/%d unexpected config error %s", tidx, len(tests), err)
t.Fatalf("%s : unexpected error '%s'", test.TestName, err)
}
}
//check we got the expected mode
if tests[tidx].cfg.Mode == "" {
tests[tidx].cfg.Mode = TAIL_MODE
switch test.TestName {
case "basic_valid_config":
mock := (*ds).Dump().(*MockSource)
assert.Equal(t, mock.Toto, "test_value1")
assert.Equal(t, mock.Mode, "cat")
assert.Equal(t, mock.logger.Logger.Level, log.InfoLevel)
assert.DeepEqual(t, mock.Labels, map[string]string{"test": "foobar"})
case "basic_debug_config":
mock := (*ds).Dump().(*MockSource)
assert.Equal(t, mock.Toto, "test_value1")
assert.Equal(t, mock.Mode, "cat")
assert.Equal(t, mock.logger.Logger.Level, log.DebugLevel)
assert.DeepEqual(t, mock.Labels, map[string]string{"test": "foobar"})
case "basic_tailmode_config":
mock := (*ds).Dump().(*MockSource)
assert.Equal(t, mock.Toto, "test_value1")
assert.Equal(t, mock.Mode, "tail")
assert.Equal(t, mock.logger.Logger.Level, log.DebugLevel)
assert.DeepEqual(t, mock.Labels, map[string]string{"test": "foobar"})
}
assert.Equal(t, srcs.Mode(), tests[tidx].cfg.Mode)
}
}
out := make(chan types.Event)
tomb := tomb.Tomb{}
go func() {
err = StartAcquisition([]DataSource{srcs}, out, &tomb)
if test.read_error != "" {
assert.Contains(t, fmt.Sprintf("%s", err), test.read_error)
log.Infof("expected read error ok : %s", test.read_error)
func TestLoadAcquisitionFromFile(t *testing.T) {
appendMockSource()
tests := []struct {
TestName string
Config csconfig.CrowdsecServiceCfg
ExpectedError string
ExpectedLen int
}{
{
TestName: "non_existent_file",
Config: csconfig.CrowdsecServiceCfg{
AcquisitionFiles: []string{"does_not_exist"},
},
ExpectedError: "can't open does_not_exist",
ExpectedLen: 0,
},
{
TestName: "invalid_yaml_file",
Config: csconfig.CrowdsecServiceCfg{
AcquisitionFiles: []string{"test_files/badyaml.yaml"},
},
ExpectedError: "failed to yaml decode test_files/badyaml.yaml: yaml: unmarshal errors",
ExpectedLen: 0,
},
{
TestName: "invalid_empty_yaml",
Config: csconfig.CrowdsecServiceCfg{
AcquisitionFiles: []string{"test_files/emptyitem.yaml"},
},
ExpectedLen: 0,
},
{
TestName: "basic_valid",
Config: csconfig.CrowdsecServiceCfg{
AcquisitionFiles: []string{"test_files/basic_filemode.yaml"},
},
ExpectedLen: 2,
},
{
TestName: "missing_labels",
Config: csconfig.CrowdsecServiceCfg{
AcquisitionFiles: []string{"test_files/missing_labels.yaml"},
},
ExpectedError: "missing labels in test_files/missing_labels.yaml",
},
{
TestName: "backward_compat",
Config: csconfig.CrowdsecServiceCfg{
AcquisitionFiles: []string{"test_files/backward_compat.yaml"},
},
ExpectedLen: 2,
},
{
TestName: "bad_type",
Config: csconfig.CrowdsecServiceCfg{
AcquisitionFiles: []string{"test_files/bad_source.yaml"},
},
ExpectedError: "unknown data source does_not_exist in test_files/bad_source.yaml",
},
{
TestName: "invalid_filetype_config",
Config: csconfig.CrowdsecServiceCfg{
AcquisitionFiles: []string{"test_files/bad_filetype.yaml"},
},
ExpectedError: "while configuring datasource of type file from test_files/bad_filetype.yaml",
},
}
for _, test := range tests {
dss, err := LoadAcquisitionFromFile(&test.Config)
if test.ExpectedError != "" {
if err == nil {
t.Fatalf("expected error %s, got none", test.ExpectedError)
}
if !strings.Contains(err.Error(), test.ExpectedError) {
t.Fatalf("%s : expected error '%s' in '%s'", test.TestName, test.ExpectedError, err.Error())
} else {
if err != nil {
log.Fatalf("%d/%d unexpected read error %s", tidx, len(tests), err)
}
continue
}
}()
log.Printf("kill iiittt")
//we're actually not interested in the result :)
tomb.Kill(nil)
time.Sleep(1 * time.Second)
if test.tomb_error != "" {
assert.Contains(t, fmt.Sprintf("%s", tomb.Err()), test.tomb_error)
log.Infof("expected tomb error ok : %s", test.read_error)
continue
} else {
if tomb.Err() != nil {
t.Fatalf("%d/%d unexpected tomb error %s", tidx, len(tests), tomb.Err())
if err != nil {
t.Fatalf("%s : unexpected error '%s'", test.TestName, err)
}
}
if len(dss) != test.ExpectedLen {
t.Fatalf("%s : expected %d datasources got %d", test.TestName, test.ExpectedLen, len(dss))
}
}
}
/*
test start acquisition :
- create mock parser in cat mode : start acquisition, check it returns, count items in chan
- create mock parser in tail mode : start acquisition, sleep, check item count, tomb kill it, wait for it to return
*/
type MockCat struct {
configuration.DataSourceCommonCfg `yaml:",inline"`
logger *log.Entry
}
func (f *MockCat) Configure(cfg []byte, logger *log.Entry) error {
f.logger = logger
if f.Mode == "" {
f.Mode = configuration.CAT_MODE
}
if f.Mode != configuration.CAT_MODE {
return fmt.Errorf("mode %s is not supported", f.Mode)
}
return nil
}
func (f *MockCat) GetName() string { return "mock_cat" }
func (f *MockCat) GetMode() string { return "cat" }
func (f *MockCat) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error {
for i := 0; i < 10; i++ {
evt := types.Event{}
evt.Line.Src = "test"
out <- evt
}
return nil
}
func (f *MockCat) StreamingAcquisition(chan types.Event, *tomb.Tomb) error {
return fmt.Errorf("can't run in tail")
}
func (f *MockCat) CanRun() error { return nil }
func (f *MockCat) GetMetrics() []prometheus.Collector { return nil }
func (f *MockCat) GetAggregMetrics() []prometheus.Collector { return nil }
func (f *MockCat) Dump() interface{} { return f }
func (f *MockCat) ConfigureByDSN(string, string, *log.Entry) error { return fmt.Errorf("not supported") }
//----
type MockTail struct {
configuration.DataSourceCommonCfg `yaml:",inline"`
logger *log.Entry
}
func (f *MockTail) Configure(cfg []byte, logger *log.Entry) error {
f.logger = logger
if f.Mode == "" {
f.Mode = configuration.TAIL_MODE
}
if f.Mode != configuration.TAIL_MODE {
return fmt.Errorf("mode %s is not supported", f.Mode)
}
return nil
}
func (f *MockTail) GetName() string { return "mock_tail" }
func (f *MockTail) GetMode() string { return "tail" }
func (f *MockTail) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error {
return fmt.Errorf("can't run in cat mode")
}
func (f *MockTail) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
for i := 0; i < 10; i++ {
evt := types.Event{}
evt.Line.Src = "test"
out <- evt
}
select {
case <-t.Dying():
return nil
}
}
func (f *MockTail) CanRun() error { return nil }
func (f *MockTail) GetMetrics() []prometheus.Collector { return nil }
func (f *MockTail) GetAggregMetrics() []prometheus.Collector { return nil }
func (f *MockTail) Dump() interface{} { return f }
func (f *MockTail) ConfigureByDSN(string, string, *log.Entry) error {
return fmt.Errorf("not supported")
}
//func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error {
func TestStartAcquisitionCat(t *testing.T) {
sources := []DataSource{
&MockCat{},
}
out := make(chan types.Event)
acquisTomb := tomb.Tomb{}
go func() {
if err := StartAcquisition(sources, out, &acquisTomb); err != nil {
t.Fatalf("unexpected error")
}
}()
count := 0
READLOOP:
for {
select {
case <-out:
count++
case <-time.After(1 * time.Second):
break READLOOP
}
}
if count != 10 {
t.Fatalf("expected 10 results, got %d", count)
}
}
func TestStartAcquisitionTail(t *testing.T) {
sources := []DataSource{
&MockTail{},
}
out := make(chan types.Event)
acquisTomb := tomb.Tomb{}
go func() {
if err := StartAcquisition(sources, out, &acquisTomb); err != nil {
t.Fatalf("unexpected error")
}
}()
count := 0
READLOOP:
for {
select {
case <-out:
count++
case <-time.After(1 * time.Second):
break READLOOP
}
}
if count != 10 {
t.Fatalf("expected 10 results, got %d", count)
}
acquisTomb.Kill(nil)
time.Sleep(1 * time.Second)
if acquisTomb.Err() != nil {
t.Fatalf("unexpected tomb error %s (should be dead)", acquisTomb.Err())
}
}
//
type MockTailError struct {
MockTail
}
func (f *MockTailError) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
for i := 0; i < 10; i++ {
evt := types.Event{}
evt.Line.Src = "test"
out <- evt
}
t.Kill(fmt.Errorf("got error (tomb)"))
return fmt.Errorf("got error")
}
func TestStartAcquisitionTailError(t *testing.T) {
sources := []DataSource{
&MockTailError{},
}
out := make(chan types.Event)
acquisTomb := tomb.Tomb{}
go func() {
if err := StartAcquisition(sources, out, &acquisTomb); err != nil && err.Error() != "got error (tomb)" {
t.Fatalf("expected error, got '%s'", err.Error())
}
}()
count := 0
READLOOP:
for {
select {
case <-out:
count++
case <-time.After(1 * time.Second):
break READLOOP
}
}
if count != 10 {
t.Fatalf("expected 10 results, got %d", count)
}
//acquisTomb.Kill(nil)
time.Sleep(1 * time.Second)
if acquisTomb.Err().Error() != "got error (tomb)" {
t.Fatalf("didn't got expected error, got '%s'", acquisTomb.Err().Error())
}
}
type MockSourceByDSN struct {
configuration.DataSourceCommonCfg `yaml:",inline"`
Toto string `yaml:"toto"`
logger *log.Entry
}
func (f *MockSourceByDSN) Configure(cfg []byte, logger *log.Entry) error { return nil }
func (f *MockSourceByDSN) GetMode() string { return f.Mode }
func (f *MockSourceByDSN) OneShotAcquisition(chan types.Event, *tomb.Tomb) error { return nil }
func (f *MockSourceByDSN) StreamingAcquisition(chan types.Event, *tomb.Tomb) error { return nil }
func (f *MockSourceByDSN) CanRun() error { return nil }
func (f *MockSourceByDSN) GetMetrics() []prometheus.Collector { return nil }
func (f *MockSourceByDSN) GetAggregMetrics() []prometheus.Collector { return nil }
func (f *MockSourceByDSN) Dump() interface{} { return f }
func (f *MockSourceByDSN) GetName() string { return "mockdsn" }
func (f *MockSourceByDSN) ConfigureByDSN(dsn string, logType string, logger *log.Entry) error {
dsn = strings.TrimPrefix(dsn, "mockdsn://")
if dsn != "test_expect" {
return fmt.Errorf("unexpected value")
}
return nil
}
func TestConfigureByDSN(t *testing.T) {
tests := []struct {
dsn string
ExpectedError string
ExpectedResLen int
}{
{
dsn: "baddsn",
ExpectedError: "baddsn isn't valid dsn (no protocol)",
},
{
dsn: "foobar://toto",
ExpectedError: "no acquisition for protocol foobar://",
},
{
dsn: "mockdsn://test_expect",
ExpectedResLen: 1,
},
{
dsn: "mockdsn://bad",
ExpectedError: "unexpected value",
},
}
if GetDataSourceIface("mockdsn") == nil {
mock := struct {
name string
iface func() DataSource
}{
name: "mockdsn",
iface: func() DataSource { return &MockSourceByDSN{} },
}
AcquisitionSources = append(AcquisitionSources, mock)
}
for _, test := range tests {
srcs, err := LoadAcquisitionFromDSN(test.dsn, "test_label")
if err != nil && test.ExpectedError != "" {
if !strings.Contains(err.Error(), test.ExpectedError) {
t.Fatalf("expected '%s', got '%s'", test.ExpectedError, err.Error())
}
} else if err != nil && test.ExpectedError == "" {
t.Fatalf("got unexpected error '%s'", err.Error())
} else if err == nil && test.ExpectedError != "" {
t.Fatalf("expected error '%s' got none", test.ExpectedError)
}
if len(srcs) != test.ExpectedResLen {
t.Fatalf("expected %d results, got %d", test.ExpectedResLen, len(srcs))
}
}
}

18
pkg/acquisition/configuration/configuration.go Normal file
View file

@ -0,0 +1,18 @@
package configuration
import (
log "github.com/sirupsen/logrus"
)
type DataSourceCommonCfg struct {
Mode string `yaml:"mode,omitempty"`
Labels map[string]string `yaml:"labels,omitempty"`
LogLevel *log.Level `yaml:"log_level,omitempty"`
Source string `yaml:"source,omitempty"`
Name string `yaml:"name,omitempty"`
Config map[string]interface{} `yaml:",inline"` //to keep the datasource-specific configuration directives
}
var TAIL_MODE = "tail"
var CAT_MODE = "cat"
var SERVER_MODE = "server" // No difference with tail, just a bit more verbose

227
pkg/acquisition/file_reader.go
View file

@ -1,227 +0,0 @@
package acquisition
import (
"bufio"
"compress/gzip"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/nxadm/tail"
log "github.com/sirupsen/logrus"
"github.com/prometheus/client_golang/prometheus"
tomb "gopkg.in/tomb.v2"
)
type FileSource struct {
Config DataSourceCfg
tails []*tail.Tail
Files []string
}
func (f *FileSource) Configure(Config DataSourceCfg) error {
f.Config = Config
if len(Config.Filename) == 0 && len(Config.Filenames) == 0 {
return fmt.Errorf("no filename or filenames")
}
//let's deal with the array no matter what
if len(Config.Filename) != 0 {
Config.Filenames = append(Config.Filenames, Config.Filename)
}
for _, fexpr := range Config.Filenames {
files, err := filepath.Glob(fexpr)
if err != nil {
return errors.Wrapf(err, "while globbing %s", fexpr)
}
if len(files) == 0 {
log.Warningf("[file datasource] no results for %s", fexpr)
continue
}
for _, file := range files {
/*check that we can read said file*/
if err := unix.Access(file, unix.R_OK); err != nil {
return fmt.Errorf("unable to open %s : %s", file, err)
}
log.Infof("[file datasource] opening file '%s'", file)
if f.Config.Mode == TAIL_MODE {
tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: true, Location: &tail.SeekInfo{Offset: 0, Whence: 2}})
if err != nil {
log.Errorf("[file datasource] skipping %s : %v", file, err)
continue
}
f.Files = append(f.Files, file)
f.tails = append(f.tails, tail)
} else if f.Config.Mode == CAT_MODE {
//simply check that the file exists, it will be read differently
if _, err := os.Stat(file); err != nil {
return fmt.Errorf("can't open file %s : %s", file, err)
}
f.Files = append(f.Files, file)
} else {
return fmt.Errorf("unknown mode %s for file acquisition", f.Config.Mode)
}
}
}
if len(f.Files) == 0 {
return fmt.Errorf("no files to read for %+v", Config.Filenames)
}
return nil
}
func (f *FileSource) Mode() string {
return f.Config.Mode
}
func (f *FileSource) StartReading(out chan types.Event, t *tomb.Tomb) error {
if f.Config.Mode == CAT_MODE {
return f.StartCat(out, t)
} else if f.Config.Mode == TAIL_MODE {
return f.StartTail(out, t)
} else {
return fmt.Errorf("unknown mode '%s' for file acquisition", f.Config.Mode)
}
}
/*A tail-mode file reader (tail) */
func (f *FileSource) StartTail(output chan types.Event, AcquisTomb *tomb.Tomb) error {
log.Debugf("starting file tail with %d items", len(f.tails))
for i := 0; i < len(f.tails); i++ {
idx := i
log.Debugf("starting %d", idx)
AcquisTomb.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/tailfile")
return f.TailOneFile(output, AcquisTomb, idx)
})
}
return nil
}
/*A one shot file reader (cat) */
func (f *FileSource) StartCat(output chan types.Event, AcquisTomb *tomb.Tomb) error {
for i := 0; i < len(f.Files); i++ {
idx := i
log.Debugf("starting %d", idx)
AcquisTomb.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/catfile")
return f.CatOneFile(output, AcquisTomb, idx)
})
}
return nil
}
/*A tail-mode file reader (tail) */
func (f *FileSource) TailOneFile(output chan types.Event, AcquisTomb *tomb.Tomb, idx int) error {
file := f.Files[idx]
tail := f.tails[idx]
clog := log.WithFields(log.Fields{
"acquisition file": f.Files[idx],
})
clog.Debugf("starting")
timeout := time.Tick(1 * time.Second)
for {
l := types.Line{}
select {
case <-AcquisTomb.Dying(): //we are being killed by main
clog.Infof("file datasource %s stopping", file)
if err := tail.Stop(); err != nil {
clog.Errorf("error in stop : %s", err)
}
return nil
case <-tail.Tomb.Dying(): //our tailer is dying
clog.Warningf("File reader of %s died", file)
AcquisTomb.Kill(fmt.Errorf("dead reader for %s", file))
return fmt.Errorf("reader for %s is dead", file)
case line := <-tail.Lines:
if line == nil {
clog.Debugf("Nil line")
return fmt.Errorf("tail for %s is empty", file)
}
if line.Err != nil {
log.Warningf("fetch error : %v", line.Err)
return line.Err
}
if line.Text == "" { //skip empty lines
continue
}
ReaderHits.With(prometheus.Labels{"source": file}).Inc()
l.Raw = line.Text
l.Labels = f.Config.Labels
l.Time = line.Time
l.Src = file
l.Process = true
//we're tailing, it must be real time logs
log.Debugf("pushing %+v", l)
output <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.LIVE}
case <-timeout:
//time out, shall we do stuff ?
clog.Debugf("timeout")
}
}
}
/*A one shot file reader (cat) */
func (f *FileSource) CatOneFile(output chan types.Event, AcquisTomb *tomb.Tomb, idx int) error {
var scanner *bufio.Scanner
log.Infof("reading %s at once", f.Files[idx])
file := f.Files[idx]
clog := log.WithFields(log.Fields{
"file": file,
})
fd, err := os.Open(file)
defer fd.Close()
if err != nil {
clog.Errorf("Failed opening file: %s", err)
return errors.Wrapf(err, "failed opening %s", f.Files[idx])
}
if strings.HasSuffix(file, ".gz") {
gz, err := gzip.NewReader(fd)
if err != nil {
clog.Errorf("Failed to read gz file: %s", err)
return errors.Wrapf(err, "failed to read gz %s", f.Files[idx])
}
defer gz.Close()
scanner = bufio.NewScanner(gz)
} else {
scanner = bufio.NewScanner(fd)
}
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
log.Tracef("line %s", scanner.Text())
l := types.Line{}
l.Raw = scanner.Text()
l.Time = time.Now()
l.Src = file
l.Labels = f.Config.Labels
l.Process = true
ReaderHits.With(prometheus.Labels{"source": file}).Inc()
//we're reading logs at once, it must be time-machine buckets
output <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.TIMEMACHINE}
}
AcquisTomb.Kill(nil)
return nil
}

383
pkg/acquisition/file_reader_test.go
View file

@ -1,383 +0,0 @@
package acquisition
import (
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
tomb "gopkg.in/tomb.v2"
)
func TestAcquisCat(t *testing.T) {
tests := []struct {
cfg DataSourceCfg
//tombState
config_error string
read_error string
tomb_error string
lines int
}{
{ //missing filename(s)
cfg: DataSourceCfg{
Mode: CAT_MODE,
},
config_error: "no filename or filenames",
},
{ //forbiden file
cfg: DataSourceCfg{
Mode: CAT_MODE,
Filename: "/etc/shadow",
},
config_error: "unable to open /etc/shadow : permission denied",
},
{ //bad regexp
cfg: DataSourceCfg{
Filename: "[a-",
Mode: CAT_MODE,
},
config_error: "while globbing [a-: syntax error in pattern",
},
{ //inexisting file
cfg: DataSourceCfg{
Filename: "/does/not/exists",
Mode: CAT_MODE,
},
config_error: "no files to read for [/does/not/exists]",
},
{ //ok file
cfg: DataSourceCfg{
Filename: "./tests/test.log",
Mode: CAT_MODE,
},
lines: 1,
},
{ //invalid gz
cfg: DataSourceCfg{
Filename: "./tests/badlog.gz",
Mode: CAT_MODE,
},
lines: 0,
tomb_error: "failed to read gz ./tests/badlog.gz: EOF",
},
{ //good gz
cfg: DataSourceCfg{
Filename: "./tests/test.log.gz",
Mode: CAT_MODE,
},
lines: 1,
},
}
for tidx, test := range tests {
fileSrc := new(FileSource)
err := fileSrc.Configure(test.cfg)
if test.config_error != "" {
assert.Contains(t, fmt.Sprintf("%s", err), test.config_error)
log.Infof("expected config error ok : %s", test.config_error)
continue
} else {
if err != nil {
t.Fatalf("%d/%d unexpected config error %s", tidx, len(tests), err)
}
}
out := make(chan types.Event)
tomb := tomb.Tomb{}
count := 0
err = fileSrc.StartReading(out, &tomb)
if test.read_error != "" {
assert.Contains(t, fmt.Sprintf("%s", err), test.read_error)
log.Infof("expected read error ok : %s", test.read_error)
continue
} else {
if err != nil {
t.Fatalf("%d/%d unexpected read error %s", tidx, len(tests), err)
}
}
READLOOP:
for {
select {
case <-out:
count++
case <-time.After(1 * time.Second):
break READLOOP
}
}
if count != test.lines {
t.Fatalf("%d/%d expected %d line read, got %d", tidx, len(tests), test.lines, count)
}
if test.tomb_error != "" {
assert.Contains(t, fmt.Sprintf("%s", tomb.Err()), test.tomb_error)
log.Infof("expected tomb error ok : %s", test.read_error)
continue
} else {
if tomb.Err() != nil {
t.Fatalf("%d/%d unexpected tomb error %s", tidx, len(tests), tomb.Err())
}
}
}
}
func TestTailKill(t *testing.T) {
cfg := DataSourceCfg{
Filename: "./tests/test.log",
Mode: TAIL_MODE,
}
fileSrc := new(FileSource)
err := fileSrc.Configure(cfg)
if err != nil {
t.Fatalf("unexpected config error %s", err)
}
out := make(chan types.Event)
tb := tomb.Tomb{}
err = fileSrc.StartReading(out, &tb)
if err != nil {
t.Fatalf("unexpected read error %s", err)
}
time.Sleep(1 * time.Second)
if tb.Err() != tomb.ErrStillAlive {
t.Fatalf("unexpected tomb error %s (should be alive)", tb.Err())
}
//kill it :>
tb.Kill(nil)
time.Sleep(1 * time.Second)
if tb.Err() != nil {
t.Fatalf("unexpected tomb error %s (should be dead)", tb.Err())
}
}
func TestTailKillBis(t *testing.T) {
cfg := DataSourceCfg{
Filename: "./tests/test.log",
Mode: TAIL_MODE,
}
fileSrc := new(FileSource)
err := fileSrc.Configure(cfg)
if err != nil {
t.Fatalf("unexpected config error %s", err)
}
out := make(chan types.Event)
tb := tomb.Tomb{}
err = fileSrc.StartReading(out, &tb)
if err != nil {
t.Fatalf("unexpected read error %s", err)
}
time.Sleep(1 * time.Second)
if tb.Err() != tomb.ErrStillAlive {
t.Fatalf("unexpected tomb error %s (should be alive)", tb.Err())
}
//kill the underlying tomb of tailer
fileSrc.tails[0].Kill(fmt.Errorf("ratata"))
time.Sleep(1 * time.Second)
//it can be two errors :
if !strings.Contains(fmt.Sprintf("%s", tb.Err()), "dead reader for ./tests/test.log") &&
!strings.Contains(fmt.Sprintf("%s", tb.Err()), "tail for ./tests/test.log is empty") {
t.Fatalf("unexpected error : %s", tb.Err())
}
}
func TestTailRuntime(t *testing.T) {
//log.SetLevel(log.TraceLevel)
cfg := DataSourceCfg{
Filename: "./tests/test.log",
Mode: TAIL_MODE,
}
fileSrc := new(FileSource)
err := fileSrc.Configure(cfg)
if err != nil {
t.Fatalf("unexpected config error %s", err)
}
out := make(chan types.Event)
tb := tomb.Tomb{}
count := 0
err = fileSrc.StartReading(out, &tb)
if err != nil {
t.Fatalf("unexpected read error %s", err)
}
time.Sleep(1 * time.Second)
//write data
f, err := os.OpenFile(cfg.Filename, os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 5; i++ {
_, err := f.WriteString(fmt.Sprintf("ratata%d\n", i))
if err != nil {
t.Fatal(err)
}
}
f.Close()
READLOOP:
for {
select {
case <-out:
count++
case <-time.After(1 * time.Second):
break READLOOP
}
}
if count != 5 {
t.Fatalf("expected %d line read, got %d", 5, count)
}
if tb.Err() != tomb.ErrStillAlive {
t.Fatalf("unexpected tomb error %s", tb.Err())
}
/*reset the file*/
f, err = os.OpenFile(cfg.Filename, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
t.Fatal(err)
}
_, err = f.WriteString("one log line\n")
if err != nil {
t.Fatal(err)
}
f.Close()
}
func TestAcquisTail(t *testing.T) {
tests := []struct {
cfg DataSourceCfg
//tombState
config_error string
read_error string
tomb_error string
lines int
}{
{ //missing filename(s)
cfg: DataSourceCfg{
Mode: TAIL_MODE,
},
config_error: "no filename or filenames",
},
{ //forbiden file
cfg: DataSourceCfg{
Mode: TAIL_MODE,
Filename: "/etc/shadow",
},
config_error: "unable to open /etc/shadow : permission denied",
},
{ //bad regexp
cfg: DataSourceCfg{
Filename: "[a-",
Mode: TAIL_MODE,
},
config_error: "while globbing [a-: syntax error in pattern",
},
{ //inexisting file
cfg: DataSourceCfg{
Filename: "/does/not/exists",
Mode: TAIL_MODE,
},
config_error: "no files to read for [/does/not/exists]",
},
{ //ok file
cfg: DataSourceCfg{
Filename: "./tests/test.log",
Mode: TAIL_MODE,
},
lines: 0,
tomb_error: "still alive",
},
{ //invalid gz
cfg: DataSourceCfg{
Filename: "./tests/badlog.gz",
Mode: TAIL_MODE,
},
lines: 0,
tomb_error: "still alive",
},
{ //good gz
cfg: DataSourceCfg{
Filename: "./tests/test.log.gz",
Mode: TAIL_MODE,
},
lines: 0,
tomb_error: "still alive",
},
}
for tidx, test := range tests {
fileSrc := new(FileSource)
err := fileSrc.Configure(test.cfg)
if test.config_error != "" {
assert.Contains(t, fmt.Sprintf("%s", err), test.config_error)
log.Infof("expected config error ok : %s", test.config_error)
continue
} else {
if err != nil {
t.Fatalf("%d/%d unexpected config error %s", tidx, len(tests), err)
}
}
out := make(chan types.Event)
tomb := tomb.Tomb{}
count := 0
err = fileSrc.StartReading(out, &tomb)
if test.read_error != "" {
assert.Contains(t, fmt.Sprintf("%s", err), test.read_error)
log.Infof("expected read error ok : %s", test.read_error)
continue
} else {
if err != nil {
t.Fatalf("%d/%d unexpected read error %s", tidx, len(tests), err)
}
}
READLOOP:
for {
select {
case <-out:
count++
case <-time.After(1 * time.Second):
break READLOOP
}
}
if count != test.lines {
t.Fatalf("%d/%d expected %d line read, got %d", tidx, len(tests), test.lines, count)
}
if test.tomb_error != "" {
assert.Contains(t, fmt.Sprintf("%s", tomb.Err()), test.tomb_error)
log.Infof("expected tomb error ok : %s", test.read_error)
continue
} else {
if tomb.Err() != nil {
t.Fatalf("%d/%d unexpected tomb error %s", tidx, len(tests), tomb.Err())
}
}
}
}

174
pkg/acquisition/journalctl_reader.go
View file

@ -1,174 +0,0 @@
package acquisition
import (
"bufio"
"encoding/json"
"fmt"
"io"
"os/exec"
"strings"
"time"
log "github.com/sirupsen/logrus"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
tomb "gopkg.in/tomb.v2"
)
/*
journald/systemd support :
systemd has its own logging system, which stores files in non-text mode.
To be able to read those, we're going to read the output of journalctl, see https://github.com/crowdsecurity/crowdsec/issues/423
TBD :
- handle journalctl errors
*/
type JournaldSource struct {
Config DataSourceCfg
Cmd *exec.Cmd
Stdout io.ReadCloser
Stderr io.ReadCloser
Decoder *json.Decoder
SrcName string
}
var JOURNALD_CMD = "journalctl"
var JOURNALD_DEFAULT_TAIL_ARGS = []string{"--follow"}
var JOURNALD_DEFAULT_CAT_ARGS = []string{}
func (j *JournaldSource) Configure(config DataSourceCfg) error {
var journalArgs []string
j.Config = config
if config.JournalctlFilters == nil {
return fmt.Errorf("journalctl_filter shouldn't be empty")
}
if j.Config.Mode == TAIL_MODE {
journalArgs = JOURNALD_DEFAULT_TAIL_ARGS
} else if j.Config.Mode == CAT_MODE {
journalArgs = JOURNALD_DEFAULT_CAT_ARGS
} else {
return fmt.Errorf("unknown mode '%s' for journald source", j.Config.Mode)
}
journalArgs = append(journalArgs, config.JournalctlFilters...)
j.Cmd = exec.Command(JOURNALD_CMD, journalArgs...)
j.Stderr, _ = j.Cmd.StderrPipe()
j.Stdout, _ = j.Cmd.StdoutPipe()
j.SrcName = fmt.Sprintf("journalctl-%s", strings.Join(config.JournalctlFilters, "."))
log.Infof("[journald datasource] Configured with filters : %+v", journalArgs)
log.Debugf("cmd path : %s", j.Cmd.Path)
log.Debugf("cmd args : %+v", j.Cmd.Args)
return nil
}
func (j *JournaldSource) Mode() string {
return j.Config.Mode
}
func (j *JournaldSource) readOutput(out chan types.Event, t *tomb.Tomb) error {
/*
todo : handle the channel
*/
clog := log.WithFields(log.Fields{
"acquisition file": j.SrcName,
})
if err := j.Cmd.Start(); err != nil {
clog.Errorf("failed to start journalctl: %s", err)
return errors.Wrapf(err, "starting journalctl (%s)", j.SrcName)
}
readErr := make(chan error)
/*read stderr*/
go func() {
scanner := bufio.NewScanner(j.Stderr)
if scanner == nil {
readErr <- fmt.Errorf("failed to create stderr scanner")
return
}
for scanner.Scan() {
txt := scanner.Text()
clog.Warningf("got stderr message : %s", txt)
readErr <- fmt.Errorf(txt)
}
}()
/*read stdout*/
go func() {
scanner := bufio.NewScanner(j.Stdout)
if scanner == nil {
readErr <- fmt.Errorf("failed to create stdout scanner")
return
}
for scanner.Scan() {
l := types.Line{}
ReaderHits.With(prometheus.Labels{"source": j.SrcName}).Inc()
l.Raw = scanner.Text()
clog.Debugf("getting one line : %s", l.Raw)
l.Labels = j.Config.Labels
l.Time = time.Now()
l.Src = j.SrcName
l.Process = true
evt := types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.LIVE}
out <- evt
}
clog.Debugf("finished reading from journalctl")
if err := scanner.Err(); err != nil {
clog.Debugf("got an error while reading %s : %s", j.SrcName, err)
readErr <- err
return
}
readErr <- nil
}()
for {
select {
case <-t.Dying():
clog.Debugf("journalctl datasource %s stopping", j.SrcName)
return nil
case err := <-readErr:
clog.Debugf("the subroutine returned, leave as well")
if err != nil {
clog.Warningf("journalctl reader error : %s", err)
t.Kill(err)
}
return err
}
}
}
func (j *JournaldSource) StartReading(out chan types.Event, t *tomb.Tomb) error {
if j.Config.Mode == CAT_MODE {
return j.StartCat(out, t)
} else if j.Config.Mode == TAIL_MODE {
return j.StartTail(out, t)
} else {
return fmt.Errorf("unknown mode '%s' for file acquisition", j.Config.Mode)
}
}
func (j *JournaldSource) StartCat(out chan types.Event, t *tomb.Tomb) error {
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/tailjournalctl")
return j.readOutput(out, t)
})
return nil
}
func (j *JournaldSource) StartTail(out chan types.Event, t *tomb.Tomb) error {
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/catjournalctl")
return j.readOutput(out, t)
})
return nil
}

238
pkg/acquisition/journalctl_reader_test.go
View file

@ -1,238 +0,0 @@
package acquisition
import (
"fmt"
"os"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
tomb "gopkg.in/tomb.v2"
)
/*
As we can't decently run journalctl in the CI but we still need to test the command execution aspect :
- we create tests 'output only' (cf. TestSimJournalctlCat) that just produce outputs
- we run ourselves (os.Args[0]) with specific args to call specific 'output only' tests
- and this is how we test the behavior
*/
//14 lines of sshd logs
var testjournalctl_output_1 string = `-- Logs begin at Fri 2019-07-26 17:13:13 CEST, end at Mon 2020-11-23 09:17:34 CET. --
Nov 22 11:22:19 zeroed sshd[1480]: Invalid user wqeqwe from 127.0.0.1 port 55818
Nov 22 11:22:23 zeroed sshd[1480]: Failed password for invalid user wqeqwe from 127.0.0.1 port 55818 ssh2
Nov 22 11:23:22 zeroed sshd[1769]: Invalid user wqeqwe1 from 127.0.0.1 port 55824
Nov 22 11:23:24 zeroed sshd[1769]: Disconnecting invalid user wqeqwe1 127.0.0.1 port 55824: Too many authentication failures [preauth]
Nov 22 11:23:24 zeroed sshd[1777]: Invalid user wqeqwe2 from 127.0.0.1 port 55826
Nov 22 11:23:25 zeroed sshd[1777]: Disconnecting invalid user wqeqwe2 127.0.0.1 port 55826: Too many authentication failures [preauth]
Nov 22 11:23:25 zeroed sshd[1780]: Invalid user wqeqwe3 from 127.0.0.1 port 55828
Nov 22 11:23:26 zeroed sshd[1780]: Disconnecting invalid user wqeqwe3 127.0.0.1 port 55828: Too many authentication failures [preauth]
Nov 22 11:23:26 zeroed sshd[1786]: Invalid user wqeqwe4 from 127.0.0.1 port 55830
Nov 22 11:23:27 zeroed sshd[1786]: Failed password for invalid user wqeqwe4 from 127.0.0.1 port 55830 ssh2
Nov 22 11:23:27 zeroed sshd[1786]: Disconnecting invalid user wqeqwe4 127.0.0.1 port 55830: Too many authentication failures [preauth]
Nov 22 11:23:27 zeroed sshd[1791]: Invalid user wqeqwe5 from 127.0.0.1 port 55834
Nov 22 11:23:27 zeroed sshd[1791]: Failed password for invalid user wqeqwe5 from 127.0.0.1 port 55834 ssh2
`
func TestSimJournalctlCat(t *testing.T) {
if os.Getenv("GO_WANT_TEST_OUTPUT") != "1" {
return
}
defer os.Exit(0)
fmt.Print(testjournalctl_output_1)
}
func TestSimJournalctlCatError(t *testing.T) {
if os.Getenv("GO_WANT_TEST_OUTPUT") != "1" {
return
}
defer os.Exit(0)
fmt.Print("this is a single line being produced")
log.Warningf("this is an error message")
}
func TestSimJournalctlCatOneLine(t *testing.T) {
if os.Getenv("GO_WANT_TEST_OUTPUT") != "1" {
return
}
defer os.Exit(0)
fmt.Print("this is a single line being produced")
}
func TestJournaldTail(t *testing.T) {
tests := []struct {
cfg DataSourceCfg
config_error string
read_error string
tomb_error string
lines int
}{
{ //missing filename(s)
cfg: DataSourceCfg{
Mode: TAIL_MODE,
},
config_error: "journalctl_filter shouldn't be empty",
},
{ //bad mode
cfg: DataSourceCfg{
Mode: "ratatata",
JournalctlFilters: []string{"-test.run=DoesNotExist", "--"},
},
/*here would actually be the journalctl error message on bad args, but you get the point*/
config_error: "unknown mode 'ratatata' for journald source",
},
{ //wrong arguments
cfg: DataSourceCfg{
Mode: TAIL_MODE,
JournalctlFilters: []string{"--this-is-bad-option", "--"},
},
/*here would actually be the journalctl error message on bad args, but you get the point*/
tomb_error: "flag provided but not defined: -this-is-bad-option",
},
}
//we're actually using tests to do this, hold my beer and watch this
JOURNALD_CMD = os.Args[0]
JOURNALD_DEFAULT_TAIL_ARGS = []string{}
for tidx, test := range tests {
journalSrc := new(JournaldSource)
err := journalSrc.Configure(test.cfg)
if test.config_error != "" {
assert.Contains(t, fmt.Sprintf("%s", err), test.config_error)
log.Infof("expected config error ok : %s", test.config_error)
continue
} else {
if err != nil {
t.Fatalf("%d/%d unexpected config error %s", tidx, len(tests), err)
}
}
assert.Equal(t, journalSrc.Mode(), test.cfg.Mode)
//this tells our fake tests to produce data
journalSrc.Cmd.Env = []string{"GO_WANT_TEST_OUTPUT=1"}
out := make(chan types.Event)
tomb := tomb.Tomb{}
count := 0
//start consuming the data before we start the prog, so that chan isn't full
go func() {
for {
select {
case <-out:
count++
case <-time.After(1 * time.Second):
return
}
}
}()
err = journalSrc.StartReading(out, &tomb)
if test.read_error != "" {
assert.Contains(t, fmt.Sprintf("%s", err), test.read_error)
log.Infof("expected read error ok : %s", test.read_error)
continue
} else {
if err != nil {
t.Fatalf("%d/%d unexpected read error %s", tidx, len(tests), err)
}
}
time.Sleep(2 * time.Second)
log.Printf("now let's check number of lines & errors")
if count != test.lines {
t.Fatalf("%d/%d expected %d line read, got %d", tidx, len(tests), test.lines, count)
}
if test.tomb_error != "" {
assert.Contains(t, fmt.Sprintf("%s", tomb.Err()), test.tomb_error)
log.Infof("expected tomb error ok : %s", test.read_error)
continue
} else {
if tomb.Err() != nil {
t.Fatalf("%d/%d unexpected tomb error %s", tidx, len(tests), tomb.Err())
}
}
}
}
func TestJournaldSimple(t *testing.T) {
JOURNALD_CMD = os.Args[0]
JOURNALD_DEFAULT_TAIL_ARGS = []string{}
jBaseCfg := DataSourceCfg{
JournalctlFilters: []string{"-test.run=TestSimJournalctlCat", "--"},
Mode: CAT_MODE,
}
journalSrc := new(JournaldSource)
err := journalSrc.Configure(jBaseCfg)
if err != nil {
t.Fatalf("configuring journalctl : %s", err)
}
journalSrc.Cmd.Env = []string{"GO_WANT_TEST_OUTPUT=1"}
out := make(chan types.Event)
tomb := tomb.Tomb{}
count := 0
//start the reading : it doesn't give hand back before it's done
err = journalSrc.StartReading(out, &tomb)
if err != nil {
t.Fatalf("unexpected read error %s", err)
}
RLOOP:
for {
select {
case <-out:
count++
case <-time.After(1 * time.Second):
break RLOOP
}
}
//we expect 14 lines to be read
assert.Equal(t, 14, count)
}
func TestJournalctlKill(t *testing.T) {
cfg := DataSourceCfg{
Mode: CAT_MODE,
JournalctlFilters: []string{"-test.run=TestSimJournalctlCatOneLine", "--"},
}
//we're actually using tests to do this, hold my beer and watch this
JOURNALD_CMD = os.Args[0]
JOURNALD_DEFAULT_TAIL_ARGS = []string{}
log.SetLevel(log.TraceLevel)
journalSrc := new(JournaldSource)
err := journalSrc.Configure(cfg)
if err != nil {
t.Fatalf("unexpected config error %s", err)
}
journalSrc.Cmd.Env = []string{"GO_WANT_TEST_OUTPUT=1"}
out := make(chan types.Event)
tb := tomb.Tomb{}
err = journalSrc.StartReading(out, &tb)
if err != nil {
t.Fatalf("unexpected read error %s", err)
}
time.Sleep(1 * time.Second)
if tb.Err() != tomb.ErrStillAlive {
t.Fatalf("unexpected tomb error %s (should be alive)", tb.Err())
}
//kill it :>
tb.Kill(nil)
time.Sleep(1 * time.Second)
if tb.Err() != nil {
t.Fatalf("unexpected tomb error %s (should be dead)", tb.Err())
}
}

681
pkg/acquisition/modules/cloudwatch/cloudwatch.go Normal file
View file

@ -0,0 +1,681 @@
package cloudwatchacquisition
import (
"context"
"fmt"
"net/url"
"os"
"regexp"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
)
var openedStreams = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "cs_cloudwatch_openstreams_total",
Help: "Number of opened stream within group.",
},
[]string{"group"},
)
var linesRead = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_cloudwatch_stream_hits_total",
Help: "Number of event read from stream.",
},
[]string{"group", "stream"},
)
//CloudwatchSource is the runtime instance keeping track of N streams within 1 cloudwatch group
type CloudwatchSource struct {
Config CloudwatchSourceConfiguration
/*runtime stuff*/
logger *log.Entry
t *tomb.Tomb
cwClient *cloudwatchlogs.CloudWatchLogs
monitoredStreams []*LogStreamTailConfig
streamIndexes map[string]string
}
//CloudwatchSourceConfiguration allows user to define one or more streams to monitor within a cloudwatch log group
type CloudwatchSourceConfiguration struct {
configuration.DataSourceCommonCfg `yaml:",inline"`
GroupName string `yaml:"group_name"` //the group name to be monitored
StreamRegexp *string `yaml:"stream_regexp,omitempty"` //allow to filter specific streams
StreamName *string `yaml:"stream_name,omitempty"`
StartTime, EndTime *time.Time `yaml:"-"`
DescribeLogStreamsLimit *int64 `yaml:"describelogstreams_limit,omitempty"` //batch size for DescribeLogStreamsPagesWithContext
GetLogEventsPagesLimit *int64 `yaml:"getlogeventspages_limit,omitempty"`
PollNewStreamInterval *time.Duration `yaml:"poll_new_stream_interval,omitempty"` //frequency at which we poll for new streams within the log group
MaxStreamAge *time.Duration `yaml:"max_stream_age,omitempty"` //monitor only streams that have been updated within $duration
PollStreamInterval *time.Duration `yaml:"poll_stream_interval,omitempty"` //frequency at which we poll each stream
StreamReadTimeout *time.Duration `yaml:"stream_read_timeout,omitempty"` //stop monitoring streams that haven't been updated within $duration, might be reopened later tho
AwsApiCallTimeout *time.Duration `yaml:"aws_api_timeout,omitempty"`
AwsProfile *string `yaml:"aws_profile,omitempty"`
PrependCloudwatchTimestamp *bool `yaml:"prepend_cloudwatch_timestamp,omitempty"`
AwsConfigDir *string `yaml:"aws_config_dir,omitempty"`
}
//LogStreamTailConfig is the configuration for one given stream within one group
type LogStreamTailConfig struct {
GroupName string
StreamName string
GetLogEventsPagesLimit int64
PollStreamInterval time.Duration
StreamReadTimeout time.Duration
PrependCloudwatchTimestamp *bool
Labels map[string]string
logger *log.Entry
ExpectMode int
t tomb.Tomb
StartTime, EndTime time.Time //only used for CatMode
}
var (
def_DescribeLogStreamsLimit = int64(50)
def_PollNewStreamInterval = 10 * time.Second
def_MaxStreamAge = 5 * time.Minute
def_PollStreamInterval = 10 * time.Second
def_AwsApiCallTimeout = 10 * time.Second
def_StreamReadTimeout = 10 * time.Minute
def_PollDeadStreamInterval = 10 * time.Second
def_GetLogEventsPagesLimit = int64(1000)
def_AwsConfigDir = "/root/.aws/"
)
func (cw *CloudwatchSource) Configure(cfg []byte, logger *log.Entry) error {
cwConfig := CloudwatchSourceConfiguration{}
targetStream := "*"
if err := yaml.UnmarshalStrict(cfg, &cwConfig); err != nil {
return errors.Wrap(err, "Cannot parse CloudwatchSource configuration")
}
cw.Config = cwConfig
if len(cw.Config.GroupName) == 0 {
return fmt.Errorf("group_name is mandatory for CloudwatchSource")
}
cw.logger = logger.WithField("group", cw.Config.GroupName)
if cw.Config.Mode == "" {
cw.Config.Mode = configuration.TAIL_MODE
}
logger.Debugf("Starting configuration for Cloudwatch group %s", cw.Config.GroupName)
if cw.Config.DescribeLogStreamsLimit == nil {
cw.Config.DescribeLogStreamsLimit = &def_DescribeLogStreamsLimit
}
logger.Tracef("describelogstreams_limit set to %d", *cw.Config.DescribeLogStreamsLimit)
if cw.Config.PollNewStreamInterval == nil {
cw.Config.PollNewStreamInterval = &def_PollNewStreamInterval
}
logger.Tracef("poll_new_stream_interval set to %v", *cw.Config.PollNewStreamInterval)
if cw.Config.MaxStreamAge == nil {
cw.Config.MaxStreamAge = &def_MaxStreamAge
}
logger.Tracef("max_stream_age set to %v", *cw.Config.MaxStreamAge)
if cw.Config.PollStreamInterval == nil {
cw.Config.PollStreamInterval = &def_PollStreamInterval
}
logger.Tracef("poll_stream_interval set to %v", *cw.Config.PollStreamInterval)
if cw.Config.StreamReadTimeout == nil {
cw.Config.StreamReadTimeout = &def_StreamReadTimeout
}
logger.Tracef("stream_read_timeout set to %v", *cw.Config.StreamReadTimeout)
if cw.Config.GetLogEventsPagesLimit == nil {
cw.Config.GetLogEventsPagesLimit = &def_GetLogEventsPagesLimit
}
logger.Tracef("getlogeventspages_limit set to %v", *cw.Config.GetLogEventsPagesLimit)
if cw.Config.AwsApiCallTimeout == nil {
cw.Config.AwsApiCallTimeout = &def_AwsApiCallTimeout
}
logger.Tracef("aws_api_timeout set to %v", *cw.Config.AwsApiCallTimeout)
if *cw.Config.MaxStreamAge > *cw.Config.StreamReadTimeout {
logger.Warningf("max_stream_age > stream_read_timeout, stream might keep being opened/closed")
}
if cw.Config.AwsConfigDir == nil {
cw.Config.AwsConfigDir = &def_AwsConfigDir
}
logger.Tracef("aws_config_dir set to %s", *cw.Config.AwsConfigDir)
_, err := os.Stat(*cw.Config.AwsConfigDir)
if os.IsNotExist(err) {
logger.Errorf("aws_config_dir '%s' : directory does not exists", *cw.Config.AwsConfigDir)
return fmt.Errorf("aws_config_dir %s does not exist", *cw.Config.AwsConfigDir)
}
os.Setenv("AWS_SDK_LOAD_CONFIG", "1")
//as aws sdk relies on $HOME, let's allow the user to override it :)
os.Setenv("AWS_CONFIG_FILE", fmt.Sprintf("%s/config", *cw.Config.AwsConfigDir))
os.Setenv("AWS_SHARED_CREDENTIALS_FILE", fmt.Sprintf("%s/credentials", *cw.Config.AwsConfigDir))
if err := cw.newClient(); err != nil {
return err
}
cw.streamIndexes = make(map[string]string)
if cw.Config.StreamRegexp != nil {
if _, err := regexp.Compile(*cw.Config.StreamRegexp); err != nil {
return errors.Wrapf(err, "error while compiling regexp '%s'", *cw.Config.StreamRegexp)
}
targetStream = *cw.Config.StreamRegexp
} else if cw.Config.StreamName != nil {
targetStream = *cw.Config.StreamName
}
logger.Infof("Adding cloudwatch group '%s' (stream:%s) to datasources", cw.Config.GroupName, targetStream)
return nil
}
func (cw *CloudwatchSource) newClient() error {
var sess *session.Session
if cw.Config.AwsProfile != nil {
sess = session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
Profile: *cw.Config.AwsProfile,
}))
} else {
sess = session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
}))
}
if sess == nil {
return fmt.Errorf("failed to create aws session")
}
if v := os.Getenv("AWS_ENDPOINT_FORCE"); v != "" {
cw.logger.Debugf("[testing] overloading endpoint with %s", v)
cw.cwClient = cloudwatchlogs.New(sess, aws.NewConfig().WithEndpoint(v))
} else {
cw.cwClient = cloudwatchlogs.New(sess)
}
if cw.cwClient == nil {
return fmt.Errorf("failed to create cloudwatch client")
}
return nil
}
func (cw *CloudwatchSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
cw.t = t
monitChan := make(chan LogStreamTailConfig)
t.Go(func() error {
return cw.LogStreamManager(monitChan, out)
})
return cw.WatchLogGroupForStreams(monitChan)
}
func (cw *CloudwatchSource) GetMetrics() []prometheus.Collector {
return []prometheus.Collector{linesRead, openedStreams}
}
func (cw *CloudwatchSource) GetAggregMetrics() []prometheus.Collector {
return []prometheus.Collector{linesRead, openedStreams}
}
func (cw *CloudwatchSource) GetMode() string {
return cw.Config.Mode
}
func (cw *CloudwatchSource) GetName() string {
return "cloudwatch"
}
func (cw *CloudwatchSource) CanRun() error {
return nil
}
func (cw *CloudwatchSource) Dump() interface{} {
return cw
}
func (cw *CloudwatchSource) WatchLogGroupForStreams(out chan LogStreamTailConfig) error {
cw.logger.Debugf("Starting to watch group (interval:%s)", cw.Config.PollNewStreamInterval)
ticker := time.NewTicker(*cw.Config.PollNewStreamInterval)
var startFrom *string
for {
select {
case <-cw.t.Dying():
cw.logger.Infof("stopping group watch")
return nil
case <-ticker.C:
hasMoreStreams := true
startFrom = nil
for hasMoreStreams {
cw.logger.Tracef("doing the call to DescribeLogStreamsPagesWithContext")
ctx := context.Background()
//there can be a lot of streams in a group, and we're only interested in those recently written to, so we sort by LastEventTime
err := cw.cwClient.DescribeLogStreamsPagesWithContext(
ctx,
&cloudwatchlogs.DescribeLogStreamsInput{
LogGroupName: aws.String(cw.Config.GroupName),
Descending: aws.Bool(true),
NextToken: startFrom,
OrderBy: aws.String(cloudwatchlogs.OrderByLastEventTime),
Limit: cw.Config.DescribeLogStreamsLimit,
},
func(page *cloudwatchlogs.DescribeLogStreamsOutput, lastPage bool) bool {
cw.logger.Tracef("in helper of of DescribeLogStreamsPagesWithContext")
for _, event := range page.LogStreams {
startFrom = page.NextToken
//we check if the stream has been written to recently enough to be monitored
if event.LastIngestionTime != nil {
//aws uses millisecond since the epoch
oldest := time.Now().UTC().Add(-*cw.Config.MaxStreamAge)
//TBD : verify that this is correct : Unix 2nd arg expects Nanoseconds, and have a code that is more explicit.
LastIngestionTime := time.Unix(0, *event.LastIngestionTime*int64(time.Millisecond))
if LastIngestionTime.Before(oldest) {
cw.logger.Tracef("stop iteration, %s reached oldest age, stop (%s < %s)", *event.LogStreamName, LastIngestionTime, time.Now().Add(-*cw.Config.MaxStreamAge))
hasMoreStreams = false
return false
}
cw.logger.Tracef("stream %s is elligible for monitoring", *event.LogStreamName)
//the stream has been update recently, check if we should monitor it
monitorStream := LogStreamTailConfig{
GroupName: cw.Config.GroupName,
StreamName: *event.LogStreamName,
GetLogEventsPagesLimit: *cw.Config.GetLogEventsPagesLimit,
PollStreamInterval: *cw.Config.PollStreamInterval,
StreamReadTimeout: *cw.Config.StreamReadTimeout,
PrependCloudwatchTimestamp: cw.Config.PrependCloudwatchTimestamp,
ExpectMode: leaky.LIVE,
Labels: cw.Config.Labels,
}
out <- monitorStream
}
}
if lastPage {
cw.logger.Tracef("reached last page")
hasMoreStreams = false
}
return true
},
)
if err != nil {
newerr := errors.Wrapf(err, "while describing group %s", cw.Config.GroupName)
return newerr
}
cw.logger.Tracef("after DescribeLogStreamsPagesWithContext")
}
}
}
}
//LogStreamManager receives the potential streams to monitor, and start a go routine when needed
func (cw *CloudwatchSource) LogStreamManager(in chan LogStreamTailConfig, outChan chan types.Event) error {
cw.logger.Debugf("starting to monitor streams for %s", cw.Config.GroupName)
pollDeadStreamInterval := time.NewTicker(def_PollDeadStreamInterval)
for {
select {
case newStream := <-in:
shouldCreate := true
cw.logger.Tracef("received new streams to monitor : %s/%s", newStream.GroupName, newStream.StreamName)
if cw.Config.StreamName != nil && newStream.StreamName != *cw.Config.StreamName {
cw.logger.Tracef("stream %s != %s", newStream.StreamName, *cw.Config.StreamName)
continue
}
if cw.Config.StreamRegexp != nil {
match, err := regexp.Match(newStream.StreamName, []byte(*cw.Config.StreamRegexp))
if err != nil {
cw.logger.Warningf("invalid regexp : %s", err)
} else {
if !match {
cw.logger.Tracef("stream %s doesn't match %s", newStream.StreamName, *cw.Config.StreamRegexp)
continue
}
}
}
for idx, stream := range cw.monitoredStreams {
if newStream.GroupName == stream.GroupName && newStream.StreamName == stream.StreamName {
//stream exists, but is dead, remove it from list
if !stream.t.Alive() {
cw.logger.Debugf("stream %s already exists, but is dead", newStream.StreamName)
cw.monitoredStreams = append(cw.monitoredStreams[:idx], cw.monitoredStreams[idx+1:]...)
openedStreams.With(prometheus.Labels{"group": newStream.GroupName}).Dec()
break
}
shouldCreate = false
break
}
}
//let's start watching this stream
if shouldCreate {
openedStreams.With(prometheus.Labels{"group": newStream.GroupName}).Inc()
newStream.t = tomb.Tomb{}
newStream.logger = cw.logger.WithFields(log.Fields{"stream": newStream.StreamName})
cw.logger.Debugf("starting tail of stream %s", newStream.StreamName)
newStream.t.Go(func() error {
return cw.TailLogStream(&newStream, outChan)
})
cw.monitoredStreams = append(cw.monitoredStreams, &newStream)
}
case <-pollDeadStreamInterval.C:
for idx, stream := range cw.monitoredStreams {
if !cw.monitoredStreams[idx].t.Alive() {
cw.logger.Debugf("remove dead stream %s", stream.StreamName)
openedStreams.With(prometheus.Labels{"group": cw.monitoredStreams[idx].GroupName}).Dec()
cw.monitoredStreams = append(cw.monitoredStreams[:idx], cw.monitoredStreams[idx+1:]...)
break
}
}
case <-cw.t.Dying():
cw.logger.Infof("LogStreamManager for %s is dying, %d alive streams", cw.Config.GroupName, len(cw.monitoredStreams))
for idx, stream := range cw.monitoredStreams {
if cw.monitoredStreams[idx].t.Alive() {
cw.logger.Debugf("killing stream %s", stream.StreamName)
cw.monitoredStreams[idx].t.Kill(nil)
if err := cw.monitoredStreams[idx].t.Wait(); err != nil {
cw.logger.Debugf("error while waiting for death of %s : %s", stream.StreamName, err)
}
} else {
cw.monitoredStreams = append(cw.monitoredStreams[:idx], cw.monitoredStreams[idx+1:]...)
}
}
cw.logger.Debugf("routine cleanup done, return")
return nil
}
}
}
func (cw *CloudwatchSource) TailLogStream(cfg *LogStreamTailConfig, outChan chan types.Event) error {
var startFrom *string
var lastReadMessage time.Time = time.Now()
startup := true
ticker := time.NewTicker(cfg.PollStreamInterval)
//resume at existing index if we already had
if v, ok := cw.streamIndexes[cfg.GroupName+"+"+cfg.StreamName]; ok && v != "" {
cfg.logger.Debugf("restarting on index %s", v)
startFrom = &v
startup = false
}
/*during first run, we want to avoid reading any message, but just get a token.
if we don't, we might end up sending the same item several times. hence the 'startup' hack */
for {
select {
case <-ticker.C:
cfg.logger.Tracef("entering loop")
hasMorePages := true
for hasMorePages {
/*for the first call, we only consume the last item*/
limit := cfg.GetLogEventsPagesLimit
if startup {
limit = 1
}
cfg.logger.Tracef("calling GetLogEventsPagesWithContext")
ctx := context.Background()
err := cw.cwClient.GetLogEventsPagesWithContext(ctx,
&cloudwatchlogs.GetLogEventsInput{
Limit: aws.Int64(limit),
LogGroupName: aws.String(cfg.GroupName),
LogStreamName: aws.String(cfg.StreamName),
NextToken: startFrom,
},
func(page *cloudwatchlogs.GetLogEventsOutput, lastPage bool) bool {
cfg.logger.Tracef("%d results, last:%t", len(page.Events), lastPage)
startFrom = page.NextForwardToken
if page.NextForwardToken != nil {
cw.streamIndexes[cfg.GroupName+"+"+cfg.StreamName] = *page.NextForwardToken
}
if startup { //we grab the NextForwardToken and we return on first iteration
return false
}
if lastPage { /*wait another ticker to check on new log availability*/
cfg.logger.Tracef("last page")
hasMorePages = false
}
if len(page.Events) > 0 {
lastReadMessage = time.Now()
}
for _, event := range page.Events {
evt, err := cwLogToEvent(event, cfg)
if err != nil {
cfg.logger.Warningf("cwLogToEvent error, discarded event : %s", err)
} else {
cfg.logger.Debugf("pushing message : %s", evt.Line.Raw)
linesRead.With(prometheus.Labels{"group": cfg.GroupName, "stream": cfg.StreamName}).Inc()
outChan <- evt
}
}
return true
},
)
if err != nil {
newerr := errors.Wrapf(err, "while reading %s/%s", cfg.GroupName, cfg.StreamName)
cfg.logger.Warningf("err : %s", newerr)
return newerr
}
if startup {
startup = false
}
cfg.logger.Tracef("done reading GetLogEventsPagesWithContext")
if time.Since(lastReadMessage) > cfg.StreamReadTimeout {
cfg.logger.Infof("%s/%s reached timeout (%s) (last message was %s)", cfg.GroupName, cfg.StreamName, time.Since(lastReadMessage),
lastReadMessage)
return nil
}
}
case <-cfg.t.Dying():
cfg.logger.Infof("logstream tail stopping")
return fmt.Errorf("killed")
}
}
}
func (cw *CloudwatchSource) ConfigureByDSN(dsn string, logtype string, logger *log.Entry) error {
cw.logger = logger
dsn = strings.TrimPrefix(dsn, cw.GetName()+"://")
args := strings.Split(dsn, "?")
if len(args) != 2 {
return fmt.Errorf("query is mandatory (at least start_date and end_date or backlog)")
}
frags := strings.Split(args[0], ":")
if len(frags) != 2 {
return fmt.Errorf("cloudwatch path must contain group and stream : /my/group/name:stream/name")
}
cw.Config.GroupName = frags[0]
cw.Config.StreamName = &frags[1]
cw.Config.Labels = make(map[string]string)
cw.Config.Labels["type"] = logtype
u, err := url.ParseQuery(args[1])
if err != nil {
return errors.Wrapf(err, "while parsing %s", dsn)
}
for k, v := range u {
switch k {
case "log_level":
if len(v) != 1 {
return fmt.Errorf("expected zero or one value for 'log_level'")
}
lvl, err := log.ParseLevel(v[0])
if err != nil {
return errors.Wrapf(err, "unknown level %s", v[0])
}
cw.logger.Logger.SetLevel(lvl)
case "profile":
if len(v) != 1 {
return fmt.Errorf("expected zero or one value for 'profile'")
}
awsprof := v[0]
cw.Config.AwsProfile = &awsprof
cw.logger.Debugf("profile set to '%s'", *cw.Config.AwsProfile)
case "start_date":
if len(v) != 1 {
return fmt.Errorf("expected zero or one argument for 'start_date'")
}
//let's reuse our parser helper so that a ton of date formats are supported
strdate, startDate := parser.GenDateParse(v[0])
cw.logger.Debugf("parsed '%s' as '%s'", v[0], strdate)
cw.Config.StartTime = &startDate
case "end_date":
if len(v) != 1 {
return fmt.Errorf("expected zero or one argument for 'end_date'")
}
//let's reuse our parser helper so that a ton of date formats are supported
strdate, endDate := parser.GenDateParse(v[0])
cw.logger.Debugf("parsed '%s' as '%s'", v[0], strdate)
cw.Config.EndTime = &endDate
case "backlog":
if len(v) != 1 {
return fmt.Errorf("expected zero or one argument for 'backlog'")
}
//let's reuse our parser helper so that a ton of date formats are supported
duration, err := time.ParseDuration(v[0])
if err != nil {
return errors.Wrapf(err, "unable to parse '%s' as duration", v[0])
}
cw.logger.Debugf("parsed '%s' as '%s'", v[0], duration)
start := time.Now().UTC().Add(-duration)
cw.Config.StartTime = &start
end := time.Now().UTC()
cw.Config.EndTime = &end
default:
return fmt.Errorf("unexpected argument %s", k)
}
}
cw.logger.Tracef("host=%s", cw.Config.GroupName)
cw.logger.Tracef("stream=%s", *cw.Config.StreamName)
cw.Config.GetLogEventsPagesLimit = &def_GetLogEventsPagesLimit
if err := cw.newClient(); err != nil {
return err
}
if cw.Config.StreamName == nil || cw.Config.GroupName == "" {
return fmt.Errorf("missing stream or group name")
}
if cw.Config.StartTime == nil || cw.Config.EndTime == nil {
return fmt.Errorf("start_date and end_date or backlog are mandatory in one-shot mode")
}
cw.Config.Mode = configuration.CAT_MODE
cw.streamIndexes = make(map[string]string)
cw.t = &tomb.Tomb{}
return nil
}
func (cw *CloudwatchSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error {
//StreamName string, Start time.Time, End time.Time
config := LogStreamTailConfig{
GroupName: cw.Config.GroupName,
StreamName: *cw.Config.StreamName,
StartTime: *cw.Config.StartTime,
EndTime: *cw.Config.EndTime,
GetLogEventsPagesLimit: *cw.Config.GetLogEventsPagesLimit,
logger: cw.logger.WithFields(log.Fields{
"group": cw.Config.GroupName,
"stream": *cw.Config.StreamName,
}),
Labels: cw.Config.Labels,
ExpectMode: leaky.TIMEMACHINE,
}
return cw.CatLogStream(&config, out)
}
func (cw *CloudwatchSource) CatLogStream(cfg *LogStreamTailConfig, outChan chan types.Event) error {
var startFrom *string
var head = true
/*convert the times*/
startTime := cfg.StartTime.UTC().Unix() * 1000
endTime := cfg.EndTime.UTC().Unix() * 1000
hasMoreEvents := true
for hasMoreEvents {
select {
default:
cfg.logger.Tracef("Calling GetLogEventsPagesWithContext(%s, %s), startTime:%d / endTime:%d",
cfg.GroupName, cfg.StreamName, startTime, endTime)
cfg.logger.Tracef("startTime:%s / endTime:%s", cfg.StartTime, cfg.EndTime)
if startFrom != nil {
cfg.logger.Tracef("next_token: %s", *startFrom)
}
ctx := context.Background()
err := cw.cwClient.GetLogEventsPagesWithContext(ctx,
&cloudwatchlogs.GetLogEventsInput{
Limit: aws.Int64(10),
LogGroupName: aws.String(cfg.GroupName),
LogStreamName: aws.String(cfg.StreamName),
StartTime: aws.Int64(startTime),
EndTime: aws.Int64(endTime),
StartFromHead: &head,
NextToken: startFrom,
},
func(page *cloudwatchlogs.GetLogEventsOutput, lastPage bool) bool {
cfg.logger.Tracef("in GetLogEventsPagesWithContext handker (%d events) (last:%t)", len(page.Events), lastPage)
for _, event := range page.Events {
evt, err := cwLogToEvent(event, cfg)
if err != nil {
cfg.logger.Warningf("discard event : %s", err)
}
cfg.logger.Debugf("pushing message : %s", evt.Line.Raw)
outChan <- evt
}
if startFrom != nil && *page.NextForwardToken == *startFrom {
cfg.logger.Debugf("reached end of available events")
hasMoreEvents = false
return false
}
startFrom = page.NextForwardToken
return true
},
)
if err != nil {
return errors.Wrapf(err, "while reading logs from %s/%s", cfg.GroupName, cfg.StreamName)
}
cfg.logger.Tracef("after GetLogEventsPagesWithContext")
case <-cw.t.Dying():
cfg.logger.Warningf("cat stream killed")
return nil
}
}
cfg.logger.Tracef("CatLogStream out")
return nil
}
func cwLogToEvent(log *cloudwatchlogs.OutputLogEvent, cfg *LogStreamTailConfig) (types.Event, error) {
l := types.Line{}
evt := types.Event{}
if log.Message == nil {
return evt, fmt.Errorf("nil message")
}
msg := *log.Message
if cfg.PrependCloudwatchTimestamp != nil && *cfg.PrependCloudwatchTimestamp {
eventTimestamp := time.Unix(0, *log.Timestamp*int64(time.Millisecond))
msg = eventTimestamp.String() + " " + msg
}
l.Raw = msg
l.Labels = cfg.Labels
l.Time = time.Now()
l.Src = fmt.Sprintf("%s/%s", cfg.GroupName, cfg.StreamName)
l.Process = true
l.Module = "cloudwatch"
evt.Line = l
evt.Process = true
evt.Type = types.LOG
evt.ExpectMode = cfg.ExpectMode
cfg.logger.Debugf("returned event labels : %+v", evt.Line.Labels)
return evt, nil
}

862
pkg/acquisition/modules/cloudwatch/cloudwatch_test.go Normal file
View file

@ -0,0 +1,862 @@
package cloudwatchacquisition
import (
"fmt"
"net"
"os"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
)
/*
test plan :
- start on bad group/bad stream
- start on good settings (oneshot) -> check expected messages
- start on good settings (stream) -> check expected messages within given time
- check shutdown/restart
*/
func checkForLocalStackAvailability() error {
if v := os.Getenv("AWS_ENDPOINT_FORCE"); v != "" {
v = strings.TrimPrefix(v, "http://")
_, err := net.Dial("tcp", v)
if err != nil {
return fmt.Errorf("while dialing %s : %s : aws endpoint isn't available", v, err)
}
} else {
return fmt.Errorf("missing aws endpoint for tests : AWS_ENDPOINT_FORCE")
}
return nil
}
func TestMain(m *testing.M) {
if err := checkForLocalStackAvailability(); err != nil {
log.Fatalf("local stack error : %s", err)
}
def_PollNewStreamInterval = 1 * time.Second
def_PollStreamInterval = 1 * time.Second
def_StreamReadTimeout = 10 * time.Second
def_MaxStreamAge = 5 * time.Second
def_PollDeadStreamInterval = 5 * time.Second
os.Exit(m.Run())
}
func TestWatchLogGroupForStreams(t *testing.T) {
var err error
log.SetLevel(log.DebugLevel)
tests := []struct {
config []byte
expectedCfgErr string
expectedStartErr string
name string
pre func(*CloudwatchSource)
run func(*CloudwatchSource)
post func(*CloudwatchSource)
expectedResLen int
expectedResMessages []string
}{
//require a group name that doesn't exist
{
name: "group_does_not_exists",
config: []byte(`
source: cloudwatch
labels:
type: test_source
group_name: b
stream_name: test_stream`),
expectedStartErr: "The specified log group does not exist",
pre: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: aws.String("test_group_not_used_1"),
}); err != nil {
t.Fatalf("failed to create log group : %s", err)
}
},
post: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
LogGroupName: aws.String("test_group_not_used_1"),
}); err != nil {
t.Fatalf("failed to delete log group : %s", err)
}
},
},
//test stream mismatch
{
name: "group_exists_bad_stream_name",
config: []byte(`
source: cloudwatch
labels:
type: test_source
group_name: test_group1
stream_name: test_stream_bad`),
pre: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: aws.String("test_group1"),
}); err != nil {
t.Fatalf("failed to create log group : %s", err)
}
if _, err := cw.cwClient.CreateLogStream(&cloudwatchlogs.CreateLogStreamInput{
LogGroupName: aws.String("test_group1"),
LogStreamName: aws.String("test_stream"),
}); err != nil {
t.Fatalf("failed to create log stream : %s", err)
}
//have a message before we start - won't be popped, but will trigger stream monitoring
if _, err := cw.cwClient.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
LogGroupName: aws.String("test_group1"),
LogStreamName: aws.String("test_stream"),
LogEvents: []*cloudwatchlogs.InputLogEvent{
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_1"),
Timestamp: aws.Int64(time.Now().UTC().Unix() * 1000),
},
},
}); err != nil {
log.Fatalf("failed to put logs")
}
},
post: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
LogGroupName: aws.String("test_group1"),
}); err != nil {
t.Fatalf("failed to delete log group : %s", err)
}
},
expectedResLen: 0,
},
//test stream mismatch
{
name: "group_exists_bad_stream_regexp",
config: []byte(`
source: cloudwatch
labels:
type: test_source
group_name: test_group1
stream_regexp: test_bad[0-9]+`),
pre: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: aws.String("test_group1"),
}); err != nil {
t.Fatalf("failed to create log group : %s", err)
}
if _, err := cw.cwClient.CreateLogStream(&cloudwatchlogs.CreateLogStreamInput{
LogGroupName: aws.String("test_group1"),
LogStreamName: aws.String("test_stream"),
}); err != nil {
t.Fatalf("failed to create log stream : %s", err)
}
//have a message before we start - won't be popped, but will trigger stream monitoring
if _, err := cw.cwClient.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
LogGroupName: aws.String("test_group1"),
LogStreamName: aws.String("test_stream"),
LogEvents: []*cloudwatchlogs.InputLogEvent{
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_1"),
Timestamp: aws.Int64(time.Now().UTC().Unix() * 1000),
},
},
}); err != nil {
t.Fatalf("failed to put logs")
}
},
post: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
LogGroupName: aws.String("test_group1"),
}); err != nil {
t.Fatalf("failed to delete log group : %s", err)
}
},
expectedResLen: 0,
},
//require a group name that does exist and contains a stream in which we gonna put events
{
name: "group_exists_stream_exists_has_events",
config: []byte(`
source: cloudwatch
labels:
type: test_source
group_name: test_log_group1
log_level: trace
stream_name: test_stream`),
//expectedStartErr: "The specified log group does not exist",
pre: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: aws.String("test_log_group1"),
}); err != nil {
t.Fatalf("failed to create log group : %s", err)
}
if _, err := cw.cwClient.CreateLogStream(&cloudwatchlogs.CreateLogStreamInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
}); err != nil {
t.Fatalf("failed to create log stream : %s", err)
}
//have a message before we start - won't be popped, but will trigger stream monitoring
if _, err := cw.cwClient.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
LogEvents: []*cloudwatchlogs.InputLogEvent{
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_1"),
Timestamp: aws.Int64(time.Now().UTC().Unix() * 1000),
},
},
}); err != nil {
t.Fatalf("failed to put logs")
}
},
run: func(cw *CloudwatchSource) {
//wait for new stream pickup + stream poll interval
time.Sleep(def_PollNewStreamInterval + (1 * time.Second))
time.Sleep(def_PollStreamInterval + (1 * time.Second))
if _, err := cw.cwClient.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
LogEvents: []*cloudwatchlogs.InputLogEvent{
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_4"),
Timestamp: aws.Int64(time.Now().UTC().Unix() * 1000),
},
//and add an event in the future that will be popped
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_5"),
Timestamp: aws.Int64(time.Now().UTC().Unix() * 1000),
},
},
}); err != nil {
t.Fatalf("failed to put logs : %s", err)
}
},
post: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.DeleteLogStream(&cloudwatchlogs.DeleteLogStreamInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
}); err != nil {
t.Fatalf("failed to delete log stream : %s", err)
}
if _, err := cw.cwClient.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
LogGroupName: aws.String("test_log_group1"),
}); err != nil {
t.Fatalf("failed to delete log group : %s", err)
}
},
expectedResLen: 2,
expectedResMessages: []string{"test_message_4", "test_message_5"},
},
//have a stream generate events, reach time-out and gets polled again
{
name: "group_exists_stream_exists_has_events+timeout",
config: []byte(`
source: cloudwatch
labels:
type: test_source
group_name: test_log_group1
log_level: trace
stream_name: test_stream`),
//expectedStartErr: "The specified log group does not exist",
pre: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: aws.String("test_log_group1"),
}); err != nil {
t.Fatalf("failed to create log group : %s", err)
}
if _, err := cw.cwClient.CreateLogStream(&cloudwatchlogs.CreateLogStreamInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
}); err != nil {
t.Fatalf("failed to create log stream : %s", err)
}
//have a message before we start - won't be popped, but will trigger stream monitoring
if _, err := cw.cwClient.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
LogEvents: []*cloudwatchlogs.InputLogEvent{
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_1"),
Timestamp: aws.Int64(time.Now().UTC().Unix() * 1000),
},
},
}); err != nil {
t.Fatalf("failed to put logs")
}
},
run: func(cw *CloudwatchSource) {
//wait for new stream pickup + stream poll interval
time.Sleep(def_PollNewStreamInterval + (1 * time.Second))
time.Sleep(def_PollStreamInterval + (1 * time.Second))
//send some events
if _, err := cw.cwClient.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
LogEvents: []*cloudwatchlogs.InputLogEvent{
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_41"),
Timestamp: aws.Int64(time.Now().UTC().Unix() * 1000),
},
},
}); err != nil {
t.Fatalf("failed to put logs : %s", err)
}
//wait for the stream to time-out
time.Sleep(def_StreamReadTimeout + (1 * time.Second))
//and send events again
if _, err := cw.cwClient.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
LogEvents: []*cloudwatchlogs.InputLogEvent{
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_51"),
Timestamp: aws.Int64(time.Now().UTC().Unix() * 1000),
},
},
}); err != nil {
t.Fatalf("failed to put logs : %s", err)
}
//wait for new stream pickup + stream poll interval
time.Sleep(def_PollNewStreamInterval + (1 * time.Second))
time.Sleep(def_PollStreamInterval + (1 * time.Second))
},
post: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.DeleteLogStream(&cloudwatchlogs.DeleteLogStreamInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
}); err != nil {
t.Fatalf("failed to delete log stream : %s", err)
}
if _, err := cw.cwClient.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
LogGroupName: aws.String("test_log_group1"),
}); err != nil {
t.Fatalf("failed to delete log group : %s", err)
}
},
expectedResLen: 2,
expectedResMessages: []string{"test_message_41", "test_message_51"},
},
//have a stream generate events, reach time-out and dead body collection
{
name: "group_exists_stream_exists_has_events+timeout+GC",
config: []byte(`
source: cloudwatch
labels:
type: test_source
group_name: test_log_group1
log_level: trace
stream_name: test_stream`),
//expectedStartErr: "The specified log group does not exist",
pre: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: aws.String("test_log_group1"),
}); err != nil {
t.Fatalf("failed to create log group : %s", err)
}
if _, err := cw.cwClient.CreateLogStream(&cloudwatchlogs.CreateLogStreamInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
}); err != nil {
t.Fatalf("failed to create log stream : %s", err)
}
//have a message before we start - won't be popped, but will trigger stream monitoring
if _, err := cw.cwClient.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
LogEvents: []*cloudwatchlogs.InputLogEvent{
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_1"),
Timestamp: aws.Int64(time.Now().UTC().Unix() * 1000),
},
},
}); err != nil {
t.Fatalf("failed to put logs")
}
},
run: func(cw *CloudwatchSource) {
//wait for new stream pickup + stream poll interval
time.Sleep(def_PollNewStreamInterval + (1 * time.Second))
time.Sleep(def_PollStreamInterval + (1 * time.Second))
time.Sleep(def_PollDeadStreamInterval + (1 * time.Second))
},
post: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.DeleteLogStream(&cloudwatchlogs.DeleteLogStreamInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
}); err != nil {
t.Fatalf("failed to delete log stream : %s", err)
}
if _, err := cw.cwClient.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
LogGroupName: aws.String("test_log_group1"),
}); err != nil {
t.Fatalf("failed to delete log stream : %s", err)
}
},
expectedResLen: 0,
},
}
for _, test := range tests {
dbgLogger := log.New().WithField("test", test.name)
dbgLogger.Logger.SetLevel(log.DebugLevel)
dbgLogger.Infof("starting test")
cw := CloudwatchSource{}
err = cw.Configure(test.config, dbgLogger)
if err != nil && test.expectedCfgErr != "" {
if !strings.Contains(err.Error(), test.expectedCfgErr) {
t.Fatalf("%s expected error '%s' got error '%s'", test.name, test.expectedCfgErr, err.Error())
}
log.Debugf("got expected error : %s", err)
continue
} else if err != nil && test.expectedCfgErr == "" {
t.Fatalf("%s unexpected error : %s", test.name, err)
continue
} else if test.expectedCfgErr != "" && err == nil {
t.Fatalf("%s expected error '%s', got none", test.name, test.expectedCfgErr)
continue
}
dbgLogger.Infof("config done test")
//run pre-routine : tests use it to set group & streams etc.
if test.pre != nil {
test.pre(&cw)
}
out := make(chan types.Event)
tmb := tomb.Tomb{}
var rcvd_evts []types.Event
dbgLogger.Infof("running StreamingAcquisition")
actmb := tomb.Tomb{}
actmb.Go(func() error {
err := cw.StreamingAcquisition(out, &actmb)
dbgLogger.Infof("acquis done")
if err != nil && test.expectedStartErr != "" && !strings.Contains(err.Error(), test.expectedStartErr) {
t.Fatalf("%s expected error '%s' got '%s'", test.name, test.expectedStartErr, err.Error())
} else if err != nil && test.expectedStartErr == "" {
t.Fatalf("%s unexpected error '%s'", test.name, err)
} else if err == nil && test.expectedStartErr != "" {
t.Fatalf("%s expected error '%s' got none", test.name, err)
}
return nil
})
//let's empty output chan
tmb.Go(func() error {
for {
select {
case in := <-out:
log.Debugf("received event %+v", in)
rcvd_evts = append(rcvd_evts, in)
case <-tmb.Dying():
log.Debugf("pumper died")
return nil
}
}
})
if test.run != nil {
test.run(&cw)
} else {
dbgLogger.Warning("no run code")
}
time.Sleep(5 * time.Second)
dbgLogger.Infof("killing collector")
tmb.Kill(nil)
<-tmb.Dead()
dbgLogger.Infof("killing datasource")
actmb.Kill(nil)
<-actmb.Dead()
//dbgLogger.Infof("collected events : %d -> %+v", len(rcvd_evts), rcvd_evts)
//check results
if test.expectedResLen != -1 {
if test.expectedResLen != len(rcvd_evts) {
t.Fatalf("%s : expected %d results got %d -> %v", test.name, test.expectedResLen, len(rcvd_evts), rcvd_evts)
} else {
dbgLogger.Debugf("got %d expected messages", len(rcvd_evts))
}
}
if len(test.expectedResMessages) != 0 {
res := test.expectedResMessages
for idx, v := range rcvd_evts {
if len(res) == 0 {
t.Fatalf("result %d/%d : received '%s', didn't expect anything (recvd:%d, expected:%d)", idx, len(rcvd_evts), v.Line.Raw, len(rcvd_evts), len(test.expectedResMessages))
}
if res[0] != v.Line.Raw {
t.Fatalf("result %d/%d : expected '%s', received '%s' (recvd:%d, expected:%d)", idx, len(rcvd_evts), res[0], v.Line.Raw, len(rcvd_evts), len(test.expectedResMessages))
} else {
dbgLogger.Debugf("got message '%s'", res[0])
}
res = res[1:]
}
if len(res) != 0 {
t.Fatalf("leftover unmatched results : %v", res)
}
}
if test.post != nil {
test.post(&cw)
}
}
}
func TestConfiguration(t *testing.T) {
var err error
log.SetLevel(log.DebugLevel)
tests := []struct {
config []byte
expectedCfgErr string
expectedStartErr string
name string
}{
{
name: "group_does_not_exists",
config: []byte(`
source: cloudwatch
labels:
type: test_source
group_name: test_group
stream_name: test_stream`),
expectedStartErr: "The specified log group does not exist",
},
{
config: []byte(`
xxx: cloudwatch
labels:
type: test_source
group_name: test_group
stream_name: test_stream`),
expectedCfgErr: "field xxx not found in type",
},
{
name: "missing_group_name",
config: []byte(`
source: cloudwatch
labels:
type: test_source
stream_name: test_stream`),
expectedCfgErr: "group_name is mandatory for CloudwatchSource",
},
}
for idx, test := range tests {
dbgLogger := log.New().WithField("test", test.name)
dbgLogger.Logger.SetLevel(log.DebugLevel)
log.Printf("%d/%d", idx, len(tests))
cw := CloudwatchSource{}
err = cw.Configure(test.config, dbgLogger)
if err != nil && test.expectedCfgErr != "" {
if !strings.Contains(err.Error(), test.expectedCfgErr) {
t.Fatalf("%s expected error '%s' got error '%s'", test.name, test.expectedCfgErr, err.Error())
}
log.Debugf("got expected error : %s", err)
continue
} else if err != nil && test.expectedCfgErr == "" {
t.Fatalf("%s unexpected error : %s", test.name, err)
continue
} else if test.expectedCfgErr != "" && err == nil {
t.Fatalf("%s expected error '%s', got none", test.name, test.expectedCfgErr)
continue
}
out := make(chan types.Event)
tmb := tomb.Tomb{}
switch cw.GetMode() {
case "tail":
err = cw.StreamingAcquisition(out, &tmb)
case "cat":
err = cw.OneShotAcquisition(out, &tmb)
}
if err != nil && test.expectedStartErr != "" && !strings.Contains(err.Error(), test.expectedStartErr) {
t.Fatalf("%s expected error '%s' got '%s'", test.name, test.expectedStartErr, err.Error())
} else if err != nil && test.expectedStartErr == "" {
t.Fatalf("%s unexpected error '%s'", test.name, err)
} else if err == nil && test.expectedStartErr != "" {
t.Fatalf("%s expected error '%s' got none", test.name, err)
}
log.Debugf("killing ...")
tmb.Kill(nil)
<-tmb.Dead()
log.Debugf("dead :)")
}
}
func TestConfigureByDSN(t *testing.T) {
var err error
log.SetLevel(log.DebugLevel)
tests := []struct {
dsn, logtype string
expectedCfgErr string
name string
}{
{
name: "missing_query",
dsn: "cloudwatch://bad_log_group:bad_stream_name",
expectedCfgErr: "query is mandatory (at least start_date and end_date or backlog)",
},
{
name: "backlog",
dsn: "cloudwatch://bad_log_group:bad_stream_name?backlog=30m&log_level=info&profile=test",
//expectedCfgErr: "query is mandatory (at least start_date and end_date or backlog)",
},
{
name: "start_date/end_date",
dsn: "cloudwatch://bad_log_group:bad_stream_name?start_date=2021/05/15 14:04&end_date=2021/05/15 15:04",
//expectedCfgErr: "query is mandatory (at least start_date and end_date or backlog)",
},
{
name: "bad_log_level",
dsn: "cloudwatch://bad_log_group:bad_stream_name?backlog=4h&log_level=",
expectedCfgErr: "unknown level : not a valid logrus Level: ",
},
}
for idx, test := range tests {
dbgLogger := log.New().WithField("test", test.name)
dbgLogger.Logger.SetLevel(log.DebugLevel)
log.Printf("%d/%d", idx, len(tests))
cw := CloudwatchSource{}
err = cw.ConfigureByDSN(test.dsn, test.logtype, dbgLogger)
if err != nil && test.expectedCfgErr != "" {
if !strings.Contains(err.Error(), test.expectedCfgErr) {
t.Fatalf("%s expected error '%s' got error '%s'", test.name, test.expectedCfgErr, err.Error())
}
log.Debugf("got expected error : %s", err)
continue
} else if err != nil && test.expectedCfgErr == "" {
t.Fatalf("%s unexpected error : %s", test.name, err)
continue
} else if test.expectedCfgErr != "" && err == nil {
t.Fatalf("%s expected error '%s', got none", test.name, test.expectedCfgErr)
continue
}
}
}
func TestOneShotAcquisition(t *testing.T) {
var err error
log.SetLevel(log.DebugLevel)
tests := []struct {
dsn string
expectedCfgErr string
expectedStartErr string
name string
pre func(*CloudwatchSource)
run func(*CloudwatchSource)
post func(*CloudwatchSource)
expectedResLen int
expectedResMessages []string
}{
//stream with no data
{
name: "empty_stream",
dsn: "cloudwatch://test_log_group1:test_stream?backlog=1h",
//expectedStartErr: "The specified log group does not exist",
pre: func(cw *CloudwatchSource) {
cw.cwClient.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: aws.String("test_log_group1"),
})
cw.cwClient.CreateLogStream(&cloudwatchlogs.CreateLogStreamInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
})
},
post: func(cw *CloudwatchSource) {
cw.cwClient.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
LogGroupName: aws.String("test_log_group1"),
})
},
expectedResLen: 0,
},
//stream with one event
{
name: "get_one_event",
dsn: "cloudwatch://test_log_group1:test_stream?backlog=1h",
//expectedStartErr: "The specified log group does not exist",
pre: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: aws.String("test_log_group1"),
}); err != nil {
t.Fatalf("error while CreateLogGroup")
}
if _, err := cw.cwClient.CreateLogStream(&cloudwatchlogs.CreateLogStreamInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
}); err != nil {
t.Fatalf("error while CreateLogStream")
}
//this one is too much in the back
if _, err := cw.cwClient.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
LogEvents: []*cloudwatchlogs.InputLogEvent{
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_1"),
Timestamp: aws.Int64(time.Now().Add(-(2 * time.Hour)).UTC().Unix() * 1000),
},
},
}); err != nil {
log.Fatalf("failed to put logs")
}
//this one can be read
if _, err := cw.cwClient.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
LogEvents: []*cloudwatchlogs.InputLogEvent{
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_2"),
Timestamp: aws.Int64(time.Now().UTC().Unix() * 1000),
},
},
}); err != nil {
log.Fatalf("failed to put logs")
}
//this one is in the past
if _, err := cw.cwClient.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
LogGroupName: aws.String("test_log_group1"),
LogStreamName: aws.String("test_stream"),
LogEvents: []*cloudwatchlogs.InputLogEvent{
&cloudwatchlogs.InputLogEvent{
Message: aws.String("test_message_3"),
Timestamp: aws.Int64(time.Now().Add(-(3 * time.Hour)).UTC().Unix() * 1000),
},
},
}); err != nil {
log.Fatalf("failed to put logs")
}
},
post: func(cw *CloudwatchSource) {
if _, err := cw.cwClient.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
LogGroupName: aws.String("test_log_group1"),
}); err != nil {
t.Fatalf("failed to delete")
}
},
expectedResLen: 1,
expectedResMessages: []string{"test_message_2"},
},
}
for _, test := range tests {
dbgLogger := log.New().WithField("test", test.name)
dbgLogger.Logger.SetLevel(log.DebugLevel)
dbgLogger.Infof("starting test")
cw := CloudwatchSource{}
err = cw.ConfigureByDSN(test.dsn, "test", dbgLogger)
if err != nil && test.expectedCfgErr != "" {
if !strings.Contains(err.Error(), test.expectedCfgErr) {
t.Fatalf("%s expected error '%s' got error '%s'", test.name, test.expectedCfgErr, err.Error())
}
log.Debugf("got expected error : %s", err)
continue
} else if err != nil && test.expectedCfgErr == "" {
t.Fatalf("%s unexpected error : %s", test.name, err)
continue
} else if test.expectedCfgErr != "" && err == nil {
t.Fatalf("%s expected error '%s', got none", test.name, test.expectedCfgErr)
continue
}
dbgLogger.Infof("config done test")
//run pre-routine : tests use it to set group & streams etc.
if test.pre != nil {
test.pre(&cw)
}
out := make(chan types.Event)
tmb := tomb.Tomb{}
var rcvd_evts []types.Event
dbgLogger.Infof("running StreamingAcquisition")
actmb := tomb.Tomb{}
actmb.Go(func() error {
err := cw.OneShotAcquisition(out, &actmb)
dbgLogger.Infof("acquis done")
if err != nil && test.expectedStartErr != "" && !strings.Contains(err.Error(), test.expectedStartErr) {
t.Fatalf("%s expected error '%s' got '%s'", test.name, test.expectedStartErr, err.Error())
} else if err != nil && test.expectedStartErr == "" {
t.Fatalf("%s unexpected error '%s'", test.name, err)
} else if err == nil && test.expectedStartErr != "" {
t.Fatalf("%s expected error '%s' got none", test.name, err)
}
return nil
})
//let's empty output chan
tmb.Go(func() error {
for {
select {
case in := <-out:
log.Debugf("received event %+v", in)
rcvd_evts = append(rcvd_evts, in)
case <-tmb.Dying():
log.Debugf("pumper died")
return nil
}
}
})
if test.run != nil {
test.run(&cw)
} else {
dbgLogger.Warning("no run code")
}
time.Sleep(5 * time.Second)
dbgLogger.Infof("killing collector")
tmb.Kill(nil)
<-tmb.Dead()
dbgLogger.Infof("killing datasource")
actmb.Kill(nil)
dbgLogger.Infof("waiting datasource death")
<-actmb.Dead()
//check results
if test.expectedResLen != -1 {
if test.expectedResLen != len(rcvd_evts) {
t.Fatalf("%s : expected %d results got %d -> %v", test.name, test.expectedResLen, len(rcvd_evts), rcvd_evts)
} else {
dbgLogger.Debugf("got %d expected messages", len(rcvd_evts))
}
}
if len(test.expectedResMessages) != 0 {
res := test.expectedResMessages
for idx, v := range rcvd_evts {
if len(res) == 0 {
t.Fatalf("result %d/%d : received '%s', didn't expect anything (recvd:%d, expected:%d)", idx, len(rcvd_evts), v.Line.Raw, len(rcvd_evts), len(test.expectedResMessages))
}
if res[0] != v.Line.Raw {
t.Fatalf("result %d/%d : expected '%s', received '%s' (recvd:%d, expected:%d)", idx, len(rcvd_evts), res[0], v.Line.Raw, len(rcvd_evts), len(test.expectedResMessages))
} else {
dbgLogger.Debugf("got message '%s'", res[0])
}
res = res[1:]
}
if len(res) != 0 {
t.Fatalf("leftover unmatched results : %v", res)
}
}
if test.post != nil {
test.post(&cw)
}
}
}

417
pkg/acquisition/modules/file/file.go Normal file
View file

@ -0,0 +1,417 @@
package fileacquisition
import (
"bufio"
"compress/gzip"
"fmt"
"io"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/fsnotify/fsnotify"
"github.com/nxadm/tail"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
)
var linesRead = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_filesource_hits_total",
Help: "Total lines that were read.",
},
[]string{"source"})
type FileConfiguration struct {
Filenames []string
Filename string
ForceInotify bool `yaml:"force_inotify"`
configuration.DataSourceCommonCfg `yaml:",inline"`
}
type FileSource struct {
config FileConfiguration
watcher *fsnotify.Watcher
watchedDirectories map[string]bool
tails map[string]bool
logger *log.Entry
files []string
}
func (f *FileSource) Configure(Config []byte, logger *log.Entry) error {
fileConfig := FileConfiguration{}
f.logger = logger
f.watchedDirectories = make(map[string]bool)
f.tails = make(map[string]bool)
err := yaml.UnmarshalStrict(Config, &fileConfig)
if err != nil {
return errors.Wrap(err, "Cannot parse FileAcquisition configuration")
}
f.logger.Tracef("FileAcquisition configuration: %+v", fileConfig)
if len(fileConfig.Filename) != 0 {
fileConfig.Filenames = append(fileConfig.Filenames, fileConfig.Filename)
}
if len(fileConfig.Filenames) == 0 {
return fmt.Errorf("no filename or filenames configuration provided")
}
f.config = fileConfig
if f.config.Mode == "" {
f.config.Mode = configuration.TAIL_MODE
}
if f.config.Mode != configuration.CAT_MODE && f.config.Mode != configuration.TAIL_MODE {
return fmt.Errorf("unsupported mode %s for file source", f.config.Mode)
}
f.watcher, err = fsnotify.NewWatcher()
if err != nil {
return errors.Wrapf(err, "Could not create fsnotify watcher")
}
f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config)
for _, pattern := range f.config.Filenames {
if f.config.ForceInotify {
directory := path.Dir(pattern)
f.logger.Infof("Force add watch on %s", directory)
if !f.watchedDirectories[directory] {
err = f.watcher.Add(directory)
if err != nil {
f.logger.Errorf("Could not create watch on directory %s : %s", directory, err)
continue
}
f.watchedDirectories[directory] = true
}
}
files, err := filepath.Glob(pattern)
if err != nil {
return errors.Wrap(err, "Glob failure")
}
if len(files) == 0 {
f.logger.Warnf("No matching files for pattern %s", pattern)
continue
}
for _, file := range files {
if files[0] != pattern && f.config.Mode == configuration.TAIL_MODE { //we have a glob pattern
directory := path.Dir(file)
if !f.watchedDirectories[directory] {
err = f.watcher.Add(directory)
if err != nil {
f.logger.Errorf("Could not create watch on directory %s : %s", directory, err)
continue
}
f.watchedDirectories[directory] = true
}
}
f.logger.Infof("Adding file %s to datasources", file)
f.files = append(f.files, file)
}
}
return nil
}
func (f *FileSource) ConfigureByDSN(dsn string, labelType string, logger *log.Entry) error {
if !strings.HasPrefix(dsn, "file://") {
return fmt.Errorf("invalid DSN %s for file source, must start with file://", dsn)
}
f.logger = logger
dsn = strings.TrimPrefix(dsn, "file://")
args := strings.Split(dsn, "?")
if len(args[0]) == 0 {
return fmt.Errorf("empty file:// DSN")
}
if len(args) == 2 && len(args[1]) != 0 {
params, err := url.ParseQuery(args[1])
if err != nil {
return fmt.Errorf("could not parse file args : %s", err)
}
for key, value := range params {
if key != "log_level" {
return fmt.Errorf("unsupported key %s in file DSN", key)
}
if len(value) != 1 {
return fmt.Errorf("expected zero or one value for 'log_level'")
}
lvl, err := log.ParseLevel(value[0])
if err != nil {
return errors.Wrapf(err, "unknown level %s", value[0])
}
f.logger.Logger.SetLevel(lvl)
}
}
f.config = FileConfiguration{}
f.config.Labels = map[string]string{"type": labelType}
f.config.Mode = configuration.CAT_MODE
f.logger.Debugf("Will try pattern %s", args[0])
files, err := filepath.Glob(args[0])
if err != nil {
return errors.Wrap(err, "Glob failure")
}
if len(files) == 0 {
return fmt.Errorf("no matching files for pattern %s", args[0])
}
if len(files) > 1 {
f.logger.Infof("Will read %d files", len(files))
}
for _, file := range files {
f.logger.Infof("Adding file %s to filelist", file)
f.files = append(f.files, file)
}
return nil
}
func (f *FileSource) GetMode() string {
return f.config.Mode
}
//SupportedModes returns the supported modes by the acquisition module
func (f *FileSource) SupportedModes() []string {
return []string{configuration.TAIL_MODE, configuration.CAT_MODE}
}
//OneShotAcquisition reads a set of file and returns when done
func (f *FileSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error {
f.logger.Debug("In oneshot")
for _, file := range f.files {
fi, err := os.Stat(file)
if err != nil {
return fmt.Errorf("could not stat file %s : %w", file, err)
}
if fi.IsDir() {
f.logger.Warnf("%s is a directory, ignoring it.", file)
continue
}
f.logger.Infof("reading %s at once", file)
err = f.readFile(file, out, t)
if err != nil {
return err
}
}
return nil
}
func (f *FileSource) GetMetrics() []prometheus.Collector {
return []prometheus.Collector{linesRead}
}
func (f *FileSource) GetAggregMetrics() []prometheus.Collector {
return []prometheus.Collector{linesRead}
}
func (f *FileSource) GetName() string {
return "file"
}
func (f *FileSource) CanRun() error {
return nil
}
func (f *FileSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
f.logger.Debug("Starting live acquisition")
t.Go(func() error {
return f.monitorNewFiles(out, t)
})
for _, file := range f.files {
err := unix.Access(file, unix.R_OK)
if err != nil {
f.logger.Errorf("unable to read %s : %s", file, err)
continue
}
fi, err := os.Stat(file)
if err != nil {
return fmt.Errorf("could not stat file %s : %w", file, err)
}
if fi.IsDir() {
f.logger.Warnf("%s is a directory, ignoring it.", file)
continue
}
tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: true, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}})
if err != nil {
f.logger.Errorf("Could not start tailing file %s : %s", file, err)
continue
}
f.tails[file] = true
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/file/live/fsnotify")
return f.tailFile(out, t, tail)
})
}
return nil
}
func (f *FileSource) Dump() interface{} {
return f
}
func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error {
logger := f.logger.WithField("goroutine", "inotify")
for {
select {
case event, ok := <-f.watcher.Events:
if !ok {
return nil
}
if event.Op&fsnotify.Create == fsnotify.Create {
fi, err := os.Stat(event.Name)
if err != nil {
logger.Errorf("Could not stat() new file %s, ignoring it : %s", event.Name, err)
continue
}
if fi.IsDir() {
continue
}
logger.Debugf("Detected new file %s", event.Name)
matched := false
for _, pattern := range f.config.Filenames {
logger.Debugf("Matching %s with %s", pattern, event.Name)
matched, err = path.Match(pattern, event.Name)
if err != nil {
logger.Errorf("Could not match pattern : %s", err)
continue
}
if matched {
break
}
}
if !matched {
continue
}
if f.tails[event.Name] {
//we already have a tail on it, do not start a new one
logger.Debugf("Already tailing file %s, not creating a new tail", event.Name)
break
}
err = unix.Access(event.Name, unix.R_OK)
if err != nil {
logger.Errorf("unable to read %s : %s", event.Name, err)
continue
}
//Slightly different parameters for Location, as we want to read the first lines of the newly created file
tail, err := tail.TailFile(event.Name, tail.Config{ReOpen: true, Follow: true, Poll: true, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}})
if err != nil {
logger.Errorf("Could not start tailing file %s : %s", event.Name, err)
break
}
f.tails[event.Name] = true
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/tailfile")
return f.tailFile(out, t, tail)
})
}
case err, ok := <-f.watcher.Errors:
if !ok {
return nil
}
logger.Errorf("Error while monitoring folder: %s", err)
case <-t.Dying():
err := f.watcher.Close()
if err != nil {
return errors.Wrapf(err, "could not remove all inotify watches")
}
return nil
}
}
}
func (f *FileSource) tailFile(out chan types.Event, t *tomb.Tomb, tail *tail.Tail) error {
logger := f.logger.WithField("tail", tail.Filename)
logger.Debugf("-> Starting tail of %s", tail.Filename)
for {
l := types.Line{}
select {
case <-t.Dying():
logger.Infof("File datasource %s stopping", tail.Filename)
if err := tail.Stop(); err != nil {
f.logger.Errorf("error in stop : %s", err)
return err
}
return nil
case <-tail.Tomb.Dying(): //our tailer is dying
logger.Warningf("File reader of %s died", tail.Filename)
t.Kill(fmt.Errorf("dead reader for %s", tail.Filename))
return fmt.Errorf("reader for %s is dead", tail.Filename)
case line := <-tail.Lines:
if line == nil {
logger.Debugf("Nil line")
return fmt.Errorf("tail for %s is empty", tail.Filename)
}
if line.Err != nil {
logger.Warningf("fetch error : %v", line.Err)
return line.Err
}
if line.Text == "" { //skip empty lines
continue
}
linesRead.With(prometheus.Labels{"source": tail.Filename}).Inc()
l.Raw = line.Text
l.Labels = f.config.Labels
l.Time = line.Time
l.Src = tail.Filename
l.Process = true
l.Module = f.GetName()
//we're tailing, it must be real time logs
logger.Debugf("pushing %+v", l)
out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.LIVE}
}
}
}
func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tomb) error {
var scanner *bufio.Scanner
logger := f.logger.WithField("oneshot", filename)
fd, err := os.Open(filename)
if err != nil {
return errors.Wrapf(err, "failed opening %s", filename)
}
defer fd.Close()
if strings.HasSuffix(filename, ".gz") {
gz, err := gzip.NewReader(fd)
if err != nil {
logger.Errorf("Failed to read gz file: %s", err)
return errors.Wrapf(err, "failed to read gz %s", filename)
}
defer gz.Close()
scanner = bufio.NewScanner(gz)
} else {
scanner = bufio.NewScanner(fd)
}
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
logger.Debugf("line %s", scanner.Text())
l := types.Line{}
l.Raw = scanner.Text()
l.Time = time.Now()
l.Src = filename
l.Labels = f.config.Labels
l.Process = true
l.Module = f.GetName()
linesRead.With(prometheus.Labels{"source": filename}).Inc()
//we're reading logs at once, it must be time-machine buckets
out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.TIMEMACHINE}
}
t.Kill(nil)
return nil
}

403
pkg/acquisition/modules/file/file_test.go Normal file
View file

@ -0,0 +1,403 @@
package fileacquisition
import (
"fmt"
"os"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"gopkg.in/tomb.v2"
)
func TestBadConfiguration(t *testing.T) {
tests := []struct {
config string
expectedErr string
}{
{
config: `foobar: asd.log`,
expectedErr: "line 1: field foobar not found in type fileacquisition.FileConfiguration",
},
{
config: `mode: tail`,
expectedErr: "no filename or filenames configuration provided",
},
{
config: `filename: "[asd-.log"`,
expectedErr: "Glob failure: syntax error in pattern",
},
}
subLogger := log.WithFields(log.Fields{
"type": "file",
})
for _, test := range tests {
f := FileSource{}
err := f.Configure([]byte(test.config), subLogger)
assert.Contains(t, err.Error(), test.expectedErr)
}
}
func TestConfigureDSN(t *testing.T) {
tests := []struct {
dsn string
expectedErr string
}{
{
dsn: "asd://",
expectedErr: "invalid DSN asd:// for file source, must start with file://",
},
{
dsn: "file://",
expectedErr: "empty file:// DSN",
},
{
dsn: "file:///etc/passwd?log_level=warn",
expectedErr: "",
},
{
dsn: "file:///etc/passwd?log_level=foobar",
expectedErr: "unknown level foobar: not a valid logrus Level:",
},
}
subLogger := log.WithFields(log.Fields{
"type": "file",
})
for _, test := range tests {
f := FileSource{}
err := f.ConfigureByDSN(test.dsn, "testtype", subLogger)
if test.expectedErr != "" {
assert.Contains(t, err.Error(), test.expectedErr)
} else {
assert.Equal(t, err, nil)
}
}
}
func TestOneShot(t *testing.T) {
tests := []struct {
config string
expectedErr string
expectedOutput string
expectedLines int
logLevel log.Level
setup func()
afterConfigure func()
teardown func()
}{
{
config: `
mode: cat
filename: /etc/shadow`,
expectedErr: "failed opening /etc/shadow: open /etc/shadow: permission denied",
expectedOutput: "",
logLevel: log.WarnLevel,
expectedLines: 0,
},
{
config: `
mode: cat
filename: /`,
expectedErr: "",
expectedOutput: "/ is a directory, ignoring it",
logLevel: log.WarnLevel,
expectedLines: 0,
},
{
config: `
mode: cat
filename: "[*-.log"`,
expectedErr: "Glob failure: syntax error in pattern",
expectedOutput: "",
logLevel: log.WarnLevel,
expectedLines: 0,
},
{
config: `
mode: cat
filename: /do/not/exist`,
expectedErr: "",
expectedOutput: "No matching files for pattern /do/not/exist",
logLevel: log.WarnLevel,
expectedLines: 0,
},
{
config: `
mode: cat
filename: test_files/test.log`,
expectedErr: "",
expectedOutput: "",
expectedLines: 5,
logLevel: log.WarnLevel,
},
{
config: `
mode: cat
filename: test_files/test.log.gz`,
expectedErr: "",
expectedOutput: "",
expectedLines: 5,
logLevel: log.WarnLevel,
},
{
config: `
mode: cat
filename: test_files/bad.gz`,
expectedErr: "failed to read gz test_files/bad.gz: unexpected EOF",
expectedOutput: "",
expectedLines: 0,
logLevel: log.WarnLevel,
},
{
config: `
mode: cat
filename: test_files/test_delete.log`,
setup: func() {
os.Create("test_files/test_delete.log")
},
afterConfigure: func() {
os.Remove("test_files/test_delete.log")
},
expectedErr: "could not stat file test_files/test_delete.log : stat test_files/test_delete.log: no such file or directory",
},
}
for _, ts := range tests {
logger, hook := test.NewNullLogger()
logger.SetLevel(ts.logLevel)
subLogger := logger.WithFields(log.Fields{
"type": "file",
})
tomb := tomb.Tomb{}
out := make(chan types.Event)
f := FileSource{}
if ts.setup != nil {
ts.setup()
}
err := f.Configure([]byte(ts.config), subLogger)
if err != nil && ts.expectedErr != "" {
assert.Contains(t, err.Error(), ts.expectedErr)
continue
} else if err != nil && ts.expectedErr == "" {
t.Fatalf("Unexpected error : %s", err)
}
if ts.afterConfigure != nil {
ts.afterConfigure()
}
actualLines := 0
if ts.expectedLines != 0 {
go func() {
READLOOP:
for {
select {
case <-out:
actualLines++
case <-time.After(1 * time.Second):
break READLOOP
}
}
}()
}
err = f.OneShotAcquisition(out, &tomb)
if ts.expectedLines != 0 {
assert.Equal(t, actualLines, ts.expectedLines)
}
if ts.expectedErr != "" {
if err == nil {
t.Fatalf("Expected error but got nothing ! %+v", ts)
}
assert.Contains(t, err.Error(), ts.expectedErr)
}
if ts.expectedOutput != "" {
assert.Contains(t, hook.LastEntry().Message, ts.expectedOutput)
hook.Reset()
}
if ts.teardown != nil {
ts.teardown()
}
}
}
func TestLiveAcquisition(t *testing.T) {
tests := []struct {
config string
expectedErr string
expectedOutput string
expectedLines int
logLevel log.Level
setup func()
afterConfigure func()
teardown func()
}{
{
config: `
mode: tail
filename: /etc/shadow`,
expectedErr: "",
expectedOutput: "unable to read /etc/shadow : permission denied",
logLevel: log.InfoLevel,
expectedLines: 0,
},
{
config: `
mode: tail
filename: /`,
expectedErr: "",
expectedOutput: "/ is a directory, ignoring it",
logLevel: log.WarnLevel,
expectedLines: 0,
},
{
config: `
mode: tail
filename: /do/not/exist`,
expectedErr: "",
expectedOutput: "No matching files for pattern /do/not/exist",
logLevel: log.WarnLevel,
expectedLines: 0,
},
{
config: `
mode: tail
filenames:
- test_files/*.log
force_inotify: true`,
expectedErr: "",
expectedOutput: "",
expectedLines: 5,
logLevel: log.DebugLevel,
},
{
config: `
mode: tail
filenames:
- test_files/*.log
force_inotify: true`,
expectedErr: "",
expectedOutput: "",
expectedLines: 0,
logLevel: log.DebugLevel,
afterConfigure: func() {
os.Create("test_files/a.log")
os.Remove("test_files/a.log")
},
},
{
config: `
mode: tail
filenames:
- test_files/*.log
force_inotify: true`,
expectedErr: "",
expectedOutput: "",
expectedLines: 5,
logLevel: log.DebugLevel,
afterConfigure: func() {
os.Create("test_files/a.log")
time.Sleep(1 * time.Second)
os.Chmod("test_files/a.log", 0000)
},
teardown: func() {
os.Chmod("test_files/a.log", 0644)
os.Remove("test_files/a.log")
},
},
{
config: `
mode: tail
filenames:
- test_files/*.log
force_inotify: true`,
expectedErr: "",
expectedOutput: "",
expectedLines: 5,
logLevel: log.DebugLevel,
afterConfigure: func() {
os.Mkdir("test_files/pouet/", 0700)
},
teardown: func() {
os.Remove("test_files/pouet/")
},
},
}
for _, ts := range tests {
logger, hook := test.NewNullLogger()
logger.SetLevel(ts.logLevel)
subLogger := logger.WithFields(log.Fields{
"type": "file",
})
tomb := tomb.Tomb{}
out := make(chan types.Event)
f := FileSource{}
if ts.setup != nil {
ts.setup()
}
err := f.Configure([]byte(ts.config), subLogger)
if err != nil {
t.Fatalf("Unexpected error : %s", err)
}
if ts.afterConfigure != nil {
ts.afterConfigure()
}
actualLines := 0
if ts.expectedLines != 0 {
go func() {
READLOOP:
for {
select {
case <-out:
actualLines++
case <-time.After(2 * time.Second):
break READLOOP
}
}
}()
}
err = f.StreamingAcquisition(out, &tomb)
if ts.expectedErr != "" {
if err == nil {
t.Fatalf("Expected error but got nothing ! %+v", ts)
}
assert.Contains(t, err.Error(), ts.expectedErr)
}
if ts.expectedLines != 0 {
fd, err := os.Create("test_files/stream.log")
if err != nil {
t.Fatalf("could not create test file : %s", err)
}
for i := 0; i < 5; i++ {
_, err = fd.WriteString(fmt.Sprintf("%d\n", i))
if err != nil {
t.Fatalf("could not write test file : %s", err)
os.Remove("test_files/stream.log")
}
}
fd.Close()
//we sleep to make sure we detect the new file
time.Sleep(1 * time.Second)
os.Remove("test_files/stream.log")
assert.Equal(t, actualLines, ts.expectedLines)
}
if ts.expectedOutput != "" {
if hook.LastEntry() == nil {
t.Fatalf("expected output %s, but got nothing", ts.expectedOutput)
}
assert.Contains(t, hook.LastEntry().Message, ts.expectedOutput)
hook.Reset()
}
if ts.teardown != nil {
ts.teardown()
}
tomb.Kill(nil)
}
}

1
pkg/acquisition/modules/file/test_files/bad.gz Normal file
View file

@ -0,0 +1 @@
42

5
pkg/acquisition/modules/file/test_files/test.log Normal file
View file

@ -0,0 +1,5 @@
1
2
3
4
5

BIN
pkg/acquisition/modules/file/test_files/test.log.gz Normal file
View file

Binary file not shown.

256
pkg/acquisition/modules/journalctl/journalctl.go Normal file
View file

@ -0,0 +1,256 @@
package journalctlacquisition
import (
"bufio"
"context"
"fmt"
"net/url"
"os/exec"
"strings"
"time"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
)
type JournalCtlConfiguration struct {
configuration.DataSourceCommonCfg `yaml:",inline"`
Filters []string `yaml:"journalctl_filter"`
}
type JournalCtlSource struct {
config JournalCtlConfiguration
logger *log.Entry
src string
args []string
}
const journalctlCmd string = "journalctl"
var (
journalctlArgsOneShot = []string{}
journalctlArgstreaming = []string{"--follow", "-n", "0"}
)
var linesRead = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_journalctlsource_hits_total",
Help: "Total lines that were read.",
},
[]string{"source"})
func readLine(scanner *bufio.Scanner, out chan string, errChan chan error) error {
for scanner.Scan() {
txt := scanner.Text()
out <- txt
}
if errChan != nil && scanner.Err() != nil {
errChan <- scanner.Err()
close(errChan)
return scanner.Err()
}
if errChan != nil {
close(errChan)
}
return nil
}
func (j *JournalCtlSource) runJournalCtl(out chan types.Event, t *tomb.Tomb) error {
ctx, cancel := context.WithCancel(context.Background())
cmd := exec.CommandContext(ctx, journalctlCmd, j.args...)
stdout, err := cmd.StdoutPipe()
if err != nil {
cancel()
return fmt.Errorf("could not get journalctl stdout: %s", err)
}
stderr, err := cmd.StderrPipe()
if err != nil {
cancel()
return fmt.Errorf("could not get journalctl stderr: %s", err)
}
stderrChan := make(chan string)
stdoutChan := make(chan string)
errChan := make(chan error)
logger := j.logger.WithField("src", j.src)
logger.Infof("Running journalctl command: %s %s", cmd.Path, cmd.Args)
err = cmd.Start()
if err != nil {
cancel()
logger.Errorf("could not start journalctl command : %s", err)
return err
}
stdoutscanner := bufio.NewScanner(stdout)
if stdoutscanner == nil {
cancel()
cmd.Wait()
return fmt.Errorf("failed to create stdout scanner")
}
stderrScanner := bufio.NewScanner(stderr)
if stderrScanner == nil {
cancel()
cmd.Wait()
return fmt.Errorf("failed to create stderr scanner")
}
t.Go(func() error {
return readLine(stdoutscanner, stdoutChan, errChan)
})
t.Go(func() error {
//looks like journalctl closes stderr quite early, so ignore its status (but not its output)
return readLine(stderrScanner, stderrChan, nil)
})
for {
select {
case <-t.Dying():
logger.Infof("journalctl datasource %s stopping", j.src)
cancel()
cmd.Wait() //avoid zombie process
return nil
case stdoutLine := <-stdoutChan:
l := types.Line{}
l.Raw = stdoutLine
logger.Debugf("getting one line : %s", l.Raw)
l.Labels = j.config.Labels
l.Time = time.Now()
l.Src = j.src
l.Process = true
l.Module = j.GetName()
linesRead.With(prometheus.Labels{"source": j.src}).Inc()
evt := types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.LIVE}
out <- evt
case stderrLine := <-stderrChan:
logger.Warnf("Got stderr message : %s", stderrLine)
err := fmt.Errorf("journalctl error : %s", stderrLine)
t.Kill(err)
case errScanner, ok := <-errChan:
if !ok {
logger.Debugf("errChan is closed, quitting")
t.Kill(nil)
}
if errScanner != nil {
t.Kill(errScanner)
}
}
}
}
func (j *JournalCtlSource) GetMetrics() []prometheus.Collector {
return []prometheus.Collector{linesRead}
}
func (j *JournalCtlSource) GetAggregMetrics() []prometheus.Collector {
return []prometheus.Collector{linesRead}
}
func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry) error {
config := JournalCtlConfiguration{}
j.logger = logger
err := yaml.UnmarshalStrict(yamlConfig, &config)
if err != nil {
return errors.Wrap(err, "Cannot parse JournalCtlSource configuration")
}
if config.Mode == "" {
config.Mode = configuration.TAIL_MODE
}
var args []string
if config.Mode == configuration.TAIL_MODE {
args = journalctlArgstreaming
} else {
args = journalctlArgsOneShot
}
if len(config.Filters) == 0 {
return fmt.Errorf("journalctl_filter is required")
}
j.args = append(args, config.Filters...)
j.src = fmt.Sprintf("journalctl-%s", strings.Join(config.Filters, "."))
j.config = config
return nil
}
func (j *JournalCtlSource) ConfigureByDSN(dsn string, labelType string, logger *log.Entry) error {
j.logger = logger
j.config = JournalCtlConfiguration{}
j.config.Mode = configuration.CAT_MODE
j.config.Labels = map[string]string{"type": labelType}
//format for the DSN is : journalctl://filters=FILTER1&filters=FILTER2
if !strings.HasPrefix(dsn, "journalctl://") {
return fmt.Errorf("invalid DSN %s for journalctl source, must start with journalctl://", dsn)
}
qs := strings.TrimPrefix(dsn, "journalctl://")
if len(qs) == 0 {
return fmt.Errorf("empty journalctl:// DSN")
}
params, err := url.ParseQuery(qs)
if err != nil {
return fmt.Errorf("could not parse journalctl DSN : %s", err)
}
for key, value := range params {
switch key {
case "filters":
j.config.Filters = append(j.config.Filters, value...)
case "log_level":
if len(value) != 1 {
return fmt.Errorf("expected zero or one value for 'log_level'")
}
lvl, err := log.ParseLevel(value[0])
if err != nil {
return errors.Wrapf(err, "unknown level %s", value[0])
}
j.logger.Logger.SetLevel(lvl)
default:
return fmt.Errorf("unsupported key %s in journalctl DSN", key)
}
}
j.args = append(j.args, j.config.Filters...)
return nil
}
func (j *JournalCtlSource) GetMode() string {
return j.config.Mode
}
func (j *JournalCtlSource) GetName() string {
return "journalctl"
}
func (j *JournalCtlSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error {
defer types.CatchPanic("crowdsec/acquis/journalctl/oneshot")
err := j.runJournalCtl(out, t)
j.logger.Debug("Oneshot journalctl acquisition is done")
return err
}
func (j *JournalCtlSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/journalctl/streaming")
return j.runJournalCtl(out, t)
})
return nil
}
func (j *JournalCtlSource) CanRun() error {
//TODO: add a more precise check on version or something ?
_, err := exec.LookPath(journalctlCmd)
return err
}
func (j *JournalCtlSource) Dump() interface{} {
return j
}

283
pkg/acquisition/modules/journalctl/journalctl_test.go Normal file
View file

@ -0,0 +1,283 @@
package journalctlacquisition
import (
"os"
"os/exec"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"gopkg.in/tomb.v2"
)
func TestBadConfiguration(t *testing.T) {
tests := []struct {
config string
expectedErr string
}{
{
config: `foobar: asd.log`,
expectedErr: "line 1: field foobar not found in type journalctlacquisition.JournalCtlConfiguration",
},
{
config: `
mode: tail
source: journalctl`,
expectedErr: "journalctl_filter is required",
},
{
config: `
mode: cat
source: journalctl
journalctl_filter:
- _UID=42`,
expectedErr: "",
},
}
subLogger := log.WithFields(log.Fields{
"type": "journalctl",
})
for _, test := range tests {
f := JournalCtlSource{}
err := f.Configure([]byte(test.config), subLogger)
if test.expectedErr != "" && err == nil {
t.Fatalf("Expected err %s but got nil !", test.expectedErr)
}
if test.expectedErr != "" {
assert.Contains(t, err.Error(), test.expectedErr)
}
}
}
func TestConfigureDSN(t *testing.T) {
tests := []struct {
dsn string
expectedErr string
}{
{
dsn: "asd://",
expectedErr: "invalid DSN asd:// for journalctl source, must start with journalctl://",
},
{
dsn: "journalctl://",
expectedErr: "empty journalctl:// DSN",
},
{
dsn: "journalctl://foobar=42",
expectedErr: "unsupported key foobar in journalctl DSN",
},
{
dsn: "journalctl://filters=%ZZ",
expectedErr: "could not parse journalctl DSN : invalid URL escape \"%ZZ\"",
},
{
dsn: "journalctl://filters=_UID=42?log_level=warn",
expectedErr: "",
},
{
dsn: "journalctl://filters=_UID=1000&log_level=foobar",
expectedErr: "unknown level foobar: not a valid logrus Level:",
},
}
subLogger := log.WithFields(log.Fields{
"type": "journalctl",
})
for _, test := range tests {
f := JournalCtlSource{}
err := f.ConfigureByDSN(test.dsn, "testtype", subLogger)
if test.expectedErr != "" {
assert.Contains(t, err.Error(), test.expectedErr)
} else {
assert.Equal(t, err, nil)
}
}
}
func TestOneShot(t *testing.T) {
tests := []struct {
config string
expectedErr string
expectedOutput string
expectedLines int
logLevel log.Level
}{
{
config: `
source: journalctl
mode: cat
journalctl_filter:
- "-_UID=42"`,
expectedErr: "",
expectedOutput: "journalctl: invalid option",
logLevel: log.WarnLevel,
expectedLines: 0,
},
{
config: `
source: journalctl
mode: cat
journalctl_filter:
- _SYSTEMD_UNIT=ssh.service`,
expectedErr: "",
expectedOutput: "",
logLevel: log.WarnLevel,
expectedLines: 14,
},
}
for _, ts := range tests {
var logger *log.Logger
var subLogger *log.Entry
var hook *test.Hook
if ts.expectedOutput != "" {
logger, hook = test.NewNullLogger()
logger.SetLevel(ts.logLevel)
subLogger = logger.WithFields(log.Fields{
"type": "journalctl",
})
} else {
subLogger = log.WithFields(log.Fields{
"type": "journalctl",
})
}
tomb := tomb.Tomb{}
out := make(chan types.Event)
j := JournalCtlSource{}
err := j.Configure([]byte(ts.config), subLogger)
if err != nil {
t.Fatalf("Unexpected error : %s", err)
}
actualLines := 0
if ts.expectedLines != 0 {
go func() {
READLOOP:
for {
select {
case <-out:
actualLines++
case <-time.After(1 * time.Second):
break READLOOP
}
}
}()
}
err = j.OneShotAcquisition(out, &tomb)
if ts.expectedErr == "" && err != nil {
t.Fatalf("Unexpected error : %s", err)
} else if ts.expectedErr != "" && err != nil {
assert.Contains(t, err.Error(), ts.expectedErr)
continue
} else if ts.expectedErr != "" && err == nil {
t.Fatalf("Expected error %s, but got nothing !", ts.expectedErr)
}
if ts.expectedLines != 0 {
assert.Equal(t, ts.expectedLines, actualLines)
}
if ts.expectedOutput != "" {
if hook.LastEntry() == nil {
t.Fatalf("Expected log output '%s' but got nothing !", ts.expectedOutput)
}
assert.Contains(t, hook.LastEntry().Message, ts.expectedOutput)
hook.Reset()
}
}
}
func TestStreaming(t *testing.T) {
tests := []struct {
config string
expectedErr string
expectedOutput string
expectedLines int
logLevel log.Level
}{
{
config: `
source: journalctl
mode: cat
journalctl_filter:
- _SYSTEMD_UNIT=ssh.service`,
expectedErr: "",
expectedOutput: "",
logLevel: log.WarnLevel,
expectedLines: 14,
},
}
for _, ts := range tests {
var logger *log.Logger
var subLogger *log.Entry
var hook *test.Hook
if ts.expectedOutput != "" {
logger, hook = test.NewNullLogger()
logger.SetLevel(ts.logLevel)
subLogger = logger.WithFields(log.Fields{
"type": "journalctl",
})
} else {
subLogger = log.WithFields(log.Fields{
"type": "journalctl",
})
}
tomb := tomb.Tomb{}
out := make(chan types.Event)
j := JournalCtlSource{}
err := j.Configure([]byte(ts.config), subLogger)
if err != nil {
t.Fatalf("Unexpected error : %s", err)
}
actualLines := 0
if ts.expectedLines != 0 {
go func() {
READLOOP:
for {
select {
case <-out:
actualLines++
case <-time.After(1 * time.Second):
break READLOOP
}
}
}()
}
err = j.StreamingAcquisition(out, &tomb)
if ts.expectedErr == "" && err != nil {
t.Fatalf("Unexpected error : %s", err)
} else if ts.expectedErr != "" && err != nil {
assert.Contains(t, err.Error(), ts.expectedErr)
continue
} else if ts.expectedErr != "" && err == nil {
t.Fatalf("Expected error %s, but got nothing !", ts.expectedErr)
}
if ts.expectedLines != 0 {
time.Sleep(1 * time.Second)
assert.Equal(t, ts.expectedLines, actualLines)
}
tomb.Kill(nil)
tomb.Wait()
output, _ := exec.Command("pgrep", "-x", "journalctl").CombinedOutput()
if string(output) != "" {
t.Fatalf("Found a journalctl process after killing the tomb !")
}
if ts.expectedOutput != "" {
if hook.LastEntry() == nil {
t.Fatalf("Expected log output '%s' but got nothing !", ts.expectedOutput)
}
assert.Contains(t, hook.LastEntry().Message, ts.expectedOutput)
hook.Reset()
}
}
}
func TestMain(m *testing.M) {
if os.Getenv("USE_SYSTEM_JOURNALCTL") == "" {
os.Setenv("PATH", "./test_files"+":"+os.Getenv("PATH"))
}
os.Exit(m.Run())
}

45
pkg/acquisition/modules/journalctl/test_files/journalctl Executable file
View file

@ -0,0 +1,45 @@
#!/usr/bin/env python3
import argparse
import time
import sys
class CustomParser(argparse.ArgumentParser):
#small hack to make argparse errors the same as journalctl
def error(self, message):
if 'unrecognized arguments:' in message:
sys.stderr.write("journalctl: invalid option -- '_'\n")
sys.stderr.flush()
exit(1)
else:
sys.stderr.write(message)
sys.stderr.flush()
exit(1)
LOGS = """-- Logs begin at Fri 2019-07-26 17:13:13 CEST, end at Mon 2020-11-23 09:17:34 CET. --
Nov 22 11:22:19 zeroed sshd[1480]: Invalid user wqeqwe from 127.0.0.1 port 55818
Nov 22 11:22:23 zeroed sshd[1480]: Failed password for invalid user wqeqwe from 127.0.0.1 port 55818 ssh2
Nov 22 11:23:22 zeroed sshd[1769]: Invalid user wqeqwe1 from 127.0.0.1 port 55824
Nov 22 11:23:24 zeroed sshd[1769]: Disconnecting invalid user wqeqwe1 127.0.0.1 port 55824: Too many authentication failures [preauth]
Nov 22 11:23:24 zeroed sshd[1777]: Invalid user wqeqwe2 from 127.0.0.1 port 55826
Nov 22 11:23:25 zeroed sshd[1777]: Disconnecting invalid user wqeqwe2 127.0.0.1 port 55826: Too many authentication failures [preauth]
Nov 22 11:23:25 zeroed sshd[1780]: Invalid user wqeqwe3 from 127.0.0.1 port 55828
Nov 22 11:23:26 zeroed sshd[1780]: Disconnecting invalid user wqeqwe3 127.0.0.1 port 55828: Too many authentication failures [preauth]
Nov 22 11:23:26 zeroed sshd[1786]: Invalid user wqeqwe4 from 127.0.0.1 port 55830
Nov 22 11:23:27 zeroed sshd[1786]: Failed password for invalid user wqeqwe4 from 127.0.0.1 port 55830 ssh2
Nov 22 11:23:27 zeroed sshd[1786]: Disconnecting invalid user wqeqwe4 127.0.0.1 port 55830: Too many authentication failures [preauth]
Nov 22 11:23:27 zeroed sshd[1791]: Invalid user wqeqwe5 from 127.0.0.1 port 55834
Nov 22 11:23:27 zeroed sshd[1791]: Failed password for invalid user wqeqwe5 from 127.0.0.1 port 55834 ssh2"""
parser = CustomParser()
parser.add_argument('filter', metavar='FILTER', type=str, nargs='?')
parser.add_argument('-n', dest='n', type=int)
parser.add_argument('--follow', dest='follow', action='store_true', default=False)
args = parser.parse_args()
for line in LOGS.split('\n'):
print(line)
if args.follow:
time.sleep(9999)

95
pkg/acquisition/modules/syslog/internal/syslogserver.go Normal file
View file

@ -0,0 +1,95 @@
package syslogserver
import (
"fmt"
"net"
"strings"
"time"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
)
type SyslogServer struct {
listenAddr string
port int
channel chan SyslogMessage
udpConn *net.UDPConn
Logger *log.Entry
MaxMessageLen int
}
type SyslogMessage struct {
Message []byte
Client string
}
func (s *SyslogServer) Listen(listenAddr string, port int) error {
s.listenAddr = listenAddr
s.port = port
udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", s.listenAddr, s.port))
if err != nil {
return errors.Wrapf(err, "could not resolve addr %s", s.listenAddr)
}
udpConn, err := net.ListenUDP("udp", udpAddr)
if err != nil {
return errors.Wrapf(err, "could not listen on port %d", s.port)
}
s.udpConn = udpConn
err = s.udpConn.SetReadBuffer(s.MaxMessageLen) // FIXME probably
if err != nil {
return errors.Wrap(err, "could not set readbuffer on UDP socket")
}
err = s.udpConn.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
if err != nil {
return errors.Wrap(err, "could not set read deadline on UDP socket")
}
return nil
}
func (s *SyslogServer) SetChannel(c chan SyslogMessage) {
s.channel = c
}
func (s *SyslogServer) StartServer() *tomb.Tomb {
t := tomb.Tomb{}
t.Go(func() error {
for {
select {
case <-t.Dying():
s.Logger.Info("syslog server tomb is dying")
err := s.KillServer()
return err
default:
//RFC3164 says 1024 bytes max
//RFC5424 says 480 bytes minimum, and should support up to 2048 bytes
b := make([]byte, s.MaxMessageLen)
n, addr, err := s.udpConn.ReadFrom(b)
if err != nil && !strings.Contains(err.Error(), "i/o timeout") {
s.Logger.Errorf("error while reading from socket : %s", err)
s.udpConn.Close()
return err
}
if err == nil {
s.channel <- SyslogMessage{Message: b[:n], Client: strings.Split(addr.String(), ":")[0]}
}
err = s.udpConn.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
if err != nil {
return err
}
}
}
})
return &t
}
func (s *SyslogServer) KillServer() error {
err := s.udpConn.Close()
if err != nil {
return errors.Wrap(err, "could not close UDP connection")
}
return nil
}

237
pkg/acquisition/modules/syslog/syslog.go Normal file
View file

@ -0,0 +1,237 @@
package syslogacquisition
import (
"fmt"
"net"
"strconv"
"time"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
syslogserver "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/influxdata/go-syslog/v3/rfc3164"
"github.com/influxdata/go-syslog/v3/rfc5424"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
)
type SyslogConfiguration struct {
Proto string `yaml:"protocol,omitempty"`
Port int `yaml:"listen_port,omitempty"`
Addr string `yaml:"listen_addr,omitempty"`
MaxMessageLen int `yaml:"max_message_len,omitempty"`
configuration.DataSourceCommonCfg `yaml:",inline"`
}
type SyslogSource struct {
config SyslogConfiguration
logger *log.Entry
server *syslogserver.SyslogServer
serverTomb *tomb.Tomb
}
var linesReceived = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_syslogsource_hits_total",
Help: "Total lines that were received.",
},
[]string{"source"})
var linesParsed = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_syslogsource_parsed_total",
Help: "Total lines that were successfully parsed",
},
[]string{"source", "type"})
func (s *SyslogSource) GetName() string {
return "syslog"
}
func (s *SyslogSource) GetMode() string {
return s.config.Mode
}
func (s *SyslogSource) Dump() interface{} {
return s
}
func (s *SyslogSource) CanRun() error {
return nil
}
func (s *SyslogSource) GetMetrics() []prometheus.Collector {
return []prometheus.Collector{linesReceived, linesParsed}
}
func (s *SyslogSource) GetAggregMetrics() []prometheus.Collector {
return []prometheus.Collector{linesReceived, linesParsed}
}
func (s *SyslogSource) ConfigureByDSN(dsn string, labelType string, logger *log.Entry) error {
return fmt.Errorf("syslog datasource does not support one shot acquisition")
}
func (s *SyslogSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error {
return fmt.Errorf("syslog datasource does not support one shot acquisition")
}
func validatePort(port int) bool {
return port > 0 && port <= 65535
}
func validateAddr(addr string) bool {
return net.ParseIP(addr) != nil
}
func (s *SyslogSource) Configure(yamlConfig []byte, logger *log.Entry) error {
s.logger = logger
s.logger.Infof("Starting syslog datasource configuration")
syslogConfig := SyslogConfiguration{}
syslogConfig.Mode = configuration.TAIL_MODE
err := yaml.UnmarshalStrict(yamlConfig, &syslogConfig)
if err != nil {
return errors.Wrap(err, "Cannot parse syslog configuration")
}
if syslogConfig.Addr == "" {
syslogConfig.Addr = "127.0.0.1" //do we want a usable or secure default ?
}
if syslogConfig.Port == 0 {
syslogConfig.Port = 514
}
if syslogConfig.MaxMessageLen == 0 {
syslogConfig.MaxMessageLen = 2048
}
if !validatePort(syslogConfig.Port) {
return fmt.Errorf("invalid port %d", syslogConfig.Port)
}
if !validateAddr(syslogConfig.Addr) {
return fmt.Errorf("invalid listen IP %s", syslogConfig.Addr)
}
s.config = syslogConfig
return nil
}
func (s *SyslogSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
c := make(chan syslogserver.SyslogMessage)
s.server = &syslogserver.SyslogServer{Logger: s.logger.WithField("syslog", "internal"), MaxMessageLen: s.config.MaxMessageLen}
s.server.SetChannel(c)
err := s.server.Listen(s.config.Addr, s.config.Port)
if err != nil {
return errors.Wrap(err, "could not start syslog server")
}
s.serverTomb = s.server.StartServer()
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/syslog/live")
return s.handleSyslogMsg(out, t, c)
})
return nil
}
func (s *SyslogSource) buildLogFromSyslog(ts *time.Time, hostname *string,
appname *string, pid *string, msg *string) (string, error) {
ret := ""
if ts != nil {
ret += ts.Format("Jan 2 15:04:05")
} else {
ret += time.Now().Format("Jan 2 15:04:05")
}
if hostname != nil {
ret += " " + *hostname
} else {
ret += " unknownhost"
}
if appname != nil {
ret += " " + *appname
} else {
return "", errors.Errorf("missing appname field in syslog message")
}
if pid != nil {
/*
!!! ugly hack !!!
Due to a bug in the syslog parser we use (https://github.com/influxdata/go-syslog/issues/31),
the ProcID field will contain garbage if the message as a ] anywhere in it.
Assume that a correctly formated ProcID only contains number, and if this is not the case, set it to an arbitrary value
*/
_, err := strconv.Atoi(*pid)
if err != nil {
ret += "[1]: "
} else {
ret += "[" + *pid + "]: "
}
} else {
ret += ": "
}
if msg != nil {
ret += *msg
} else {
return "", errors.Errorf("missing message field in syslog message")
}
return ret, nil
}
func (s *SyslogSource) handleSyslogMsg(out chan types.Event, t *tomb.Tomb, c chan syslogserver.SyslogMessage) error {
for {
select {
case <-t.Dying():
s.logger.Info("Syslog datasource is dying")
s.serverTomb.Kill(nil)
return s.serverTomb.Wait()
case <-s.serverTomb.Dying():
s.logger.Info("Syslog server is dying, exiting")
return nil
case <-s.serverTomb.Dead():
s.logger.Info("Syslog server has exited")
return nil
case syslogLine := <-c:
var line string
var ts time.Time
logger := s.logger.WithField("client", syslogLine.Client)
linesReceived.With(prometheus.Labels{"source": syslogLine.Client}).Inc()
p := rfc5424.NewParser()
m, err := p.Parse(syslogLine.Message)
if err != nil {
logger.Debugf("could not parse message as RFC5424, falling back to RFC3164 : %s", err)
p = rfc3164.NewParser(rfc3164.WithYear(rfc3164.CurrentYear{}))
m, err = p.Parse(syslogLine.Message)
if err != nil {
logger.Errorf("could not parse message: %s", err)
continue
}
msg := m.(*rfc3164.SyslogMessage)
line, err = s.buildLogFromSyslog(msg.Timestamp, msg.Hostname, msg.Appname, msg.ProcID, msg.Message)
if err != nil {
logger.Error(err)
continue
}
linesParsed.With(prometheus.Labels{"source": syslogLine.Client,
"type": "RFC3164"}).Inc()
} else {
msg := m.(*rfc5424.SyslogMessage)
line, err = s.buildLogFromSyslog(msg.Timestamp, msg.Hostname, msg.Appname, msg.ProcID, msg.Message)
if err != nil {
logger.Error(err)
continue
}
linesParsed.With(prometheus.Labels{"source": syslogLine.Client,
"type": "RFC5424"}).Inc()
}
l := types.Line{}
l.Raw = line
l.Module = s.GetName()
l.Labels = s.config.Labels
l.Time = ts
l.Src = syslogLine.Client
l.Process = true
out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.LIVE}
}
}
}

149
pkg/acquisition/modules/syslog/syslog_test.go Normal file
View file

@ -0,0 +1,149 @@
package syslogacquisition
import (
"fmt"
"net"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"github.com/stretchr/testify/assert"
)
func TestConfigure(t *testing.T) {
tests := []struct {
config string
expectedErr string
}{
{
config: `
foobar: bla
source: syslog`,
expectedErr: "line 2: field foobar not found in type syslogacquisition.SyslogConfiguration",
},
{
config: `source: syslog`,
expectedErr: "",
},
{
config: `
source: syslog
listen_port: asd`,
expectedErr: "cannot unmarshal !!str `asd` into int",
},
{
config: `
source: syslog
listen_port: 424242`,
expectedErr: "invalid port 424242",
},
{
config: `
source: syslog
listen_addr: 10.0.0`,
expectedErr: "invalid listen IP 10.0.0",
},
}
subLogger := log.WithFields(log.Fields{
"type": "syslog",
})
for _, test := range tests {
s := SyslogSource{}
err := s.Configure([]byte(test.config), subLogger)
if test.expectedErr != "" {
if err == nil {
t.Fatalf("Expected error but got nothing : %+v", test)
}
assert.Contains(t, err.Error(), test.expectedErr)
}
}
}
func writeToSyslog(logs []string) {
conn, err := net.Dial("udp", "127.0.0.1:4242")
if err != nil {
fmt.Printf("could not establish connection to syslog server : %s", err)
return
}
for _, log := range logs {
fmt.Fprint(conn, log)
}
}
func TestStreamingAcquisition(t *testing.T) {
tests := []struct {
config string
expectedErr string
logs []string
expectedLines int
}{
{
config: `source: syslog`,
expectedErr: "could not start syslog server: could not listen on port 514: listen udp 127.0.0.1:514: bind: permission denied",
},
{
config: `
source: syslog
listen_port: 4242
listen_addr: 127.0.0.1`,
logs: []string{"foobar", "bla", "pouet"},
},
{
config: `
source: syslog
listen_port: 4242
listen_addr: 127.0.0.1`,
expectedLines: 2,
logs: []string{`<13>1 2021-05-18T11:58:40.828081+02:00 mantis sshd 49340 - [timeQuality isSynced="0" tzKnown="1"] blabla`,
`<13>1 2021-05-18T12:12:37.560695+02:00 mantis sshd 49340 - [timeQuality isSynced="0" tzKnown="1"] blabla2[foobar]`},
},
{
config: `
source: syslog
listen_port: 4242
listen_addr: 127.0.0.1`,
expectedLines: 3,
logs: []string{`<13>May 18 12:37:56 mantis sshd[49340]: blabla2[foobar]`,
`<13>May 18 12:37:56 mantis sshd[49340]: blabla2`,
`<13>May 18 12:37:56 mantis sshd: blabla2`,
`<13>May 18 12:37:56 mantis sshd`},
},
}
for _, ts := range tests {
subLogger := log.WithFields(log.Fields{
"type": "syslog",
})
s := SyslogSource{}
_ = s.Configure([]byte(ts.config), subLogger)
tomb := tomb.Tomb{}
out := make(chan types.Event)
err := s.StreamingAcquisition(out, &tomb)
if ts.expectedErr != "" && err == nil {
t.Fatalf("expected error but got nothing : %+v", ts)
} else if ts.expectedErr == "" && err != nil {
t.Fatalf("unexpected error : %s", err)
} else if ts.expectedErr != "" && err != nil {
assert.Contains(t, err.Error(), ts.expectedErr)
continue
}
actualLines := 0
go writeToSyslog(ts.logs)
READLOOP:
for {
select {
case <-out:
actualLines++
case <-time.After(2 * time.Second):
break READLOOP
}
}
assert.Equal(t, ts.expectedLines, actualLines)
tomb.Kill(nil)
tomb.Wait()
}
}

15
pkg/acquisition/test_files/backward_compat.yaml Normal file
View file

@ -0,0 +1,15 @@
filename: /tmp/test.log
labels:
type: syslog
---
filenames:
- /tmp/test*.log
labels:
type: syslog
---
# to be uncommented when we reimplement back journalctl
# journalctl_filter:
# - "_SYSTEMD_UNIT=ssh.service"
# labels:
# type: syslog
---

5
pkg/acquisition/test_files/bad_filetype.yaml Normal file
View file

@ -0,0 +1,5 @@
type: file
filenames: /tmp/tltlt.log #it should be an array
labels:
type: syslog

4
pkg/acquisition/test_files/bad_source.yaml Normal file
View file

@ -0,0 +1,4 @@
source: does_not_exist
labels:
type: syslog
foobar: toto

1
pkg/acquisition/test_files/badyaml.yaml Normal file
View file

@ -0,0 +1 @@
<aaaa

11
pkg/acquisition/test_files/basic_filemode.yaml Normal file
View file

@ -0,0 +1,11 @@
#type: file
filename: /tmp/test.log
labels:
type: syslog
---
#type: file
filenames:
- /tmp/test*.log
labels:
type: syslog
---

1
pkg/acquisition/test_files/emptyitem.yaml Normal file
View file

@ -0,0 +1 @@

2
pkg/acquisition/test_files/missing_labels.yaml Normal file
View file

@ -0,0 +1,2 @@
type: file
filename: /tmp/test.log

5
pkg/acquisition/tests/acquis_test.yaml
View file

@ -1,5 +0,0 @@
filenames:
- ./tests/test.log
mode: tail
labels:
type: my_test_log

0
pkg/acquisition/tests/badlog.gz
View file

1
pkg/acquisition/tests/test.log
View file

@ -1 +0,0 @@
one log line

BIN
pkg/acquisition/tests/test.log.gz
View file

Binary file not shown.

4
pkg/leakybucket/bucket.go
View file

@ -77,7 +77,7 @@ var BucketsPour = prometheus.NewCounterVec(
Name: "cs_bucket_poured_total",
Help: "Total events were poured in bucket.",
},
[]string{"source", "name"},
[]string{"source", "type", "name"},
)
var BucketsOverflow = prometheus.NewCounterVec(
@ -226,7 +226,7 @@ func LeakRoutine(leaky *Leaky) error {
if leaky.logger.Level >= log.TraceLevel {
leaky.logger.Tracef("Pour event: %s", spew.Sdump(msg))
}
BucketsPour.With(prometheus.Labels{"name": leaky.Name, "source": msg.Line.Src}).Inc()
BucketsPour.With(prometheus.Labels{"name": leaky.Name, "source": msg.Line.Src, "type": msg.Line.Module}).Inc()
leaky.Pour(leaky, msg) // glue for now
//Clear cache on behalf of pour

6
pkg/parser/node.go
View file

@ -147,7 +147,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
}
if n.Name != "" {
NodesHits.With(prometheus.Labels{"source": p.Line.Src, "name": n.Name}).Inc()
NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
}
isWhitelisted := false
hasWhitelist := false
@ -308,14 +308,14 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
//grok or leafs failed, don't process statics
if !NodeState {
if n.Name != "" {
NodesHitsKo.With(prometheus.Labels{"source": p.Line.Src, "name": n.Name}).Inc()
NodesHitsKo.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
}
clog.Debugf("Event leaving node : ko")
return NodeState, nil
}
if n.Name != "" {
NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "name": n.Name}).Inc()
NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
}
/*
Please kill me. this is to apply statics when the node *has* whitelists that successfully matched the node.

6
pkg/parser/runtime.go
View file

@ -195,7 +195,7 @@ var NodesHits = prometheus.NewCounterVec(
Name: "cs_node_hits_total",
Help: "Total events entered node.",
},
[]string{"source", "name"},
[]string{"source", "type", "name"},
)
var NodesHitsOk = prometheus.NewCounterVec(
@ -203,7 +203,7 @@ var NodesHitsOk = prometheus.NewCounterVec(
Name: "cs_node_hits_ok_total",
Help: "Total events successfuly exited node.",
},
[]string{"source", "name"},
[]string{"source", "type", "name"},
)
var NodesHitsKo = prometheus.NewCounterVec(
@ -211,7 +211,7 @@ var NodesHitsKo = prometheus.NewCounterVec(
Name: "cs_node_hits_ko_total",
Help: "Total events unsuccessfuly exited node.",
},
[]string{"source", "name"},
[]string{"source", "type", "name"},
)
func stageidx(stage string, stages []string) int {

1
pkg/types/line.go
View file

@ -8,4 +8,5 @@ type Line struct {
Time time.Time //acquis time
Labels map[string]string `yaml:"Labels,omitempty"`
Process bool
Module string `yaml:"Module,omitempty"`
}

2
scripts/func_tests/tests_post-install_4cold-logs.sh
View file

@ -18,7 +18,7 @@ for i in `seq 1 10` ; do
echo `date '+%b %d %H:%M:%S '`'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424' >> ssh-bf.log
done;
${CROWDSEC} -file ./ssh-bf.log -type syslog -no-api
${CROWDSEC} -dsn "file://./ssh-bf.log" -type syslog -no-api
${CSCLI} decisions list -o=json | ${JQ} '. | length == 1' || fail "expected exactly one decision"
${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].value == "1.1.1.172"' || fail "(exact) expected ban on 1.1.1.172"

6
scripts/func_tests/tests_post-install_5simulation.sh
View file

@ -20,7 +20,7 @@ for i in `seq 1 10` ; do
echo `date '+%b %d %H:%M:%S '`'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.174 port 35424' >> ssh-bf.log
done;
${CROWDSEC} -file ./ssh-bf.log -type syslog -no-api
${CROWDSEC} -dsn file://./ssh-bf.log -type syslog -no-api
${CSCLI} decisions list -o=json | ${JQ} '. | length == 1' || fail "expected exactly one decision"
${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].value == "1.1.1.174"' || fail "(exact) expected ban on 1.1.1.174"
@ -32,7 +32,7 @@ ${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].simulated == false' |
${CSCLI} decisions delete --all
${CSCLI} simulation enable $SCENARIO
${CROWDSEC} -file ./ssh-bf.log -type syslog -no-api
${CROWDSEC} -dsn file://./ssh-bf.log -type syslog -no-api
${CSCLI} decisions list --no-simu -o=json | ${JQ} '. == null' || fail "expected no decision (listing only non-simulated decisions)"
@ -42,6 +42,6 @@ ${CSCLI} decisions delete --all
${CSCLI} simulation disable $SCENARIO
${CSCLI} simulation enable --global
${CROWDSEC} -file ./ssh-bf.log -type syslog -no-api
${CROWDSEC} -dsn file://./ssh-bf.log -type syslog -no-api
${CSCLI} decisions list --no-simu -o=json | ${JQ} '. == null' || fail "expected no decision (listing only non-simulated decisions)"