acquisition: validate datasources before configuration (static checks) (#1841)

* acquisition: validate datasources before configuration (allow static configuration checks)

* remove comment

* import reviser, format

* error wrap
This commit is contained in:
mmetc 2022-11-30 17:36:56 +01:00 committed by GitHub
parent f2528f3e29
commit 4a6a9c4355
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 361 additions and 251 deletions

View file

@ -1,11 +1,17 @@
package acquisition package acquisition
import ( import (
"errors"
"fmt" "fmt"
"io" "io"
"os" "os"
"strings" "strings"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
tomb "gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch" cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch"
dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker" dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker"
@ -17,19 +23,14 @@ import (
wineventlogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/wineventlog" wineventlogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/wineventlog"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
tomb "gopkg.in/tomb.v2"
) )
// The interface each datasource must implement // The interface each datasource must implement
type DataSource interface { type DataSource interface {
GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module
GetAggregMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module (aggregated mode, limits cardinality) GetAggregMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module (aggregated mode, limits cardinality)
Configure([]byte, *log.Entry) error // Configure the datasource UnmarshalConfig([]byte) error // Decode and pre-validate the YAML datasource - anything that can be checked before runtime
Configure([]byte, *log.Entry) error // Complete the YAML datasource configuration and perform runtime checks.
ConfigureByDSN(string, map[string]string, *log.Entry) error // Configure the datasource ConfigureByDSN(string, map[string]string, *log.Entry) error // Configure the datasource
GetMode() string // Get the mode (TAIL, CAT or SERVER) GetMode() string // Get the mode (TAIL, CAT or SERVER)
GetName() string // Get the name of the module GetName() string // Get the name of the module
@ -39,66 +40,37 @@ type DataSource interface {
Dump() interface{} Dump() interface{}
} }
var AcquisitionSources = []struct { var AcquisitionSources = map[string]func() DataSource{
name string "file": func() DataSource { return &fileacquisition.FileSource{} },
iface func() DataSource "journalctl": func() DataSource { return &journalctlacquisition.JournalCtlSource{} },
}{ "cloudwatch": func() DataSource { return &cloudwatchacquisition.CloudwatchSource{} },
{ "syslog": func() DataSource { return &syslogacquisition.SyslogSource{} },
name: "file", "docker": func() DataSource { return &dockeracquisition.DockerSource{} },
iface: func() DataSource { return &fileacquisition.FileSource{} }, "kinesis": func() DataSource { return &kinesisacquisition.KinesisSource{} },
}, "wineventlog": func() DataSource { return &wineventlogacquisition.WinEventLogSource{} },
{ "kafka": func() DataSource { return &kafkaacquisition.KafkaSource{} },
name: "journalctl",
iface: func() DataSource { return &journalctlacquisition.JournalCtlSource{} },
},
{
name: "cloudwatch",
iface: func() DataSource { return &cloudwatchacquisition.CloudwatchSource{} },
},
{
name: "syslog",
iface: func() DataSource { return &syslogacquisition.SyslogSource{} },
},
{
name: "docker",
iface: func() DataSource { return &dockeracquisition.DockerSource{} },
},
{
name: "kinesis",
iface: func() DataSource { return &kinesisacquisition.KinesisSource{} },
},
{
name: "wineventlog",
iface: func() DataSource { return &wineventlogacquisition.WinEventLogSource{} },
},
{
name: "kafka",
iface: func() DataSource { return &kafkaacquisition.KafkaSource{} },
},
} }
func GetDataSourceIface(dataSourceType string) DataSource { func GetDataSourceIface(dataSourceType string) DataSource {
for _, source := range AcquisitionSources { source := AcquisitionSources[dataSourceType]
if source.name == dataSourceType { if source == nil {
return source.iface() return nil
}
} }
return nil return source()
} }
func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) { func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
// we dump it back to []byte, because we want to decode the yaml blob twice:
//we dump it back to []byte, because we want to decode the yaml blob twice : // once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
//once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
yamlConfig, err := yaml.Marshal(commonConfig) yamlConfig, err := yaml.Marshal(commonConfig)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "unable to marshal back interface") return nil, fmt.Errorf("unable to marshal back interface: %w", err)
} }
if dataSrc := GetDataSourceIface(commonConfig.Source); dataSrc != nil { if dataSrc := GetDataSourceIface(commonConfig.Source); dataSrc != nil {
/* this logger will then be used by the datasource at runtime */ /* this logger will then be used by the datasource at runtime */
clog := log.New() clog := log.New()
if err := types.ConfigureLogger(clog); err != nil { if err := types.ConfigureLogger(clog); err != nil {
return nil, errors.Wrap(err, "while configuring datasource logger") return nil, fmt.Errorf("while configuring datasource logger: %w", err)
} }
if commonConfig.LogLevel != nil { if commonConfig.LogLevel != nil {
clog.SetLevel(*commonConfig.LogLevel) clog.SetLevel(*commonConfig.LogLevel)
@ -112,11 +84,11 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
subLogger := clog.WithFields(customLog) subLogger := clog.WithFields(customLog)
/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */ /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
if err := dataSrc.CanRun(); err != nil { if err := dataSrc.CanRun(); err != nil {
return nil, errors.Wrapf(err, "datasource %s cannot be run", commonConfig.Source) return nil, fmt.Errorf("datasource %s cannot be run: %w", commonConfig.Source, err)
} }
/* configure the actual datasource */ /* configure the actual datasource */
if err := dataSrc.Configure(yamlConfig, subLogger); err != nil { if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
return nil, errors.Wrapf(err, "failed to configure datasource %s", commonConfig.Source) return nil, fmt.Errorf("failed to configure datasource %s: %w", commonConfig.Source, err)
} }
return &dataSrc, nil return &dataSrc, nil
@ -124,9 +96,8 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
return nil, fmt.Errorf("cannot find source %s", commonConfig.Source) return nil, fmt.Errorf("cannot find source %s", commonConfig.Source)
} }
//detectBackwardCompatAcquis : try to magically detect the type for backward compat (type was not mandatory then) // detectBackwardCompatAcquis: try to magically detect the type for backward compat (type was not mandatory then)
func detectBackwardCompatAcquis(sub configuration.DataSourceCommonCfg) string { func detectBackwardCompatAcquis(sub configuration.DataSourceCommonCfg) string {
if _, ok := sub.Config["filename"]; ok { if _, ok := sub.Config["filename"]; ok {
return "file" return "file"
} }
@ -153,14 +124,14 @@ func LoadAcquisitionFromDSN(dsn string, labels map[string]string) ([]DataSource,
/* this logger will then be used by the datasource at runtime */ /* this logger will then be used by the datasource at runtime */
clog := log.New() clog := log.New()
if err := types.ConfigureLogger(clog); err != nil { if err := types.ConfigureLogger(clog); err != nil {
return nil, errors.Wrap(err, "while configuring datasource logger") return nil, fmt.Errorf("while configuring datasource logger: %w", err)
} }
subLogger := clog.WithFields(log.Fields{ subLogger := clog.WithFields(log.Fields{
"type": dsn, "type": dsn,
}) })
err := dataSrc.ConfigureByDSN(dsn, labels, subLogger) err := dataSrc.ConfigureByDSN(dsn, labels, subLogger)
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "while configuration datasource for %s", dsn) return nil, fmt.Errorf("while configuration datasource for %s: %w", dsn, err)
} }
sources = append(sources, dataSrc) sources = append(sources, dataSrc)
return sources, nil return sources, nil
@ -168,7 +139,6 @@ func LoadAcquisitionFromDSN(dsn string, labels map[string]string) ([]DataSource,
// LoadAcquisitionFromFile unmarshals the configuration item and checks its availability // LoadAcquisitionFromFile unmarshals the configuration item and checks its availability
func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, error) { func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, error) {
var sources []DataSource var sources []DataSource
for _, acquisFile := range config.AcquisitionFiles { for _, acquisFile := range config.AcquisitionFiles {
@ -184,8 +154,8 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
var idx int var idx int
err = dec.Decode(&sub) err = dec.Decode(&sub)
if err != nil { if err != nil {
if ! errors.Is(err, io.EOF) { if !errors.Is(err, io.EOF) {
return nil, errors.Wrapf(err, "failed to yaml decode %s", acquisFile) return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err)
} }
log.Tracef("End of yaml file") log.Tracef("End of yaml file")
break break
@ -212,7 +182,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
} }
src, err := DataSourceConfigure(sub) src, err := DataSourceConfigure(sub)
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "while configuring datasource of type %s from %s (position: %d)", sub.Source, acquisFile, idx) return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err)
} }
sources = append(sources, *src) sources = append(sources, *src)
idx += 1 idx += 1
@ -232,9 +202,9 @@ func GetMetrics(sources []DataSource, aggregated bool) error {
for _, metric := range metrics { for _, metric := range metrics {
if err := prometheus.Register(metric); err != nil { if err := prometheus.Register(metric); err != nil {
if _, ok := err.(prometheus.AlreadyRegisteredError); !ok { if _, ok := err.(prometheus.AlreadyRegisteredError); !ok {
return errors.Wrapf(err, "could not register metrics for datasource %s", sources[i].GetName()) return fmt.Errorf("could not register metrics for datasource %s: %w", sources[i].GetName(), err)
} }
//ignore the error // ignore the error
} }
} }

View file

@ -6,7 +6,6 @@ import (
"testing" "testing"
"time" "time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -26,10 +25,19 @@ type MockSource struct {
logger *log.Entry logger *log.Entry
} }
func (f *MockSource) UnmarshalConfig(cfg []byte) error {
err := yaml.UnmarshalStrict(cfg, &f)
if err != nil {
return err
}
return nil
}
func (f *MockSource) Configure(cfg []byte, logger *log.Entry) error { func (f *MockSource) Configure(cfg []byte, logger *log.Entry) error {
f.logger = logger f.logger = logger
if err := yaml.UnmarshalStrict(cfg, &f); err != nil { if err := f.UnmarshalConfig(cfg); err != nil {
return errors.Wrap(err, "while unmarshaling to reader specific config") return err
} }
if f.Mode == "" { if f.Mode == "" {
f.Mode = configuration.CAT_MODE f.Mode = configuration.CAT_MODE
@ -65,24 +73,10 @@ func (f *MockSourceCantRun) GetName() string { return "mock_cant_run" }
// appendMockSource is only used to add mock source for tests // appendMockSource is only used to add mock source for tests
func appendMockSource() { func appendMockSource() {
if GetDataSourceIface("mock") == nil { if GetDataSourceIface("mock") == nil {
mock := struct { AcquisitionSources["mock"] = func() DataSource { return &MockSource{} }
name string
iface func() DataSource
}{
name: "mock",
iface: func() DataSource { return &MockSource{} },
}
AcquisitionSources = append(AcquisitionSources, mock)
} }
if GetDataSourceIface("mock_cant_run") == nil { if GetDataSourceIface("mock_cant_run") == nil {
mock := struct { AcquisitionSources["mock_cant_run"] = func() DataSource { return &MockSourceCantRun{} }
name string
iface func() DataSource
}{
name: "mock_cant_run",
iface: func() DataSource { return &MockSourceCantRun{} },
}
AcquisitionSources = append(AcquisitionSources, mock)
} }
} }
@ -313,8 +307,10 @@ func (f *MockCat) Configure(cfg []byte, logger *log.Entry) error {
} }
return nil return nil
} }
func (f *MockCat) GetName() string { return "mock_cat" }
func (f *MockCat) GetMode() string { return "cat" } func (f *MockCat) UnmarshalConfig(cfg []byte) error { return nil }
func (f *MockCat) GetName() string { return "mock_cat" }
func (f *MockCat) GetMode() string { return "cat" }
func (f *MockCat) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error { func (f *MockCat) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error {
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
evt := types.Event{} evt := types.Event{}
@ -351,8 +347,10 @@ func (f *MockTail) Configure(cfg []byte, logger *log.Entry) error {
} }
return nil return nil
} }
func (f *MockTail) GetName() string { return "mock_tail" }
func (f *MockTail) GetMode() string { return "tail" } func (f *MockTail) UnmarshalConfig(cfg []byte) error { return nil }
func (f *MockTail) GetName() string { return "mock_tail" }
func (f *MockTail) GetMode() string { return "tail" }
func (f *MockTail) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error { func (f *MockTail) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error {
return fmt.Errorf("can't run in cat mode") return fmt.Errorf("can't run in cat mode")
} }
@ -482,6 +480,7 @@ type MockSourceByDSN struct {
logger *log.Entry //nolint: unused logger *log.Entry //nolint: unused
} }
func (f *MockSourceByDSN) UnmarshalConfig(cfg []byte) error { return nil }
func (f *MockSourceByDSN) Configure(cfg []byte, logger *log.Entry) error { return nil } func (f *MockSourceByDSN) Configure(cfg []byte, logger *log.Entry) error { return nil }
func (f *MockSourceByDSN) GetMode() string { return f.Mode } func (f *MockSourceByDSN) GetMode() string { return f.Mode }
func (f *MockSourceByDSN) OneShotAcquisition(chan types.Event, *tomb.Tomb) error { return nil } func (f *MockSourceByDSN) OneShotAcquisition(chan types.Event, *tomb.Tomb) error { return nil }
@ -524,14 +523,7 @@ func TestConfigureByDSN(t *testing.T) {
} }
if GetDataSourceIface("mockdsn") == nil { if GetDataSourceIface("mockdsn") == nil {
mock := struct { AcquisitionSources["mockdsn"] = func() DataSource { return &MockSourceByDSN{} }
name string
iface func() DataSource
}{
name: "mockdsn",
iface: func() DataSource { return &MockSourceByDSN{} },
}
AcquisitionSources = append(AcquisitionSources, mock)
} }
for _, tc := range tests { for _, tc := range tests {

View file

@ -12,17 +12,17 @@ import (
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/aws/session"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/aws/aws-sdk-go/service/cloudwatchlogs"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2" "gopkg.in/tomb.v2"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
) )
var openedStreams = prometheus.NewGaugeVec( var openedStreams = prometheus.NewGaugeVec(
@ -101,61 +101,81 @@ var (
def_AwsConfigDir = "" def_AwsConfigDir = ""
) )
func (cw *CloudwatchSource) Configure(cfg []byte, logger *log.Entry) error { func (cw *CloudwatchSource) UnmarshalConfig(yamlConfig []byte) error {
cwConfig := CloudwatchSourceConfiguration{} cw.Config = CloudwatchSourceConfiguration{}
targetStream := "*" if err := yaml.UnmarshalStrict(yamlConfig, &cw.Config); err != nil {
if err := yaml.UnmarshalStrict(cfg, &cwConfig); err != nil { return fmt.Errorf("cannot parse CloudwatchSource configuration: %w", err)
return errors.Wrap(err, "Cannot parse CloudwatchSource configuration")
} }
cw.Config = cwConfig
if len(cw.Config.GroupName) == 0 { if len(cw.Config.GroupName) == 0 {
return fmt.Errorf("group_name is mandatory for CloudwatchSource") return fmt.Errorf("group_name is mandatory for CloudwatchSource")
} }
cw.logger = logger.WithField("group", cw.Config.GroupName)
if cw.Config.Mode == "" { if cw.Config.Mode == "" {
cw.Config.Mode = configuration.TAIL_MODE cw.Config.Mode = configuration.TAIL_MODE
} }
logger.Debugf("Starting configuration for Cloudwatch group %s", cw.Config.GroupName)
if cw.Config.DescribeLogStreamsLimit == nil { if cw.Config.DescribeLogStreamsLimit == nil {
cw.Config.DescribeLogStreamsLimit = &def_DescribeLogStreamsLimit cw.Config.DescribeLogStreamsLimit = &def_DescribeLogStreamsLimit
} }
logger.Tracef("describelogstreams_limit set to %d", *cw.Config.DescribeLogStreamsLimit)
if cw.Config.PollNewStreamInterval == nil { if cw.Config.PollNewStreamInterval == nil {
cw.Config.PollNewStreamInterval = &def_PollNewStreamInterval cw.Config.PollNewStreamInterval = &def_PollNewStreamInterval
} }
logger.Tracef("poll_new_stream_interval set to %v", *cw.Config.PollNewStreamInterval)
if cw.Config.MaxStreamAge == nil { if cw.Config.MaxStreamAge == nil {
cw.Config.MaxStreamAge = &def_MaxStreamAge cw.Config.MaxStreamAge = &def_MaxStreamAge
} }
logger.Tracef("max_stream_age set to %v", *cw.Config.MaxStreamAge)
if cw.Config.PollStreamInterval == nil { if cw.Config.PollStreamInterval == nil {
cw.Config.PollStreamInterval = &def_PollStreamInterval cw.Config.PollStreamInterval = &def_PollStreamInterval
} }
logger.Tracef("poll_stream_interval set to %v", *cw.Config.PollStreamInterval)
if cw.Config.StreamReadTimeout == nil { if cw.Config.StreamReadTimeout == nil {
cw.Config.StreamReadTimeout = &def_StreamReadTimeout cw.Config.StreamReadTimeout = &def_StreamReadTimeout
} }
logger.Tracef("stream_read_timeout set to %v", *cw.Config.StreamReadTimeout)
if cw.Config.GetLogEventsPagesLimit == nil { if cw.Config.GetLogEventsPagesLimit == nil {
cw.Config.GetLogEventsPagesLimit = &def_GetLogEventsPagesLimit cw.Config.GetLogEventsPagesLimit = &def_GetLogEventsPagesLimit
} }
logger.Tracef("getlogeventspages_limit set to %v", *cw.Config.GetLogEventsPagesLimit)
if cw.Config.AwsApiCallTimeout == nil { if cw.Config.AwsApiCallTimeout == nil {
cw.Config.AwsApiCallTimeout = &def_AwsApiCallTimeout cw.Config.AwsApiCallTimeout = &def_AwsApiCallTimeout
} }
logger.Tracef("aws_api_timeout set to %v", *cw.Config.AwsApiCallTimeout)
if *cw.Config.MaxStreamAge > *cw.Config.StreamReadTimeout {
logger.Warningf("max_stream_age > stream_read_timeout, stream might keep being opened/closed")
}
if cw.Config.AwsConfigDir == nil { if cw.Config.AwsConfigDir == nil {
cw.Config.AwsConfigDir = &def_AwsConfigDir cw.Config.AwsConfigDir = &def_AwsConfigDir
} }
logger.Tracef("aws_config_dir set to %s", *cw.Config.AwsConfigDir)
return nil
}
func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry) error {
err := cw.UnmarshalConfig(yamlConfig)
if err != nil {
return err
}
cw.logger = logger.WithField("group", cw.Config.GroupName)
cw.logger.Debugf("Starting configuration for Cloudwatch group %s", cw.Config.GroupName)
cw.logger.Tracef("describelogstreams_limit set to %d", *cw.Config.DescribeLogStreamsLimit)
cw.logger.Tracef("poll_new_stream_interval set to %v", *cw.Config.PollNewStreamInterval)
cw.logger.Tracef("max_stream_age set to %v", *cw.Config.MaxStreamAge)
cw.logger.Tracef("poll_stream_interval set to %v", *cw.Config.PollStreamInterval)
cw.logger.Tracef("stream_read_timeout set to %v", *cw.Config.StreamReadTimeout)
cw.logger.Tracef("getlogeventspages_limit set to %v", *cw.Config.GetLogEventsPagesLimit)
cw.logger.Tracef("aws_api_timeout set to %v", *cw.Config.AwsApiCallTimeout)
if *cw.Config.MaxStreamAge > *cw.Config.StreamReadTimeout {
cw.logger.Warningf("max_stream_age > stream_read_timeout, stream might keep being opened/closed")
}
cw.logger.Tracef("aws_config_dir set to %s", *cw.Config.AwsConfigDir)
if *cw.Config.AwsConfigDir != "" { if *cw.Config.AwsConfigDir != "" {
_, err := os.Stat(*cw.Config.AwsConfigDir) _, err := os.Stat(*cw.Config.AwsConfigDir)
if err != nil { if err != nil {
logger.Errorf("can't read aws_config_dir '%s' got err %s", *cw.Config.AwsConfigDir, err) cw.logger.Errorf("can't read aws_config_dir '%s' got err %s", *cw.Config.AwsConfigDir, err)
return fmt.Errorf("can't read aws_config_dir %s got err %s ", *cw.Config.AwsConfigDir, err) return fmt.Errorf("can't read aws_config_dir %s got err %s ", *cw.Config.AwsConfigDir, err)
} }
os.Setenv("AWS_SDK_LOAD_CONFIG", "1") os.Setenv("AWS_SDK_LOAD_CONFIG", "1")
@ -164,7 +184,7 @@ func (cw *CloudwatchSource) Configure(cfg []byte, logger *log.Entry) error {
os.Setenv("AWS_SHARED_CREDENTIALS_FILE", fmt.Sprintf("%s/credentials", *cw.Config.AwsConfigDir)) os.Setenv("AWS_SHARED_CREDENTIALS_FILE", fmt.Sprintf("%s/credentials", *cw.Config.AwsConfigDir))
} else { } else {
if cw.Config.AwsRegion == nil { if cw.Config.AwsRegion == nil {
logger.Errorf("aws_region is not specified, specify it or aws_config_dir") cw.logger.Errorf("aws_region is not specified, specify it or aws_config_dir")
return fmt.Errorf("aws_region is not specified, specify it or aws_config_dir") return fmt.Errorf("aws_region is not specified, specify it or aws_config_dir")
} }
os.Setenv("AWS_REGION", *cw.Config.AwsRegion) os.Setenv("AWS_REGION", *cw.Config.AwsRegion)
@ -174,6 +194,8 @@ func (cw *CloudwatchSource) Configure(cfg []byte, logger *log.Entry) error {
return err return err
} }
cw.streamIndexes = make(map[string]string) cw.streamIndexes = make(map[string]string)
targetStream := "*"
if cw.Config.StreamRegexp != nil { if cw.Config.StreamRegexp != nil {
if _, err := regexp.Compile(*cw.Config.StreamRegexp); err != nil { if _, err := regexp.Compile(*cw.Config.StreamRegexp); err != nil {
return errors.Wrapf(err, "error while compiling regexp '%s'", *cw.Config.StreamRegexp) return errors.Wrapf(err, "error while compiling regexp '%s'", *cw.Config.StreamRegexp)
@ -183,7 +205,7 @@ func (cw *CloudwatchSource) Configure(cfg []byte, logger *log.Entry) error {
targetStream = *cw.Config.StreamName targetStream = *cw.Config.StreamName
} }
logger.Infof("Adding cloudwatch group '%s' (stream:%s) to datasources", cw.Config.GroupName, targetStream) cw.logger.Infof("Adding cloudwatch group '%s' (stream:%s) to datasources", cw.Config.GroupName, targetStream)
return nil return nil
} }

View file

@ -10,18 +10,19 @@ import (
"strings" "strings"
"time" "time"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/dlog"
dockerTypes "github.com/docker/docker/api/types" dockerTypes "github.com/docker/docker/api/types"
"github.com/docker/docker/client" "github.com/docker/docker/client"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2" "gopkg.in/tomb.v2"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"github.com/crowdsecurity/dlog"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
) )
var linesRead = prometheus.NewCounterVec( var linesRead = prometheus.NewCounterVec(
@ -67,24 +68,22 @@ type ContainerConfig struct {
Tty bool Tty bool
} }
func (d *DockerSource) Configure(Config []byte, logger *log.Entry) error { func (d *DockerSource) UnmarshalConfig(yamlConfig []byte) error {
var err error
d.Config = DockerConfiguration{ d.Config = DockerConfiguration{
FollowStdout: true, // default FollowStdout: true, // default
FollowStdErr: true, // default FollowStdErr: true, // default
CheckInterval: "1s", // default CheckInterval: "1s", // default
} }
d.logger = logger
d.runningContainerState = make(map[string]*ContainerConfig) err := yaml.UnmarshalStrict(yamlConfig, &d.Config)
err = yaml.UnmarshalStrict(Config, &d.Config)
if err != nil { if err != nil {
return errors.Wrap(err, "Cannot parse DockerAcquisition configuration") return errors.Wrap(err, "Cannot parse DockerAcquisition configuration")
} }
d.logger.Tracef("DockerAcquisition configuration: %+v", d.Config) if d.logger != nil {
d.logger.Tracef("DockerAcquisition configuration: %+v", d.Config)
}
if len(d.Config.ContainerName) == 0 && len(d.Config.ContainerID) == 0 && len(d.Config.ContainerIDRegexp) == 0 && len(d.Config.ContainerNameRegexp) == 0 { if len(d.Config.ContainerName) == 0 && len(d.Config.ContainerID) == 0 && len(d.Config.ContainerIDRegexp) == 0 && len(d.Config.ContainerNameRegexp) == 0 {
return fmt.Errorf("no containers names or containers ID configuration provided") return fmt.Errorf("no containers names or containers ID configuration provided")
} }
@ -100,7 +99,6 @@ func (d *DockerSource) Configure(Config []byte, logger *log.Entry) error {
if d.Config.Mode != configuration.CAT_MODE && d.Config.Mode != configuration.TAIL_MODE { if d.Config.Mode != configuration.CAT_MODE && d.Config.Mode != configuration.TAIL_MODE {
return fmt.Errorf("unsupported mode %s for docker datasource", d.Config.Mode) return fmt.Errorf("unsupported mode %s for docker datasource", d.Config.Mode)
} }
d.logger.Tracef("Actual DockerAcquisition configuration %+v", d.Config)
for _, cont := range d.Config.ContainerNameRegexp { for _, cont := range d.Config.ContainerNameRegexp {
d.compiledContainerName = append(d.compiledContainerName, regexp.MustCompile(cont)) d.compiledContainerName = append(d.compiledContainerName, regexp.MustCompile(cont))
@ -110,11 +108,6 @@ func (d *DockerSource) Configure(Config []byte, logger *log.Entry) error {
d.compiledContainerID = append(d.compiledContainerID, regexp.MustCompile(cont)) d.compiledContainerID = append(d.compiledContainerID, regexp.MustCompile(cont))
} }
dockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return err
}
if d.Config.Since == "" { if d.Config.Since == "" {
d.Config.Since = time.Now().UTC().Format(time.RFC3339) d.Config.Since = time.Now().UTC().Format(time.RFC3339)
} }
@ -130,17 +123,37 @@ func (d *DockerSource) Configure(Config []byte, logger *log.Entry) error {
d.containerLogsOptions.Until = d.Config.Until d.containerLogsOptions.Until = d.Config.Until
} }
return nil
}
func (d *DockerSource) Configure(yamlConfig []byte, logger *log.Entry) error {
d.logger = logger
err := d.UnmarshalConfig(yamlConfig)
if err != nil {
return err
}
d.runningContainerState = make(map[string]*ContainerConfig)
d.logger.Tracef("Actual DockerAcquisition configuration %+v", d.Config)
dockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return err
}
if d.Config.DockerHost != "" { if d.Config.DockerHost != "" {
if err := client.WithHost(d.Config.DockerHost)(dockerClient); err != nil { err = client.WithHost(d.Config.DockerHost)(dockerClient)
if err != nil {
return err return err
} }
} }
d.Client = dockerClient d.Client = dockerClient
_, err = d.Client.Info(context.Background()) _, err = d.Client.Info(context.Background())
if err != nil { if err != nil {
return errors.Wrapf(err, "failed to configure docker datasource %s", d.Config.DockerHost) return fmt.Errorf("failed to configure docker datasource %s: %w", d.Config.DockerHost, err)
} }
return nil return nil

View file

@ -13,9 +13,6 @@ import (
"strings" "strings"
"time" "time"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/fsnotify/fsnotify" "github.com/fsnotify/fsnotify"
"github.com/nxadm/tail" "github.com/nxadm/tail"
"github.com/pkg/errors" "github.com/pkg/errors"
@ -23,6 +20,10 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2" "gopkg.in/tomb.v2"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
) )
var linesRead = prometheus.NewCounterVec( var linesRead = prometheus.NewCounterVec(
@ -50,41 +51,62 @@ type FileSource struct {
exclude_regexps []*regexp.Regexp exclude_regexps []*regexp.Regexp
} }
func (f *FileSource) Configure(Config []byte, logger *log.Entry) error { func (f *FileSource) UnmarshalConfig(yamlConfig []byte) error {
fileConfig := FileConfiguration{} f.config = FileConfiguration{}
f.logger = logger err := yaml.UnmarshalStrict(yamlConfig, &f.config)
f.watchedDirectories = make(map[string]bool)
f.tails = make(map[string]bool)
err := yaml.UnmarshalStrict(Config, &fileConfig)
if err != nil { if err != nil {
return errors.Wrap(err, "Cannot parse FileAcquisition configuration") return fmt.Errorf("cannot parse FileAcquisition configuration: %w", err)
} }
f.logger.Tracef("FileAcquisition configuration: %+v", fileConfig)
if len(fileConfig.Filename) != 0 { if f.logger != nil {
fileConfig.Filenames = append(fileConfig.Filenames, fileConfig.Filename) f.logger.Tracef("FileAcquisition configuration: %+v", f.config)
} }
if len(fileConfig.Filenames) == 0 {
if len(f.config.Filename) != 0 {
f.config.Filenames = append(f.config.Filenames, f.config.Filename)
}
if len(f.config.Filenames) == 0 {
return fmt.Errorf("no filename or filenames configuration provided") return fmt.Errorf("no filename or filenames configuration provided")
} }
f.config = fileConfig
if f.config.Mode == "" { if f.config.Mode == "" {
f.config.Mode = configuration.TAIL_MODE f.config.Mode = configuration.TAIL_MODE
} }
if f.config.Mode != configuration.CAT_MODE && f.config.Mode != configuration.TAIL_MODE { if f.config.Mode != configuration.CAT_MODE && f.config.Mode != configuration.TAIL_MODE {
return fmt.Errorf("unsupported mode %s for file source", f.config.Mode) return fmt.Errorf("unsupported mode %s for file source", f.config.Mode)
} }
for _, exclude := range f.config.ExcludeRegexps {
re, err := regexp.Compile(exclude)
if err != nil {
return fmt.Errorf("could not compile regexp %s: %w", exclude, err)
}
f.exclude_regexps = append(f.exclude_regexps, re)
}
return nil
}
func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error {
f.logger = logger
err := f.UnmarshalConfig(yamlConfig)
if err != nil {
return err
}
f.watchedDirectories = make(map[string]bool)
f.tails = make(map[string]bool)
f.watcher, err = fsnotify.NewWatcher() f.watcher, err = fsnotify.NewWatcher()
if err != nil { if err != nil {
return errors.Wrapf(err, "Could not create fsnotify watcher") return errors.Wrapf(err, "Could not create fsnotify watcher")
} }
for _, exclude := range f.config.ExcludeRegexps {
re, err := regexp.Compile(exclude)
if err != nil {
return errors.Wrapf(err, "Could not compile regexp %s", exclude)
}
f.exclude_regexps = append(f.exclude_regexps, re)
}
f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config) f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config)
for _, pattern := range f.config.Filenames { for _, pattern := range f.config.Filenames {
if f.config.ForceInotify { if f.config.ForceInotify {
directory := filepath.Dir(pattern) directory := filepath.Dir(pattern)

View file

@ -7,14 +7,15 @@ import (
"testing" "testing"
"time" "time"
fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file"
"github.com/crowdsecurity/crowdsec/pkg/cstest"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test" "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"gopkg.in/tomb.v2" "gopkg.in/tomb.v2"
fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file"
"github.com/crowdsecurity/crowdsec/pkg/cstest"
"github.com/crowdsecurity/crowdsec/pkg/types"
) )
func TestBadConfiguration(t *testing.T) { func TestBadConfiguration(t *testing.T) {
@ -42,7 +43,7 @@ func TestBadConfiguration(t *testing.T) {
name: "bad exclude regexp", name: "bad exclude regexp",
config: `filenames: ["asd.log"] config: `filenames: ["asd.log"]
exclude_regexps: ["as[a-$d"]`, exclude_regexps: ["as[a-$d"]`,
expectedErr: "Could not compile regexp as", expectedErr: "could not compile regexp as",
}, },
} }

View file

@ -9,15 +9,15 @@ import (
"strings" "strings"
"time" "time"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2" "gopkg.in/tomb.v2"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
) )
type JournalCtlConfiguration struct { type JournalCtlConfiguration struct {
@ -163,28 +163,41 @@ func (j *JournalCtlSource) GetAggregMetrics() []prometheus.Collector {
return []prometheus.Collector{linesRead} return []prometheus.Collector{linesRead}
} }
func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry) error { func (j *JournalCtlSource) UnmarshalConfig(yamlConfig []byte) error {
config := JournalCtlConfiguration{} j.config = JournalCtlConfiguration{}
j.logger = logger err := yaml.UnmarshalStrict(yamlConfig, &j.config)
err := yaml.UnmarshalStrict(yamlConfig, &config)
if err != nil { if err != nil {
return errors.Wrap(err, "Cannot parse JournalCtlSource configuration") return fmt.Errorf("cannot parse JournalCtlSource configuration: %w", err)
} }
if config.Mode == "" {
config.Mode = configuration.TAIL_MODE if j.config.Mode == "" {
j.config.Mode = configuration.TAIL_MODE
} }
var args []string var args []string
if config.Mode == configuration.TAIL_MODE { if j.config.Mode == configuration.TAIL_MODE {
args = journalctlArgstreaming args = journalctlArgstreaming
} else { } else {
args = journalctlArgsOneShot args = journalctlArgsOneShot
} }
if len(config.Filters) == 0 {
if len(j.config.Filters) == 0 {
return fmt.Errorf("journalctl_filter is required") return fmt.Errorf("journalctl_filter is required")
} }
j.args = append(args, config.Filters...) j.args = append(args, j.config.Filters...)
j.src = fmt.Sprintf("journalctl-%s", strings.Join(config.Filters, ".")) j.src = fmt.Sprintf("journalctl-%s", strings.Join(j.config.Filters, "."))
j.config = config
return nil
}
func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry) error {
j.logger = logger
err := j.UnmarshalConfig(yamlConfig)
if err != nil {
return err
}
return nil return nil
} }

View file

@ -10,15 +10,16 @@ import (
"strconv" "strconv"
"time" "time"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/segmentio/kafka-go" "github.com/segmentio/kafka-go"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2" "gopkg.in/tomb.v2"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
) )
var ( var (
@ -54,35 +55,51 @@ type KafkaSource struct {
Reader *kafka.Reader Reader *kafka.Reader
} }
func (k *KafkaSource) Configure(Config []byte, logger *log.Entry) error { func (k *KafkaSource) UnmarshalConfig(yamlConfig []byte) error {
var err error
k.Config = KafkaConfiguration{} k.Config = KafkaConfiguration{}
k.logger = logger
err = yaml.UnmarshalStrict(Config, &k.Config) err := yaml.UnmarshalStrict(yamlConfig, &k.Config)
if err != nil { if err != nil {
return errors.Wrapf(err, "cannot parse %s datasource configuration", dataSourceName) return fmt.Errorf("cannot parse %s datasource configuration: %w", dataSourceName, err)
} }
if len(k.Config.Brokers) == 0 { if len(k.Config.Brokers) == 0 {
return fmt.Errorf("cannot create a %s reader with an empty list of broker addresses", dataSourceName) return fmt.Errorf("cannot create a %s reader with an empty list of broker addresses", dataSourceName)
} }
if k.Config.Topic == "" { if k.Config.Topic == "" {
return fmt.Errorf("cannot create a %s reader with am empty topic", dataSourceName) return fmt.Errorf("cannot create a %s reader with am empty topic", dataSourceName)
} }
if k.Config.Mode == "" { if k.Config.Mode == "" {
k.Config.Mode = configuration.TAIL_MODE k.Config.Mode = configuration.TAIL_MODE
} }
return err
}
func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry) error {
k.logger = logger
err := k.UnmarshalConfig(yamlConfig)
if err != nil {
return err
}
dialer, err := k.Config.NewDialer() dialer, err := k.Config.NewDialer()
if err != nil { if err != nil {
return errors.Wrapf(err, "cannot create %s dialer", dataSourceName) return errors.Wrapf(err, "cannot create %s dialer", dataSourceName)
} }
k.Reader, err = k.Config.NewReader(dialer) k.Reader, err = k.Config.NewReader(dialer)
if err != nil { if err != nil {
return errors.Wrapf(err, "cannote create %s reader", dataSourceName) return errors.Wrapf(err, "cannote create %s reader", dataSourceName)
} }
if k.Reader == nil { if k.Reader == nil {
return fmt.Errorf("cannot create %s reader", dataSourceName) return fmt.Errorf("cannot create %s reader", dataSourceName)
} }
return nil return nil
} }

View file

@ -13,14 +13,15 @@ import (
"github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/kinesis" "github.com/aws/aws-sdk-go/service/kinesis"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2" "gopkg.in/tomb.v2"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
) )
type KinesisConfiguration struct { type KinesisConfiguration struct {
@ -113,17 +114,18 @@ func (k *KinesisSource) GetAggregMetrics() []prometheus.Collector {
return []prometheus.Collector{linesRead, linesReadShards} return []prometheus.Collector{linesRead, linesReadShards}
} }
func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry) error { func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error {
config := KinesisConfiguration{} k.Config = KinesisConfiguration{}
k.logger = logger
err := yaml.UnmarshalStrict(yamlConfig, &config) err := yaml.UnmarshalStrict(yamlConfig, &k.Config)
if err != nil { if err != nil {
return errors.Wrap(err, "Cannot parse kinesis datasource configuration") return errors.Wrap(err, "Cannot parse kinesis datasource configuration")
} }
if config.Mode == "" {
config.Mode = configuration.TAIL_MODE if k.Config.Mode == "" {
k.Config.Mode = configuration.TAIL_MODE
} }
k.Config = config
if k.Config.StreamName == "" && !k.Config.UseEnhancedFanOut { if k.Config.StreamName == "" && !k.Config.UseEnhancedFanOut {
return fmt.Errorf("stream_name is mandatory when use_enhanced_fanout is false") return fmt.Errorf("stream_name is mandatory when use_enhanced_fanout is false")
} }
@ -139,10 +141,23 @@ func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry) error {
if k.Config.MaxRetries <= 0 { if k.Config.MaxRetries <= 0 {
k.Config.MaxRetries = 10 k.Config.MaxRetries = 10
} }
return nil
}
func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry) error {
k.logger = logger
err := k.UnmarshalConfig(yamlConfig)
if err != nil {
return err
}
err = k.newClient() err = k.newClient()
if err != nil { if err != nil {
return errors.Wrap(err, "Cannot create kinesis client") return fmt.Errorf("cannot create kinesis client: %w", err)
} }
k.shardReaderTomb = &tomb.Tomb{} k.shardReaderTomb = &tomb.Tomb{}
return nil return nil
} }

View file

@ -6,18 +6,18 @@ import (
"strings" "strings"
"time" "time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/parser/rfc3164" "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/parser/rfc3164"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/parser/rfc5424" "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/parser/rfc5424"
syslogserver "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/server" syslogserver "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/server"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
) )
type SyslogConfiguration struct { type SyslogConfiguration struct {
@ -89,31 +89,43 @@ func validateAddr(addr string) bool {
return net.ParseIP(addr) != nil return net.ParseIP(addr) != nil
} }
func (s *SyslogSource) Configure(yamlConfig []byte, logger *log.Entry) error { func (s *SyslogSource) UnmarshalConfig(yamlConfig []byte) error {
s.logger = logger s.config = SyslogConfiguration{}
s.logger.Infof("Starting syslog datasource configuration") s.config.Mode = configuration.TAIL_MODE
syslogConfig := SyslogConfiguration{}
syslogConfig.Mode = configuration.TAIL_MODE err := yaml.UnmarshalStrict(yamlConfig, &s.config)
err := yaml.UnmarshalStrict(yamlConfig, &syslogConfig)
if err != nil { if err != nil {
return errors.Wrap(err, "Cannot parse syslog configuration") return errors.Wrap(err, "Cannot parse syslog configuration")
} }
if syslogConfig.Addr == "" {
syslogConfig.Addr = "127.0.0.1" //do we want a usable or secure default ? if s.config.Addr == "" {
s.config.Addr = "127.0.0.1" //do we want a usable or secure default ?
} }
if syslogConfig.Port == 0 { if s.config.Port == 0 {
syslogConfig.Port = 514 s.config.Port = 514
} }
if syslogConfig.MaxMessageLen == 0 { if s.config.MaxMessageLen == 0 {
syslogConfig.MaxMessageLen = 2048 s.config.MaxMessageLen = 2048
} }
if !validatePort(syslogConfig.Port) { if !validatePort(s.config.Port) {
return fmt.Errorf("invalid port %d", syslogConfig.Port) return fmt.Errorf("invalid port %d", s.config.Port)
} }
if !validateAddr(syslogConfig.Addr) { if !validateAddr(s.config.Addr) {
return fmt.Errorf("invalid listen IP %s", syslogConfig.Addr) return fmt.Errorf("invalid listen IP %s", s.config.Addr)
} }
s.config = syslogConfig
return nil
}
func (s *SyslogSource) Configure(yamlConfig []byte, logger *log.Entry) error {
s.logger = logger
s.logger.Infof("Starting syslog datasource configuration")
err := s.UnmarshalConfig(yamlConfig)
if err != nil {
return err
}
return nil return nil
} }

View file

@ -5,15 +5,20 @@ package wineventlogacquisition
import ( import (
"errors" "errors"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2" "gopkg.in/tomb.v2"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
) )
type WinEventLogSource struct{} type WinEventLogSource struct{}
func (w *WinEventLogSource) UnmarshalConfig(yamlConfig []byte) error {
return nil
}
func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry) error { func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry) error {
return nil return nil
} }

View file

@ -9,9 +9,6 @@ import (
"syscall" "syscall"
"time" "time"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/google/winops/winlog" "github.com/google/winops/winlog"
"github.com/google/winops/winlog/wevtapi" "github.com/google/winops/winlog/wevtapi"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
@ -19,6 +16,10 @@ import (
"golang.org/x/sys/windows" "golang.org/x/sys/windows"
"gopkg.in/tomb.v2" "gopkg.in/tomb.v2"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
) )
type WinEventLogConfiguration struct { type WinEventLogConfiguration struct {
@ -228,29 +229,26 @@ func (w *WinEventLogSource) generateConfig(query string) (*winlog.SubscribeConfi
return &config, nil return &config, nil
} }
func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry) error { func (w *WinEventLogSource) UnmarshalConfig(yamlConfig []byte) error {
w.config = WinEventLogConfiguration{}
config := WinEventLogConfiguration{}
w.logger = logger
err := yaml.UnmarshalStrict(yamlConfig, &config)
err := yaml.UnmarshalStrict(yamlConfig, &w.config)
if err != nil { if err != nil {
return fmt.Errorf("unable to parse configuration: %v", err) return fmt.Errorf("unable to parse configuration: %v", err)
} }
if config.EventChannel != "" && config.XPathQuery != "" { if w.config.EventChannel != "" && w.config.XPathQuery != "" {
return fmt.Errorf("event_channel and xpath_query are mutually exclusive") return fmt.Errorf("event_channel and xpath_query are mutually exclusive")
} }
if config.EventChannel == "" && config.XPathQuery == "" { if w.config.EventChannel == "" && w.config.XPathQuery == "" {
return fmt.Errorf("event_channel or xpath_query must be set") return fmt.Errorf("event_channel or xpath_query must be set")
} }
config.Mode = configuration.TAIL_MODE w.config.Mode = configuration.TAIL_MODE
w.config = config
if config.XPathQuery != "" { if w.config.XPathQuery != "" {
w.query = config.XPathQuery w.query = w.config.XPathQuery
} else { } else {
w.query, err = w.buildXpathQuery() w.query, err = w.buildXpathQuery()
if err != nil { if err != nil {
@ -258,15 +256,26 @@ func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry) erro
} }
} }
w.evtConfig, err = w.generateConfig(w.query) if w.config.PrettyName != "" {
w.name = w.config.PrettyName
} else {
w.name = w.query
}
return nil
}
func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry) error {
w.logger = logger
err := w.UnmarshalConfig(yamlConfig)
if err != nil { if err != nil {
return err return err
} }
if config.PrettyName != "" { w.evtConfig, err = w.generateConfig(w.query)
w.name = config.PrettyName if err != nil {
} else { return err
w.name = w.query
} }
return nil return nil

View file

@ -1,7 +1,9 @@
package cstest package cstest
import ( import (
"strings"
"testing" "testing"
"text/template"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -28,3 +30,20 @@ func RequireErrorContains(t *testing.T, err error, expectedErr string) {
require.NoError(t, err) require.NoError(t, err)
} }
// Interpolate fills a string template with the given values, can be map or struct.
// example: Interpolate("{{.Name}}", map[string]string{"Name": "JohnDoe"})
func Interpolate(s string, data interface{}) (string, error) {
tmpl, err := template.New("").Parse(s)
if err != nil {
return "", err
}
var b strings.Builder
err = tmpl.Execute(&b, data)
if err != nil {
return "", err
}
return b.String(), nil
}