non-fatal error if some datasource can't be run (i.e. journalctl but systemd is missing) (#2309)
This on the other hand, gives a new fatal error when there are no valid datasources. In the previous version, crowdsec kept running with just a warning if no acquisition yaml or dir were specified.
This commit is contained in:
parent
d26e17f505
commit
a910b7beca
|
@ -24,22 +24,22 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
// Populate cwhub package tools
|
// Populate cwhub package tools
|
||||||
if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
|
if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
|
||||||
return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
|
return nil, fmt.Errorf("while loading hub index: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start loading configs
|
// Start loading configs
|
||||||
csParsers := parser.NewParsers()
|
csParsers := parser.NewParsers()
|
||||||
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
|
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
|
||||||
return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
|
return nil, fmt.Errorf("while loading parsers: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := LoadBuckets(cConfig); err != nil {
|
if err := LoadBuckets(cConfig); err != nil {
|
||||||
return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
|
return nil, fmt.Errorf("while loading scenarios: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := LoadAcquisition(cConfig); err != nil {
|
if err := LoadAcquisition(cConfig); err != nil {
|
||||||
return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
|
return nil, fmt.Errorf("while loading acquisition config: %w", err)
|
||||||
}
|
}
|
||||||
return csParsers, nil
|
return csParsers, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -119,6 +119,10 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(dataSources) == 0 {
|
||||||
|
return fmt.Errorf("no datasource enabled")
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -34,6 +34,20 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type DataSourceUnavailableError struct {
|
||||||
|
Name string
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *DataSourceUnavailableError) Error() string {
|
||||||
|
return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *DataSourceUnavailableError) Unwrap() error {
|
||||||
|
return e.Err
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// The interface each datasource must implement
|
// The interface each datasource must implement
|
||||||
type DataSource interface {
|
type DataSource interface {
|
||||||
GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module
|
GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module
|
||||||
|
@ -73,6 +87,10 @@ func GetDataSourceIface(dataSourceType string) DataSource {
|
||||||
return source()
|
return source()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DataSourceConfigure creates and returns a DataSource object from a configuration,
|
||||||
|
// if the configuration is not valid it returns an error.
|
||||||
|
// If the datasource can't be run (eg. journalctl not available), it still returns an error which
|
||||||
|
// can be checked for the appropriate action.
|
||||||
func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
|
func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
|
||||||
// we dump it back to []byte, because we want to decode the yaml blob twice:
|
// we dump it back to []byte, because we want to decode the yaml blob twice:
|
||||||
// once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
|
// once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
|
||||||
|
@ -98,7 +116,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
|
||||||
subLogger := clog.WithFields(customLog)
|
subLogger := clog.WithFields(customLog)
|
||||||
/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
|
/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
|
||||||
if err := dataSrc.CanRun(); err != nil {
|
if err := dataSrc.CanRun(); err != nil {
|
||||||
return nil, fmt.Errorf("datasource %s cannot be run: %w", commonConfig.Source, err)
|
return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
|
||||||
}
|
}
|
||||||
/* configure the actual datasource */
|
/* configure the actual datasource */
|
||||||
if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
|
if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
|
||||||
|
@ -171,10 +189,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
|
||||||
}
|
}
|
||||||
dec := yaml.NewDecoder(yamlFile)
|
dec := yaml.NewDecoder(yamlFile)
|
||||||
dec.SetStrict(true)
|
dec.SetStrict(true)
|
||||||
|
idx := -1
|
||||||
for {
|
for {
|
||||||
var sub configuration.DataSourceCommonCfg
|
var sub configuration.DataSourceCommonCfg
|
||||||
var idx int
|
|
||||||
err = dec.Decode(&sub)
|
err = dec.Decode(&sub)
|
||||||
|
idx += 1
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, io.EOF) {
|
if !errors.Is(err, io.EOF) {
|
||||||
return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err)
|
return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err)
|
||||||
|
@ -191,7 +210,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
|
||||||
if len(sub.Labels) == 0 {
|
if len(sub.Labels) == 0 {
|
||||||
if sub.Source == "" {
|
if sub.Source == "" {
|
||||||
log.Debugf("skipping empty item in %s", acquisFile)
|
log.Debugf("skipping empty item in %s", acquisFile)
|
||||||
idx += 1
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
|
return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
|
||||||
|
@ -206,6 +224,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
|
||||||
sub.UniqueId = uniqueId
|
sub.UniqueId = uniqueId
|
||||||
src, err := DataSourceConfigure(sub)
|
src, err := DataSourceConfigure(sub)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
var dserr *DataSourceUnavailableError
|
||||||
|
if errors.As(err, &dserr) {
|
||||||
|
log.Error(err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err)
|
return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err)
|
||||||
}
|
}
|
||||||
if sub.TransformExpr != "" {
|
if sub.TransformExpr != "" {
|
||||||
|
@ -216,7 +239,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
|
||||||
transformRuntimes[uniqueId] = vm
|
transformRuntimes[uniqueId] = vm
|
||||||
}
|
}
|
||||||
sources = append(sources, *src)
|
sources = append(sources, *src)
|
||||||
idx += 1
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return sources, nil
|
return sources, nil
|
||||||
|
@ -293,6 +315,11 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo
|
||||||
}
|
}
|
||||||
|
|
||||||
func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error {
|
func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error {
|
||||||
|
// Don't wait if we have no sources, as it will hang forever
|
||||||
|
if len(sources) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
for i := 0; i < len(sources); i++ {
|
for i := 0; i < len(sources); i++ {
|
||||||
subsrc := sources[i] //ensure its a copy
|
subsrc := sources[i] //ensure its a copy
|
||||||
log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc)
|
log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc)
|
||||||
|
@ -328,11 +355,8 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
// Don't wait if we have no sources, as it will hang forever
|
|
||||||
if len(sources) > 0 {
|
|
||||||
/*return only when acquisition is over (cat) or never (tail)*/
|
/*return only when acquisition is over (cat) or never (tail)*/
|
||||||
err := AcquisTomb.Wait()
|
err := AcquisTomb.Wait()
|
||||||
return err
|
return err
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -167,7 +167,7 @@ log_level: debug
|
||||||
source: mock_cant_run
|
source: mock_cant_run
|
||||||
wowo: ajsajasjas
|
wowo: ajsajasjas
|
||||||
`,
|
`,
|
||||||
ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
|
ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -151,9 +151,10 @@ teardown() {
|
||||||
rm -f "$ACQUIS_DIR"
|
rm -f "$ACQUIS_DIR"
|
||||||
|
|
||||||
config_set '.common.log_media="stdout"'
|
config_set '.common.log_media="stdout"'
|
||||||
rune -124 timeout 2s "${CROWDSEC}"
|
rune -1 timeout 2s "${CROWDSEC}"
|
||||||
# check warning
|
# check warning
|
||||||
assert_stderr_line --partial "no acquisition file found"
|
assert_stderr --partial "no acquisition file found"
|
||||||
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" {
|
@test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" {
|
||||||
|
@ -166,19 +167,56 @@ teardown() {
|
||||||
config_set '.crowdsec_service.acquisition_dir=""'
|
config_set '.crowdsec_service.acquisition_dir=""'
|
||||||
|
|
||||||
config_set '.common.log_media="stdout"'
|
config_set '.common.log_media="stdout"'
|
||||||
rune -124 timeout 2s "${CROWDSEC}"
|
rune -1 timeout 2s "${CROWDSEC}"
|
||||||
# check warning
|
# check warning
|
||||||
assert_stderr_line --partial "no acquisition_path or acquisition_dir specified"
|
assert_stderr --partial "no acquisition_path or acquisition_dir specified"
|
||||||
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
|
@test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
|
||||||
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
||||||
rm -f "$ACQUIS_YAML"
|
|
||||||
config_set '.crowdsec_service.acquisition_path=""'
|
config_set '.crowdsec_service.acquisition_path=""'
|
||||||
|
|
||||||
ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
|
ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
|
||||||
mkdir -p "$ACQUIS_DIR"
|
mkdir -p "$ACQUIS_DIR"
|
||||||
touch "$ACQUIS_DIR"/foo.yaml
|
mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
|
||||||
|
|
||||||
rune -124 timeout 2s "${CROWDSEC}"
|
rune -124 timeout 2s "${CROWDSEC}"
|
||||||
|
|
||||||
|
# now, if foo.yaml is empty instead, there won't be valid datasources.
|
||||||
|
|
||||||
|
cat /dev/null >"$ACQUIS_DIR"/foo.yaml
|
||||||
|
|
||||||
|
rune -1 timeout 2s "${CROWDSEC}"
|
||||||
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@test "crowdsec (disabled datasources)" {
|
||||||
|
config_set '.common.log_media="stdout"'
|
||||||
|
|
||||||
|
# a datasource cannot run - missing journalctl command
|
||||||
|
|
||||||
|
ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
|
||||||
|
mkdir -p "$ACQUIS_DIR"
|
||||||
|
cat >"$ACQUIS_DIR"/foo.yaml <<-EOT
|
||||||
|
source: journalctl
|
||||||
|
journalctl_filter:
|
||||||
|
- "_SYSTEMD_UNIT=ssh.service"
|
||||||
|
labels:
|
||||||
|
type: syslog
|
||||||
|
EOT
|
||||||
|
|
||||||
|
rune -124 timeout 2s env PATH='' "${CROWDSEC}"
|
||||||
|
#shellcheck disable=SC2016
|
||||||
|
assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
|
||||||
|
|
||||||
|
# if all datasources are disabled, crowdsec should exit
|
||||||
|
|
||||||
|
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
||||||
|
rm -f "$ACQUIS_YAML"
|
||||||
|
config_set '.crowdsec_service.acquisition_path=""'
|
||||||
|
|
||||||
|
rune -1 timeout 2s env PATH='' "${CROWDSEC}"
|
||||||
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue