non-fatal error if some datasource can't be run (i.e. journalctl but systemd is missing) (#2309)
This on the other hand, gives a new fatal error when there are no valid datasources. In the previous version, crowdsec kept running with just a warning if no acquisition yaml or dir were specified.
This commit is contained in:
parent
d26e17f505
commit
a910b7beca
|
@ -24,22 +24,22 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
|
|||
var err error
|
||||
|
||||
// Populate cwhub package tools
|
||||
if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
|
||||
return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
|
||||
if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
|
||||
return nil, fmt.Errorf("while loading hub index: %w", err)
|
||||
}
|
||||
|
||||
// Start loading configs
|
||||
csParsers := parser.NewParsers()
|
||||
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
|
||||
return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
|
||||
return nil, fmt.Errorf("while loading parsers: %w", err)
|
||||
}
|
||||
|
||||
if err := LoadBuckets(cConfig); err != nil {
|
||||
return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
|
||||
return nil, fmt.Errorf("while loading scenarios: %w", err)
|
||||
}
|
||||
|
||||
if err := LoadAcquisition(cConfig); err != nil {
|
||||
return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
|
||||
return nil, fmt.Errorf("while loading acquisition config: %w", err)
|
||||
}
|
||||
return csParsers, nil
|
||||
}
|
||||
|
|
|
@ -119,6 +119,10 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
|
|||
}
|
||||
}
|
||||
|
||||
if len(dataSources) == 0 {
|
||||
return fmt.Errorf("no datasource enabled")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -34,6 +34,20 @@ import (
|
|||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||
)
|
||||
|
||||
type DataSourceUnavailableError struct {
|
||||
Name string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *DataSourceUnavailableError) Error() string {
|
||||
return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err)
|
||||
}
|
||||
|
||||
func (e *DataSourceUnavailableError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
|
||||
// The interface each datasource must implement
|
||||
type DataSource interface {
|
||||
GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module
|
||||
|
@ -73,6 +87,10 @@ func GetDataSourceIface(dataSourceType string) DataSource {
|
|||
return source()
|
||||
}
|
||||
|
||||
// DataSourceConfigure creates and returns a DataSource object from a configuration,
|
||||
// if the configuration is not valid it returns an error.
|
||||
// If the datasource can't be run (eg. journalctl not available), it still returns an error which
|
||||
// can be checked for the appropriate action.
|
||||
func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
|
||||
// we dump it back to []byte, because we want to decode the yaml blob twice:
|
||||
// once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
|
||||
|
@ -98,7 +116,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
|
|||
subLogger := clog.WithFields(customLog)
|
||||
/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
|
||||
if err := dataSrc.CanRun(); err != nil {
|
||||
return nil, fmt.Errorf("datasource %s cannot be run: %w", commonConfig.Source, err)
|
||||
return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
|
||||
}
|
||||
/* configure the actual datasource */
|
||||
if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
|
||||
|
@ -171,10 +189,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
|
|||
}
|
||||
dec := yaml.NewDecoder(yamlFile)
|
||||
dec.SetStrict(true)
|
||||
idx := -1
|
||||
for {
|
||||
var sub configuration.DataSourceCommonCfg
|
||||
var idx int
|
||||
err = dec.Decode(&sub)
|
||||
idx += 1
|
||||
if err != nil {
|
||||
if !errors.Is(err, io.EOF) {
|
||||
return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err)
|
||||
|
@ -191,7 +210,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
|
|||
if len(sub.Labels) == 0 {
|
||||
if sub.Source == "" {
|
||||
log.Debugf("skipping empty item in %s", acquisFile)
|
||||
idx += 1
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
|
||||
|
@ -206,6 +224,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
|
|||
sub.UniqueId = uniqueId
|
||||
src, err := DataSourceConfigure(sub)
|
||||
if err != nil {
|
||||
var dserr *DataSourceUnavailableError
|
||||
if errors.As(err, &dserr) {
|
||||
log.Error(err)
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err)
|
||||
}
|
||||
if sub.TransformExpr != "" {
|
||||
|
@ -216,7 +239,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
|
|||
transformRuntimes[uniqueId] = vm
|
||||
}
|
||||
sources = append(sources, *src)
|
||||
idx += 1
|
||||
}
|
||||
}
|
||||
return sources, nil
|
||||
|
@ -293,6 +315,11 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo
|
|||
}
|
||||
|
||||
func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error {
|
||||
// Don't wait if we have no sources, as it will hang forever
|
||||
if len(sources) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for i := 0; i < len(sources); i++ {
|
||||
subsrc := sources[i] //ensure its a copy
|
||||
log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc)
|
||||
|
@ -328,11 +355,8 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb
|
|||
return nil
|
||||
})
|
||||
}
|
||||
// Don't wait if we have no sources, as it will hang forever
|
||||
if len(sources) > 0 {
|
||||
|
||||
/*return only when acquisition is over (cat) or never (tail)*/
|
||||
err := AcquisTomb.Wait()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -167,7 +167,7 @@ log_level: debug
|
|||
source: mock_cant_run
|
||||
wowo: ajsajasjas
|
||||
`,
|
||||
ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
|
||||
ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro",
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
@ -151,9 +151,10 @@ teardown() {
|
|||
rm -f "$ACQUIS_DIR"
|
||||
|
||||
config_set '.common.log_media="stdout"'
|
||||
rune -124 timeout 2s "${CROWDSEC}"
|
||||
rune -1 timeout 2s "${CROWDSEC}"
|
||||
# check warning
|
||||
assert_stderr_line --partial "no acquisition file found"
|
||||
assert_stderr --partial "no acquisition file found"
|
||||
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||
}
|
||||
|
||||
@test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" {
|
||||
|
@ -166,19 +167,56 @@ teardown() {
|
|||
config_set '.crowdsec_service.acquisition_dir=""'
|
||||
|
||||
config_set '.common.log_media="stdout"'
|
||||
rune -124 timeout 2s "${CROWDSEC}"
|
||||
rune -1 timeout 2s "${CROWDSEC}"
|
||||
# check warning
|
||||
assert_stderr_line --partial "no acquisition_path or acquisition_dir specified"
|
||||
assert_stderr --partial "no acquisition_path or acquisition_dir specified"
|
||||
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||
}
|
||||
|
||||
@test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
|
||||
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
||||
rm -f "$ACQUIS_YAML"
|
||||
config_set '.crowdsec_service.acquisition_path=""'
|
||||
|
||||
ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
|
||||
mkdir -p "$ACQUIS_DIR"
|
||||
touch "$ACQUIS_DIR"/foo.yaml
|
||||
mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
|
||||
|
||||
rune -124 timeout 2s "${CROWDSEC}"
|
||||
|
||||
# now, if foo.yaml is empty instead, there won't be valid datasources.
|
||||
|
||||
cat /dev/null >"$ACQUIS_DIR"/foo.yaml
|
||||
|
||||
rune -1 timeout 2s "${CROWDSEC}"
|
||||
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||
}
|
||||
|
||||
@test "crowdsec (disabled datasources)" {
|
||||
config_set '.common.log_media="stdout"'
|
||||
|
||||
# a datasource cannot run - missing journalctl command
|
||||
|
||||
ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
|
||||
mkdir -p "$ACQUIS_DIR"
|
||||
cat >"$ACQUIS_DIR"/foo.yaml <<-EOT
|
||||
source: journalctl
|
||||
journalctl_filter:
|
||||
- "_SYSTEMD_UNIT=ssh.service"
|
||||
labels:
|
||||
type: syslog
|
||||
EOT
|
||||
|
||||
rune -124 timeout 2s env PATH='' "${CROWDSEC}"
|
||||
#shellcheck disable=SC2016
|
||||
assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
|
||||
|
||||
# if all datasources are disabled, crowdsec should exit
|
||||
|
||||
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
||||
rm -f "$ACQUIS_YAML"
|
||||
config_set '.crowdsec_service.acquisition_path=""'
|
||||
|
||||
rune -1 timeout 2s env PATH='' "${CROWDSEC}"
|
||||
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue