non-fatal error if some datasource can't be run (i.e. journalctl but systemd is missing) (#2309)

This on the other hand, gives a new fatal error when there are no valid datasources.
In the previous version, crowdsec kept running with just a warning if no
acquisition yaml or dir were specified.

cmd/crowdsec/crowdsec.go

@@ -24,22 +24,22 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
 	var err error
 
 	// Populate cwhub package tools
-	if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
+	if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
+		return nil, fmt.Errorf("while loading hub index: %w", err)
 	}
 
 	// Start loading configs
 	csParsers := parser.NewParsers()
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
+		return nil, fmt.Errorf("while loading parsers: %w", err)
 	}
 
 	if err := LoadBuckets(cConfig); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
+		return nil, fmt.Errorf("while loading scenarios: %w", err)
 	}
 
 	if err := LoadAcquisition(cConfig); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
+		return nil, fmt.Errorf("while loading acquisition config: %w", err)
 	}
 	return csParsers, nil
 }

cmd/crowdsec/main.go

@@ -119,6 +119,10 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
 		}
 	}
 
+	if len(dataSources) == 0 {
+		return fmt.Errorf("no datasource enabled")
+	}
+
 	return nil
 }
 

pkg/acquisition/acquisition.go

@@ -34,6 +34,20 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
+type DataSourceUnavailableError struct {
+	Name string
+	Err error
+}
+
+func (e *DataSourceUnavailableError) Error() string {
+	return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err)
+}
+
+func (e *DataSourceUnavailableError) Unwrap() error {
+	return e.Err
+}
+
+
 // The interface each datasource must implement
 type DataSource interface {
 	GetMetrics() []prometheus.Collector                                 // Returns pointers to metrics that are managed by the module
@@ -73,6 +87,10 @@ func GetDataSourceIface(dataSourceType string) DataSource {
 	return source()
 }
 
+// DataSourceConfigure creates and returns a DataSource object from a configuration,
+// if the configuration is not valid it returns an error.
+// If the datasource can't be run (eg. journalctl not available), it still returns an error which
+// can be checked for the appropriate action.
 func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
 	// we dump it back to []byte, because we want to decode the yaml blob twice:
 	// once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
@@ -98,7 +116,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
 		subLogger := clog.WithFields(customLog)
 		/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
 		if err := dataSrc.CanRun(); err != nil {
-			return nil, fmt.Errorf("datasource %s cannot be run: %w", commonConfig.Source, err)
+			return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
 		}
 		/* configure the actual datasource */
 		if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
@@ -171,10 +189,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 		}
 		dec := yaml.NewDecoder(yamlFile)
 		dec.SetStrict(true)
+		idx := -1
 		for {
 			var sub configuration.DataSourceCommonCfg
-			var idx int
 			err = dec.Decode(&sub)
+			idx += 1
 			if err != nil {
 				if !errors.Is(err, io.EOF) {
 					return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err)
@@ -191,7 +210,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 			if len(sub.Labels) == 0 {
 				if sub.Source == "" {
 					log.Debugf("skipping empty item in %s", acquisFile)
-					idx += 1
 					continue
 				}
 				return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
@@ -206,6 +224,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 			sub.UniqueId = uniqueId
 			src, err := DataSourceConfigure(sub)
 			if err != nil {
+				var dserr *DataSourceUnavailableError
+				if errors.As(err, &dserr) {
+					log.Error(err)
+					continue
+				}
 				return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err)
 			}
 			if sub.TransformExpr != "" {
@@ -216,7 +239,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 				transformRuntimes[uniqueId] = vm
 			}
 			sources = append(sources, *src)
-			idx += 1
 		}
 	}
 	return sources, nil
@@ -293,6 +315,11 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo
 }
 
 func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error {
+	// Don't wait if we have no sources, as it will hang forever
+	if len(sources) == 0 {
+		return nil
+	}
+
 	for i := 0; i < len(sources); i++ {
 		subsrc := sources[i] //ensure its a copy
 		log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc)
@@ -328,11 +355,8 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb
 			return nil
 		})
 	}
-	// Don't wait if we have no sources, as it will hang forever
-	if len(sources) > 0 {
-		/*return only when acquisition is over (cat) or never (tail)*/
-		err := AcquisTomb.Wait()
-		return err
-	}
-	return nil
+
+	/*return only when acquisition is over (cat) or never (tail)*/
+	err := AcquisTomb.Wait()
+	return err
 }

pkg/acquisition/acquisition_test.go

@@ -167,7 +167,7 @@ log_level: debug
 source: mock_cant_run
 wowo: ajsajasjas
 `,
-			ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
+			ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro",
 		},
 	}
 

test/bats/01_crowdsec.bats

@@ -151,9 +151,10 @@ teardown() {
     rm -f "$ACQUIS_DIR"
 
     config_set '.common.log_media="stdout"'
-    rune -124 timeout 2s "${CROWDSEC}"
+    rune -1 timeout 2s "${CROWDSEC}"
     # check warning
-    assert_stderr_line --partial "no acquisition file found"
+    assert_stderr --partial "no acquisition file found"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
 @test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" {
@@ -166,19 +167,56 @@ teardown() {
     config_set '.crowdsec_service.acquisition_dir=""'
 
     config_set '.common.log_media="stdout"'
-    rune -124 timeout 2s "${CROWDSEC}"
+    rune -1 timeout 2s "${CROWDSEC}"
     # check warning
-    assert_stderr_line --partial "no acquisition_path or acquisition_dir specified"
+    assert_stderr --partial "no acquisition_path or acquisition_dir specified"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
 @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
     ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
-    rm -f "$ACQUIS_YAML"
     config_set '.crowdsec_service.acquisition_path=""'
 
     ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
     mkdir -p "$ACQUIS_DIR"
-    touch "$ACQUIS_DIR"/foo.yaml
+    mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
 
     rune -124 timeout 2s "${CROWDSEC}"
+
+    # now, if foo.yaml is empty instead, there won't be valid datasources.
+
+    cat /dev/null >"$ACQUIS_DIR"/foo.yaml
+
+    rune -1 timeout 2s "${CROWDSEC}"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
+
+@test "crowdsec (disabled datasources)" {
+    config_set '.common.log_media="stdout"'
+
+    # a datasource cannot run - missing journalctl command
+
+    ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
+    mkdir -p "$ACQUIS_DIR"
+    cat >"$ACQUIS_DIR"/foo.yaml <<-EOT
+	source: journalctl
+	journalctl_filter:
+	 - "_SYSTEMD_UNIT=ssh.service"
+	labels:
+	  type: syslog
+	EOT
+
+    rune -124 timeout 2s env PATH='' "${CROWDSEC}"
+    #shellcheck disable=SC2016
+    assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
+
+    # if all datasources are disabled, crowdsec should exit
+
+    ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
+    rm -f "$ACQUIS_YAML"
+    config_set '.crowdsec_service.acquisition_path=""'
+
+    rune -1 timeout 2s env PATH='' "${CROWDSEC}"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
+}
+