Refact pkg/csconfig, HubCfg (#2552)

- rename csconfig.Hub -> HubCfg
 - move some Load*() functions to NewConfig()
 - config.yaml: optional common section
 - remove unused working_dir

cmd/crowdsec-cli/config_restore.go

@@ -48,10 +48,6 @@ func silentInstallItem(name string, obtype string) (string, error) {
 func restoreHub(dirPath string) error {
 	var err error
 
-	if err := csConfig.LoadHub(); err != nil {
-		return err
-	}
-
 	cwhub.SetHubBranch()
 
 	for _, itype := range cwhub.ItemTypes {

cmd/crowdsec-cli/hub.go

@@ -88,10 +88,6 @@ func NewHubListCmd() *cobra.Command {
 }
 
 func runHubUpdate(cmd *cobra.Command, args []string) error {
-	if err := csConfig.LoadHub(); err != nil {
-		return err
-	}
-
 	if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
 		if !errors.Is(err, cwhub.ErrIndexNotFound) {
 			return fmt.Errorf("failed to get Hub index : %w", err)

cmd/crowdsec-cli/main.go

@@ -51,10 +51,8 @@ func initConfig() {
 		if err != nil {
 			log.Fatal(err)
 		}
-		if err := csConfig.LoadCSCLI(); err != nil {
-			log.Fatal(err)
-		}
 	} else {
+		// XXX: check all the defaults
 		csConfig = csconfig.NewDefaultConfig()
 	}
 

cmd/crowdsec-cli/require/require.go

@@ -65,10 +65,6 @@ func Notifications(c *csconfig.Config) error {
 }
 
 func Hub (c *csconfig.Config) error {
-	if err := c.LoadHub(); err != nil {
-		return err
-	}
-
 	if c.Hub == nil {
 		return fmt.Errorf("you must configure cli before interacting with hub")
 	}

cmd/crowdsec/main.go

@@ -212,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
 func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
 	cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("while loading configuration file: %w", err)
-	}
-
-	if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
-		return nil, fmt.Errorf("unable to load configuration: common section is empty")
 	}
 
 	cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@@ -228,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		dumpStates = true
 	}
 
-	// Configuration paths are dependency to load crowdsec configuration
-	if err := cConfig.LoadConfigurationPaths(); err != nil {
-		return nil, err
-	}
-
 	if flags.SingleFileType != "" && flags.OneShotDSN != "" {
 		// if we're in time-machine mode, we don't want to log to file
 		cConfig.Common.LogMedia = "stdout"

config/config.yaml

@@ -6,7 +6,6 @@ common:
   log_max_size: 20
   compress_logs: true
   log_max_files: 10
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data/

config/config_win.yaml

@@ -3,7 +3,6 @@ common:
   log_media: file
   log_level: info
   log_dir:  C:\ProgramData\CrowdSec\log\
-  working_dir: .
 config_paths:
   config_dir:  C:\ProgramData\CrowdSec\config\
   data_dir:  C:\ProgramData\CrowdSec\data\

config/config_win_no_lapi.yaml

@@ -3,7 +3,6 @@ common:
   log_media: file
   log_level: info
   log_dir:  C:\ProgramData\CrowdSec\log\
-  working_dir: .
 config_paths:
   config_dir:  C:\ProgramData\CrowdSec\config\
   data_dir:  C:\ProgramData\CrowdSec\data\

config/dev.yaml

@@ -2,7 +2,6 @@ common:
   daemonize: true
   log_media: stdout
   log_level: info
-  working_dir: .
 config_paths:
   config_dir: ./config
   data_dir: ./data/   

config/user.yaml

@@ -3,7 +3,6 @@ common:
   log_media: stdout
   log_level: info
   log_dir: /var/log/
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data

docker/config.yaml

@@ -3,7 +3,6 @@ common:
   log_media: stdout
   log_level: info
   log_dir: /var/log/
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data/

pkg/csconfig/api.go

@@ -286,10 +286,6 @@ func (c *Config) LoadAPIServer() error {
 		log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs))
 	}
 
-	if err := c.LoadCommon(); err != nil {
-		return fmt.Errorf("loading common configuration: %s", err)
-	}
-
 	c.API.Server.LogDir = c.Common.LogDir
 	c.API.Server.LogMedia = c.Common.LogMedia
 	c.API.Server.CompressLogs = c.Common.CompressLogs

pkg/csconfig/api_test.go

@@ -3,7 +3,6 @@ package csconfig
 import (
 	"net"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -142,9 +141,6 @@ func TestLoadAPIServer(t *testing.T) {
 	err := tmpLAPI.LoadProfiles()
 	require.NoError(t, err)
 
-	LogDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
 	logLevel := log.InfoLevel
 	config := &Config{}
 	fcontent, err := os.ReadFile("./testdata/config.yaml")
@@ -179,7 +175,7 @@ func TestLoadAPIServer(t *testing.T) {
 					DbPath: "./testdata/test.db",
 				},
 				Common: &CommonCfg{
-					LogDir:   "./testdata/",
+					LogDir:   "./testdata",
 					LogMedia: "stdout",
 				},
 				DisableAPI: false,
@@ -202,7 +198,7 @@ func TestLoadAPIServer(t *testing.T) {
 					ShareContext:          ptr.Of(false),
 					ConsoleManagement:     ptr.Of(false),
 				},
-				LogDir:   LogDirFullPath,
+				LogDir:   "./testdata",
 				LogMedia: "stdout",
 				OnlineClient: &OnlineApiClientCfg{
 					CredentialsFilePath: "./testdata/online-api-secrets.yaml",

pkg/csconfig/common.go

@@ -14,7 +14,7 @@ type CommonCfg struct {
 	LogMedia       string     `yaml:"log_media"`
 	LogDir         string     `yaml:"log_dir,omitempty"` //if LogMedia = file
 	LogLevel       *log.Level `yaml:"log_level"`
-	WorkingDir     string     `yaml:"working_dir,omitempty"` ///var/run
+	WorkingDir     string     `yaml:"working_dir,omitempty"` // TODO: This is just for backward compat. Remove this later
 	CompressLogs   *bool      `yaml:"compress_logs,omitempty"`
 	LogMaxSize     int        `yaml:"log_max_size,omitempty"`
 	LogMaxAge      int        `yaml:"log_max_age,omitempty"`
@@ -22,15 +22,18 @@ type CommonCfg struct {
 	ForceColorLogs bool       `yaml:"force_color_logs,omitempty"`
 }
 
-func (c *Config) LoadCommon() error {
+func (c *Config) loadCommon() error {
 	var err error
 	if c.Common == nil {
-		return fmt.Errorf("no common block provided in configuration file")
+		c.Common = &CommonCfg{}
+	}
+
+	if c.Common.LogMedia == "" {
+		c.Common.LogMedia = "stdout"
 	}
 
 	var CommonCleanup = []*string{
 		&c.Common.LogDir,
-		&c.Common.WorkingDir,
 	}
 	for _, k := range CommonCleanup {
 		if *k == "" {

pkg/csconfig/common_test.go

@@ -1,83 +0,0 @@
-package csconfig
-
-import (
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/crowdsecurity/go-cs-lib/cstest"
-)
-
-func TestLoadCommon(t *testing.T) {
-	pidDirPath := "./testdata"
-	LogDirFullPath, err := filepath.Abs("./testdata/log/")
-	require.NoError(t, err)
-
-	WorkingDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	tests := []struct {
-		name        string
-		input       *Config
-		expected    *CommonCfg
-		expectedErr string
-	}{
-		{
-			name: "basic valid configuration",
-			input: &Config{
-				Common: &CommonCfg{
-					Daemonize:  true,
-					PidDir:     "./testdata",
-					LogMedia:   "file",
-					LogDir:     "./testdata/log/",
-					WorkingDir: "./testdata/",
-				},
-			},
-			expected: &CommonCfg{
-				Daemonize:  true,
-				PidDir:     pidDirPath,
-				LogMedia:   "file",
-				LogDir:     LogDirFullPath,
-				WorkingDir: WorkingDirFullPath,
-			},
-		},
-		{
-			name: "empty working dir",
-			input: &Config{
-				Common: &CommonCfg{
-					Daemonize: true,
-					PidDir:    "./testdata",
-					LogMedia:  "file",
-					LogDir:    "./testdata/log/",
-				},
-			},
-			expected: &CommonCfg{
-				Daemonize: true,
-				PidDir:    pidDirPath,
-				LogMedia:  "file",
-				LogDir:    LogDirFullPath,
-			},
-		},
-		{
-			name:        "no common",
-			input:       &Config{},
-			expected:    nil,
-			expectedErr: "no common block provided in configuration file",
-		},
-	}
-
-	for _, tc := range tests {
-		tc := tc
-		t.Run(tc.name, func(t *testing.T) {
-			err := tc.input.LoadCommon()
-			cstest.RequireErrorContains(t, err, tc.expectedErr)
-			if tc.expectedErr != "" {
-				return
-			}
-
-			assert.Equal(t, tc.expected, tc.input.Common)
-		})
-	}
-}

pkg/csconfig/config.go

@@ -36,7 +36,7 @@ type Config struct {
 	PluginConfig *PluginCfg          `yaml:"plugin_config,omitempty"`
 	DisableAPI   bool                `yaml:"-"`
 	DisableAgent bool                `yaml:"-"`
-	Hub          *Hub                `yaml:"-"`
+	Hub          *HubCfg             `yaml:"-"`
 }
 
 func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {
@@ -73,18 +73,34 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool
 		log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort)
 	}
 
+	if err = cfg.loadCommon(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadConfigurationPaths(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadHub(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadCSCLI(); err != nil {
+		return nil, "", err
+	}
+
 	return &cfg, configData, nil
 }
 
+// XXX: We must not not have a different behavior with an empty vs a missing configuration file.
+// XXX: For this reason, all defaults have to come from NewConfig(). The following function should
+// XXX: be replaced
 func NewDefaultConfig() *Config {
 	logLevel := log.InfoLevel
 	commonCfg := CommonCfg{
 		Daemonize: false,
-		PidDir:    "/tmp/",
 		LogMedia:  "stdout",
-		//LogDir unneeded
 		LogLevel:   &logLevel,
-		WorkingDir: ".",
 	}
 	prometheus := PrometheusCfg{
 		Enabled: true,

pkg/csconfig/config_paths.go

@@ -15,21 +15,25 @@ type ConfigurationPaths struct {
 	NotificationDir    string `yaml:"notification_dir,omitempty"`
 }
 
-func (c *Config) LoadConfigurationPaths() error {
+func (c *Config) loadConfigurationPaths() error {
 	var err error
 	if c.ConfigPaths == nil {
+		// XXX: test me
 		return fmt.Errorf("no configuration paths provided")
 	}
 
 	if c.ConfigPaths.DataDir == "" {
+		// XXX: test me
 		return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section")
 	}
 
 	if c.ConfigPaths.HubDir == "" {
+		// XXX: test me
 		c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub")
 	}
 
 	if c.ConfigPaths.HubIndexFile == "" {
+		// XXX: test me
 		c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json")
 	}
 

pkg/csconfig/crowdsec_service.go

@@ -145,10 +145,6 @@ func (c *Config) LoadCrowdsec() error {
 		return fmt.Errorf("loading api client: %s", err)
 	}
 
-	if err := c.LoadHub(); err != nil {
-		return fmt.Errorf("while loading hub: %w", err)
-	}
-
 	c.Crowdsec.ContextToSend = make(map[string][]string, 0)
 	fallback := false
 	if c.Crowdsec.ConsoleContextPath == "" {

pkg/csconfig/crowdsec_service_test.go

@@ -20,18 +20,6 @@ func TestLoadCrowdsec(t *testing.T) {
 	acquisDirFullPath, err := filepath.Abs("./testdata/acquis")
 	require.NoError(t, err)
 
-	hubFullPath, err := filepath.Abs("./hub")
-	require.NoError(t, err)
-
-	dataFullPath, err := filepath.Abs("./data")
-	require.NoError(t, err)
-
-	configDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	require.NoError(t, err)
-
 	contextFileFullPath, err := filepath.Abs("./testdata/context.yaml")
 	require.NoError(t, err)
 
@@ -66,10 +54,11 @@ func TestLoadCrowdsec(t *testing.T) {
 				AcquisitionDirPath:        "",
 				ConsoleContextPath:        contextFileFullPath,
 				AcquisitionFilePath:       acquisFullPath,
-				ConfigDir:                 configDirFullPath,
+				ConfigDir:                 "./testdata",
-				DataDir:                   dataFullPath,
+				DataDir:                   "./data",
-				HubDir:                    hubFullPath,
+				HubDir:                    "./hub",
-				HubIndexFile:              hubIndexFileFullPath,
+				// XXX: need to ensure a default here
+				HubIndexFile:              "",
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
@@ -109,10 +98,11 @@ func TestLoadCrowdsec(t *testing.T) {
 				AcquisitionDirPath:        acquisDirFullPath,
 				AcquisitionFilePath:       acquisFullPath,
 				ConsoleContextPath:        contextFileFullPath,
-				ConfigDir:                 configDirFullPath,
+				ConfigDir:                 "./testdata",
-				HubIndexFile:              hubIndexFileFullPath,
+				// XXX: need to ensure a default here
-				DataDir:                   dataFullPath,
+				HubIndexFile:              "",
-				HubDir:                    hubFullPath,
+				DataDir:                   "./data",
+				HubDir:                    "./hub",
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
@@ -141,7 +131,7 @@ func TestLoadCrowdsec(t *testing.T) {
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
-					ConsoleContextPath:        contextFileFullPath,
+					ConsoleContextPath:        "./testdata/context.yaml",
 					ConsoleContextValueLength: 10,
 				},
 			},
@@ -149,10 +139,11 @@ func TestLoadCrowdsec(t *testing.T) {
 				Enable:                    ptr.Of(true),
 				AcquisitionDirPath:        "",
 				AcquisitionFilePath:       "",
-				ConfigDir:                 configDirFullPath,
+				ConfigDir:                 "./testdata",
-				HubIndexFile:              hubIndexFileFullPath,
+				// XXX: need to ensure a default here
-				DataDir:                   dataFullPath,
+				HubIndexFile:              "",
-				HubDir:                    hubFullPath,
+				DataDir:                   "./data",
+				HubDir:                    "./hub",
 				ConsoleContextPath:        contextFileFullPath,
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,

pkg/csconfig/cscli.go

@@ -19,13 +19,10 @@ type CscliCfg struct {
 	PrometheusUrl      string            `yaml:"prometheus_uri"`
 }
 
-func (c *Config) LoadCSCLI() error {
+func (c *Config) loadCSCLI() error {
 	if c.Cscli == nil {
 		c.Cscli = &CscliCfg{}
 	}
-	if err := c.LoadConfigurationPaths(); err != nil {
-		return err
-	}
 	c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
 	c.Cscli.DataDir = c.ConfigPaths.DataDir
 	c.Cscli.HubDir = c.ConfigPaths.HubDir

pkg/csconfig/cscli_test.go

@@ -1,28 +1,14 @@
 package csconfig
 
 import (
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 
 	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadCSCLI(t *testing.T) {
-	hubFullPath, err := filepath.Abs("./hub")
-	require.NoError(t, err)
-
-	dataFullPath, err := filepath.Abs("./data")
-	require.NoError(t, err)
-
-	configDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	require.NoError(t, err)
-
 	tests := []struct {
 		name        string
 		input       *Config
@@ -46,25 +32,19 @@ func TestLoadCSCLI(t *testing.T) {
 				},
 			},
 			expected: &CscliCfg{
-				ConfigDir:     configDirFullPath,
+				ConfigDir:     "./testdata",
-				DataDir:       dataFullPath,
+				DataDir:       "./data",
-				HubDir:        hubFullPath,
+				HubDir:        "./hub",
-				HubIndexFile:  hubIndexFileFullPath,
+				HubIndexFile:  "./hub/.index.json",
 				PrometheusUrl: "http://127.0.0.1:6060/metrics",
 			},
 		},
-		{
-			name:        "no configuration path",
-			input:       &Config{},
-			expected:    &CscliCfg{},
-			expectedErr: "no configuration paths provided",
-		},
 	}
 
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			err := tc.input.LoadCSCLI()
+			err := tc.input.loadCSCLI()
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			if tc.expectedErr != "" {
 				return

pkg/csconfig/hub.go

@@ -1,19 +1,15 @@
 package csconfig
 
-/*cscli specific config, such as hub directory*/
+// HubConfig holds the configuration for a hub
-type Hub struct {
+type HubCfg struct {
-	HubIndexFile   string
+	HubIndexFile   string	// Path to the local index file
-	HubDir         string
+	HubDir         string	// Where the hub items are downloaded
-	InstallDir     string
+	InstallDir     string	// Where to install items
-	InstallDataDir string
+	InstallDataDir string	// Where to install data
 }
 
-func (c *Config) LoadHub() error {
+func (c *Config) loadHub() error {
-	if err := c.LoadConfigurationPaths(); err != nil {
+	c.Hub = &HubCfg{
-		return err
-	}
-
-	c.Hub = &Hub{
 		HubIndexFile:   c.ConfigPaths.HubIndexFile,
 		HubDir:         c.ConfigPaths.HubDir,
 		InstallDir:     c.ConfigPaths.ConfigDir,

pkg/csconfig/hub_test.go

@@ -1,32 +1,18 @@
 package csconfig
 
 import (
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 
 	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadHub(t *testing.T) {
-	hubFullPath, err := filepath.Abs("./hub")
-	require.NoError(t, err)
-
-	dataFullPath, err := filepath.Abs("./data")
-	require.NoError(t, err)
-
-	configDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	require.NoError(t, err)
-
 	tests := []struct {
 		name        string
 		input       *Config
-		expected    *Hub
+		expected    *HubCfg
 		expectedErr string
 	}{
 		{
@@ -39,35 +25,19 @@ func TestLoadHub(t *testing.T) {
 					HubIndexFile: "./hub/.index.json",
 				},
 			},
-			expected: &Hub{
+			expected: &HubCfg{
-				HubDir:         hubFullPath,
+				HubDir:         "./hub",
-				HubIndexFile:   hubIndexFileFullPath,
+				HubIndexFile:   "./hub/.index.json",
-				InstallDir:     configDirFullPath,
+				InstallDir:     "./testdata",
-				InstallDataDir: dataFullPath,
+				InstallDataDir: "./data",
 			},
 		},
-		{
-			name: "no data dir",
-			input: &Config{
-				ConfigPaths: &ConfigurationPaths{
-					ConfigDir:    "./testdata",
-					HubDir:       "./hub",
-					HubIndexFile: "./hub/.index.json",
-				},
-			},
-			expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section",
-		},
-		{
-			name:        "no configuration path",
-			input:       &Config{},
-			expectedErr: "no configuration paths provided",
-		},
 	}
 
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			err := tc.input.LoadHub()
+			err := tc.input.loadHub()
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			if tc.expectedErr != "" {
 				return

pkg/csconfig/simulation.go

@@ -30,11 +30,6 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool {
 }
 
 func (c *Config) LoadSimulation() error {
-
-	if err := c.LoadConfigurationPaths(); err != nil {
-		return err
-	}
-
 	simCfg := SimulationConfig{}
 	if c.ConfigPaths.SimulationFilePath == "" {
 		c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")

pkg/csconfig/simulation_test.go

@@ -2,7 +2,6 @@ package csconfig
 
 import (
 	"fmt"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -12,12 +11,6 @@ import (
 )
 
 func TestSimulationLoading(t *testing.T) {
-	testXXFullPath, err := filepath.Abs("./testdata/xxx.yaml")
-	require.NoError(t, err)
-
-	badYamlFullPath, err := filepath.Abs("./testdata/config.yaml")
-	require.NoError(t, err)
-
 	tests := []struct {
 		name        string
 		input       *Config
@@ -56,7 +49,7 @@ func TestSimulationLoading(t *testing.T) {
 				},
 				Crowdsec: &CrowdsecServiceCfg{},
 			},
-			expectedErr: fmt.Sprintf("while reading yaml file: open %s: %s", testXXFullPath, cstest.FileNotFoundMessage),
+			expectedErr: fmt.Sprintf("while reading yaml file: open ./testdata/xxx.yaml: %s", cstest.FileNotFoundMessage),
 		},
 		{
 			name: "basic bad file content",
@@ -67,7 +60,7 @@ func TestSimulationLoading(t *testing.T) {
 				},
 				Crowdsec: &CrowdsecServiceCfg{},
 			},
-			expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
+			expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
 		},
 		{
 			name: "basic bad file content",
@@ -78,7 +71,7 @@ func TestSimulationLoading(t *testing.T) {
 				},
 				Crowdsec: &CrowdsecServiceCfg{},
 			},
-			expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
+			expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
 		},
 	}
 

pkg/csconfig/testdata/config.yaml

@@ -2,7 +2,6 @@ common:
   daemonize: false
   log_media: stdout
   log_level: info
-  working_dir: .
 prometheus:
   enabled: true
   level: full

pkg/cwhub/cwhub_test.go

@@ -121,7 +121,7 @@ func TestIndexDownload(t *testing.T) {
 }
 
 func getTestCfg() *csconfig.Config {
-	cfg := &csconfig.Config{Hub: &csconfig.Hub{}}
+	cfg := &csconfig.Config{Hub: &csconfig.HubCfg{}}
 	cfg.Hub.InstallDir, _ = filepath.Abs("./install")
 	cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
 	cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
@@ -172,7 +172,7 @@ func envTearDown(cfg *csconfig.Config) {
 	}
 }
 
-func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
+func testInstallItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
 	// Install the parser
 	err := DownloadLatest(cfg, &item, false, false)
 	require.NoError(t, err, "failed to download %s", item.Name)
@@ -193,7 +193,7 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
 }
 
-func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
+func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
 	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
 
 	f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
@@ -211,7 +211,7 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	assert.True(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
 }
 
-func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
+func testUpdateItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
 	assert.False(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
 
 	// Update it + check status
@@ -226,7 +226,7 @@ func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
 }
 
-func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
+func testDisableItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
 	assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
 
 	// Remove

pkg/cwhub/download.go

@@ -20,7 +20,7 @@ import (
 var ErrIndexNotFound = fmt.Errorf("index not found")
 
 // UpdateHubIdx downloads the latest version of the index and updates the one in memory
-func UpdateHubIdx(hub *csconfig.Hub) error {
+func UpdateHubIdx(hub *csconfig.HubCfg) error {
 	bidx, err := DownloadHubIdx(hub)
 	if err != nil {
 		return fmt.Errorf("failed to download index: %w", err)
@@ -43,7 +43,7 @@ func UpdateHubIdx(hub *csconfig.Hub) error {
 }
 
 // DownloadHubIdx downloads the latest version of the index and returns the content
-func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
+func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) {
 	log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
 
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
@@ -98,7 +98,7 @@ func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
 }
 
 // DownloadLatest will download the latest version of Item to the tdir directory
-func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly bool) error {
+func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOnly bool) error {
 	var err error
 
 	log.Debugf("Downloading %s %s", target.Type, target.Name)
@@ -165,7 +165,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly
 	return nil
 }
 
-func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error {
+func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error {
 	tdir := hub.HubDir
 
 	// if user didn't --force, don't overwrite local, tainted, up-to-date files
@@ -273,7 +273,7 @@ func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error {
 }
 
 // DownloadDataIfNeeded downloads the data files for an item
-func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
+func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error {
 	itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName)
 
 	itemFile, err := os.Open(itemFilePath)

pkg/cwhub/download_test.go

@@ -17,7 +17,7 @@ func TestDownloadHubIdx(t *testing.T) {
 
 	RawFileURLTemplate = "x"
 
-	ret, err := DownloadHubIdx(&csconfig.Hub{})
+	ret, err := DownloadHubIdx(&csconfig.HubCfg{})
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
 		log.Errorf("unexpected error %s", err)
 	}
@@ -29,7 +29,7 @@ func TestDownloadHubIdx(t *testing.T) {
 
 	RawFileURLTemplate = "https://baddomain/%s/%s"
 
-	ret, err = DownloadHubIdx(&csconfig.Hub{})
+	ret, err = DownloadHubIdx(&csconfig.HubCfg{})
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
 		log.Errorf("unexpected error %s", err)
 	}
@@ -41,7 +41,7 @@ func TestDownloadHubIdx(t *testing.T) {
 
 	RawFileURLTemplate = back
 
-	ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
+	ret, err = DownloadHubIdx(&csconfig.HubCfg{HubIndexFile: "/does/not/exist/index.json"})
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
 		log.Errorf("unexpected error %s", err)
 	}

pkg/cwhub/hubindex.go

@@ -29,6 +29,8 @@ var (
 
 type HubItems map[string]map[string]Item
 
+// HubIndex represents the runtime status of the hub (parsed items, etc.)
+// XXX: this could be renamed "Hub" tout court once the confusion with HubCfg is cleared
 type HubIndex struct {
 	Items HubItems
 	skippedLocal   int

pkg/cwhub/install.go

@@ -10,7 +10,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
-func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
+func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) {
 	itempath := hub.HubDir + "/" + target.RemotePath
 
 	// disable hub file
@@ -26,7 +26,7 @@ func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
 }
 
 // DisableItem to disable an item managed by the hub, removes the symlink if purge is true
-func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error {
+func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) error {
 	var err error
 
 	// already disabled, noop unless purge
@@ -137,7 +137,7 @@ func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error
 
 // creates symlink between actual config file at hub.HubDir and hub.ConfigDir
 // Handles collections recursively
-func EnableItem(hub *csconfig.Hub, target *Item) error {
+func EnableItem(hub *csconfig.HubCfg, target *Item) error {
 	var err error
 
 	parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/")

pkg/cwhub/loader.go

@@ -66,7 +66,7 @@ type Walker struct {
 	installdir string
 }
 
-func NewWalker(hub *csconfig.Hub) Walker {
+func NewWalker(hub *csconfig.HubCfg) Walker {
 	return Walker{
 		hubdir:     hub.HubDir,
 		installdir: hub.InstallDir,
@@ -404,7 +404,7 @@ func CollecDepsCheck(v *Item) error {
 	return nil
 }
 
-func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
+func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) {
 	warnings := []string{}
 
 	// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
@@ -445,7 +445,7 @@ func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
 }
 
 // Updates the info from HubInit() with the local state
-func LocalSync(hub *csconfig.Hub) ([]string, error) {
+func LocalSync(hub *csconfig.HubCfg) ([]string, error) {
 	hubIdx.skippedLocal = 0
 	hubIdx.skippedTainted = 0
 
@@ -462,7 +462,7 @@ func LocalSync(hub *csconfig.Hub) ([]string, error) {
 	return warnings, nil
 }
 
-func GetHubIdx(hub *csconfig.Hub) error {
+func GetHubIdx(hub *csconfig.HubCfg) error {
 	if hub == nil {
 		return fmt.Errorf("no configuration found for hub")
 	}

pkg/hubtest/hubtest_item.go

@@ -39,7 +39,7 @@ type HubTestItem struct {
 	RuntimeConfigFilePath     string
 	RuntimeProfileFilePath    string
 	RuntimeSimulationFilePath string
-	RuntimeHubConfig          *csconfig.Hub
+	RuntimeHubConfig          *csconfig.HubCfg
 
 	ResultsPath          string
 	ParserResultFile     string
@@ -117,7 +117,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
 		ParserResultFile:          filepath.Join(resultPath, ParserResultFileName),
 		ScenarioResultFile:        filepath.Join(resultPath, ScenarioResultFileName),
 		BucketPourResultFile:      filepath.Join(resultPath, BucketPourResultFileName),
-		RuntimeHubConfig: &csconfig.Hub{
+		RuntimeHubConfig: &csconfig.HubCfg{
 			HubDir:         runtimeHubFolder,
 			HubIndexFile:   hubTest.HubIndexFile,
 			InstallDir:     runtimeFolder,

pkg/setup/install.go

@@ -52,10 +52,6 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 		return err
 	}
 
-	if err := csConfig.LoadHub(); err != nil {
-		return fmt.Errorf("loading hub: %w", err)
-	}
-
 	cwhub.SetHubBranch()
 
 	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {

test/bats/01_crowdsec.bats

@@ -55,16 +55,16 @@ teardown() {
     assert_stderr --partial "unable to create database client: unknown database type 'meh'"
 }
 
-@test "crowdsec - bad configuration (empty/missing common section)" {
+@test "crowdsec - default logging configuration (empty/missing common section)" {
     config_set '.common={}'
-    rune -1 "${CROWDSEC}"
+    rune -124 timeout 1s "${CROWDSEC}"
     refute_output
-    assert_stderr --partial "unable to load configuration: common section is empty"
+    assert_stderr --partial "Starting processing data"
 
     config_set 'del(.common)'
-    rune -1 "${CROWDSEC}"
+    rune -124 timeout 1s "${CROWDSEC}"
     refute_output
-    assert_stderr --partial "unable to load configuration: common section is empty"
+    assert_stderr --partial "Starting processing data"
 }
 
 @test "CS_LAPI_SECRET not strong enough" {