beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Refact pkg/csconfig, HubCfg (#2552)

Browse source 
- rename csconfig.Hub -> HubCfg
 - move some Load*() functions to NewConfig()
 - config.yaml: optional common section
 - remove unused working_dir
This commit is contained in:
mmetc 2023-10-18 09:38:33 +02:00 committed by GitHub
parent 4eae40865e
commit be6555e46c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 105 additions and 287 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmd/crowdsec-cli/config_restore.go
View file

@ -48,10 +48,6 @@ func silentInstallItem(name string, obtype string) (string, error) {
func restoreHub(dirPath string) error {
var err error
if err := csConfig.LoadHub(); err != nil {
return err
}
cwhub.SetHubBranch()
for _, itype := range cwhub.ItemTypes {

4
cmd/crowdsec-cli/hub.go
View file

@ -88,10 +88,6 @@ func NewHubListCmd() *cobra.Command {
}
func runHubUpdate(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
if !errors.Is(err, cwhub.ErrIndexNotFound) {
return fmt.Errorf("failed to get Hub index : %w", err)

4
cmd/crowdsec-cli/main.go
View file

@ -51,10 +51,8 @@ func initConfig() {
if err != nil {
log.Fatal(err)
}
if err := csConfig.LoadCSCLI(); err != nil {
log.Fatal(err)
}
} else {
// XXX: check all the defaults
csConfig = csconfig.NewDefaultConfig()
}

4
cmd/crowdsec-cli/require/require.go
View file

@ -65,10 +65,6 @@ func Notifications(c *csconfig.Config) error {
}
func Hub (c *csconfig.Config) error {
if err := c.LoadHub(); err != nil {
return err
}
if c.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}

11
cmd/crowdsec/main.go
View file

@ -212,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
if err != nil {
return nil, err
}
if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
return nil, fmt.Errorf("unable to load configuration: common section is empty")
return nil, fmt.Errorf("while loading configuration file: %w", err)
}
cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@ -228,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
dumpStates = true
}
// Configuration paths are dependency to load crowdsec configuration
if err := cConfig.LoadConfigurationPaths(); err != nil {
return nil, err
}
if flags.SingleFileType != "" && flags.OneShotDSN != "" {
// if we're in time-machine mode, we don't want to log to file
cConfig.Common.LogMedia = "stdout"

1
config/config.yaml
View file

@ -6,7 +6,6 @@ common:
log_max_size: 20
compress_logs: true
log_max_files: 10
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/

1
config/config_win.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: file
log_level: info
log_dir: C:\ProgramData\CrowdSec\log\
working_dir: .
config_paths:
config_dir: C:\ProgramData\CrowdSec\config\
data_dir: C:\ProgramData\CrowdSec\data\

1
config/config_win_no_lapi.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: file
log_level: info
log_dir: C:\ProgramData\CrowdSec\log\
working_dir: .
config_paths:
config_dir: C:\ProgramData\CrowdSec\config\
data_dir: C:\ProgramData\CrowdSec\data\

1
config/dev.yaml
View file

@ -2,7 +2,6 @@ common:
daemonize: true
log_media: stdout
log_level: info
working_dir: .
config_paths:
config_dir: ./config
data_dir: ./data/

1
config/user.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: stdout
log_level: info
log_dir: /var/log/
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data

1
docker/config.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: stdout
log_level: info
log_dir: /var/log/
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/

4
pkg/csconfig/api.go
View file

@ -286,10 +286,6 @@ func (c *Config) LoadAPIServer() error {
log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs))
}
if err := c.LoadCommon(); err != nil {
return fmt.Errorf("loading common configuration: %s", err)
}
c.API.Server.LogDir = c.Common.LogDir
c.API.Server.LogMedia = c.Common.LogMedia
c.API.Server.CompressLogs = c.Common.CompressLogs

8
pkg/csconfig/api_test.go
View file

@ -3,7 +3,6 @@ package csconfig
import (
"net"
"os"
"path/filepath"
"strings"
"testing"
@ -142,9 +141,6 @@ func TestLoadAPIServer(t *testing.T) {
err := tmpLAPI.LoadProfiles()
require.NoError(t, err)
LogDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
logLevel := log.InfoLevel
config := &Config{}
fcontent, err := os.ReadFile("./testdata/config.yaml")
@ -179,7 +175,7 @@ func TestLoadAPIServer(t *testing.T) {
DbPath: "./testdata/test.db",
},
Common: &CommonCfg{
LogDir: "./testdata/",
LogDir: "./testdata",
LogMedia: "stdout",
},
DisableAPI: false,
@ -202,7 +198,7 @@ func TestLoadAPIServer(t *testing.T) {
ShareContext: ptr.Of(false),
ConsoleManagement: ptr.Of(false),
},
LogDir: LogDirFullPath,
LogDir: "./testdata",
LogMedia: "stdout",
OnlineClient: &OnlineApiClientCfg{
CredentialsFilePath: "./testdata/online-api-secrets.yaml",

11
pkg/csconfig/common.go
View file

@ -14,7 +14,7 @@ type CommonCfg struct {
LogMedia string `yaml:"log_media"`
LogDir string `yaml:"log_dir,omitempty"` //if LogMedia = file
LogLevel *log.Level `yaml:"log_level"`
WorkingDir string `yaml:"working_dir,omitempty"` ///var/run
WorkingDir string `yaml:"working_dir,omitempty"` // TODO: This is just for backward compat. Remove this later
CompressLogs *bool `yaml:"compress_logs,omitempty"`
LogMaxSize int `yaml:"log_max_size,omitempty"`
LogMaxAge int `yaml:"log_max_age,omitempty"`
@ -22,15 +22,18 @@ type CommonCfg struct {
ForceColorLogs bool `yaml:"force_color_logs,omitempty"`
}
func (c *Config) LoadCommon() error {
func (c *Config) loadCommon() error {
var err error
if c.Common == nil {
return fmt.Errorf("no common block provided in configuration file")
c.Common = &CommonCfg{}
}
if c.Common.LogMedia == "" {
c.Common.LogMedia = "stdout"
}
var CommonCleanup = []*string{
&c.Common.LogDir,
&c.Common.WorkingDir,
}
for _, k := range CommonCleanup {
if *k == "" {

83
pkg/csconfig/common_test.go
View file

@ -1,83 +0,0 @@
package csconfig
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadCommon(t *testing.T) {
pidDirPath := "./testdata"
LogDirFullPath, err := filepath.Abs("./testdata/log/")
require.NoError(t, err)
WorkingDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
tests := []struct {
name string
input *Config
expected *CommonCfg
expectedErr string
}{
{
name: "basic valid configuration",
input: &Config{
Common: &CommonCfg{
Daemonize: true,
PidDir: "./testdata",
LogMedia: "file",
LogDir: "./testdata/log/",
WorkingDir: "./testdata/",
},
},
expected: &CommonCfg{
Daemonize: true,
PidDir: pidDirPath,
LogMedia: "file",
LogDir: LogDirFullPath,
WorkingDir: WorkingDirFullPath,
},
},
{
name: "empty working dir",
input: &Config{
Common: &CommonCfg{
Daemonize: true,
PidDir: "./testdata",
LogMedia: "file",
LogDir: "./testdata/log/",
},
},
expected: &CommonCfg{
Daemonize: true,
PidDir: pidDirPath,
LogMedia: "file",
LogDir: LogDirFullPath,
},
},
{
name: "no common",
input: &Config{},
expected: nil,
expectedErr: "no common block provided in configuration file",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadCommon()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return
}
assert.Equal(t, tc.expected, tc.input.Common)
})
}
}

24
pkg/csconfig/config.go
View file

@ -36,7 +36,7 @@ type Config struct {
PluginConfig *PluginCfg `yaml:"plugin_config,omitempty"`
DisableAPI bool `yaml:"-"`
DisableAgent bool `yaml:"-"`
Hub *Hub `yaml:"-"`
Hub *HubCfg `yaml:"-"`
}
func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {
@ -73,18 +73,34 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool
log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort)
}
if err = cfg.loadCommon(); err != nil {
return nil, "", err
}
if err = cfg.loadConfigurationPaths(); err != nil {
return nil, "", err
}
if err = cfg.loadHub(); err != nil {
return nil, "", err
}
if err = cfg.loadCSCLI(); err != nil {
return nil, "", err
}
return &cfg, configData, nil
}
// XXX: We must not not have a different behavior with an empty vs a missing configuration file.
// XXX: For this reason, all defaults have to come from NewConfig(). The following function should
// XXX: be replaced
func NewDefaultConfig() *Config {
logLevel := log.InfoLevel
commonCfg := CommonCfg{
Daemonize: false,
PidDir: "/tmp/",
LogMedia: "stdout",
//LogDir unneeded
LogLevel: &logLevel,
WorkingDir: ".",
}
prometheus := PrometheusCfg{
Enabled: true,

6
pkg/csconfig/config_paths.go
View file

@ -15,21 +15,25 @@ type ConfigurationPaths struct {
NotificationDir string `yaml:"notification_dir,omitempty"`
}
func (c *Config) LoadConfigurationPaths() error {
func (c *Config) loadConfigurationPaths() error {
var err error
if c.ConfigPaths == nil {
// XXX: test me
return fmt.Errorf("no configuration paths provided")
}
if c.ConfigPaths.DataDir == "" {
// XXX: test me
return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section")
}
if c.ConfigPaths.HubDir == "" {
// XXX: test me
c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub")
}
if c.ConfigPaths.HubIndexFile == "" {
// XXX: test me
c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json")
}

4
pkg/csconfig/crowdsec_service.go
View file

@ -145,10 +145,6 @@ func (c *Config) LoadCrowdsec() error {
return fmt.Errorf("loading api client: %s", err)
}
if err := c.LoadHub(); err != nil {
return fmt.Errorf("while loading hub: %w", err)
}
c.Crowdsec.ContextToSend = make(map[string][]string, 0)
fallback := false
if c.Crowdsec.ConsoleContextPath == "" {

41
pkg/csconfig/crowdsec_service_test.go
View file

@ -20,18 +20,6 @@ func TestLoadCrowdsec(t *testing.T) {
acquisDirFullPath, err := filepath.Abs("./testdata/acquis")
require.NoError(t, err)
hubFullPath, err := filepath.Abs("./hub")
require.NoError(t, err)
dataFullPath, err := filepath.Abs("./data")
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
require.NoError(t, err)
contextFileFullPath, err := filepath.Abs("./testdata/context.yaml")
require.NoError(t, err)
@ -66,10 +54,11 @@ func TestLoadCrowdsec(t *testing.T) {
AcquisitionDirPath: "",
ConsoleContextPath: contextFileFullPath,
AcquisitionFilePath: acquisFullPath,
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
ConfigDir: "./testdata",
DataDir: "./data",
HubDir: "./hub",
// XXX: need to ensure a default here
HubIndexFile: "",
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
@ -109,10 +98,11 @@ func TestLoadCrowdsec(t *testing.T) {
AcquisitionDirPath: acquisDirFullPath,
AcquisitionFilePath: acquisFullPath,
ConsoleContextPath: contextFileFullPath,
ConfigDir: configDirFullPath,
HubIndexFile: hubIndexFileFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
ConfigDir: "./testdata",
// XXX: need to ensure a default here
HubIndexFile: "",
DataDir: "./data",
HubDir: "./hub",
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
@ -141,7 +131,7 @@ func TestLoadCrowdsec(t *testing.T) {
},
},
Crowdsec: &CrowdsecServiceCfg{
ConsoleContextPath: contextFileFullPath,
ConsoleContextPath: "./testdata/context.yaml",
ConsoleContextValueLength: 10,
},
},
@ -149,10 +139,11 @@ func TestLoadCrowdsec(t *testing.T) {
Enable: ptr.Of(true),
AcquisitionDirPath: "",
AcquisitionFilePath: "",
ConfigDir: configDirFullPath,
HubIndexFile: hubIndexFileFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
ConfigDir: "./testdata",
// XXX: need to ensure a default here
HubIndexFile: "",
DataDir: "./data",
HubDir: "./hub",
ConsoleContextPath: contextFileFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,

5
pkg/csconfig/cscli.go
View file

@ -19,13 +19,10 @@ type CscliCfg struct {
PrometheusUrl string `yaml:"prometheus_uri"`
}
func (c *Config) LoadCSCLI() error {
func (c *Config) loadCSCLI() error {
if c.Cscli == nil {
c.Cscli = &CscliCfg{}
}
if err := c.LoadConfigurationPaths(); err != nil {
return err
}
c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
c.Cscli.DataDir = c.ConfigPaths.DataDir
c.Cscli.HubDir = c.ConfigPaths.HubDir

30
pkg/csconfig/cscli_test.go
View file

@ -1,28 +1,14 @@
package csconfig
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadCSCLI(t *testing.T) {
hubFullPath, err := filepath.Abs("./hub")
require.NoError(t, err)
dataFullPath, err := filepath.Abs("./data")
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
require.NoError(t, err)
tests := []struct {
name string
input *Config
@ -46,25 +32,19 @@ func TestLoadCSCLI(t *testing.T) {
},
},
expected: &CscliCfg{
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
ConfigDir: "./testdata",
DataDir: "./data",
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
PrometheusUrl: "http://127.0.0.1:6060/metrics",
},
},
{
name: "no configuration path",
input: &Config{},
expected: &CscliCfg{},
expectedErr: "no configuration paths provided",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadCSCLI()
err := tc.input.loadCSCLI()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return

20
pkg/csconfig/hub.go
View file

@ -1,19 +1,15 @@
package csconfig
/*cscli specific config, such as hub directory*/
type Hub struct {
HubIndexFile string
HubDir string
InstallDir string
InstallDataDir string
// HubConfig holds the configuration for a hub
type HubCfg struct {
HubIndexFile string // Path to the local index file
HubDir string // Where the hub items are downloaded
InstallDir string // Where to install items
InstallDataDir string // Where to install data
}
func (c *Config) LoadHub() error {
if err := c.LoadConfigurationPaths(); err != nil {
return err
}
c.Hub = &Hub{
func (c *Config) loadHub() error {
c.Hub = &HubCfg{
HubIndexFile: c.ConfigPaths.HubIndexFile,
HubDir: c.ConfigPaths.HubDir,
InstallDir: c.ConfigPaths.ConfigDir,

44
pkg/csconfig/hub_test.go
View file

@ -1,32 +1,18 @@
package csconfig
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadHub(t *testing.T) {
hubFullPath, err := filepath.Abs("./hub")
require.NoError(t, err)
dataFullPath, err := filepath.Abs("./data")
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
require.NoError(t, err)
tests := []struct {
name string
input *Config
expected *Hub
expected *HubCfg
expectedErr string
}{
{
@ -39,35 +25,19 @@ func TestLoadHub(t *testing.T) {
HubIndexFile: "./hub/.index.json",
},
},
expected: &Hub{
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
InstallDir: configDirFullPath,
InstallDataDir: dataFullPath,
expected: &HubCfg{
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
InstallDir: "./testdata",
InstallDataDir: "./data",
},
},
{
name: "no data dir",
input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./testdata",
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
},
},
expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section",
},
{
name: "no configuration path",
input: &Config{},
expectedErr: "no configuration paths provided",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadHub()
err := tc.input.loadHub()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return

5
pkg/csconfig/simulation.go
View file

@ -30,11 +30,6 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool {
}
func (c *Config) LoadSimulation() error {
if err := c.LoadConfigurationPaths(); err != nil {
return err
}
simCfg := SimulationConfig{}
if c.ConfigPaths.SimulationFilePath == "" {
c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")

13
pkg/csconfig/simulation_test.go
View file

@ -2,7 +2,6 @@ package csconfig
import (
"fmt"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
@ -12,12 +11,6 @@ import (
)
func TestSimulationLoading(t *testing.T) {
testXXFullPath, err := filepath.Abs("./testdata/xxx.yaml")
require.NoError(t, err)
badYamlFullPath, err := filepath.Abs("./testdata/config.yaml")
require.NoError(t, err)
tests := []struct {
name string
input *Config
@ -56,7 +49,7 @@ func TestSimulationLoading(t *testing.T) {
},
Crowdsec: &CrowdsecServiceCfg{},
},
expectedErr: fmt.Sprintf("while reading yaml file: open %s: %s", testXXFullPath, cstest.FileNotFoundMessage),
expectedErr: fmt.Sprintf("while reading yaml file: open ./testdata/xxx.yaml: %s", cstest.FileNotFoundMessage),
},
{
name: "basic bad file content",
@ -67,7 +60,7 @@ func TestSimulationLoading(t *testing.T) {
},
Crowdsec: &CrowdsecServiceCfg{},
},
expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
},
{
name: "basic bad file content",
@ -78,7 +71,7 @@ func TestSimulationLoading(t *testing.T) {
},
Crowdsec: &CrowdsecServiceCfg{},
},
expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
},
}

1
pkg/csconfig/testdata/config.yaml vendored
View file

@ -2,7 +2,6 @@ common:
daemonize: false
log_media: stdout
log_level: info
working_dir: .
prometheus:
enabled: true
level: full

10
pkg/cwhub/cwhub_test.go
View file

@ -121,7 +121,7 @@ func TestIndexDownload(t *testing.T) {
}
func getTestCfg() *csconfig.Config {
cfg := &csconfig.Config{Hub: &csconfig.Hub{}}
cfg := &csconfig.Config{Hub: &csconfig.HubCfg{}}
cfg.Hub.InstallDir, _ = filepath.Abs("./install")
cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
@ -172,7 +172,7 @@ func envTearDown(cfg *csconfig.Config) {
}
}
func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
func testInstallItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
// Install the parser
err := DownloadLatest(cfg, &item, false, false)
require.NoError(t, err, "failed to download %s", item.Name)
@ -193,7 +193,7 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
}
func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
@ -211,7 +211,7 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
assert.True(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
}
func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
func testUpdateItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
assert.False(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
// Update it + check status
@ -226,7 +226,7 @@ func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
}
func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
func testDisableItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
// Remove

10
pkg/cwhub/download.go
View file

@ -20,7 +20,7 @@ import (
var ErrIndexNotFound = fmt.Errorf("index not found")
// UpdateHubIdx downloads the latest version of the index and updates the one in memory
func UpdateHubIdx(hub *csconfig.Hub) error {
func UpdateHubIdx(hub *csconfig.HubCfg) error {
bidx, err := DownloadHubIdx(hub)
if err != nil {
return fmt.Errorf("failed to download index: %w", err)
@ -43,7 +43,7 @@ func UpdateHubIdx(hub *csconfig.Hub) error {
}
// DownloadHubIdx downloads the latest version of the index and returns the content
func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) {
log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
@ -98,7 +98,7 @@ func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
}
// DownloadLatest will download the latest version of Item to the tdir directory
func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly bool) error {
func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOnly bool) error {
var err error
log.Debugf("Downloading %s %s", target.Type, target.Name)
@ -165,7 +165,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly
return nil
}
func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error {
func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error {
tdir := hub.HubDir
// if user didn't --force, don't overwrite local, tainted, up-to-date files
@ -273,7 +273,7 @@ func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error {
}
// DownloadDataIfNeeded downloads the data files for an item
func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error {
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName)
itemFile, err := os.Open(itemFilePath)

6
pkg/cwhub/download_test.go
View file

@ -17,7 +17,7 @@ func TestDownloadHubIdx(t *testing.T) {
RawFileURLTemplate = "x"
ret, err := DownloadHubIdx(&csconfig.Hub{})
ret, err := DownloadHubIdx(&csconfig.HubCfg{})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
log.Errorf("unexpected error %s", err)
}
@ -29,7 +29,7 @@ func TestDownloadHubIdx(t *testing.T) {
RawFileURLTemplate = "https://baddomain/%s/%s"
ret, err = DownloadHubIdx(&csconfig.Hub{})
ret, err = DownloadHubIdx(&csconfig.HubCfg{})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
log.Errorf("unexpected error %s", err)
}
@ -41,7 +41,7 @@ func TestDownloadHubIdx(t *testing.T) {
RawFileURLTemplate = back
ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
ret, err = DownloadHubIdx(&csconfig.HubCfg{HubIndexFile: "/does/not/exist/index.json"})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
log.Errorf("unexpected error %s", err)
}

2
pkg/cwhub/hubindex.go
View file

@ -29,6 +29,8 @@ var (
type HubItems map[string]map[string]Item
// HubIndex represents the runtime status of the hub (parsed items, etc.)
// XXX: this could be renamed "Hub" tout court once the confusion with HubCfg is cleared
type HubIndex struct {
Items HubItems
skippedLocal int

6
pkg/cwhub/install.go
View file

@ -10,7 +10,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) {
itempath := hub.HubDir + "/" + target.RemotePath
// disable hub file
@ -26,7 +26,7 @@ func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
}
// DisableItem to disable an item managed by the hub, removes the symlink if purge is true
func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error {
func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) error {
var err error
// already disabled, noop unless purge
@ -137,7 +137,7 @@ func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error
// creates symlink between actual config file at hub.HubDir and hub.ConfigDir
// Handles collections recursively
func EnableItem(hub *csconfig.Hub, target *Item) error {
func EnableItem(hub *csconfig.HubCfg, target *Item) error {
var err error
parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/")

8
pkg/cwhub/loader.go
View file

@ -66,7 +66,7 @@ type Walker struct {
installdir string
}
func NewWalker(hub *csconfig.Hub) Walker {
func NewWalker(hub *csconfig.HubCfg) Walker {
return Walker{
hubdir: hub.HubDir,
installdir: hub.InstallDir,
@ -404,7 +404,7 @@ func CollecDepsCheck(v *Item) error {
return nil
}
func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) {
warnings := []string{}
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
@ -445,7 +445,7 @@ func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
}
// Updates the info from HubInit() with the local state
func LocalSync(hub *csconfig.Hub) ([]string, error) {
func LocalSync(hub *csconfig.HubCfg) ([]string, error) {
hubIdx.skippedLocal = 0
hubIdx.skippedTainted = 0
@ -462,7 +462,7 @@ func LocalSync(hub *csconfig.Hub) ([]string, error) {
return warnings, nil
}
func GetHubIdx(hub *csconfig.Hub) error {
func GetHubIdx(hub *csconfig.HubCfg) error {
if hub == nil {
return fmt.Errorf("no configuration found for hub")
}

4
pkg/hubtest/hubtest_item.go
View file

@ -39,7 +39,7 @@ type HubTestItem struct {
RuntimeConfigFilePath string
RuntimeProfileFilePath string
RuntimeSimulationFilePath string
RuntimeHubConfig *csconfig.Hub
RuntimeHubConfig *csconfig.HubCfg
ResultsPath string
ParserResultFile string
@ -117,7 +117,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
ParserResultFile: filepath.Join(resultPath, ParserResultFileName),
ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName),
BucketPourResultFile: filepath.Join(resultPath, BucketPourResultFileName),
RuntimeHubConfig: &csconfig.Hub{
RuntimeHubConfig: &csconfig.HubCfg{
HubDir: runtimeHubFolder,
HubIndexFile: hubTest.HubIndexFile,
InstallDir: runtimeFolder,

4
pkg/setup/install.go
View file

@ -52,10 +52,6 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
return err
}
if err := csConfig.LoadHub(); err != nil {
return fmt.Errorf("loading hub: %w", err)
}
cwhub.SetHubBranch()
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {

10
test/bats/01_crowdsec.bats
View file

@ -55,16 +55,16 @@ teardown() {
assert_stderr --partial "unable to create database client: unknown database type 'meh'"
}
@test "crowdsec - bad configuration (empty/missing common section)" {
@test "crowdsec - default logging configuration (empty/missing common section)" {
config_set '.common={}'
rune -1 "${CROWDSEC}"
rune -124 timeout 1s "${CROWDSEC}"
refute_output
assert_stderr --partial "unable to load configuration: common section is empty"
assert_stderr --partial "Starting processing data"
config_set 'del(.common)'
rune -1 "${CROWDSEC}"
rune -124 timeout 1s "${CROWDSEC}"
refute_output
assert_stderr --partial "unable to load configuration: common section is empty"
assert_stderr --partial "Starting processing data"
}
@test "CS_LAPI_SECRET not strong enough" {