Refact cscli hub / pkg/cwhub (part 5) (#2521)

* remove unused yaml tags
* cscli/cwhub: deduplicate, remove dead code
* log.Fatal -> fmt.Errorf
* deflate utils.go by moving functions to respective files
* indexOf() -> slices.Index()
* ItemStatus() + toEmoji() -> Item.status()
* Item.versionStatus()
* move getSHA256() to loader.go

cmd/crowdsec-cli/capi.go

@@ -60,16 +60,16 @@ func NewCapiRegisterCmd() *cobra.Command {
 		Short:             "Register to Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			var err error
 			capiUser, err := generateID(capiUserPrefix)
 			if err != nil {
-				log.Fatalf("unable to generate machine id: %s", err)
+				return fmt.Errorf("unable to generate machine id: %s", err)
 			}
 			password := strfmt.Password(generatePassword(passwordLength))
 			apiurl, err := url.Parse(types.CAPIBaseURL)
 			if err != nil {
-				log.Fatalf("unable to parse api url %s : %s", types.CAPIBaseURL, err)
+				return fmt.Errorf("unable to parse api url %s: %w", types.CAPIBaseURL, err)
 			}
 			_, err = apiclient.RegisterClient(&apiclient.Config{
 				MachineID:     capiUser,
@@ -80,7 +80,7 @@ func NewCapiRegisterCmd() *cobra.Command {
 			}, nil)
 
 			if err != nil {
-				log.Fatalf("api client register ('%s'): %s", types.CAPIBaseURL, err)
+				return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err)
 			}
 			log.Printf("Successfully registered to Central API (CAPI)")
 
@@ -103,12 +103,12 @@ func NewCapiRegisterCmd() *cobra.Command {
 			}
 			apiConfigDump, err := yaml.Marshal(apiCfg)
 			if err != nil {
-				log.Fatalf("unable to marshal api credentials: %s", err)
+				return fmt.Errorf("unable to marshal api credentials: %w", err)
 			}
 			if dumpFile != "" {
 				err = os.WriteFile(dumpFile, apiConfigDump, 0600)
 				if err != nil {
-					log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
+					return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
 				}
 				log.Printf("Central API credentials dumped to '%s'", dumpFile)
 			} else {
@@ -116,6 +116,8 @@ func NewCapiRegisterCmd() *cobra.Command {
 			}
 
 			log.Warning(ReloadMessage())
+
+			return nil
 		},
 	}
 	cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
@@ -133,53 +135,56 @@ func NewCapiStatusCmd() *cobra.Command {
 		Short:             "Check status with the Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if csConfig.API.Server.OnlineClient == nil {
-				log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+				return fmt.Errorf("please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 			}
 
 			if csConfig.API.Server.OnlineClient.Credentials == nil {
-				log.Fatalf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+				return fmt.Errorf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 			}
 
 			password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
+
 			apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
 			if err != nil {
-				log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
+				return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
 			}
 
-			if err := csConfig.LoadHub(); err != nil {
+			if err := require.Hub(csConfig); err != nil {
-				log.Fatal(err)
+				return err
 			}
 
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to load hub index : %s", err)
-			}
 			scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 			if err != nil {
-				log.Fatalf("failed to get scenarios : %s", err)
+				return fmt.Errorf("failed to get scenarios: %w", err)
 			}
+
 			if len(scenarios) == 0 {
-				log.Fatalf("no scenarios installed, abort")
+				return fmt.Errorf("no scenarios installed, abort")
 			}
 
 			Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
 			if err != nil {
-				log.Fatalf("init default client: %s", err)
+				return fmt.Errorf("init default client: %w", err)
 			}
+
 			t := models.WatcherAuthRequest{
 				MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login,
 				Password:  &password,
 				Scenarios: scenarios,
 			}
+
 			log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 			log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl)
+
 			_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
 			if err != nil {
-				log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err)
+				return fmt.Errorf("failed to authenticate to Central API (CAPI): %w", err)
 			}
 			log.Infof("You can successfully interact with Central API (CAPI)")
+
+			return nil
 		},
 	}
 

cmd/crowdsec-cli/config_backup.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -8,9 +9,75 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
+func backupHub(dirPath string) error {
+	var err error
+	var itemDirectory string
+	var upstreamParsers []string
+
+	for _, itemType := range cwhub.ItemTypes {
+		clog := log.WithFields(log.Fields{
+			"type": itemType,
+		})
+		itemMap := cwhub.GetItemMap(itemType)
+		if itemMap == nil {
+			clog.Infof("No %s to backup.", itemType)
+			continue
+		}
+		itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
+		if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
+			return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
+		}
+		upstreamParsers = []string{}
+		for k, v := range itemMap {
+			clog = clog.WithFields(log.Fields{
+				"file": v.Name,
+			})
+			if !v.Installed { //only backup installed ones
+				clog.Debugf("[%s] : not installed", k)
+				continue
+			}
+
+			//for the local/tainted ones, we backup the full file
+			if v.Tainted || v.Local || !v.UpToDate {
+				//we need to backup stages for parsers
+				if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
+					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
+					if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
+						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
+					}
+				}
+				clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
+				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
+				if err = CopyFile(v.LocalPath, tfile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
+				}
+				clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
+				continue
+			}
+			clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
+			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
+			upstreamParsers = append(upstreamParsers, v.Name)
+		}
+		//write the upstream items
+		upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
+		upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
+		if err != nil {
+			return fmt.Errorf("failed marshaling upstream parsers : %s", err)
+		}
+		err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
+		if err != nil {
+			return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
+		}
+		clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
+	}
+
+	return nil
+}
+
 /*
 	Backup crowdsec configurations to directory <dirPath>:
 
@@ -122,7 +189,7 @@ func backupConfigToDirectory(dirPath string) error {
 		log.Infof("Saved profiles to %s", backupProfiles)
 	}
 
-	if err = BackupHub(dirPath); err != nil {
+	if err = backupHub(dirPath); err != nil {
 		return fmt.Errorf("failed to backup hub config: %s", err)
 	}
 

cmd/crowdsec-cli/config_restore.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
 type OldAPICfg struct {
@@ -20,6 +21,125 @@ type OldAPICfg struct {
 	Password  string `json:"password"`
 }
 
+// it's a rip of the cli version, but in silent-mode
+func silentInstallItem(name string, obtype string) (string, error) {
+	var item = cwhub.GetItem(obtype, name)
+	if item == nil {
+		return "", fmt.Errorf("error retrieving item")
+	}
+	it := *item
+	if downloadOnly && it.Downloaded && it.UpToDate {
+		return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
+	}
+	it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
+	if err != nil {
+		return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
+	}
+	if err := cwhub.AddItem(obtype, it); err != nil {
+		return "", err
+	}
+
+	if downloadOnly {
+		return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
+	}
+	it, err = cwhub.EnableItem(csConfig.Hub, it)
+	if err != nil {
+		return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
+	}
+	if err := cwhub.AddItem(obtype, it); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("Enabled %s", it.Name), nil
+}
+
+func restoreHub(dirPath string) error {
+	var err error
+
+	if err := csConfig.LoadHub(); err != nil {
+		return err
+	}
+
+	cwhub.SetHubBranch()
+
+	for _, itype := range cwhub.ItemTypes {
+		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
+		if _, err = os.Stat(itemDirectory); err != nil {
+			log.Infof("no %s in backup", itype)
+			continue
+		}
+		/*restore the upstream items*/
+		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
+		file, err := os.ReadFile(upstreamListFN)
+		if err != nil {
+			return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
+		}
+		var upstreamList []string
+		err = json.Unmarshal(file, &upstreamList)
+		if err != nil {
+			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
+		}
+		for _, toinstall := range upstreamList {
+			label, err := silentInstallItem(toinstall, itype)
+			if err != nil {
+				log.Errorf("Error while installing %s : %s", toinstall, err)
+			} else if label != "" {
+				log.Infof("Installed %s : %s", toinstall, label)
+			} else {
+				log.Printf("Installed %s : ok", toinstall)
+			}
+		}
+
+		/*restore the local and tainted items*/
+		files, err := os.ReadDir(itemDirectory)
+		if err != nil {
+			return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
+		}
+		for _, file := range files {
+			//this was the upstream data
+			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
+				continue
+			}
+			if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
+				//we expect a stage here
+				if !file.IsDir() {
+					continue
+				}
+				stage := file.Name()
+				stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
+				log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
+				if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
+					return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
+				}
+				/*find items*/
+				ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
+				if err != nil {
+					return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
+				}
+				//finally copy item
+				for _, tfile := range ifiles {
+					log.Infof("Going to restore local/tainted [%s]", tfile.Name())
+					sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
+					destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
+					if err = CopyFile(sourceFile, destinationFile); err != nil {
+						return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
+					}
+					log.Infof("restored %s to %s", sourceFile, destinationFile)
+				}
+			} else {
+				log.Infof("Going to restore local/tainted [%s]", file.Name())
+				sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
+				destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
+				if err = CopyFile(sourceFile, destinationFile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
+				}
+				log.Infof("restored %s to %s", sourceFile, destinationFile)
+			}
+
+		}
+	}
+	return nil
+}
+
 /*
 	Restore crowdsec configurations to directory <dirPath>:
 
@@ -168,7 +288,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 		}
 	}
 
-	if err = RestoreHub(dirPath); err != nil {
+	if err = restoreHub(dirPath); err != nil {
 		return fmt.Errorf("failed to restore hub config : %s", err)
 	}
 

cmd/crowdsec-cli/hub.go

@@ -88,9 +88,8 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := cwhub.SetHubBranch(); err != nil {
+			cwhub.SetHubBranch()
-				return fmt.Errorf("error while setting hub branch: %s", err)
+
-			}
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -134,9 +133,8 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := cwhub.SetHubBranch(); err != nil {
+			cwhub.SetHubBranch()
-				return fmt.Errorf("error while setting hub branch: %s", err)
+
-			}
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {

cmd/crowdsec-cli/require/require.go

@@ -73,9 +73,7 @@ func Hub (c *csconfig.Config) error {
 		return fmt.Errorf("you must configure cli before interacting with hub")
 	}
 
-	if err := cwhub.SetHubBranch(); err != nil {
+	cwhub.SetHubBranch()
-		return fmt.Errorf("while setting hub branch: %w", err)
-	}
 
 	if err := cwhub.GetHubIdx(c.Hub); err != nil {
 		return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)

cmd/crowdsec-cli/simulation.go

@@ -19,7 +19,7 @@ func addToExclusion(name string) error {
 }
 
 func removeFromExclusion(name string) error {
-	index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
+	index := slices.Index(csConfig.Cscli.SimulationConfig.Exclusions, name)
 
 	// Remove element from the slice
 	csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]

cmd/crowdsec-cli/utils.go

@@ -8,7 +8,6 @@ import (
 	"math"
 	"net"
 	"net/http"
-	"os"
 	"slices"
 	"strconv"
 	"strings"
@@ -24,6 +23,7 @@ import (
 
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -38,34 +38,6 @@ func printHelp(cmd *cobra.Command) {
 	}
 }
 
-func indexOf(s string, slice []string) int {
-	for i, elem := range slice {
-		if s == elem {
-			return i
-		}
-	}
-	return -1
-}
-
-func LoadHub() error {
-	if err := csConfig.LoadHub(); err != nil {
-		log.Fatal(err)
-	}
-	if csConfig.Hub == nil {
-		return fmt.Errorf("unable to load hub")
-	}
-
-	if err := cwhub.SetHubBranch(); err != nil {
-		log.Warningf("unable to set hub branch (%s), default to master", err)
-	}
-
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
-	}
-
-	return nil
-}
-
 func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
 	errMsg := ""
 	if score < MaxDistance {
@@ -100,7 +72,7 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
 }
 
 func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := LoadHub(); err != nil {
+	if err := require.Hub(csConfig); err != nil {
 		return nil, cobra.ShellCompDirectiveDefault
 	}
 
@@ -116,7 +88,7 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string,
 }
 
 func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := LoadHub(); err != nil {
+	if err := require.Hub(csConfig); err != nil {
 		return nil, cobra.ShellCompDirectiveDefault
 	}
 
@@ -453,37 +425,6 @@ func GetScenarioMetric(url string, itemName string) map[string]int {
 	return stats
 }
 
-// it's a rip of the cli version, but in silent-mode
-func silenceInstallItem(name string, obtype string) (string, error) {
-	var item = cwhub.GetItem(obtype, name)
-	if item == nil {
-		return "", fmt.Errorf("error retrieving item")
-	}
-	it := *item
-	if downloadOnly && it.Downloaded && it.UpToDate {
-		return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
-	}
-	it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
-	if err != nil {
-		return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
-	}
-	if err := cwhub.AddItem(obtype, it); err != nil {
-		return "", err
-	}
-
-	if downloadOnly {
-		return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
-	}
-	it, err = cwhub.EnableItem(csConfig.Hub, it)
-	if err != nil {
-		return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
-	}
-	if err := cwhub.AddItem(obtype, it); err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("Enabled %s", it.Name), nil
-}
-
 func GetPrometheusMetric(url string) []*prom2json.Family {
 	mfChan := make(chan *dto.MetricFamily, 1024)
 
@@ -512,160 +453,6 @@ func GetPrometheusMetric(url string) []*prom2json.Family {
 	return result
 }
 
-func RestoreHub(dirPath string) error {
-	var err error
-
-	if err := csConfig.LoadHub(); err != nil {
-		return err
-	}
-	if err := cwhub.SetHubBranch(); err != nil {
-		return fmt.Errorf("error while setting hub branch: %s", err)
-	}
-
-	for _, itype := range cwhub.ItemTypes {
-		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
-		if _, err = os.Stat(itemDirectory); err != nil {
-			log.Infof("no %s in backup", itype)
-			continue
-		}
-		/*restore the upstream items*/
-		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
-		file, err := os.ReadFile(upstreamListFN)
-		if err != nil {
-			return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
-		}
-		var upstreamList []string
-		err = json.Unmarshal(file, &upstreamList)
-		if err != nil {
-			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
-		}
-		for _, toinstall := range upstreamList {
-			label, err := silenceInstallItem(toinstall, itype)
-			if err != nil {
-				log.Errorf("Error while installing %s : %s", toinstall, err)
-			} else if label != "" {
-				log.Infof("Installed %s : %s", toinstall, label)
-			} else {
-				log.Printf("Installed %s : ok", toinstall)
-			}
-		}
-
-		/*restore the local and tainted items*/
-		files, err := os.ReadDir(itemDirectory)
-		if err != nil {
-			return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
-		}
-		for _, file := range files {
-			//this was the upstream data
-			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
-				continue
-			}
-			if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
-				//we expect a stage here
-				if !file.IsDir() {
-					continue
-				}
-				stage := file.Name()
-				stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
-				log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
-				if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
-					return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
-				}
-				/*find items*/
-				ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
-				if err != nil {
-					return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
-				}
-				//finally copy item
-				for _, tfile := range ifiles {
-					log.Infof("Going to restore local/tainted [%s]", tfile.Name())
-					sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
-					destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
-					if err = CopyFile(sourceFile, destinationFile); err != nil {
-						return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
-					}
-					log.Infof("restored %s to %s", sourceFile, destinationFile)
-				}
-			} else {
-				log.Infof("Going to restore local/tainted [%s]", file.Name())
-				sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
-				destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
-				if err = CopyFile(sourceFile, destinationFile); err != nil {
-					return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
-				}
-				log.Infof("restored %s to %s", sourceFile, destinationFile)
-			}
-
-		}
-	}
-	return nil
-}
-
-func BackupHub(dirPath string) error {
-	var err error
-	var itemDirectory string
-	var upstreamParsers []string
-
-	for _, itemType := range cwhub.ItemTypes {
-		clog := log.WithFields(log.Fields{
-			"type": itemType,
-		})
-		itemMap := cwhub.GetItemMap(itemType)
-		if itemMap == nil {
-			clog.Infof("No %s to backup.", itemType)
-			continue
-		}
-		itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
-		if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
-			return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
-		}
-		upstreamParsers = []string{}
-		for k, v := range itemMap {
-			clog = clog.WithFields(log.Fields{
-				"file": v.Name,
-			})
-			if !v.Installed { //only backup installed ones
-				clog.Debugf("[%s] : not installed", k)
-				continue
-			}
-
-			//for the local/tainted ones, we backup the full file
-			if v.Tainted || v.Local || !v.UpToDate {
-				//we need to backup stages for parsers
-				if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
-					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
-					if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
-						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
-					}
-				}
-				clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
-				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
-				if err = CopyFile(v.LocalPath, tfile); err != nil {
-					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
-				}
-				clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
-				continue
-			}
-			clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
-			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
-			upstreamParsers = append(upstreamParsers, v.Name)
-		}
-		//write the upstream items
-		upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
-		upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
-		if err != nil {
-			return fmt.Errorf("failed marshaling upstream parsers : %s", err)
-		}
-		err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
-		if err != nil {
-			return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
-		}
-		clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
-	}
-
-	return nil
-}
-
 type unit struct {
 	value  int64
 	symbol string

cmd/crowdsec-cli/utils_table.go

@@ -17,7 +17,7 @@ func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatu
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	for _, status := range statuses {
-		t.AddRow(status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath)
+		t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
 	}
 	renderTableTitle(out, title)
 	t.Render()

pkg/csconfig/hub.go

@@ -2,10 +2,10 @@ package csconfig
 
 /*cscli specific config, such as hub directory*/
 type Hub struct {
-	HubDir       string `yaml:"-"`
+	HubDir       string
-	ConfigDir    string `yaml:"-"`
+	ConfigDir    string
-	HubIndexFile string `yaml:"-"`
+	HubIndexFile string
-	DataDir      string `yaml:"-"`
+	DataDir      string
 }
 
 func (c *Config) LoadHub() error {

pkg/cwhub/cwhub.go

@@ -1,9 +1,7 @@
 package cwhub
 
 import (
-	"crypto/sha256"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"sort"
@@ -40,7 +38,7 @@ type ItemHubStatus struct {
 	LocalVersion string `json:"local_version"`
 	LocalPath    string `json:"local_path"`
 	Description  string `json:"description"`
-	UTF8_Status  string `json:"utf8_status"`
+	UTF8Status   string `json:"utf8_status"`
 	Status       string `json:"status"`
 }
 
@@ -62,7 +60,7 @@ type Item struct {
 	Versions   map[string]ItemVersion `json:"versions,omitempty"  yaml:"-"`                     // the list of existing versions
 
 	// local (deployed) info
-	LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
+	LocalPath    string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
 	LocalVersion string `json:"local_version,omitempty"`
 	LocalHash    string `json:"local_hash,omitempty"` // the local meow
 	Installed    bool   `json:"installed,omitempty"`
@@ -78,29 +76,48 @@ type Item struct {
 	Collections   []string `json:"collections,omitempty"   yaml:"collections,omitempty"`
 }
 
-func toEmoji(managed bool, installed bool, warning bool, ok bool) emoji.Emoji {
+func (i *Item) status() (string, emoji.Emoji) {
-	if !managed {
+	status := "disabled"
-		return emoji.House
+	ok := false
+
+	if i.Installed {
+		ok = true
+		status = "enabled"
 	}
 
-	if !installed {
+	managed := true
-		return emoji.Prohibited
+	if i.Local {
+		managed = false
+		status += ",local"
 	}
 
-	if warning {
+	warning := false
-		return emoji.Warning
+	if i.Tainted {
+		warning = true
+		status += ",tainted"
+	} else if !i.UpToDate && !i.Local {
+		warning = true
+		status += ",update-available"
 	}
 
-	if ok {
+	emo := emoji.QuestionMark
-		return emoji.CheckMark
+
+	switch {
+	case !managed:
+		emo = emoji.House
+	case !i.Installed:
+		emo = emoji.Prohibited
+	case warning:
+		emo = emoji.Warning
+	case ok:
+		emo = emoji.CheckMark
 	}
 
-	// XXX: this is new
+	return status, emo
-	return emoji.QuestionMark
 }
 
-func (i *Item) toHubStatus() ItemHubStatus {
+func (i *Item) hubStatus() ItemHubStatus {
-	status, ok, warning, managed := ItemStatus(*i)
+	status, emo := i.status()
 
 	return ItemHubStatus{
 		Name:         i.Name,
@@ -108,37 +125,21 @@ func (i *Item) toHubStatus() ItemHubStatus {
 		LocalPath:    i.LocalPath,
 		Description:  i.Description,
 		Status:       status,
-		UTF8_Status:  fmt.Sprintf("%v  %s", toEmoji(managed, i.Installed, warning, ok), status),
+		UTF8Status:   fmt.Sprintf("%v  %s", emo, status),
 	}
 }
 
+// versionStatus: semver requires 'v' prefix
+func (i *Item) versionStatus() int {
+	return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
+}
+
 // XXX: can we remove these globals?
 var skippedLocal = 0
 var skippedTainted = 0
 
 var ReferenceMissingError = errors.New("Reference(s) missing in collection")
 
-// GetVersionStatus: semver requires 'v' prefix
-func GetVersionStatus(v *Item) int {
-	return semver.Compare("v"+v.Version, "v"+v.LocalVersion)
-}
-
-func getSHA256(filepath string) (string, error) {
-	f, err := os.Open(filepath)
-	if err != nil {
-		return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
-	}
-
-	defer f.Close()
-
-	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
-		return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
-	}
-
-	return fmt.Sprintf("%x", h.Sum(nil)), nil
-}
-
 func GetItemMap(itemType string) map[string]Item {
 	m, ok := hubIdx[itemType]
 	if !ok {
@@ -223,35 +224,6 @@ func DisplaySummary() {
 	}
 }
 
-// returns: human-text, Enabled, Warning, Unmanaged
-func ItemStatus(v Item) (string, bool, bool, bool) {
-	strret := "disabled"
-	Ok := false
-
-	if v.Installed {
-		Ok = true
-		strret = "enabled"
-	}
-
-	Managed := true
-	if v.Local {
-		Managed = false
-		strret += ",local"
-	}
-
-	// tainted or out of date
-	Warning := false
-	if v.Tainted {
-		Warning = true
-		strret += ",tainted"
-	} else if !v.UpToDate && !v.Local {
-		Warning = true
-		strret += ",update-available"
-	}
-
-	return strret, Ok, Warning, Managed
-}
-
 func GetInstalledItems(itemType string) ([]Item, error) {
 	var retItems []Item
 
@@ -305,7 +277,7 @@ func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubSt
 			continue
 		}
 		// Check the item status
-		ret = append(ret, item.toHubStatus())
+		ret = append(ret, item.hubStatus())
 	}
 
 	sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })

pkg/cwhub/cwhub_test.go

@@ -57,7 +57,7 @@ func TestItemStatus(t *testing.T) {
 		item.Local = false
 		item.Tainted = false
 
-		txt, _, _, _ := ItemStatus(*item)
+		txt, _ := item.status()
 		if txt != "enabled,update-available" {
 			t.Fatalf("got '%s'", txt)
 		}
@@ -67,7 +67,7 @@ func TestItemStatus(t *testing.T) {
 		item.Local = true
 		item.Tainted = false
 
-		txt, _, _, _ = ItemStatus(*item)
+		txt, _ = item.status()
 		if txt != "disabled,local" {
 			t.Fatalf("got '%s'", txt)
 		}

pkg/cwhub/helpers.go

@@ -13,29 +13,29 @@ import (
 )
 
 // pick a hub branch corresponding to the current crowdsec version.
-func chooseHubBranch() (string, error) {
+func chooseHubBranch() string {
 	latest, err := cwversion.Latest()
 	if err != nil {
 		log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
 		//lint:ignore nilerr
-		return "master", nil
+		return "master"
 	}
 
 	csVersion := cwversion.VersionStrip()
 	if csVersion == latest {
 		log.Debugf("current version is equal to latest (%s)", csVersion)
-		return "master", nil
+		return "master"
 	}
 
 	// if current version is greater than the latest we are in pre-release
 	if semver.Compare(csVersion, latest) == 1 {
 		log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
-		return "master", nil
+		return "master"
 	}
 
 	if csVersion == "" {
 		log.Warning("Crowdsec version is not set, using master branch for the hub")
-		return "master", nil
+		return "master"
 	}
 
 	log.Warnf("Crowdsec is not the latest version. "+
@@ -45,26 +45,20 @@ func chooseHubBranch() (string, error) {
 	log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
 		"added to Crowdsec Hub after CrowdSec %s", latest)
 
-	return csVersion, nil
+	return csVersion
 }
 
 // SetHubBranch sets the package variable that points to the hub branch.
-func SetHubBranch() error {
+func SetHubBranch() {
 	// a branch is already set, or specified from the flags
 	if HubBranch != "" {
-		return nil
+		return
 	}
 
 	// use the branch corresponding to the crowdsec version
-	branch, err := chooseHubBranch()
+	HubBranch = chooseHubBranch()
-	if err != nil {
-		return err
-	}
 
-	HubBranch = branch
 	log.Debugf("Using branch '%s' for the hub", HubBranch)
-
-	return nil
 }
 
 func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error {

pkg/cwhub/loader.go

@@ -1,9 +1,11 @@
 package cwhub
 
 import (
+	"crypto/sha256"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"sort"
@@ -43,6 +45,22 @@ func handleSymlink(path string) (string, error) {
 	return hubpath, nil
 }
 
+func getSHA256(filepath string) (string, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
+	}
+
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
+	}
+
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
 type walker struct {
 	// the walk/parserVisit function can't receive extra args
 	hubdir     string
@@ -317,7 +335,7 @@ func (w walker) parserVisit(path string, f os.DirEntry, err error) error {
 }
 
 func CollecDepsCheck(v *Item) error {
-	if GetVersionStatus(v) != 0 { // not up-to-date
+	if v.versionStatus() != 0 { // not up-to-date
 		log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
 		return nil
 	}
@@ -415,11 +433,11 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
 			continue
 		}
 
-		versionStatus := GetVersionStatus(&item)
+		vs := item.versionStatus()
-		switch versionStatus {
+		switch vs {
 		case 0: // latest
 			if err := CollecDepsCheck(&item); err != nil {
-				warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", item.Name, err))
+				warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
 				hubIdx[COLLECTIONS][name] = item
 			}
 		case 1: // not up-to-date
@@ -428,7 +446,7 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
 			warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
 		}
 
-		log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, versionStatus, item.LocalVersion, item.Version, item.Versions)
+		log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions)
 	}
 
 	return nil, warnings

pkg/setup/install.go

@@ -56,9 +56,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 		return fmt.Errorf("loading hub: %w", err)
 	}
 
-	if err := cwhub.SetHubBranch(); err != nil {
+	cwhub.SetHubBranch()
-		return fmt.Errorf("setting hub branch: %w", err)
-	}
 
 	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
 		return fmt.Errorf("getting hub index: %w", err)