cwhub: export SetHubBranch (#1559)

cmd/crowdsec-cli/collections.go

@@ -27,7 +27,7 @@ func NewCollectionsCmd() *cobra.Command {
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := setHubBranch(); err != nil {
+			if err := cwhub.SetHubBranch(); err != nil {
 				return fmt.Errorf("error while setting hub branch: %s", err)
 			}
 

cmd/crowdsec-cli/hub.go

@@ -18,7 +18,7 @@ func NewHubCmd() *cobra.Command {
 Hub management
 
 List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
-Hub is manage by cscli, to get latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
+The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
 		`,
 		Example: `
 cscli hub list   # List all installed configurations
@@ -77,7 +77,7 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := setHubBranch(); err != nil {
+			if err := cwhub.SetHubBranch(); err != nil {
 				return fmt.Errorf("error while setting hub branch: %s", err)
 			}
 			return nil
@@ -111,7 +111,7 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := setHubBranch(); err != nil {
+			if err := cwhub.SetHubBranch(); err != nil {
 				return fmt.Errorf("error while setting hub branch: %s", err)
 			}
 			return nil

cmd/crowdsec-cli/hubtest.go

@@ -29,11 +29,9 @@ func NewHubTestCmd() *cobra.Command {
 	var cscliPath string
 
 	var cmdHubTest = &cobra.Command{
-		Use:   "hubtest",
+		Use:               "hubtest",
-		Short: "Run functional tests on hub configurations",
+		Short:             "Run functional tests on hub configurations",
-		Long: `
+		Long:              "Run functional tests on hub configurations (parsers, scenarios, collections...)",
-		Run functional tests on hub configurations (parsers, scenarios, collections...)
-		`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {

cmd/crowdsec-cli/parsers.go

@@ -31,7 +31,7 @@ cscli parsers remove crowdsecurity/sshd-logs
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := setHubBranch(); err != nil {
+			if err := cwhub.SetHubBranch(); err != nil {
 				return fmt.Errorf("error while setting hub branch: %s", err)
 			}
 

cmd/crowdsec-cli/postoverflows.go

@@ -30,7 +30,7 @@ func NewPostOverflowsCmd() *cobra.Command {
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := setHubBranch(); err != nil {
+			if err := cwhub.SetHubBranch(); err != nil {
 				return fmt.Errorf("error while setting hub branch: %s", err)
 			}
 

cmd/crowdsec-cli/scenarios.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/pkg/errors"
 
 	log "github.com/sirupsen/logrus"
 
@@ -32,13 +33,13 @@ cscli scenarios remove crowdsecurity/ssh-bf
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			if err := setHubBranch(); err != nil {
+			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
+				return errors.Wrap(err, "while setting hub branch")
 			}
 
 			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
 				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
+				log.Fatalf("Failed to get Hub index : %v", err)
 			}
 
 			return nil

cmd/crowdsec-cli/utils.go

@@ -14,7 +14,6 @@ import (
 	"time"
 
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/enescakir/emoji"
 	"github.com/olekukonko/tablewriter"
@@ -22,7 +21,6 @@ import (
 	"github.com/prometheus/prom2json"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/mod/semver"
 	"gopkg.in/yaml.v2"
 )
 
@@ -59,12 +57,12 @@ func LoadHub() error {
 		return fmt.Errorf("unable to load hub")
 	}
 
-	if err := setHubBranch(); err != nil {
+	if err := cwhub.SetHubBranch(); err != nil {
 		log.Warningf("unable to set hub branch (%s), default to master", err)
 	}
 
 	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		return fmt.Errorf("Failed to get Hub index : '%v'. Run 'sudo cscli hub update' to get the hub index", err)
+		return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
 	}
 
 	return nil
@@ -281,35 +279,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
 	return nil
 }
 
-func setHubBranch() error {
-	/*
-		if no branch has been specified in flags for the hub, then use the one corresponding to crowdsec version
-	*/
-	if cwhub.HubBranch == "" {
-		latest, err := cwversion.Latest()
-		if err != nil {
-			cwhub.HubBranch = "master"
-			return err
-		}
-		csVersion := cwversion.VersionStrip()
-		if csVersion == latest {
-			cwhub.HubBranch = "master"
-		} else if semver.Compare(csVersion, latest) == 1 { // if current version is greater than the latest we are in pre-release
-			log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
-			cwhub.HubBranch = "master"
-		} else if csVersion == "" {
-			log.Warningf("Crowdsec version is '', using master branch for the hub")
-			cwhub.HubBranch = "master"
-		} else {
-			log.Warnf("Crowdsec is not the latest version. Current version is '%s' and the latest stable version is '%s'. Please update it!", csVersion, latest)
-			log.Warnf("As a result, you will not be able to use parsers/scenarios/collections added to Crowdsec Hub after CrowdSec %s", latest)
-			cwhub.HubBranch = csVersion
-		}
-		log.Debugf("Using branch '%s' for the hub", cwhub.HubBranch)
-	}
-	return nil
-}
-
 func ShowMetrics(hubItem *cwhub.Item) {
 	switch hubItem.Type {
 	case cwhub.PARSERS:
@@ -564,7 +533,7 @@ func RestoreHub(dirPath string) error {
 	if err := csConfig.LoadHub(); err != nil {
 		return err
 	}
-	if err := setHubBranch(); err != nil {
+	if err := cwhub.SetHubBranch(); err != nil {
 		return fmt.Errorf("error while setting hub branch: %s", err)
 	}
 

pkg/cwhub/cwhub.go

@@ -2,20 +2,17 @@ package cwhub
 
 import (
 	"crypto/sha256"
+	"fmt"
+	"io"
+	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 
-	//"errors"
-	"fmt"
-	"io"
-	"os"
-
 	"github.com/enescakir/emoji"
 	"github.com/pkg/errors"
-	"golang.org/x/mod/semver"
-
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/mod/semver"
 )
 
 /*managed configuration types*/

pkg/cwhub/download.go

@@ -3,22 +3,18 @@ package cwhub
 import (
 	"bytes"
 	"crypto/sha256"
-	"path"
-	"path/filepath"
-
-	//"errors"
-	"github.com/pkg/errors"
-
-	//"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
+	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 )
@@ -80,61 +76,61 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b
 	var err error
 
 	log.Debugf("Downloading %s %s", target.Type, target.Name)
-	if target.Type == COLLECTIONS {
+	if target.Type != COLLECTIONS {
-		var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
-		for idx, ptr := range tmp {
-			ptrtype := ItemTypes[idx]
-			for _, p := range ptr {
-				val, ok := hubIdx[ptrtype][p]
-				if !ok {
-					return target, fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
-				}
-
-				if !val.Installed && updateOnly && val.Downloaded {
-					log.Debugf("skipping upgrade of %s : not installed", target.Name)
-					continue
-				}
-
-				log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly)
-				//recurse as it's a collection
-				if ptrtype == COLLECTIONS {
-					log.Tracef("collection, recurse")
-					hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly)
-					if err != nil {
-						return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
-					}
-				}
-				item, err := DownloadItem(hub, val, overwrite)
-				if err != nil {
-					return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
-				}
-
-				// We need to enable an item when it has been added to a collection since latest release of the collection.
-				// We check if val.Downloaded is false because maybe the item has been disabled by the user.
-				if !item.Installed && !val.Downloaded {
-					if item, err = EnableItem(hub, item); err != nil {
-						return target, errors.Wrapf(err, "enabling '%s'", item.Name)
-					}
-				}
-				hubIdx[ptrtype][p] = item
-			}
-		}
-		target, err = DownloadItem(hub, target, overwrite)
-		if err != nil {
-			return target, fmt.Errorf("failed to download item : %s", err)
-		}
-	} else {
 		if !target.Installed && updateOnly && target.Downloaded {
 			log.Debugf("skipping upgrade of %s : not installed", target.Name)
 			return target, nil
 		}
 		return DownloadItem(hub, target, overwrite)
 	}
+
+	// collection
+	var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
+	for idx, ptr := range tmp {
+		ptrtype := ItemTypes[idx]
+		for _, p := range ptr {
+			val, ok := hubIdx[ptrtype][p]
+			if !ok {
+				return target, fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
+			}
+
+			if !val.Installed && updateOnly && val.Downloaded {
+				log.Debugf("skipping upgrade of %s : not installed", target.Name)
+				continue
+			}
+
+			log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly)
+			//recurse as it's a collection
+			if ptrtype == COLLECTIONS {
+				log.Tracef("collection, recurse")
+				hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly)
+				if err != nil {
+					return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
+				}
+			}
+			item, err := DownloadItem(hub, val, overwrite)
+			if err != nil {
+				return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
+			}
+
+			// We need to enable an item when it has been added to a collection since latest release of the collection.
+			// We check if val.Downloaded is false because maybe the item has been disabled by the user.
+			if !item.Installed && !val.Downloaded {
+				if item, err = EnableItem(hub, item); err != nil {
+					return target, errors.Wrapf(err, "enabling '%s'", item.Name)
+				}
+			}
+			hubIdx[ptrtype][p] = item
+		}
+	}
+	target, err = DownloadItem(hub, target, overwrite)
+	if err != nil {
+		return target, fmt.Errorf("failed to download item : %s", err)
+	}
 	return target, nil
 }
 
 func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) {
-
 	var tdir = hub.HubDir
 	var dataFolder = hub.DataDir
 	/*if user didn't --force, don't overwrite local, tainted, up-to-date files*/

pkg/cwhub/helpers.go

@@ -2,17 +2,70 @@ package cwhub
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 	"github.com/enescakir/emoji"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/mod/semver"
 )
 
+// pick a hub branch corresponding to the current crowdsec version.
+func chooseHubBranch() (string, error) {
+	latest, err := cwversion.Latest()
+	if err != nil {
+		return "master", err
+	}
+
+	csVersion := cwversion.VersionStrip()
+	if csVersion == latest {
+		return "master", nil
+	}
+
+	// if current version is greater than the latest we are in pre-release
+	if semver.Compare(csVersion, latest) == 1 {
+		log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
+		return "master", nil
+	}
+
+	if csVersion == "" {
+		log.Warningf("Crowdsec version is not set, using master branch for the hub")
+		return "master", nil
+	}
+
+	log.Warnf("Crowdsec is not the latest version. "+
+		"Current version is '%s' and the latest stable version is '%s'. Please update it!",
+		csVersion, latest)
+	log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
+		"added to Crowdsec Hub after CrowdSec %s", latest)
+	return csVersion, nil
+}
+
+// SetHubBranch sets the package variable that points to the hub branch.
+func SetHubBranch() error {
+	// a branch is already set, or specified from the flags
+	if HubBranch != "" {
+		return nil
+	}
+
+	// use the branch corresponding to the crowdsec version
+	branch, err := chooseHubBranch()
+	if err != nil {
+		return err
+	}
+	HubBranch = branch
+	log.Debugf("Using branch '%s' for the hub", HubBranch)
+	return nil
+}
+
 func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error {
 	it := GetItem(obtype, name)
 	if it == nil {
-		return fmt.Errorf("unable to retrieve item : %s", name)
+		return fmt.Errorf("unable to retrieve item: %s", name)
 	}
+
 	item := *it
 	if downloadOnly && item.Downloaded && item.UpToDate {
 		log.Warningf("%s is already downloaded and up-to-date", item.Name)
@@ -20,69 +73,85 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo
 			return nil
 		}
 	}
+
 	item, err := DownloadLatest(csConfig.Hub, item, force, true)
 	if err != nil {
-		return fmt.Errorf("error while downloading %s : %v", item.Name, err)
+		return errors.Wrapf(err, "while downloading %s", item.Name)
 	}
+
 	if err := AddItem(obtype, item); err != nil {
-		return fmt.Errorf("error while adding %s : %v", item.Name, err)
+		return errors.Wrapf(err, "while adding %s", item.Name)
 	}
+
 	if downloadOnly {
-		log.Infof("Downloaded %s to %s", item.Name, csConfig.Hub.HubDir+"/"+item.RemotePath)
+		log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath))
 		return nil
 	}
+
 	item, err = EnableItem(csConfig.Hub, item)
 	if err != nil {
-		return fmt.Errorf("error while enabling  %s : %v.", item.Name, err)
+		return errors.Wrapf(err, "while enabling %s", item.Name)
 	}
+
 	if err := AddItem(obtype, item); err != nil {
-		return fmt.Errorf("error while adding %s : %v", item.Name, err)
+		return errors.Wrapf(err, "while adding %s", item.Name)
 	}
+
 	log.Infof("Enabled %s", item.Name)
 
 	return nil
 }
 
+// XXX this must return errors instead of log.Fatal
 func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) {
-	var err error
+	var (
-	var disabled int
+		err      error
+		disabled int
+	)
+
 	if name != "" {
 		it := GetItem(itemType, name)
 		if it == nil {
 			log.Fatalf("unable to retrieve: %s", name)
 		}
+
 		item := *it
 		item, err = DisableItem(csConfig.Hub, item, purge, forceAction)
 		if err != nil {
 			log.Fatalf("unable to disable %s : %v", item.Name, err)
 		}
+
 		if err := AddItem(itemType, item); err != nil {
 			log.Fatalf("unable to add %s: %v", item.Name, err)
 		}
 		return
-	} else if name == "" && all {
-		for _, v := range GetItemMap(itemType) {
-			v, err = DisableItem(csConfig.Hub, v, purge, forceAction)
-			if err != nil {
-				log.Fatalf("unable to disable %s : %v", v.Name, err)
-			}
-			if err := AddItem(itemType, v); err != nil {
-				log.Fatalf("unable to add %s: %v", v.Name, err)
-			}
-			disabled++
-		}
 	}
-	if name != "" && !all {
+
-		log.Errorf("%s not found", name)
+	if !all {
-		return
+		log.Fatal("removing item: no item specified")
+	}
+
+	// remove all
+	for _, v := range GetItemMap(itemType) {
+		v, err = DisableItem(csConfig.Hub, v, purge, forceAction)
+		if err != nil {
+			log.Fatalf("unable to disable %s : %v", v.Name, err)
+		}
+
+		if err := AddItem(itemType, v); err != nil {
+			log.Fatalf("unable to add %s: %v", v.Name, err)
+		}
+		disabled++
 	}
 	log.Infof("Disabled %d items", disabled)
 }
 
 func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) {
-	var err error
+	var (
-	var updated int
+		err     error
-	var found bool
+		updated int
+		found   bool
+	)
 
 	for _, v := range GetItemMap(itemType) {
 		if name != "" && name != v.Name {
@@ -100,6 +169,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
 		}
 
 		found = true
+
 		if v.UpToDate {
 			log.Infof("%s : up-to-date", v.Name)
 
@@ -111,10 +181,12 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
 				continue
 			}
 		}
+
 		v, err = DownloadLatest(csConfig.Hub, v, force, true)
 		if err != nil {
 			log.Fatalf("%s : download failed : %v", v.Name, err)
 		}
+
 		if !v.UpToDate {
 			if v.Tainted {
 				log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name)
@@ -125,10 +197,12 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
 			log.Infof("%v %s : updated", emoji.Package, v.Name)
 			updated++
 		}
+
 		if err := AddItem(itemType, v); err != nil {
 			log.Fatalf("unable to add %s: %v", v.Name, err)
 		}
 	}
+
 	if !found && name == "" {
 		log.Infof("No %s installed, nothing to upgrade", itemType)
 	} else if !found {
@@ -142,5 +216,4 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
 	} else if updated != 0 {
 		log.Infof("Upgraded %d items", updated)
 	}
-
 }

pkg/cwhub/install.go

@@ -19,6 +19,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
 	if err != nil {
 		return Item{}, err
 	}
+
 	if target.Local {
 		return target, fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name)
 	}
@@ -136,38 +137,41 @@ func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
 		for idx, ptr := range tmp {
 			ptrtype := ItemTypes[idx]
 			for _, p := range ptr {
-				if val, ok := hubIdx[ptrtype][p]; ok {
+				val, ok := hubIdx[ptrtype][p]
-					hubIdx[ptrtype][p], err = EnableItem(hub, val)
+				if !ok {
-					if err != nil {
-						return target, errors.Wrap(err, fmt.Sprintf("while installing %s", p))
-					}
-				} else {
 					return target, fmt.Errorf("required %s %s of %s doesn't exist, abort.", ptrtype, p, target.Name)
 				}
+
+				hubIdx[ptrtype][p], err = EnableItem(hub, val)
+				if err != nil {
+					return target, errors.Wrap(err, fmt.Sprintf("while installing %s", p))
+				}
 			}
 		}
 	}
 
 	// check if file already exists where it should in configdir (eg /etc/crowdsec/collections/)
-	if _, err := os.Lstat(parent_dir + "/" + target.FileName); os.IsNotExist(err) {
+	if _, err := os.Lstat(parent_dir + "/" + target.FileName); !os.IsNotExist(err) {
-		//tdir+target.RemotePath
-		srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath)
-		if err != nil {
-			return target, errors.Wrap(err, "while getting source path")
-		}
-		dstPath, err := filepath.Abs(parent_dir + "/" + target.FileName)
-		if err != nil {
-			return target, errors.Wrap(err, "while getting destination path")
-		}
-		err = os.Symlink(srcPath, dstPath)
-		if err != nil {
-			return target, errors.Wrap(err, fmt.Sprintf("while creating symlink from %s to %s", srcPath, dstPath))
-		}
-		log.Printf("Enabled %s : %s", target.Type, target.Name)
-	} else {
 		log.Printf("%s already exists.", parent_dir+"/"+target.FileName)
 		return target, nil
 	}
+
+	//tdir+target.RemotePath
+	srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath)
+	if err != nil {
+		return target, errors.Wrap(err, "while getting source path")
+	}
+
+	dstPath, err := filepath.Abs(parent_dir + "/" + target.FileName)
+	if err != nil {
+		return target, errors.Wrap(err, "while getting destination path")
+	}
+
+	if err = os.Symlink(srcPath, dstPath); err != nil {
+		return target, errors.Wrap(err, fmt.Sprintf("while creating symlink from %s to %s", srcPath, dstPath))
+	}
+
+	log.Printf("Enabled %s : %s", target.Type, target.Name)
 	target.Installed = true
 	hubIdx[target.Type][target.Name] = target
 	return target, nil

pkg/cwhub/loader.go

@@ -2,22 +2,17 @@ package cwhub
 
 import (
 	"encoding/json"
-	//"errors"
 	"fmt"
 	"io/ioutil"
-	"sort"
-
-	"github.com/pkg/errors"
-	"golang.org/x/mod/semver"
-
-	//"log"
-
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/mod/semver"
 )
 
 /*the walk/parser_visit function can't receive extra args*/