From 1a293a2a275be89cd56299f230c8eb145769f181 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 24 May 2022 15:46:48 +0200 Subject: [PATCH] cwhub: export SetHubBranch (#1559) --- cmd/crowdsec-cli/collections.go | 2 +- cmd/crowdsec-cli/hub.go | 6 +- cmd/crowdsec-cli/hubtest.go | 8 +- cmd/crowdsec-cli/parsers.go | 2 +- cmd/crowdsec-cli/postoverflows.go | 2 +- cmd/crowdsec-cli/scenarios.go | 7 +- cmd/crowdsec-cli/utils.go | 37 +-------- pkg/cwhub/cwhub.go | 11 +-- pkg/cwhub/download.go | 100 ++++++++++++------------ pkg/cwhub/helpers.go | 125 +++++++++++++++++++++++------- pkg/cwhub/install.go | 48 ++++++------ pkg/cwhub/loader.go | 11 +-- 12 files changed, 196 insertions(+), 163 deletions(-) diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go index 3dc079504..c25cc5105 100644 --- a/cmd/crowdsec-cli/collections.go +++ b/cmd/crowdsec-cli/collections.go @@ -27,7 +27,7 @@ func NewCollectionsCmd() *cobra.Command { return fmt.Errorf("you must configure cli before interacting with hub") } - if err := setHubBranch(); err != nil { + if err := cwhub.SetHubBranch(); err != nil { return fmt.Errorf("error while setting hub branch: %s", err) } diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 1da6e6770..bf1ace9a0 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -18,7 +18,7 @@ func NewHubCmd() *cobra.Command { Hub management List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net). -Hub is manage by cscli, to get latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update. +The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update. `, Example: ` cscli hub list # List all installed configurations @@ -77,7 +77,7 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde return fmt.Errorf("you must configure cli before interacting with hub") } - if err := setHubBranch(); err != nil { + if err := cwhub.SetHubBranch(); err != nil { return fmt.Errorf("error while setting hub branch: %s", err) } return nil @@ -111,7 +111,7 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if return fmt.Errorf("you must configure cli before interacting with hub") } - if err := setHubBranch(); err != nil { + if err := cwhub.SetHubBranch(); err != nil { return fmt.Errorf("error while setting hub branch: %s", err) } return nil diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index 00a544437..9c3a8b372 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -29,11 +29,9 @@ func NewHubTestCmd() *cobra.Command { var cscliPath string var cmdHubTest = &cobra.Command{ - Use: "hubtest", - Short: "Run functional tests on hub configurations", - Long: ` - Run functional tests on hub configurations (parsers, scenarios, collections...) - `, + Use: "hubtest", + Short: "Run functional tests on hub configurations", + Long: "Run functional tests on hub configurations (parsers, scenarios, collections...)", Args: cobra.ExactArgs(0), DisableAutoGenTag: true, PersistentPreRun: func(cmd *cobra.Command, args []string) { diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go index 077f9863f..0854ba309 100644 --- a/cmd/crowdsec-cli/parsers.go +++ b/cmd/crowdsec-cli/parsers.go @@ -31,7 +31,7 @@ cscli parsers remove crowdsecurity/sshd-logs return fmt.Errorf("you must configure cli before interacting with hub") } - if err := setHubBranch(); err != nil { + if err := cwhub.SetHubBranch(); err != nil { return fmt.Errorf("error while setting hub branch: %s", err) } diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go index e56cb7b8f..e09af9b28 100644 --- a/cmd/crowdsec-cli/postoverflows.go +++ b/cmd/crowdsec-cli/postoverflows.go @@ -30,7 +30,7 @@ func NewPostOverflowsCmd() *cobra.Command { return fmt.Errorf("you must configure cli before interacting with hub") } - if err := setHubBranch(); err != nil { + if err := cwhub.SetHubBranch(); err != nil { return fmt.Errorf("error while setting hub branch: %s", err) } diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index d5d705c5b..c1d2896ae 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -32,13 +33,13 @@ cscli scenarios remove crowdsecurity/ssh-bf return fmt.Errorf("you must configure cli before interacting with hub") } - if err := setHubBranch(); err != nil { - return fmt.Errorf("error while setting hub branch: %s", err) + if err := cwhub.SetHubBranch(); err != nil { + return errors.Wrap(err, "while setting hub branch") } if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { - log.Fatalf("Failed to get Hub index : %v", err) log.Infoln("Run 'sudo cscli hub update' to get the hub index") + log.Fatalf("Failed to get Hub index : %v", err) } return nil diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index e19eb3fa9..c3e120273 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -14,7 +14,6 @@ import ( "time" "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/enescakir/emoji" "github.com/olekukonko/tablewriter" @@ -22,7 +21,6 @@ import ( "github.com/prometheus/prom2json" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "golang.org/x/mod/semver" "gopkg.in/yaml.v2" ) @@ -59,12 +57,12 @@ func LoadHub() error { return fmt.Errorf("unable to load hub") } - if err := setHubBranch(); err != nil { + if err := cwhub.SetHubBranch(); err != nil { log.Warningf("unable to set hub branch (%s), default to master", err) } if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { - return fmt.Errorf("Failed to get Hub index : '%v'. Run 'sudo cscli hub update' to get the hub index", err) + return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err) } return nil @@ -281,35 +279,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value * return nil } -func setHubBranch() error { - /* - if no branch has been specified in flags for the hub, then use the one corresponding to crowdsec version - */ - if cwhub.HubBranch == "" { - latest, err := cwversion.Latest() - if err != nil { - cwhub.HubBranch = "master" - return err - } - csVersion := cwversion.VersionStrip() - if csVersion == latest { - cwhub.HubBranch = "master" - } else if semver.Compare(csVersion, latest) == 1 { // if current version is greater than the latest we are in pre-release - log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion) - cwhub.HubBranch = "master" - } else if csVersion == "" { - log.Warningf("Crowdsec version is '', using master branch for the hub") - cwhub.HubBranch = "master" - } else { - log.Warnf("Crowdsec is not the latest version. Current version is '%s' and the latest stable version is '%s'. Please update it!", csVersion, latest) - log.Warnf("As a result, you will not be able to use parsers/scenarios/collections added to Crowdsec Hub after CrowdSec %s", latest) - cwhub.HubBranch = csVersion - } - log.Debugf("Using branch '%s' for the hub", cwhub.HubBranch) - } - return nil -} - func ShowMetrics(hubItem *cwhub.Item) { switch hubItem.Type { case cwhub.PARSERS: @@ -564,7 +533,7 @@ func RestoreHub(dirPath string) error { if err := csConfig.LoadHub(); err != nil { return err } - if err := setHubBranch(); err != nil { + if err := cwhub.SetHubBranch(); err != nil { return fmt.Errorf("error while setting hub branch: %s", err) } diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 3208836c7..d5a9caec7 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -2,20 +2,17 @@ package cwhub import ( "crypto/sha256" + "fmt" + "io" + "os" "path/filepath" "sort" "strings" - //"errors" - "fmt" - "io" - "os" - "github.com/enescakir/emoji" "github.com/pkg/errors" - "golang.org/x/mod/semver" - log "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" ) /*managed configuration types*/ diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 582d39d4a..1bda76e5e 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -3,22 +3,18 @@ package cwhub import ( "bytes" "crypto/sha256" - "path" - "path/filepath" - - //"errors" - "github.com/pkg/errors" - - //"errors" "fmt" "io" "io/ioutil" "net/http" "os" + "path" + "path/filepath" "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -80,61 +76,61 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b var err error log.Debugf("Downloading %s %s", target.Type, target.Name) - if target.Type == COLLECTIONS { - var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} - for idx, ptr := range tmp { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - val, ok := hubIdx[ptrtype][p] - if !ok { - return target, fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) - } - - if !val.Installed && updateOnly && val.Downloaded { - log.Debugf("skipping upgrade of %s : not installed", target.Name) - continue - } - - log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly) - //recurse as it's a collection - if ptrtype == COLLECTIONS { - log.Tracef("collection, recurse") - hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly) - if err != nil { - return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name)) - } - } - item, err := DownloadItem(hub, val, overwrite) - if err != nil { - return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name)) - } - - // We need to enable an item when it has been added to a collection since latest release of the collection. - // We check if val.Downloaded is false because maybe the item has been disabled by the user. - if !item.Installed && !val.Downloaded { - if item, err = EnableItem(hub, item); err != nil { - return target, errors.Wrapf(err, "enabling '%s'", item.Name) - } - } - hubIdx[ptrtype][p] = item - } - } - target, err = DownloadItem(hub, target, overwrite) - if err != nil { - return target, fmt.Errorf("failed to download item : %s", err) - } - } else { + if target.Type != COLLECTIONS { if !target.Installed && updateOnly && target.Downloaded { log.Debugf("skipping upgrade of %s : not installed", target.Name) return target, nil } return DownloadItem(hub, target, overwrite) } + + // collection + var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} + for idx, ptr := range tmp { + ptrtype := ItemTypes[idx] + for _, p := range ptr { + val, ok := hubIdx[ptrtype][p] + if !ok { + return target, fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) + } + + if !val.Installed && updateOnly && val.Downloaded { + log.Debugf("skipping upgrade of %s : not installed", target.Name) + continue + } + + log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly) + //recurse as it's a collection + if ptrtype == COLLECTIONS { + log.Tracef("collection, recurse") + hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly) + if err != nil { + return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name)) + } + } + item, err := DownloadItem(hub, val, overwrite) + if err != nil { + return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name)) + } + + // We need to enable an item when it has been added to a collection since latest release of the collection. + // We check if val.Downloaded is false because maybe the item has been disabled by the user. + if !item.Installed && !val.Downloaded { + if item, err = EnableItem(hub, item); err != nil { + return target, errors.Wrapf(err, "enabling '%s'", item.Name) + } + } + hubIdx[ptrtype][p] = item + } + } + target, err = DownloadItem(hub, target, overwrite) + if err != nil { + return target, fmt.Errorf("failed to download item : %s", err) + } return target, nil } func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) { - var tdir = hub.HubDir var dataFolder = hub.DataDir /*if user didn't --force, don't overwrite local, tainted, up-to-date files*/ diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index d7cf7ba4b..13eb90b2c 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -2,17 +2,70 @@ package cwhub import ( "fmt" + "path/filepath" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/enescakir/emoji" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" ) +// pick a hub branch corresponding to the current crowdsec version. +func chooseHubBranch() (string, error) { + latest, err := cwversion.Latest() + if err != nil { + return "master", err + } + + csVersion := cwversion.VersionStrip() + if csVersion == latest { + return "master", nil + } + + // if current version is greater than the latest we are in pre-release + if semver.Compare(csVersion, latest) == 1 { + log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion) + return "master", nil + } + + if csVersion == "" { + log.Warningf("Crowdsec version is not set, using master branch for the hub") + return "master", nil + } + + log.Warnf("Crowdsec is not the latest version. "+ + "Current version is '%s' and the latest stable version is '%s'. Please update it!", + csVersion, latest) + log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+ + "added to Crowdsec Hub after CrowdSec %s", latest) + return csVersion, nil +} + +// SetHubBranch sets the package variable that points to the hub branch. +func SetHubBranch() error { + // a branch is already set, or specified from the flags + if HubBranch != "" { + return nil + } + + // use the branch corresponding to the crowdsec version + branch, err := chooseHubBranch() + if err != nil { + return err + } + HubBranch = branch + log.Debugf("Using branch '%s' for the hub", HubBranch) + return nil +} + func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error { it := GetItem(obtype, name) if it == nil { - return fmt.Errorf("unable to retrieve item : %s", name) + return fmt.Errorf("unable to retrieve item: %s", name) } + item := *it if downloadOnly && item.Downloaded && item.UpToDate { log.Warningf("%s is already downloaded and up-to-date", item.Name) @@ -20,69 +73,85 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo return nil } } + item, err := DownloadLatest(csConfig.Hub, item, force, true) if err != nil { - return fmt.Errorf("error while downloading %s : %v", item.Name, err) + return errors.Wrapf(err, "while downloading %s", item.Name) } + if err := AddItem(obtype, item); err != nil { - return fmt.Errorf("error while adding %s : %v", item.Name, err) + return errors.Wrapf(err, "while adding %s", item.Name) } + if downloadOnly { - log.Infof("Downloaded %s to %s", item.Name, csConfig.Hub.HubDir+"/"+item.RemotePath) + log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath)) return nil } + item, err = EnableItem(csConfig.Hub, item) if err != nil { - return fmt.Errorf("error while enabling %s : %v.", item.Name, err) + return errors.Wrapf(err, "while enabling %s", item.Name) } + if err := AddItem(obtype, item); err != nil { - return fmt.Errorf("error while adding %s : %v", item.Name, err) + return errors.Wrapf(err, "while adding %s", item.Name) } + log.Infof("Enabled %s", item.Name) return nil } +// XXX this must return errors instead of log.Fatal func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) { - var err error - var disabled int + var ( + err error + disabled int + ) + if name != "" { it := GetItem(itemType, name) if it == nil { log.Fatalf("unable to retrieve: %s", name) } + item := *it item, err = DisableItem(csConfig.Hub, item, purge, forceAction) if err != nil { log.Fatalf("unable to disable %s : %v", item.Name, err) } + if err := AddItem(itemType, item); err != nil { log.Fatalf("unable to add %s: %v", item.Name, err) } return - } else if name == "" && all { - for _, v := range GetItemMap(itemType) { - v, err = DisableItem(csConfig.Hub, v, purge, forceAction) - if err != nil { - log.Fatalf("unable to disable %s : %v", v.Name, err) - } - if err := AddItem(itemType, v); err != nil { - log.Fatalf("unable to add %s: %v", v.Name, err) - } - disabled++ - } } - if name != "" && !all { - log.Errorf("%s not found", name) - return + + if !all { + log.Fatal("removing item: no item specified") + } + + // remove all + for _, v := range GetItemMap(itemType) { + v, err = DisableItem(csConfig.Hub, v, purge, forceAction) + if err != nil { + log.Fatalf("unable to disable %s : %v", v.Name, err) + } + + if err := AddItem(itemType, v); err != nil { + log.Fatalf("unable to add %s: %v", v.Name, err) + } + disabled++ } log.Infof("Disabled %d items", disabled) } func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) { - var err error - var updated int - var found bool + var ( + err error + updated int + found bool + ) for _, v := range GetItemMap(itemType) { if name != "" && name != v.Name { @@ -100,6 +169,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } found = true + if v.UpToDate { log.Infof("%s : up-to-date", v.Name) @@ -111,10 +181,12 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc continue } } + v, err = DownloadLatest(csConfig.Hub, v, force, true) if err != nil { log.Fatalf("%s : download failed : %v", v.Name, err) } + if !v.UpToDate { if v.Tainted { log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name) @@ -125,10 +197,12 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc log.Infof("%v %s : updated", emoji.Package, v.Name) updated++ } + if err := AddItem(itemType, v); err != nil { log.Fatalf("unable to add %s: %v", v.Name, err) } } + if !found && name == "" { log.Infof("No %s installed, nothing to upgrade", itemType) } else if !found { @@ -142,5 +216,4 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } else if updated != 0 { log.Infof("Upgraded %d items", updated) } - } diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go index fa3275063..e2d1062b9 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/install.go @@ -19,6 +19,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, if err != nil { return Item{}, err } + if target.Local { return target, fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name) } @@ -136,38 +137,41 @@ func EnableItem(hub *csconfig.Hub, target Item) (Item, error) { for idx, ptr := range tmp { ptrtype := ItemTypes[idx] for _, p := range ptr { - if val, ok := hubIdx[ptrtype][p]; ok { - hubIdx[ptrtype][p], err = EnableItem(hub, val) - if err != nil { - return target, errors.Wrap(err, fmt.Sprintf("while installing %s", p)) - } - } else { + val, ok := hubIdx[ptrtype][p] + if !ok { return target, fmt.Errorf("required %s %s of %s doesn't exist, abort.", ptrtype, p, target.Name) } + + hubIdx[ptrtype][p], err = EnableItem(hub, val) + if err != nil { + return target, errors.Wrap(err, fmt.Sprintf("while installing %s", p)) + } } } } // check if file already exists where it should in configdir (eg /etc/crowdsec/collections/) - if _, err := os.Lstat(parent_dir + "/" + target.FileName); os.IsNotExist(err) { - //tdir+target.RemotePath - srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath) - if err != nil { - return target, errors.Wrap(err, "while getting source path") - } - dstPath, err := filepath.Abs(parent_dir + "/" + target.FileName) - if err != nil { - return target, errors.Wrap(err, "while getting destination path") - } - err = os.Symlink(srcPath, dstPath) - if err != nil { - return target, errors.Wrap(err, fmt.Sprintf("while creating symlink from %s to %s", srcPath, dstPath)) - } - log.Printf("Enabled %s : %s", target.Type, target.Name) - } else { + if _, err := os.Lstat(parent_dir + "/" + target.FileName); !os.IsNotExist(err) { log.Printf("%s already exists.", parent_dir+"/"+target.FileName) return target, nil } + + //tdir+target.RemotePath + srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath) + if err != nil { + return target, errors.Wrap(err, "while getting source path") + } + + dstPath, err := filepath.Abs(parent_dir + "/" + target.FileName) + if err != nil { + return target, errors.Wrap(err, "while getting destination path") + } + + if err = os.Symlink(srcPath, dstPath); err != nil { + return target, errors.Wrap(err, fmt.Sprintf("while creating symlink from %s to %s", srcPath, dstPath)) + } + + log.Printf("Enabled %s : %s", target.Type, target.Name) target.Installed = true hubIdx[target.Type][target.Name] = target return target, nil diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 36a3f2e65..e9bab9f00 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -2,22 +2,17 @@ package cwhub import ( "encoding/json" - //"errors" "fmt" "io/ioutil" - "sort" - - "github.com/pkg/errors" - "golang.org/x/mod/semver" - - //"log" - "os" "path/filepath" + "sort" "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" ) /*the walk/parser_visit function can't receive extra args*/