beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

cwhub: export SetHubBranch (#1559)

Browse source
This commit is contained in:
mmetc 2022-05-24 15:46:48 +02:00 committed by GitHub
parent f1cfcfe7cc
commit 1a293a2a27
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 196 additions and 163 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/crowdsec-cli/collections.go
View file

@ -27,7 +27,7 @@ func NewCollectionsCmd() *cobra.Command {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := setHubBranch(); err != nil {
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}

6
cmd/crowdsec-cli/hub.go
View file

@ -18,7 +18,7 @@ func NewHubCmd() *cobra.Command {
Hub management
List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
Hub is manage by cscli, to get latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
`,
Example: `
cscli hub list # List all installed configurations
@ -77,7 +77,7 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := setHubBranch(); err != nil {
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
return nil
@ -111,7 +111,7 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := setHubBranch(); err != nil {
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
return nil

8
cmd/crowdsec-cli/hubtest.go
View file

@ -29,11 +29,9 @@ func NewHubTestCmd() *cobra.Command {
var cscliPath string
var cmdHubTest = &cobra.Command{
Use: "hubtest",
Short: "Run functional tests on hub configurations",
Long: `
Run functional tests on hub configurations (parsers, scenarios, collections...)
`,
Use: "hubtest",
Short: "Run functional tests on hub configurations",
Long: "Run functional tests on hub configurations (parsers, scenarios, collections...)",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRun: func(cmd *cobra.Command, args []string) {

2
cmd/crowdsec-cli/parsers.go
View file

@ -31,7 +31,7 @@ cscli parsers remove crowdsecurity/sshd-logs
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := setHubBranch(); err != nil {
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}

2
cmd/crowdsec-cli/postoverflows.go
View file

@ -30,7 +30,7 @@ func NewPostOverflowsCmd() *cobra.Command {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := setHubBranch(); err != nil {
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}

7
cmd/crowdsec-cli/scenarios.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
@ -32,13 +33,13 @@ cscli scenarios remove crowdsecurity/ssh-bf
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := setHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
if err := cwhub.SetHubBranch(); err != nil {
return errors.Wrap(err, "while setting hub branch")
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
return nil

37
cmd/crowdsec-cli/utils.go
View file

@ -14,7 +14,6 @@ import (
"time"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/enescakir/emoji"
"github.com/olekukonko/tablewriter"
@ -22,7 +21,6 @@ import (
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/mod/semver"
"gopkg.in/yaml.v2"
)
@ -59,12 +57,12 @@ func LoadHub() error {
return fmt.Errorf("unable to load hub")
}
if err := setHubBranch(); err != nil {
if err := cwhub.SetHubBranch(); err != nil {
log.Warningf("unable to set hub branch (%s), default to master", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("Failed to get Hub index : '%v'. Run 'sudo cscli hub update' to get the hub index", err)
return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
}
return nil
@ -281,35 +279,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
return nil
}
func setHubBranch() error {
/*
if no branch has been specified in flags for the hub, then use the one corresponding to crowdsec version
*/
if cwhub.HubBranch == "" {
latest, err := cwversion.Latest()
if err != nil {
cwhub.HubBranch = "master"
return err
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
cwhub.HubBranch = "master"
} else if semver.Compare(csVersion, latest) == 1 { // if current version is greater than the latest we are in pre-release
log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
cwhub.HubBranch = "master"
} else if csVersion == "" {
log.Warningf("Crowdsec version is '', using master branch for the hub")
cwhub.HubBranch = "master"
} else {
log.Warnf("Crowdsec is not the latest version. Current version is '%s' and the latest stable version is '%s'. Please update it!", csVersion, latest)
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections added to Crowdsec Hub after CrowdSec %s", latest)
cwhub.HubBranch = csVersion
}
log.Debugf("Using branch '%s' for the hub", cwhub.HubBranch)
}
return nil
}
func ShowMetrics(hubItem *cwhub.Item) {
switch hubItem.Type {
case cwhub.PARSERS:
@ -564,7 +533,7 @@ func RestoreHub(dirPath string) error {
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := setHubBranch(); err != nil {
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}

11
pkg/cwhub/cwhub.go
View file

@ -2,20 +2,17 @@ package cwhub
import (
"crypto/sha256"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
//"errors"
"fmt"
"io"
"os"
"github.com/enescakir/emoji"
"github.com/pkg/errors"
"golang.org/x/mod/semver"
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
)
/*managed configuration types*/

100
pkg/cwhub/download.go
View file

@ -3,22 +3,18 @@ package cwhub
import (
"bytes"
"crypto/sha256"
"path"
"path/filepath"
//"errors"
"github.com/pkg/errors"
//"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
@ -80,61 +76,61 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b
var err error
log.Debugf("Downloading %s %s", target.Type, target.Name)
if target.Type == COLLECTIONS {
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
for idx, ptr := range tmp {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
val, ok := hubIdx[ptrtype][p]
if !ok {
return target, fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
}
if !val.Installed && updateOnly && val.Downloaded {
log.Debugf("skipping upgrade of %s : not installed", target.Name)
continue
}
log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly)
//recurse as it's a collection
if ptrtype == COLLECTIONS {
log.Tracef("collection, recurse")
hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly)
if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
}
}
item, err := DownloadItem(hub, val, overwrite)
if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
}
// We need to enable an item when it has been added to a collection since latest release of the collection.
// We check if val.Downloaded is false because maybe the item has been disabled by the user.
if !item.Installed && !val.Downloaded {
if item, err = EnableItem(hub, item); err != nil {
return target, errors.Wrapf(err, "enabling '%s'", item.Name)
}
}
hubIdx[ptrtype][p] = item
}
}
target, err = DownloadItem(hub, target, overwrite)
if err != nil {
return target, fmt.Errorf("failed to download item : %s", err)
}
} else {
if target.Type != COLLECTIONS {
if !target.Installed && updateOnly && target.Downloaded {
log.Debugf("skipping upgrade of %s : not installed", target.Name)
return target, nil
}
return DownloadItem(hub, target, overwrite)
}
// collection
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
for idx, ptr := range tmp {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
val, ok := hubIdx[ptrtype][p]
if !ok {
return target, fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
}
if !val.Installed && updateOnly && val.Downloaded {
log.Debugf("skipping upgrade of %s : not installed", target.Name)
continue
}
log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly)
//recurse as it's a collection
if ptrtype == COLLECTIONS {
log.Tracef("collection, recurse")
hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly)
if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
}
}
item, err := DownloadItem(hub, val, overwrite)
if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
}
// We need to enable an item when it has been added to a collection since latest release of the collection.
// We check if val.Downloaded is false because maybe the item has been disabled by the user.
if !item.Installed && !val.Downloaded {
if item, err = EnableItem(hub, item); err != nil {
return target, errors.Wrapf(err, "enabling '%s'", item.Name)
}
}
hubIdx[ptrtype][p] = item
}
}
target, err = DownloadItem(hub, target, overwrite)
if err != nil {
return target, fmt.Errorf("failed to download item : %s", err)
}
return target, nil
}
func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) {
var tdir = hub.HubDir
var dataFolder = hub.DataDir
/*if user didn't --force, don't overwrite local, tainted, up-to-date files*/

125
pkg/cwhub/helpers.go
View file

@ -2,17 +2,70 @@ package cwhub
import (
"fmt"
"path/filepath"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/enescakir/emoji"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
)
// pick a hub branch corresponding to the current crowdsec version.
func chooseHubBranch() (string, error) {
latest, err := cwversion.Latest()
if err != nil {
return "master", err
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
return "master", nil
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
return "master", nil
}
if csVersion == "" {
log.Warningf("Crowdsec version is not set, using master branch for the hub")
return "master", nil
}
log.Warnf("Crowdsec is not the latest version. "+
"Current version is '%s' and the latest stable version is '%s'. Please update it!",
csVersion, latest)
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
"added to Crowdsec Hub after CrowdSec %s", latest)
return csVersion, nil
}
// SetHubBranch sets the package variable that points to the hub branch.
func SetHubBranch() error {
// a branch is already set, or specified from the flags
if HubBranch != "" {
return nil
}
// use the branch corresponding to the crowdsec version
branch, err := chooseHubBranch()
if err != nil {
return err
}
HubBranch = branch
log.Debugf("Using branch '%s' for the hub", HubBranch)
return nil
}
func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error {
it := GetItem(obtype, name)
if it == nil {
return fmt.Errorf("unable to retrieve item : %s", name)
return fmt.Errorf("unable to retrieve item: %s", name)
}
item := *it
if downloadOnly && item.Downloaded && item.UpToDate {
log.Warningf("%s is already downloaded and up-to-date", item.Name)
@ -20,69 +73,85 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo
return nil
}
}
item, err := DownloadLatest(csConfig.Hub, item, force, true)
if err != nil {
return fmt.Errorf("error while downloading %s : %v", item.Name, err)
return errors.Wrapf(err, "while downloading %s", item.Name)
}
if err := AddItem(obtype, item); err != nil {
return fmt.Errorf("error while adding %s : %v", item.Name, err)
return errors.Wrapf(err, "while adding %s", item.Name)
}
if downloadOnly {
log.Infof("Downloaded %s to %s", item.Name, csConfig.Hub.HubDir+"/"+item.RemotePath)
log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath))
return nil
}
item, err = EnableItem(csConfig.Hub, item)
if err != nil {
return fmt.Errorf("error while enabling %s : %v.", item.Name, err)
return errors.Wrapf(err, "while enabling %s", item.Name)
}
if err := AddItem(obtype, item); err != nil {
return fmt.Errorf("error while adding %s : %v", item.Name, err)
return errors.Wrapf(err, "while adding %s", item.Name)
}
log.Infof("Enabled %s", item.Name)
return nil
}
// XXX this must return errors instead of log.Fatal
func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) {
var err error
var disabled int
var (
err error
disabled int
)
if name != "" {
it := GetItem(itemType, name)
if it == nil {
log.Fatalf("unable to retrieve: %s", name)
}
item := *it
item, err = DisableItem(csConfig.Hub, item, purge, forceAction)
if err != nil {
log.Fatalf("unable to disable %s : %v", item.Name, err)
}
if err := AddItem(itemType, item); err != nil {
log.Fatalf("unable to add %s: %v", item.Name, err)
}
return
} else if name == "" && all {
for _, v := range GetItemMap(itemType) {
v, err = DisableItem(csConfig.Hub, v, purge, forceAction)
if err != nil {
log.Fatalf("unable to disable %s : %v", v.Name, err)
}
if err := AddItem(itemType, v); err != nil {
log.Fatalf("unable to add %s: %v", v.Name, err)
}
disabled++
}
}
if name != "" && !all {
log.Errorf("%s not found", name)
return
if !all {
log.Fatal("removing item: no item specified")
}
// remove all
for _, v := range GetItemMap(itemType) {
v, err = DisableItem(csConfig.Hub, v, purge, forceAction)
if err != nil {
log.Fatalf("unable to disable %s : %v", v.Name, err)
}
if err := AddItem(itemType, v); err != nil {
log.Fatalf("unable to add %s: %v", v.Name, err)
}
disabled++
}
log.Infof("Disabled %d items", disabled)
}
func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) {
var err error
var updated int
var found bool
var (
err error
updated int
found bool
)
for _, v := range GetItemMap(itemType) {
if name != "" && name != v.Name {
@ -100,6 +169,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
}
found = true
if v.UpToDate {
log.Infof("%s : up-to-date", v.Name)
@ -111,10 +181,12 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
continue
}
}
v, err = DownloadLatest(csConfig.Hub, v, force, true)
if err != nil {
log.Fatalf("%s : download failed : %v", v.Name, err)
}
if !v.UpToDate {
if v.Tainted {
log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name)
@ -125,10 +197,12 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
log.Infof("%v %s : updated", emoji.Package, v.Name)
updated++
}
if err := AddItem(itemType, v); err != nil {
log.Fatalf("unable to add %s: %v", v.Name, err)
}
}
if !found && name == "" {
log.Infof("No %s installed, nothing to upgrade", itemType)
} else if !found {
@ -142,5 +216,4 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
} else if updated != 0 {
log.Infof("Upgraded %d items", updated)
}
}

48
pkg/cwhub/install.go
View file

@ -19,6 +19,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
if err != nil {
return Item{}, err
}
if target.Local {
return target, fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name)
}
@ -136,38 +137,41 @@ func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
for idx, ptr := range tmp {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
if val, ok := hubIdx[ptrtype][p]; ok {
hubIdx[ptrtype][p], err = EnableItem(hub, val)
if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while installing %s", p))
}
} else {
val, ok := hubIdx[ptrtype][p]
if !ok {
return target, fmt.Errorf("required %s %s of %s doesn't exist, abort.", ptrtype, p, target.Name)
}
hubIdx[ptrtype][p], err = EnableItem(hub, val)
if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while installing %s", p))
}
}
}
}
// check if file already exists where it should in configdir (eg /etc/crowdsec/collections/)
if _, err := os.Lstat(parent_dir + "/" + target.FileName); os.IsNotExist(err) {
//tdir+target.RemotePath
srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath)
if err != nil {
return target, errors.Wrap(err, "while getting source path")
}
dstPath, err := filepath.Abs(parent_dir + "/" + target.FileName)
if err != nil {
return target, errors.Wrap(err, "while getting destination path")
}
err = os.Symlink(srcPath, dstPath)
if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while creating symlink from %s to %s", srcPath, dstPath))
}
log.Printf("Enabled %s : %s", target.Type, target.Name)
} else {
if _, err := os.Lstat(parent_dir + "/" + target.FileName); !os.IsNotExist(err) {
log.Printf("%s already exists.", parent_dir+"/"+target.FileName)
return target, nil
}
//tdir+target.RemotePath
srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath)
if err != nil {
return target, errors.Wrap(err, "while getting source path")
}
dstPath, err := filepath.Abs(parent_dir + "/" + target.FileName)
if err != nil {
return target, errors.Wrap(err, "while getting destination path")
}
if err = os.Symlink(srcPath, dstPath); err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while creating symlink from %s to %s", srcPath, dstPath))
}
log.Printf("Enabled %s : %s", target.Type, target.Name)
target.Installed = true
hubIdx[target.Type][target.Name] = target
return target, nil

11
pkg/cwhub/loader.go
View file

@ -2,22 +2,17 @@ package cwhub
import (
"encoding/json"
//"errors"
"fmt"
"io/ioutil"
"sort"
"github.com/pkg/errors"
"golang.org/x/mod/semver"
//"log"
"os"
"path/filepath"
"sort"
"strings"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
)
/*the walk/parser_visit function can't receive extra args*/