beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Refact cscli hub / pkg/cwhub (part 5) (#2521)

Browse source 
* remove unused yaml tags
* cscli/cwhub: deduplicate, remove dead code
* log.Fatal -> fmt.Errorf
* deflate utils.go by moving functions to respective files
* indexOf() -> slices.Index()
* ItemStatus() + toEmoji() -> Item.status()
* Item.versionStatus()
* move getSHA256() to loader.go
This commit is contained in:
mmetc 2023-10-06 13:59:51 +02:00 committed by GitHub
parent 9235f55c47
commit 338141f067
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 304 additions and 347 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
cmd/crowdsec-cli/capi.go
View file

@ -60,16 +60,16 @@ func NewCapiRegisterCmd() *cobra.Command {
Short: "Register to Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
var err error
capiUser, err := generateID(capiUserPrefix)
if err != nil {
log.Fatalf("unable to generate machine id: %s", err)
return fmt.Errorf("unable to generate machine id: %s", err)
}
password := strfmt.Password(generatePassword(passwordLength))
apiurl, err := url.Parse(types.CAPIBaseURL)
if err != nil {
log.Fatalf("unable to parse api url %s : %s", types.CAPIBaseURL, err)
return fmt.Errorf("unable to parse api url %s: %w", types.CAPIBaseURL, err)
}
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: capiUser,
@ -80,7 +80,7 @@ func NewCapiRegisterCmd() *cobra.Command {
}, nil)
if err != nil {
log.Fatalf("api client register ('%s'): %s", types.CAPIBaseURL, err)
return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err)
}
log.Printf("Successfully registered to Central API (CAPI)")
@ -103,12 +103,12 @@ func NewCapiRegisterCmd() *cobra.Command {
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
log.Fatalf("unable to marshal api credentials: %s", err)
return fmt.Errorf("unable to marshal api credentials: %w", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0600)
if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
}
log.Printf("Central API credentials dumped to '%s'", dumpFile)
} else {
@ -116,6 +116,8 @@ func NewCapiRegisterCmd() *cobra.Command {
}
log.Warning(ReloadMessage())
return nil
},
}
cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
@ -133,53 +135,56 @@ func NewCapiStatusCmd() *cobra.Command {
Short: "Check status with the Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
return fmt.Errorf("please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
}
if csConfig.API.Server.OnlineClient.Credentials == nil {
log.Fatalf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
return fmt.Errorf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
}
password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
}
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
if err := require.Hub(csConfig); err != nil {
return err
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to load hub index : %s", err)
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
return fmt.Errorf("failed to get scenarios: %w", err)
}
if len(scenarios) == 0 {
log.Fatalf("no scenarios installed, abort")
return fmt.Errorf("no scenarios installed, abort")
}
Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
if err != nil {
log.Fatalf("init default client: %s", err)
return fmt.Errorf("init default client: %w", err)
}
t := models.WatcherAuthRequest{
MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login,
Password: &password,
Scenarios: scenarios,
}
log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil {
log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err)
return fmt.Errorf("failed to authenticate to Central API (CAPI): %w", err)
}
log.Infof("You can successfully interact with Central API (CAPI)")
return nil
},
}

69
cmd/crowdsec-cli/config_backup.go
View file

@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
@ -8,9 +9,75 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func backupHub(dirPath string) error {
var err error
var itemDirectory string
var upstreamParsers []string
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := cwhub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers = []string{}
for k, v := range itemMap {
clog = clog.WithFields(log.Fields{
"file": v.Name,
})
if !v.Installed { //only backup installed ones
clog.Debugf("[%s] : not installed", k)
continue
}
//for the local/tainted ones, we backup the full file
if v.Tainted || v.Local || !v.UpToDate {
//we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
continue
}
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
//write the upstream items
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
if err != nil {
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
}
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
if err != nil {
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
}
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
}
return nil
}
/*
Backup crowdsec configurations to directory <dirPath>:
@ -122,7 +189,7 @@ func backupConfigToDirectory(dirPath string) error {
log.Infof("Saved profiles to %s", backupProfiles)
}
if err = BackupHub(dirPath); err != nil {
if err = backupHub(dirPath); err != nil {
return fmt.Errorf("failed to backup hub config: %s", err)
}

122
cmd/crowdsec-cli/config_restore.go
View file

@ -13,6 +13,7 @@ import (
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type OldAPICfg struct {
@ -20,6 +21,125 @@ type OldAPICfg struct {
Password string `json:"password"`
}
// it's a rip of the cli version, but in silent-mode
func silentInstallItem(name string, obtype string) (string, error) {
var item = cwhub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
it := *item
if downloadOnly && it.Downloaded && it.UpToDate {
return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
}
it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
if downloadOnly {
return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
}
it, err = cwhub.EnableItem(csConfig.Hub, it)
if err != nil {
return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
return fmt.Sprintf("Enabled %s", it.Name), nil
}
func restoreHub(dirPath string) error {
var err error
if err := csConfig.LoadHub(); err != nil {
return err
}
cwhub.SetHubBranch()
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
log.Infof("no %s in backup", itype)
continue
}
/*restore the upstream items*/
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
file, err := os.ReadFile(upstreamListFN)
if err != nil {
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
}
var upstreamList []string
err = json.Unmarshal(file, &upstreamList)
if err != nil {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
label, err := silentInstallItem(toinstall, itype)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {
log.Infof("Installed %s : %s", toinstall, label)
} else {
log.Printf("Installed %s : ok", toinstall)
}
}
/*restore the local and tainted items*/
files, err := os.ReadDir(itemDirectory)
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
}
for _, file := range files {
//this was the upstream data
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
//we expect a stage here
if !file.IsDir() {
continue
}
stage := file.Name()
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
}
/*find items*/
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
}
//finally copy item
for _, tfile := range ifiles {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
} else {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
}
}
return nil
}
/*
Restore crowdsec configurations to directory <dirPath>:
@ -168,7 +288,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
}
}
if err = RestoreHub(dirPath); err != nil {
if err = restoreHub(dirPath); err != nil {
return fmt.Errorf("failed to restore hub config : %s", err)
}

10
cmd/crowdsec-cli/hub.go
View file

@ -88,9 +88,8 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
cwhub.SetHubBranch()
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
@ -134,9 +133,8 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
cwhub.SetHubBranch()
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {

4
cmd/crowdsec-cli/require/require.go
View file

@ -73,9 +73,7 @@ func Hub (c *csconfig.Config) error {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("while setting hub branch: %w", err)
}
cwhub.SetHubBranch()
if err := cwhub.GetHubIdx(c.Hub); err != nil {
return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)

2
cmd/crowdsec-cli/simulation.go
View file

@ -19,7 +19,7 @@ func addToExclusion(name string) error {
}
func removeFromExclusion(name string) error {
index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
index := slices.Index(csConfig.Cscli.SimulationConfig.Exclusions, name)
// Remove element from the slice
csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]

219
cmd/crowdsec-cli/utils.go
View file

@ -8,7 +8,6 @@ import (
"math"
"net"
"net/http"
"os"
"slices"
"strconv"
"strings"
@ -24,6 +23,7 @@ import (
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -38,34 +38,6 @@ func printHelp(cmd *cobra.Command) {
}
}
func indexOf(s string, slice []string) int {
for i, elem := range slice {
if s == elem {
return i
}
}
return -1
}
func LoadHub() error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("unable to load hub")
}
if err := cwhub.SetHubBranch(); err != nil {
log.Warningf("unable to set hub branch (%s), default to master", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
}
return nil
}
func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
errMsg := ""
if score < MaxDistance {
@ -100,7 +72,7 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := LoadHub(); err != nil {
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
@ -116,7 +88,7 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string,
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := LoadHub(); err != nil {
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
@ -453,37 +425,6 @@ func GetScenarioMetric(url string, itemName string) map[string]int {
return stats
}
// it's a rip of the cli version, but in silent-mode
func silenceInstallItem(name string, obtype string) (string, error) {
var item = cwhub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
it := *item
if downloadOnly && it.Downloaded && it.UpToDate {
return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
}
it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
if downloadOnly {
return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
}
it, err = cwhub.EnableItem(csConfig.Hub, it)
if err != nil {
return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
return fmt.Sprintf("Enabled %s", it.Name), nil
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
@ -512,160 +453,6 @@ func GetPrometheusMetric(url string) []*prom2json.Family {
return result
}
func RestoreHub(dirPath string) error {
var err error
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
log.Infof("no %s in backup", itype)
continue
}
/*restore the upstream items*/
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
file, err := os.ReadFile(upstreamListFN)
if err != nil {
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
}
var upstreamList []string
err = json.Unmarshal(file, &upstreamList)
if err != nil {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
label, err := silenceInstallItem(toinstall, itype)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {
log.Infof("Installed %s : %s", toinstall, label)
} else {
log.Printf("Installed %s : ok", toinstall)
}
}
/*restore the local and tainted items*/
files, err := os.ReadDir(itemDirectory)
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
}
for _, file := range files {
//this was the upstream data
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
//we expect a stage here
if !file.IsDir() {
continue
}
stage := file.Name()
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
}
/*find items*/
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
}
//finally copy item
for _, tfile := range ifiles {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
} else {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
}
}
return nil
}
func BackupHub(dirPath string) error {
var err error
var itemDirectory string
var upstreamParsers []string
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := cwhub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers = []string{}
for k, v := range itemMap {
clog = clog.WithFields(log.Fields{
"file": v.Name,
})
if !v.Installed { //only backup installed ones
clog.Debugf("[%s] : not installed", k)
continue
}
//for the local/tainted ones, we backup the full file
if v.Tainted || v.Local || !v.UpToDate {
//we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
continue
}
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
//write the upstream items
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
if err != nil {
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
}
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
if err != nil {
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
}
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
}
return nil
}
type unit struct {
value int64
symbol string

2
cmd/crowdsec-cli/utils_table.go
View file

@ -17,7 +17,7 @@ func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatu
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, status := range statuses {
t.AddRow(status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath)
t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
}
renderTableTitle(out, title)
t.Render()

8
pkg/csconfig/hub.go
View file

@ -2,10 +2,10 @@ package csconfig
/*cscli specific config, such as hub directory*/
type Hub struct {
HubDir string `yaml:"-"`
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
DataDir string `yaml:"-"`
HubDir string
ConfigDir string
HubIndexFile string
DataDir string
}
func (c *Config) LoadHub() error {

110
pkg/cwhub/cwhub.go
View file

@ -1,9 +1,7 @@
package cwhub
import (
"crypto/sha256"
"fmt"
"io"
"os"
"path/filepath"
"sort"
@ -40,7 +38,7 @@ type ItemHubStatus struct {
LocalVersion string `json:"local_version"`
LocalPath string `json:"local_path"`
Description string `json:"description"`
UTF8_Status string `json:"utf8_status"`
UTF8Status string `json:"utf8_status"`
Status string `json:"status"`
}
@ -62,7 +60,7 @@ type Item struct {
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions
// local (deployed) info
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
LocalVersion string `json:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty"` // the local meow
Installed bool `json:"installed,omitempty"`
@ -78,29 +76,48 @@ type Item struct {
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
}
func toEmoji(managed bool, installed bool, warning bool, ok bool) emoji.Emoji {
if !managed {
return emoji.House
func (i *Item) status() (string, emoji.Emoji) {
status := "disabled"
ok := false
if i.Installed {
ok = true
status = "enabled"
}
if !installed {
return emoji.Prohibited
managed := true
if i.Local {
managed = false
status += ",local"
}
if warning {
return emoji.Warning
warning := false
if i.Tainted {
warning = true
status += ",tainted"
} else if !i.UpToDate && !i.Local {
warning = true
status += ",update-available"
}
if ok {
return emoji.CheckMark
emo := emoji.QuestionMark
switch {
case !managed:
emo = emoji.House
case !i.Installed:
emo = emoji.Prohibited
case warning:
emo = emoji.Warning
case ok:
emo = emoji.CheckMark
}
// XXX: this is new
return emoji.QuestionMark
return status, emo
}
func (i *Item) toHubStatus() ItemHubStatus {
status, ok, warning, managed := ItemStatus(*i)
func (i *Item) hubStatus() ItemHubStatus {
status, emo := i.status()
return ItemHubStatus{
Name: i.Name,
@ -108,37 +125,21 @@ func (i *Item) toHubStatus() ItemHubStatus {
LocalPath: i.LocalPath,
Description: i.Description,
Status: status,
UTF8_Status: fmt.Sprintf("%v %s", toEmoji(managed, i.Installed, warning, ok), status),
UTF8Status: fmt.Sprintf("%v %s", emo, status),
}
}
// versionStatus: semver requires 'v' prefix
func (i *Item) versionStatus() int {
return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
}
// XXX: can we remove these globals?
var skippedLocal = 0
var skippedTainted = 0
var ReferenceMissingError = errors.New("Reference(s) missing in collection")
// GetVersionStatus: semver requires 'v' prefix
func GetVersionStatus(v *Item) int {
return semver.Compare("v"+v.Version, "v"+v.LocalVersion)
}
func getSHA256(filepath string) (string, error) {
f, err := os.Open(filepath)
if err != nil {
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
func GetItemMap(itemType string) map[string]Item {
m, ok := hubIdx[itemType]
if !ok {
@ -223,35 +224,6 @@ func DisplaySummary() {
}
}
// returns: human-text, Enabled, Warning, Unmanaged
func ItemStatus(v Item) (string, bool, bool, bool) {
strret := "disabled"
Ok := false
if v.Installed {
Ok = true
strret = "enabled"
}
Managed := true
if v.Local {
Managed = false
strret += ",local"
}
// tainted or out of date
Warning := false
if v.Tainted {
Warning = true
strret += ",tainted"
} else if !v.UpToDate && !v.Local {
Warning = true
strret += ",update-available"
}
return strret, Ok, Warning, Managed
}
func GetInstalledItems(itemType string) ([]Item, error) {
var retItems []Item
@ -305,7 +277,7 @@ func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubSt
continue
}
// Check the item status
ret = append(ret, item.toHubStatus())
ret = append(ret, item.hubStatus())
}
sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })

4
pkg/cwhub/cwhub_test.go
View file

@ -57,7 +57,7 @@ func TestItemStatus(t *testing.T) {
item.Local = false
item.Tainted = false
txt, _, _, _ := ItemStatus(*item)
txt, _ := item.status()
if txt != "enabled,update-available" {
t.Fatalf("got '%s'", txt)
}
@ -67,7 +67,7 @@ func TestItemStatus(t *testing.T) {
item.Local = true
item.Tainted = false
txt, _, _, _ = ItemStatus(*item)
txt, _ = item.status()
if txt != "disabled,local" {
t.Fatalf("got '%s'", txt)
}

24
pkg/cwhub/helpers.go
View file

@ -13,29 +13,29 @@ import (
)
// pick a hub branch corresponding to the current crowdsec version.
func chooseHubBranch() (string, error) {
func chooseHubBranch() string {
latest, err := cwversion.Latest()
if err != nil {
log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
//lint:ignore nilerr
return "master", nil
return "master"
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
log.Debugf("current version is equal to latest (%s)", csVersion)
return "master", nil
return "master"
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
return "master", nil
return "master"
}
if csVersion == "" {
log.Warning("Crowdsec version is not set, using master branch for the hub")
return "master", nil
return "master"
}
log.Warnf("Crowdsec is not the latest version. "+
@ -45,26 +45,20 @@ func chooseHubBranch() (string, error) {
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
"added to Crowdsec Hub after CrowdSec %s", latest)
return csVersion, nil
return csVersion
}
// SetHubBranch sets the package variable that points to the hub branch.
func SetHubBranch() error {
func SetHubBranch() {
// a branch is already set, or specified from the flags
if HubBranch != "" {
return nil
return
}
// use the branch corresponding to the crowdsec version
branch, err := chooseHubBranch()
if err != nil {
return err
}
HubBranch = chooseHubBranch()
HubBranch = branch
log.Debugf("Using branch '%s' for the hub", HubBranch)
return nil
}
func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error {

28
pkg/cwhub/loader.go
View file

@ -1,9 +1,11 @@
package cwhub
import (
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
@ -43,6 +45,22 @@ func handleSymlink(path string) (string, error) {
return hubpath, nil
}
func getSHA256(filepath string) (string, error) {
f, err := os.Open(filepath)
if err != nil {
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
type walker struct {
// the walk/parserVisit function can't receive extra args
hubdir string
@ -317,7 +335,7 @@ func (w walker) parserVisit(path string, f os.DirEntry, err error) error {
}
func CollecDepsCheck(v *Item) error {
if GetVersionStatus(v) != 0 { // not up-to-date
if v.versionStatus() != 0 { // not up-to-date
log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
return nil
}
@ -415,11 +433,11 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
continue
}
versionStatus := GetVersionStatus(&item)
switch versionStatus {
vs := item.versionStatus()
switch vs {
case 0: // latest
if err := CollecDepsCheck(&item); err != nil {
warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", item.Name, err))
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
hubIdx[COLLECTIONS][name] = item
}
case 1: // not up-to-date
@ -428,7 +446,7 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
}
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, versionStatus, item.LocalVersion, item.Version, item.Versions)
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions)
}
return nil, warnings

4
pkg/setup/install.go
View file

@ -56,9 +56,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
return fmt.Errorf("loading hub: %w", err)
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("setting hub branch: %w", err)
}
cwhub.SetHubBranch()
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("getting hub index: %w", err)