From 338141f0671ebf12105f3f57ab6e10456861bc6d Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Fri, 6 Oct 2023 13:59:51 +0200 Subject: [PATCH] Refact cscli hub / pkg/cwhub (part 5) (#2521) * remove unused yaml tags * cscli/cwhub: deduplicate, remove dead code * log.Fatal -> fmt.Errorf * deflate utils.go by moving functions to respective files * indexOf() -> slices.Index() * ItemStatus() + toEmoji() -> Item.status() * Item.versionStatus() * move getSHA256() to loader.go --- cmd/crowdsec-cli/capi.go | 45 +++--- cmd/crowdsec-cli/config_backup.go | 69 ++++++++- cmd/crowdsec-cli/config_restore.go | 122 +++++++++++++++- cmd/crowdsec-cli/hub.go | 10 +- cmd/crowdsec-cli/require/require.go | 4 +- cmd/crowdsec-cli/simulation.go | 2 +- cmd/crowdsec-cli/utils.go | 219 +--------------------------- cmd/crowdsec-cli/utils_table.go | 2 +- pkg/csconfig/hub.go | 8 +- pkg/cwhub/cwhub.go | 110 ++++++-------- pkg/cwhub/cwhub_test.go | 4 +- pkg/cwhub/helpers.go | 24 ++- pkg/cwhub/loader.go | 28 +++- pkg/setup/install.go | 4 +- 14 files changed, 304 insertions(+), 347 deletions(-) diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index bc390c717..0e0f217aa 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -60,16 +60,16 @@ func NewCapiRegisterCmd() *cobra.Command { Short: "Register to Central API (CAPI)", Args: cobra.MinimumNArgs(0), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { var err error capiUser, err := generateID(capiUserPrefix) if err != nil { - log.Fatalf("unable to generate machine id: %s", err) + return fmt.Errorf("unable to generate machine id: %s", err) } password := strfmt.Password(generatePassword(passwordLength)) apiurl, err := url.Parse(types.CAPIBaseURL) if err != nil { - log.Fatalf("unable to parse api url %s : %s", types.CAPIBaseURL, err) + return fmt.Errorf("unable to parse api url %s: %w", types.CAPIBaseURL, err) } _, err = apiclient.RegisterClient(&apiclient.Config{ MachineID: capiUser, @@ -80,7 +80,7 @@ func NewCapiRegisterCmd() *cobra.Command { }, nil) if err != nil { - log.Fatalf("api client register ('%s'): %s", types.CAPIBaseURL, err) + return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err) } log.Printf("Successfully registered to Central API (CAPI)") @@ -103,12 +103,12 @@ func NewCapiRegisterCmd() *cobra.Command { } apiConfigDump, err := yaml.Marshal(apiCfg) if err != nil { - log.Fatalf("unable to marshal api credentials: %s", err) + return fmt.Errorf("unable to marshal api credentials: %w", err) } if dumpFile != "" { err = os.WriteFile(dumpFile, apiConfigDump, 0600) if err != nil { - log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err) + return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err) } log.Printf("Central API credentials dumped to '%s'", dumpFile) } else { @@ -116,6 +116,8 @@ func NewCapiRegisterCmd() *cobra.Command { } log.Warning(ReloadMessage()) + + return nil }, } cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination") @@ -133,53 +135,56 @@ func NewCapiStatusCmd() *cobra.Command { Short: "Check status with the Central API (CAPI)", Args: cobra.MinimumNArgs(0), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { if csConfig.API.Server.OnlineClient == nil { - log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath) + return fmt.Errorf("please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath) } if csConfig.API.Server.OnlineClient.Credentials == nil { - log.Fatalf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath) + return fmt.Errorf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath) } password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password) + apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL) if err != nil { - log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err) + return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err) } - if err := csConfig.LoadHub(); err != nil { - log.Fatal(err) + if err := require.Hub(csConfig); err != nil { + return err } - if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { - log.Info("Run 'sudo cscli hub update' to get the hub index") - log.Fatalf("Failed to load hub index : %s", err) - } scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { - log.Fatalf("failed to get scenarios : %s", err) + return fmt.Errorf("failed to get scenarios: %w", err) } + if len(scenarios) == 0 { - log.Fatalf("no scenarios installed, abort") + return fmt.Errorf("no scenarios installed, abort") } Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil) if err != nil { - log.Fatalf("init default client: %s", err) + return fmt.Errorf("init default client: %w", err) } + t := models.WatcherAuthRequest{ MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login, Password: &password, Scenarios: scenarios, } + log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath) log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl) + _, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t) if err != nil { - log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err) + return fmt.Errorf("failed to authenticate to Central API (CAPI): %w", err) } log.Infof("You can successfully interact with Central API (CAPI)") + + return nil }, } diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index a2ac41502..436eff8aa 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -8,9 +9,75 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" ) +func backupHub(dirPath string) error { + var err error + var itemDirectory string + var upstreamParsers []string + + for _, itemType := range cwhub.ItemTypes { + clog := log.WithFields(log.Fields{ + "type": itemType, + }) + itemMap := cwhub.GetItemMap(itemType) + if itemMap == nil { + clog.Infof("No %s to backup.", itemType) + continue + } + itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType) + if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil { + return fmt.Errorf("error while creating %s : %s", itemDirectory, err) + } + upstreamParsers = []string{} + for k, v := range itemMap { + clog = clog.WithFields(log.Fields{ + "file": v.Name, + }) + if !v.Installed { //only backup installed ones + clog.Debugf("[%s] : not installed", k) + continue + } + + //for the local/tainted ones, we backup the full file + if v.Tainted || v.Local || !v.UpToDate { + //we need to backup stages for parsers + if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW { + fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) + if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil { + return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err) + } + } + clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate) + tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) + if err = CopyFile(v.LocalPath, tfile); err != nil { + return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) + } + clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile) + continue + } + clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate) + clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate) + upstreamParsers = append(upstreamParsers, v.Name) + } + //write the upstream items + upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType) + upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ") + if err != nil { + return fmt.Errorf("failed marshaling upstream parsers : %s", err) + } + err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644) + if err != nil { + return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err) + } + clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname) + } + + return nil +} + /* Backup crowdsec configurations to directory : @@ -122,7 +189,7 @@ func backupConfigToDirectory(dirPath string) error { log.Infof("Saved profiles to %s", backupProfiles) } - if err = BackupHub(dirPath); err != nil { + if err = backupHub(dirPath); err != nil { return fmt.Errorf("failed to backup hub config: %s", err) } diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index fc7206235..a9b88f308 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -13,6 +13,7 @@ import ( "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) type OldAPICfg struct { @@ -20,6 +21,125 @@ type OldAPICfg struct { Password string `json:"password"` } +// it's a rip of the cli version, but in silent-mode +func silentInstallItem(name string, obtype string) (string, error) { + var item = cwhub.GetItem(obtype, name) + if item == nil { + return "", fmt.Errorf("error retrieving item") + } + it := *item + if downloadOnly && it.Downloaded && it.UpToDate { + return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil + } + it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false) + if err != nil { + return "", fmt.Errorf("error while downloading %s : %v", it.Name, err) + } + if err := cwhub.AddItem(obtype, it); err != nil { + return "", err + } + + if downloadOnly { + return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil + } + it, err = cwhub.EnableItem(csConfig.Hub, it) + if err != nil { + return "", fmt.Errorf("error while enabling %s : %v", it.Name, err) + } + if err := cwhub.AddItem(obtype, it); err != nil { + return "", err + } + return fmt.Sprintf("Enabled %s", it.Name), nil +} + +func restoreHub(dirPath string) error { + var err error + + if err := csConfig.LoadHub(); err != nil { + return err + } + + cwhub.SetHubBranch() + + for _, itype := range cwhub.ItemTypes { + itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) + if _, err = os.Stat(itemDirectory); err != nil { + log.Infof("no %s in backup", itype) + continue + } + /*restore the upstream items*/ + upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype) + file, err := os.ReadFile(upstreamListFN) + if err != nil { + return fmt.Errorf("error while opening %s : %s", upstreamListFN, err) + } + var upstreamList []string + err = json.Unmarshal(file, &upstreamList) + if err != nil { + return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err) + } + for _, toinstall := range upstreamList { + label, err := silentInstallItem(toinstall, itype) + if err != nil { + log.Errorf("Error while installing %s : %s", toinstall, err) + } else if label != "" { + log.Infof("Installed %s : %s", toinstall, label) + } else { + log.Printf("Installed %s : ok", toinstall) + } + } + + /*restore the local and tainted items*/ + files, err := os.ReadDir(itemDirectory) + if err != nil { + return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err) + } + for _, file := range files { + //this was the upstream data + if file.Name() == fmt.Sprintf("upstream-%s.json", itype) { + continue + } + if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW { + //we expect a stage here + if !file.IsDir() { + continue + } + stage := file.Name() + stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage) + log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir) + if err = os.MkdirAll(stagedir, os.ModePerm); err != nil { + return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err) + } + /*find items*/ + ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/") + if err != nil { + return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err) + } + //finally copy item + for _, tfile := range ifiles { + log.Infof("Going to restore local/tainted [%s]", tfile.Name()) + sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name()) + destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name()) + if err = CopyFile(sourceFile, destinationFile); err != nil { + return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) + } + log.Infof("restored %s to %s", sourceFile, destinationFile) + } + } else { + log.Infof("Going to restore local/tainted [%s]", file.Name()) + sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name()) + destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name()) + if err = CopyFile(sourceFile, destinationFile); err != nil { + return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) + } + log.Infof("restored %s to %s", sourceFile, destinationFile) + } + + } + } + return nil +} + /* Restore crowdsec configurations to directory : @@ -168,7 +288,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { } } - if err = RestoreHub(dirPath); err != nil { + if err = restoreHub(dirPath); err != nil { return fmt.Errorf("failed to restore hub config : %s", err) } diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index b85785633..d56aaa176 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -88,9 +88,8 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde return fmt.Errorf("you must configure cli before interacting with hub") } - if err := cwhub.SetHubBranch(); err != nil { - return fmt.Errorf("error while setting hub branch: %s", err) - } + cwhub.SetHubBranch() + return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -134,9 +133,8 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if return fmt.Errorf("you must configure cli before interacting with hub") } - if err := cwhub.SetHubBranch(); err != nil { - return fmt.Errorf("error while setting hub branch: %s", err) - } + cwhub.SetHubBranch() + return nil }, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index 039cc6862..f4129a44f 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -73,9 +73,7 @@ func Hub (c *csconfig.Config) error { return fmt.Errorf("you must configure cli before interacting with hub") } - if err := cwhub.SetHubBranch(); err != nil { - return fmt.Errorf("while setting hub branch: %w", err) - } + cwhub.SetHubBranch() if err := cwhub.GetHubIdx(c.Hub); err != nil { return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err) diff --git a/cmd/crowdsec-cli/simulation.go b/cmd/crowdsec-cli/simulation.go index 945756d62..890785a2d 100644 --- a/cmd/crowdsec-cli/simulation.go +++ b/cmd/crowdsec-cli/simulation.go @@ -19,7 +19,7 @@ func addToExclusion(name string) error { } func removeFromExclusion(name string) error { - index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions) + index := slices.Index(csConfig.Cscli.SimulationConfig.Exclusions, name) // Remove element from the slice csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1] diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 419f6cfec..503653f82 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -8,7 +8,6 @@ import ( "math" "net" "net/http" - "os" "slices" "strconv" "strings" @@ -24,6 +23,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -38,34 +38,6 @@ func printHelp(cmd *cobra.Command) { } } -func indexOf(s string, slice []string) int { - for i, elem := range slice { - if s == elem { - return i - } - } - return -1 -} - -func LoadHub() error { - if err := csConfig.LoadHub(); err != nil { - log.Fatal(err) - } - if csConfig.Hub == nil { - return fmt.Errorf("unable to load hub") - } - - if err := cwhub.SetHubBranch(); err != nil { - log.Warningf("unable to set hub branch (%s), default to master", err) - } - - if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { - return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err) - } - - return nil -} - func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) { errMsg := "" if score < MaxDistance { @@ -100,7 +72,7 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { } func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := LoadHub(); err != nil { + if err := require.Hub(csConfig); err != nil { return nil, cobra.ShellCompDirectiveDefault } @@ -116,7 +88,7 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string, } func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := LoadHub(); err != nil { + if err := require.Hub(csConfig); err != nil { return nil, cobra.ShellCompDirectiveDefault } @@ -453,37 +425,6 @@ func GetScenarioMetric(url string, itemName string) map[string]int { return stats } -// it's a rip of the cli version, but in silent-mode -func silenceInstallItem(name string, obtype string) (string, error) { - var item = cwhub.GetItem(obtype, name) - if item == nil { - return "", fmt.Errorf("error retrieving item") - } - it := *item - if downloadOnly && it.Downloaded && it.UpToDate { - return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil - } - it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false) - if err != nil { - return "", fmt.Errorf("error while downloading %s : %v", it.Name, err) - } - if err := cwhub.AddItem(obtype, it); err != nil { - return "", err - } - - if downloadOnly { - return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil - } - it, err = cwhub.EnableItem(csConfig.Hub, it) - if err != nil { - return "", fmt.Errorf("error while enabling %s : %v", it.Name, err) - } - if err := cwhub.AddItem(obtype, it); err != nil { - return "", err - } - return fmt.Sprintf("Enabled %s", it.Name), nil -} - func GetPrometheusMetric(url string) []*prom2json.Family { mfChan := make(chan *dto.MetricFamily, 1024) @@ -512,160 +453,6 @@ func GetPrometheusMetric(url string) []*prom2json.Family { return result } -func RestoreHub(dirPath string) error { - var err error - - if err := csConfig.LoadHub(); err != nil { - return err - } - if err := cwhub.SetHubBranch(); err != nil { - return fmt.Errorf("error while setting hub branch: %s", err) - } - - for _, itype := range cwhub.ItemTypes { - itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) - if _, err = os.Stat(itemDirectory); err != nil { - log.Infof("no %s in backup", itype) - continue - } - /*restore the upstream items*/ - upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype) - file, err := os.ReadFile(upstreamListFN) - if err != nil { - return fmt.Errorf("error while opening %s : %s", upstreamListFN, err) - } - var upstreamList []string - err = json.Unmarshal(file, &upstreamList) - if err != nil { - return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err) - } - for _, toinstall := range upstreamList { - label, err := silenceInstallItem(toinstall, itype) - if err != nil { - log.Errorf("Error while installing %s : %s", toinstall, err) - } else if label != "" { - log.Infof("Installed %s : %s", toinstall, label) - } else { - log.Printf("Installed %s : ok", toinstall) - } - } - - /*restore the local and tainted items*/ - files, err := os.ReadDir(itemDirectory) - if err != nil { - return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err) - } - for _, file := range files { - //this was the upstream data - if file.Name() == fmt.Sprintf("upstream-%s.json", itype) { - continue - } - if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW { - //we expect a stage here - if !file.IsDir() { - continue - } - stage := file.Name() - stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage) - log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir) - if err = os.MkdirAll(stagedir, os.ModePerm); err != nil { - return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err) - } - /*find items*/ - ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/") - if err != nil { - return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err) - } - //finally copy item - for _, tfile := range ifiles { - log.Infof("Going to restore local/tainted [%s]", tfile.Name()) - sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name()) - destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name()) - if err = CopyFile(sourceFile, destinationFile); err != nil { - return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) - } - log.Infof("restored %s to %s", sourceFile, destinationFile) - } - } else { - log.Infof("Going to restore local/tainted [%s]", file.Name()) - sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name()) - destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name()) - if err = CopyFile(sourceFile, destinationFile); err != nil { - return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) - } - log.Infof("restored %s to %s", sourceFile, destinationFile) - } - - } - } - return nil -} - -func BackupHub(dirPath string) error { - var err error - var itemDirectory string - var upstreamParsers []string - - for _, itemType := range cwhub.ItemTypes { - clog := log.WithFields(log.Fields{ - "type": itemType, - }) - itemMap := cwhub.GetItemMap(itemType) - if itemMap == nil { - clog.Infof("No %s to backup.", itemType) - continue - } - itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType) - if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil { - return fmt.Errorf("error while creating %s : %s", itemDirectory, err) - } - upstreamParsers = []string{} - for k, v := range itemMap { - clog = clog.WithFields(log.Fields{ - "file": v.Name, - }) - if !v.Installed { //only backup installed ones - clog.Debugf("[%s] : not installed", k) - continue - } - - //for the local/tainted ones, we backup the full file - if v.Tainted || v.Local || !v.UpToDate { - //we need to backup stages for parsers - if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW { - fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) - if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil { - return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err) - } - } - clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate) - tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) - if err = CopyFile(v.LocalPath, tfile); err != nil { - return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) - } - clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile) - continue - } - clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate) - clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate) - upstreamParsers = append(upstreamParsers, v.Name) - } - //write the upstream items - upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType) - upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ") - if err != nil { - return fmt.Errorf("failed marshaling upstream parsers : %s", err) - } - err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644) - if err != nil { - return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err) - } - clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname) - } - - return nil -} - type unit struct { value int64 symbol string diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index aef1e94f7..16f42d72a 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -17,7 +17,7 @@ func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatu t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) for _, status := range statuses { - t.AddRow(status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath) + t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath) } renderTableTitle(out, title) t.Render() diff --git a/pkg/csconfig/hub.go b/pkg/csconfig/hub.go index eb3bd7c42..a48c7f034 100644 --- a/pkg/csconfig/hub.go +++ b/pkg/csconfig/hub.go @@ -2,10 +2,10 @@ package csconfig /*cscli specific config, such as hub directory*/ type Hub struct { - HubDir string `yaml:"-"` - ConfigDir string `yaml:"-"` - HubIndexFile string `yaml:"-"` - DataDir string `yaml:"-"` + HubDir string + ConfigDir string + HubIndexFile string + DataDir string } func (c *Config) LoadHub() error { diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index a8102804e..c180354e8 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -1,9 +1,7 @@ package cwhub import ( - "crypto/sha256" "fmt" - "io" "os" "path/filepath" "sort" @@ -40,7 +38,7 @@ type ItemHubStatus struct { LocalVersion string `json:"local_version"` LocalPath string `json:"local_path"` Description string `json:"description"` - UTF8_Status string `json:"utf8_status"` + UTF8Status string `json:"utf8_status"` Status string `json:"status"` } @@ -62,7 +60,7 @@ type Item struct { Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions // local (deployed) info - LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} + LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} LocalVersion string `json:"local_version,omitempty"` LocalHash string `json:"local_hash,omitempty"` // the local meow Installed bool `json:"installed,omitempty"` @@ -78,29 +76,48 @@ type Item struct { Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` } -func toEmoji(managed bool, installed bool, warning bool, ok bool) emoji.Emoji { - if !managed { - return emoji.House +func (i *Item) status() (string, emoji.Emoji) { + status := "disabled" + ok := false + + if i.Installed { + ok = true + status = "enabled" } - if !installed { - return emoji.Prohibited + managed := true + if i.Local { + managed = false + status += ",local" } - if warning { - return emoji.Warning + warning := false + if i.Tainted { + warning = true + status += ",tainted" + } else if !i.UpToDate && !i.Local { + warning = true + status += ",update-available" } - if ok { - return emoji.CheckMark + emo := emoji.QuestionMark + + switch { + case !managed: + emo = emoji.House + case !i.Installed: + emo = emoji.Prohibited + case warning: + emo = emoji.Warning + case ok: + emo = emoji.CheckMark } - // XXX: this is new - return emoji.QuestionMark + return status, emo } -func (i *Item) toHubStatus() ItemHubStatus { - status, ok, warning, managed := ItemStatus(*i) +func (i *Item) hubStatus() ItemHubStatus { + status, emo := i.status() return ItemHubStatus{ Name: i.Name, @@ -108,37 +125,21 @@ func (i *Item) toHubStatus() ItemHubStatus { LocalPath: i.LocalPath, Description: i.Description, Status: status, - UTF8_Status: fmt.Sprintf("%v %s", toEmoji(managed, i.Installed, warning, ok), status), + UTF8Status: fmt.Sprintf("%v %s", emo, status), } } +// versionStatus: semver requires 'v' prefix +func (i *Item) versionStatus() int { + return semver.Compare("v"+i.Version, "v"+i.LocalVersion) +} + // XXX: can we remove these globals? var skippedLocal = 0 var skippedTainted = 0 var ReferenceMissingError = errors.New("Reference(s) missing in collection") -// GetVersionStatus: semver requires 'v' prefix -func GetVersionStatus(v *Item) int { - return semver.Compare("v"+v.Version, "v"+v.LocalVersion) -} - -func getSHA256(filepath string) (string, error) { - f, err := os.Open(filepath) - if err != nil { - return "", fmt.Errorf("unable to open '%s': %w", filepath, err) - } - - defer f.Close() - - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err) - } - - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - func GetItemMap(itemType string) map[string]Item { m, ok := hubIdx[itemType] if !ok { @@ -223,35 +224,6 @@ func DisplaySummary() { } } -// returns: human-text, Enabled, Warning, Unmanaged -func ItemStatus(v Item) (string, bool, bool, bool) { - strret := "disabled" - Ok := false - - if v.Installed { - Ok = true - strret = "enabled" - } - - Managed := true - if v.Local { - Managed = false - strret += ",local" - } - - // tainted or out of date - Warning := false - if v.Tainted { - Warning = true - strret += ",tainted" - } else if !v.UpToDate && !v.Local { - Warning = true - strret += ",update-available" - } - - return strret, Ok, Warning, Managed -} - func GetInstalledItems(itemType string) ([]Item, error) { var retItems []Item @@ -305,7 +277,7 @@ func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubSt continue } // Check the item status - ret = append(ret, item.toHubStatus()) + ret = append(ret, item.hubStatus()) } sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 730808c70..5de7b5f4c 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -57,7 +57,7 @@ func TestItemStatus(t *testing.T) { item.Local = false item.Tainted = false - txt, _, _, _ := ItemStatus(*item) + txt, _ := item.status() if txt != "enabled,update-available" { t.Fatalf("got '%s'", txt) } @@ -67,7 +67,7 @@ func TestItemStatus(t *testing.T) { item.Local = true item.Tainted = false - txt, _, _, _ = ItemStatus(*item) + txt, _ = item.status() if txt != "disabled,local" { t.Fatalf("got '%s'", txt) } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index ec3c6d13e..c1a9042d3 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -13,29 +13,29 @@ import ( ) // pick a hub branch corresponding to the current crowdsec version. -func chooseHubBranch() (string, error) { +func chooseHubBranch() string { latest, err := cwversion.Latest() if err != nil { log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err) //lint:ignore nilerr - return "master", nil + return "master" } csVersion := cwversion.VersionStrip() if csVersion == latest { log.Debugf("current version is equal to latest (%s)", csVersion) - return "master", nil + return "master" } // if current version is greater than the latest we are in pre-release if semver.Compare(csVersion, latest) == 1 { log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion) - return "master", nil + return "master" } if csVersion == "" { log.Warning("Crowdsec version is not set, using master branch for the hub") - return "master", nil + return "master" } log.Warnf("Crowdsec is not the latest version. "+ @@ -45,26 +45,20 @@ func chooseHubBranch() (string, error) { log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+ "added to Crowdsec Hub after CrowdSec %s", latest) - return csVersion, nil + return csVersion } // SetHubBranch sets the package variable that points to the hub branch. -func SetHubBranch() error { +func SetHubBranch() { // a branch is already set, or specified from the flags if HubBranch != "" { - return nil + return } // use the branch corresponding to the crowdsec version - branch, err := chooseHubBranch() - if err != nil { - return err - } + HubBranch = chooseHubBranch() - HubBranch = branch log.Debugf("Using branch '%s' for the hub", HubBranch) - - return nil } func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error { diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 2fec1f693..84200ca1d 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -1,9 +1,11 @@ package cwhub import ( + "crypto/sha256" "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "sort" @@ -43,6 +45,22 @@ func handleSymlink(path string) (string, error) { return hubpath, nil } +func getSHA256(filepath string) (string, error) { + f, err := os.Open(filepath) + if err != nil { + return "", fmt.Errorf("unable to open '%s': %w", filepath, err) + } + + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err) + } + + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + type walker struct { // the walk/parserVisit function can't receive extra args hubdir string @@ -317,7 +335,7 @@ func (w walker) parserVisit(path string, f os.DirEntry, err error) error { } func CollecDepsCheck(v *Item) error { - if GetVersionStatus(v) != 0 { // not up-to-date + if v.versionStatus() != 0 { // not up-to-date log.Debugf("%s dependencies not checked : not up-to-date", v.Name) return nil } @@ -415,11 +433,11 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) { continue } - versionStatus := GetVersionStatus(&item) - switch versionStatus { + vs := item.versionStatus() + switch vs { case 0: // latest if err := CollecDepsCheck(&item); err != nil { - warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", item.Name, err)) + warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) hubIdx[COLLECTIONS][name] = item } case 1: // not up-to-date @@ -428,7 +446,7 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) { warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) } - log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, versionStatus, item.LocalVersion, item.Version, item.Versions) + log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions) } return nil, warnings diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 5d3bfdbc9..92a1968c8 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -56,9 +56,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error return fmt.Errorf("loading hub: %w", err) } - if err := cwhub.SetHubBranch(); err != nil { - return fmt.Errorf("setting hub branch: %w", err) - } + cwhub.SetHubBranch() if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { return fmt.Errorf("getting hub index: %w", err)