Refact cscli hub / pkg/cwhub (part 5) (#2521)
* remove unused yaml tags * cscli/cwhub: deduplicate, remove dead code * log.Fatal -> fmt.Errorf * deflate utils.go by moving functions to respective files * indexOf() -> slices.Index() * ItemStatus() + toEmoji() -> Item.status() * Item.versionStatus() * move getSHA256() to loader.go
This commit is contained in:
parent
9235f55c47
commit
338141f067
|
@ -60,16 +60,16 @@ func NewCapiRegisterCmd() *cobra.Command {
|
|||
Short: "Register to Central API (CAPI)",
|
||||
Args: cobra.MinimumNArgs(0),
|
||||
DisableAutoGenTag: true,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var err error
|
||||
capiUser, err := generateID(capiUserPrefix)
|
||||
if err != nil {
|
||||
log.Fatalf("unable to generate machine id: %s", err)
|
||||
return fmt.Errorf("unable to generate machine id: %s", err)
|
||||
}
|
||||
password := strfmt.Password(generatePassword(passwordLength))
|
||||
apiurl, err := url.Parse(types.CAPIBaseURL)
|
||||
if err != nil {
|
||||
log.Fatalf("unable to parse api url %s : %s", types.CAPIBaseURL, err)
|
||||
return fmt.Errorf("unable to parse api url %s: %w", types.CAPIBaseURL, err)
|
||||
}
|
||||
_, err = apiclient.RegisterClient(&apiclient.Config{
|
||||
MachineID: capiUser,
|
||||
|
@ -80,7 +80,7 @@ func NewCapiRegisterCmd() *cobra.Command {
|
|||
}, nil)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("api client register ('%s'): %s", types.CAPIBaseURL, err)
|
||||
return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err)
|
||||
}
|
||||
log.Printf("Successfully registered to Central API (CAPI)")
|
||||
|
||||
|
@ -103,12 +103,12 @@ func NewCapiRegisterCmd() *cobra.Command {
|
|||
}
|
||||
apiConfigDump, err := yaml.Marshal(apiCfg)
|
||||
if err != nil {
|
||||
log.Fatalf("unable to marshal api credentials: %s", err)
|
||||
return fmt.Errorf("unable to marshal api credentials: %w", err)
|
||||
}
|
||||
if dumpFile != "" {
|
||||
err = os.WriteFile(dumpFile, apiConfigDump, 0600)
|
||||
if err != nil {
|
||||
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
|
||||
return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
|
||||
}
|
||||
log.Printf("Central API credentials dumped to '%s'", dumpFile)
|
||||
} else {
|
||||
|
@ -116,6 +116,8 @@ func NewCapiRegisterCmd() *cobra.Command {
|
|||
}
|
||||
|
||||
log.Warning(ReloadMessage())
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
|
||||
|
@ -133,53 +135,56 @@ func NewCapiStatusCmd() *cobra.Command {
|
|||
Short: "Check status with the Central API (CAPI)",
|
||||
Args: cobra.MinimumNArgs(0),
|
||||
DisableAutoGenTag: true,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if csConfig.API.Server.OnlineClient == nil {
|
||||
log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
|
||||
return fmt.Errorf("please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
|
||||
}
|
||||
|
||||
if csConfig.API.Server.OnlineClient.Credentials == nil {
|
||||
log.Fatalf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
|
||||
return fmt.Errorf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
|
||||
}
|
||||
|
||||
password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
|
||||
|
||||
apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
|
||||
if err != nil {
|
||||
log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
|
||||
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
|
||||
}
|
||||
|
||||
if err := csConfig.LoadHub(); err != nil {
|
||||
log.Fatal(err)
|
||||
if err := require.Hub(csConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
|
||||
log.Info("Run 'sudo cscli hub update' to get the hub index")
|
||||
log.Fatalf("Failed to load hub index : %s", err)
|
||||
}
|
||||
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to get scenarios : %s", err)
|
||||
return fmt.Errorf("failed to get scenarios: %w", err)
|
||||
}
|
||||
|
||||
if len(scenarios) == 0 {
|
||||
log.Fatalf("no scenarios installed, abort")
|
||||
return fmt.Errorf("no scenarios installed, abort")
|
||||
}
|
||||
|
||||
Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
|
||||
if err != nil {
|
||||
log.Fatalf("init default client: %s", err)
|
||||
return fmt.Errorf("init default client: %w", err)
|
||||
}
|
||||
|
||||
t := models.WatcherAuthRequest{
|
||||
MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login,
|
||||
Password: &password,
|
||||
Scenarios: scenarios,
|
||||
}
|
||||
|
||||
log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
|
||||
log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl)
|
||||
|
||||
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err)
|
||||
return fmt.Errorf("failed to authenticate to Central API (CAPI): %w", err)
|
||||
}
|
||||
log.Infof("You can successfully interact with Central API (CAPI)")
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
@ -8,9 +9,75 @@ import (
|
|||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
||||
)
|
||||
|
||||
func backupHub(dirPath string) error {
|
||||
var err error
|
||||
var itemDirectory string
|
||||
var upstreamParsers []string
|
||||
|
||||
for _, itemType := range cwhub.ItemTypes {
|
||||
clog := log.WithFields(log.Fields{
|
||||
"type": itemType,
|
||||
})
|
||||
itemMap := cwhub.GetItemMap(itemType)
|
||||
if itemMap == nil {
|
||||
clog.Infof("No %s to backup.", itemType)
|
||||
continue
|
||||
}
|
||||
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
|
||||
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
|
||||
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
|
||||
}
|
||||
upstreamParsers = []string{}
|
||||
for k, v := range itemMap {
|
||||
clog = clog.WithFields(log.Fields{
|
||||
"file": v.Name,
|
||||
})
|
||||
if !v.Installed { //only backup installed ones
|
||||
clog.Debugf("[%s] : not installed", k)
|
||||
continue
|
||||
}
|
||||
|
||||
//for the local/tainted ones, we backup the full file
|
||||
if v.Tainted || v.Local || !v.UpToDate {
|
||||
//we need to backup stages for parsers
|
||||
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
|
||||
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
|
||||
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
|
||||
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
|
||||
}
|
||||
}
|
||||
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
|
||||
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
|
||||
if err = CopyFile(v.LocalPath, tfile); err != nil {
|
||||
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
|
||||
}
|
||||
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
|
||||
continue
|
||||
}
|
||||
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
|
||||
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
|
||||
upstreamParsers = append(upstreamParsers, v.Name)
|
||||
}
|
||||
//write the upstream items
|
||||
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
|
||||
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
|
||||
}
|
||||
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
|
||||
}
|
||||
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
Backup crowdsec configurations to directory <dirPath>:
|
||||
|
||||
|
@ -122,7 +189,7 @@ func backupConfigToDirectory(dirPath string) error {
|
|||
log.Infof("Saved profiles to %s", backupProfiles)
|
||||
}
|
||||
|
||||
if err = BackupHub(dirPath); err != nil {
|
||||
if err = backupHub(dirPath); err != nil {
|
||||
return fmt.Errorf("failed to backup hub config: %s", err)
|
||||
}
|
||||
|
||||
|
|
|
@ -13,6 +13,7 @@ import (
|
|||
|
||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||
)
|
||||
|
||||
type OldAPICfg struct {
|
||||
|
@ -20,6 +21,125 @@ type OldAPICfg struct {
|
|||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
// it's a rip of the cli version, but in silent-mode
|
||||
func silentInstallItem(name string, obtype string) (string, error) {
|
||||
var item = cwhub.GetItem(obtype, name)
|
||||
if item == nil {
|
||||
return "", fmt.Errorf("error retrieving item")
|
||||
}
|
||||
it := *item
|
||||
if downloadOnly && it.Downloaded && it.UpToDate {
|
||||
return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
|
||||
}
|
||||
it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
|
||||
}
|
||||
if err := cwhub.AddItem(obtype, it); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if downloadOnly {
|
||||
return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
|
||||
}
|
||||
it, err = cwhub.EnableItem(csConfig.Hub, it)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
|
||||
}
|
||||
if err := cwhub.AddItem(obtype, it); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("Enabled %s", it.Name), nil
|
||||
}
|
||||
|
||||
func restoreHub(dirPath string) error {
|
||||
var err error
|
||||
|
||||
if err := csConfig.LoadHub(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cwhub.SetHubBranch()
|
||||
|
||||
for _, itype := range cwhub.ItemTypes {
|
||||
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
|
||||
if _, err = os.Stat(itemDirectory); err != nil {
|
||||
log.Infof("no %s in backup", itype)
|
||||
continue
|
||||
}
|
||||
/*restore the upstream items*/
|
||||
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
|
||||
file, err := os.ReadFile(upstreamListFN)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
|
||||
}
|
||||
var upstreamList []string
|
||||
err = json.Unmarshal(file, &upstreamList)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
|
||||
}
|
||||
for _, toinstall := range upstreamList {
|
||||
label, err := silentInstallItem(toinstall, itype)
|
||||
if err != nil {
|
||||
log.Errorf("Error while installing %s : %s", toinstall, err)
|
||||
} else if label != "" {
|
||||
log.Infof("Installed %s : %s", toinstall, label)
|
||||
} else {
|
||||
log.Printf("Installed %s : ok", toinstall)
|
||||
}
|
||||
}
|
||||
|
||||
/*restore the local and tainted items*/
|
||||
files, err := os.ReadDir(itemDirectory)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
|
||||
}
|
||||
for _, file := range files {
|
||||
//this was the upstream data
|
||||
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
|
||||
continue
|
||||
}
|
||||
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
|
||||
//we expect a stage here
|
||||
if !file.IsDir() {
|
||||
continue
|
||||
}
|
||||
stage := file.Name()
|
||||
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
|
||||
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
|
||||
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
|
||||
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
|
||||
}
|
||||
/*find items*/
|
||||
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
|
||||
}
|
||||
//finally copy item
|
||||
for _, tfile := range ifiles {
|
||||
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
|
||||
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
|
||||
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
|
||||
if err = CopyFile(sourceFile, destinationFile); err != nil {
|
||||
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
|
||||
}
|
||||
log.Infof("restored %s to %s", sourceFile, destinationFile)
|
||||
}
|
||||
} else {
|
||||
log.Infof("Going to restore local/tainted [%s]", file.Name())
|
||||
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
|
||||
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
|
||||
if err = CopyFile(sourceFile, destinationFile); err != nil {
|
||||
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
|
||||
}
|
||||
log.Infof("restored %s to %s", sourceFile, destinationFile)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
Restore crowdsec configurations to directory <dirPath>:
|
||||
|
||||
|
@ -168,7 +288,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
|
|||
}
|
||||
}
|
||||
|
||||
if err = RestoreHub(dirPath); err != nil {
|
||||
if err = restoreHub(dirPath); err != nil {
|
||||
return fmt.Errorf("failed to restore hub config : %s", err)
|
||||
}
|
||||
|
||||
|
|
|
@ -88,9 +88,8 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
|
|||
return fmt.Errorf("you must configure cli before interacting with hub")
|
||||
}
|
||||
|
||||
if err := cwhub.SetHubBranch(); err != nil {
|
||||
return fmt.Errorf("error while setting hub branch: %s", err)
|
||||
}
|
||||
cwhub.SetHubBranch()
|
||||
|
||||
return nil
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
|
@ -134,9 +133,8 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
|
|||
return fmt.Errorf("you must configure cli before interacting with hub")
|
||||
}
|
||||
|
||||
if err := cwhub.SetHubBranch(); err != nil {
|
||||
return fmt.Errorf("error while setting hub branch: %s", err)
|
||||
}
|
||||
cwhub.SetHubBranch()
|
||||
|
||||
return nil
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
|
|
|
@ -73,9 +73,7 @@ func Hub (c *csconfig.Config) error {
|
|||
return fmt.Errorf("you must configure cli before interacting with hub")
|
||||
}
|
||||
|
||||
if err := cwhub.SetHubBranch(); err != nil {
|
||||
return fmt.Errorf("while setting hub branch: %w", err)
|
||||
}
|
||||
cwhub.SetHubBranch()
|
||||
|
||||
if err := cwhub.GetHubIdx(c.Hub); err != nil {
|
||||
return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
|
||||
|
|
|
@ -19,7 +19,7 @@ func addToExclusion(name string) error {
|
|||
}
|
||||
|
||||
func removeFromExclusion(name string) error {
|
||||
index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
|
||||
index := slices.Index(csConfig.Cscli.SimulationConfig.Exclusions, name)
|
||||
|
||||
// Remove element from the slice
|
||||
csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
|
||||
|
|
|
@ -8,7 +8,6 @@ import (
|
|||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
@ -24,6 +23,7 @@ import (
|
|||
|
||||
"github.com/crowdsecurity/go-cs-lib/trace"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/database"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||
|
@ -38,34 +38,6 @@ func printHelp(cmd *cobra.Command) {
|
|||
}
|
||||
}
|
||||
|
||||
func indexOf(s string, slice []string) int {
|
||||
for i, elem := range slice {
|
||||
if s == elem {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func LoadHub() error {
|
||||
if err := csConfig.LoadHub(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if csConfig.Hub == nil {
|
||||
return fmt.Errorf("unable to load hub")
|
||||
}
|
||||
|
||||
if err := cwhub.SetHubBranch(); err != nil {
|
||||
log.Warningf("unable to set hub branch (%s), default to master", err)
|
||||
}
|
||||
|
||||
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
|
||||
return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
|
||||
errMsg := ""
|
||||
if score < MaxDistance {
|
||||
|
@ -100,7 +72,7 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
|
|||
}
|
||||
|
||||
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||
if err := LoadHub(); err != nil {
|
||||
if err := require.Hub(csConfig); err != nil {
|
||||
return nil, cobra.ShellCompDirectiveDefault
|
||||
}
|
||||
|
||||
|
@ -116,7 +88,7 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string,
|
|||
}
|
||||
|
||||
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||
if err := LoadHub(); err != nil {
|
||||
if err := require.Hub(csConfig); err != nil {
|
||||
return nil, cobra.ShellCompDirectiveDefault
|
||||
}
|
||||
|
||||
|
@ -453,37 +425,6 @@ func GetScenarioMetric(url string, itemName string) map[string]int {
|
|||
return stats
|
||||
}
|
||||
|
||||
// it's a rip of the cli version, but in silent-mode
|
||||
func silenceInstallItem(name string, obtype string) (string, error) {
|
||||
var item = cwhub.GetItem(obtype, name)
|
||||
if item == nil {
|
||||
return "", fmt.Errorf("error retrieving item")
|
||||
}
|
||||
it := *item
|
||||
if downloadOnly && it.Downloaded && it.UpToDate {
|
||||
return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
|
||||
}
|
||||
it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
|
||||
}
|
||||
if err := cwhub.AddItem(obtype, it); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if downloadOnly {
|
||||
return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
|
||||
}
|
||||
it, err = cwhub.EnableItem(csConfig.Hub, it)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
|
||||
}
|
||||
if err := cwhub.AddItem(obtype, it); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("Enabled %s", it.Name), nil
|
||||
}
|
||||
|
||||
func GetPrometheusMetric(url string) []*prom2json.Family {
|
||||
mfChan := make(chan *dto.MetricFamily, 1024)
|
||||
|
||||
|
@ -512,160 +453,6 @@ func GetPrometheusMetric(url string) []*prom2json.Family {
|
|||
return result
|
||||
}
|
||||
|
||||
func RestoreHub(dirPath string) error {
|
||||
var err error
|
||||
|
||||
if err := csConfig.LoadHub(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cwhub.SetHubBranch(); err != nil {
|
||||
return fmt.Errorf("error while setting hub branch: %s", err)
|
||||
}
|
||||
|
||||
for _, itype := range cwhub.ItemTypes {
|
||||
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
|
||||
if _, err = os.Stat(itemDirectory); err != nil {
|
||||
log.Infof("no %s in backup", itype)
|
||||
continue
|
||||
}
|
||||
/*restore the upstream items*/
|
||||
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
|
||||
file, err := os.ReadFile(upstreamListFN)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
|
||||
}
|
||||
var upstreamList []string
|
||||
err = json.Unmarshal(file, &upstreamList)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
|
||||
}
|
||||
for _, toinstall := range upstreamList {
|
||||
label, err := silenceInstallItem(toinstall, itype)
|
||||
if err != nil {
|
||||
log.Errorf("Error while installing %s : %s", toinstall, err)
|
||||
} else if label != "" {
|
||||
log.Infof("Installed %s : %s", toinstall, label)
|
||||
} else {
|
||||
log.Printf("Installed %s : ok", toinstall)
|
||||
}
|
||||
}
|
||||
|
||||
/*restore the local and tainted items*/
|
||||
files, err := os.ReadDir(itemDirectory)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
|
||||
}
|
||||
for _, file := range files {
|
||||
//this was the upstream data
|
||||
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
|
||||
continue
|
||||
}
|
||||
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
|
||||
//we expect a stage here
|
||||
if !file.IsDir() {
|
||||
continue
|
||||
}
|
||||
stage := file.Name()
|
||||
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
|
||||
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
|
||||
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
|
||||
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
|
||||
}
|
||||
/*find items*/
|
||||
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
|
||||
}
|
||||
//finally copy item
|
||||
for _, tfile := range ifiles {
|
||||
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
|
||||
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
|
||||
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
|
||||
if err = CopyFile(sourceFile, destinationFile); err != nil {
|
||||
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
|
||||
}
|
||||
log.Infof("restored %s to %s", sourceFile, destinationFile)
|
||||
}
|
||||
} else {
|
||||
log.Infof("Going to restore local/tainted [%s]", file.Name())
|
||||
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
|
||||
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
|
||||
if err = CopyFile(sourceFile, destinationFile); err != nil {
|
||||
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
|
||||
}
|
||||
log.Infof("restored %s to %s", sourceFile, destinationFile)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func BackupHub(dirPath string) error {
|
||||
var err error
|
||||
var itemDirectory string
|
||||
var upstreamParsers []string
|
||||
|
||||
for _, itemType := range cwhub.ItemTypes {
|
||||
clog := log.WithFields(log.Fields{
|
||||
"type": itemType,
|
||||
})
|
||||
itemMap := cwhub.GetItemMap(itemType)
|
||||
if itemMap == nil {
|
||||
clog.Infof("No %s to backup.", itemType)
|
||||
continue
|
||||
}
|
||||
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
|
||||
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
|
||||
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
|
||||
}
|
||||
upstreamParsers = []string{}
|
||||
for k, v := range itemMap {
|
||||
clog = clog.WithFields(log.Fields{
|
||||
"file": v.Name,
|
||||
})
|
||||
if !v.Installed { //only backup installed ones
|
||||
clog.Debugf("[%s] : not installed", k)
|
||||
continue
|
||||
}
|
||||
|
||||
//for the local/tainted ones, we backup the full file
|
||||
if v.Tainted || v.Local || !v.UpToDate {
|
||||
//we need to backup stages for parsers
|
||||
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
|
||||
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
|
||||
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
|
||||
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
|
||||
}
|
||||
}
|
||||
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
|
||||
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
|
||||
if err = CopyFile(v.LocalPath, tfile); err != nil {
|
||||
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
|
||||
}
|
||||
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
|
||||
continue
|
||||
}
|
||||
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
|
||||
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
|
||||
upstreamParsers = append(upstreamParsers, v.Name)
|
||||
}
|
||||
//write the upstream items
|
||||
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
|
||||
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
|
||||
}
|
||||
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
|
||||
}
|
||||
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type unit struct {
|
||||
value int64
|
||||
symbol string
|
||||
|
|
|
@ -17,7 +17,7 @@ func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatu
|
|||
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
||||
|
||||
for _, status := range statuses {
|
||||
t.AddRow(status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath)
|
||||
t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
|
||||
}
|
||||
renderTableTitle(out, title)
|
||||
t.Render()
|
||||
|
|
|
@ -2,10 +2,10 @@ package csconfig
|
|||
|
||||
/*cscli specific config, such as hub directory*/
|
||||
type Hub struct {
|
||||
HubDir string `yaml:"-"`
|
||||
ConfigDir string `yaml:"-"`
|
||||
HubIndexFile string `yaml:"-"`
|
||||
DataDir string `yaml:"-"`
|
||||
HubDir string
|
||||
ConfigDir string
|
||||
HubIndexFile string
|
||||
DataDir string
|
||||
}
|
||||
|
||||
func (c *Config) LoadHub() error {
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
package cwhub
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
|
@ -40,7 +38,7 @@ type ItemHubStatus struct {
|
|||
LocalVersion string `json:"local_version"`
|
||||
LocalPath string `json:"local_path"`
|
||||
Description string `json:"description"`
|
||||
UTF8_Status string `json:"utf8_status"`
|
||||
UTF8Status string `json:"utf8_status"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
|
@ -78,29 +76,48 @@ type Item struct {
|
|||
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
|
||||
}
|
||||
|
||||
func toEmoji(managed bool, installed bool, warning bool, ok bool) emoji.Emoji {
|
||||
if !managed {
|
||||
return emoji.House
|
||||
func (i *Item) status() (string, emoji.Emoji) {
|
||||
status := "disabled"
|
||||
ok := false
|
||||
|
||||
if i.Installed {
|
||||
ok = true
|
||||
status = "enabled"
|
||||
}
|
||||
|
||||
if !installed {
|
||||
return emoji.Prohibited
|
||||
managed := true
|
||||
if i.Local {
|
||||
managed = false
|
||||
status += ",local"
|
||||
}
|
||||
|
||||
if warning {
|
||||
return emoji.Warning
|
||||
warning := false
|
||||
if i.Tainted {
|
||||
warning = true
|
||||
status += ",tainted"
|
||||
} else if !i.UpToDate && !i.Local {
|
||||
warning = true
|
||||
status += ",update-available"
|
||||
}
|
||||
|
||||
if ok {
|
||||
return emoji.CheckMark
|
||||
emo := emoji.QuestionMark
|
||||
|
||||
switch {
|
||||
case !managed:
|
||||
emo = emoji.House
|
||||
case !i.Installed:
|
||||
emo = emoji.Prohibited
|
||||
case warning:
|
||||
emo = emoji.Warning
|
||||
case ok:
|
||||
emo = emoji.CheckMark
|
||||
}
|
||||
|
||||
// XXX: this is new
|
||||
return emoji.QuestionMark
|
||||
return status, emo
|
||||
}
|
||||
|
||||
func (i *Item) toHubStatus() ItemHubStatus {
|
||||
status, ok, warning, managed := ItemStatus(*i)
|
||||
func (i *Item) hubStatus() ItemHubStatus {
|
||||
status, emo := i.status()
|
||||
|
||||
return ItemHubStatus{
|
||||
Name: i.Name,
|
||||
|
@ -108,37 +125,21 @@ func (i *Item) toHubStatus() ItemHubStatus {
|
|||
LocalPath: i.LocalPath,
|
||||
Description: i.Description,
|
||||
Status: status,
|
||||
UTF8_Status: fmt.Sprintf("%v %s", toEmoji(managed, i.Installed, warning, ok), status),
|
||||
UTF8Status: fmt.Sprintf("%v %s", emo, status),
|
||||
}
|
||||
}
|
||||
|
||||
// versionStatus: semver requires 'v' prefix
|
||||
func (i *Item) versionStatus() int {
|
||||
return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
|
||||
}
|
||||
|
||||
// XXX: can we remove these globals?
|
||||
var skippedLocal = 0
|
||||
var skippedTainted = 0
|
||||
|
||||
var ReferenceMissingError = errors.New("Reference(s) missing in collection")
|
||||
|
||||
// GetVersionStatus: semver requires 'v' prefix
|
||||
func GetVersionStatus(v *Item) int {
|
||||
return semver.Compare("v"+v.Version, "v"+v.LocalVersion)
|
||||
}
|
||||
|
||||
func getSHA256(filepath string) (string, error) {
|
||||
f, err := os.Open(filepath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%x", h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func GetItemMap(itemType string) map[string]Item {
|
||||
m, ok := hubIdx[itemType]
|
||||
if !ok {
|
||||
|
@ -223,35 +224,6 @@ func DisplaySummary() {
|
|||
}
|
||||
}
|
||||
|
||||
// returns: human-text, Enabled, Warning, Unmanaged
|
||||
func ItemStatus(v Item) (string, bool, bool, bool) {
|
||||
strret := "disabled"
|
||||
Ok := false
|
||||
|
||||
if v.Installed {
|
||||
Ok = true
|
||||
strret = "enabled"
|
||||
}
|
||||
|
||||
Managed := true
|
||||
if v.Local {
|
||||
Managed = false
|
||||
strret += ",local"
|
||||
}
|
||||
|
||||
// tainted or out of date
|
||||
Warning := false
|
||||
if v.Tainted {
|
||||
Warning = true
|
||||
strret += ",tainted"
|
||||
} else if !v.UpToDate && !v.Local {
|
||||
Warning = true
|
||||
strret += ",update-available"
|
||||
}
|
||||
|
||||
return strret, Ok, Warning, Managed
|
||||
}
|
||||
|
||||
func GetInstalledItems(itemType string) ([]Item, error) {
|
||||
var retItems []Item
|
||||
|
||||
|
@ -305,7 +277,7 @@ func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubSt
|
|||
continue
|
||||
}
|
||||
// Check the item status
|
||||
ret = append(ret, item.toHubStatus())
|
||||
ret = append(ret, item.hubStatus())
|
||||
}
|
||||
|
||||
sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
|
||||
|
|
|
@ -57,7 +57,7 @@ func TestItemStatus(t *testing.T) {
|
|||
item.Local = false
|
||||
item.Tainted = false
|
||||
|
||||
txt, _, _, _ := ItemStatus(*item)
|
||||
txt, _ := item.status()
|
||||
if txt != "enabled,update-available" {
|
||||
t.Fatalf("got '%s'", txt)
|
||||
}
|
||||
|
@ -67,7 +67,7 @@ func TestItemStatus(t *testing.T) {
|
|||
item.Local = true
|
||||
item.Tainted = false
|
||||
|
||||
txt, _, _, _ = ItemStatus(*item)
|
||||
txt, _ = item.status()
|
||||
if txt != "disabled,local" {
|
||||
t.Fatalf("got '%s'", txt)
|
||||
}
|
||||
|
|
|
@ -13,29 +13,29 @@ import (
|
|||
)
|
||||
|
||||
// pick a hub branch corresponding to the current crowdsec version.
|
||||
func chooseHubBranch() (string, error) {
|
||||
func chooseHubBranch() string {
|
||||
latest, err := cwversion.Latest()
|
||||
if err != nil {
|
||||
log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
|
||||
//lint:ignore nilerr
|
||||
return "master", nil
|
||||
return "master"
|
||||
}
|
||||
|
||||
csVersion := cwversion.VersionStrip()
|
||||
if csVersion == latest {
|
||||
log.Debugf("current version is equal to latest (%s)", csVersion)
|
||||
return "master", nil
|
||||
return "master"
|
||||
}
|
||||
|
||||
// if current version is greater than the latest we are in pre-release
|
||||
if semver.Compare(csVersion, latest) == 1 {
|
||||
log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
|
||||
return "master", nil
|
||||
return "master"
|
||||
}
|
||||
|
||||
if csVersion == "" {
|
||||
log.Warning("Crowdsec version is not set, using master branch for the hub")
|
||||
return "master", nil
|
||||
return "master"
|
||||
}
|
||||
|
||||
log.Warnf("Crowdsec is not the latest version. "+
|
||||
|
@ -45,26 +45,20 @@ func chooseHubBranch() (string, error) {
|
|||
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
|
||||
"added to Crowdsec Hub after CrowdSec %s", latest)
|
||||
|
||||
return csVersion, nil
|
||||
return csVersion
|
||||
}
|
||||
|
||||
// SetHubBranch sets the package variable that points to the hub branch.
|
||||
func SetHubBranch() error {
|
||||
func SetHubBranch() {
|
||||
// a branch is already set, or specified from the flags
|
||||
if HubBranch != "" {
|
||||
return nil
|
||||
return
|
||||
}
|
||||
|
||||
// use the branch corresponding to the crowdsec version
|
||||
branch, err := chooseHubBranch()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
HubBranch = chooseHubBranch()
|
||||
|
||||
HubBranch = branch
|
||||
log.Debugf("Using branch '%s' for the hub", HubBranch)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error {
|
||||
|
|
|
@ -1,9 +1,11 @@
|
|||
package cwhub
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
|
@ -43,6 +45,22 @@ func handleSymlink(path string) (string, error) {
|
|||
return hubpath, nil
|
||||
}
|
||||
|
||||
func getSHA256(filepath string) (string, error) {
|
||||
f, err := os.Open(filepath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%x", h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
type walker struct {
|
||||
// the walk/parserVisit function can't receive extra args
|
||||
hubdir string
|
||||
|
@ -317,7 +335,7 @@ func (w walker) parserVisit(path string, f os.DirEntry, err error) error {
|
|||
}
|
||||
|
||||
func CollecDepsCheck(v *Item) error {
|
||||
if GetVersionStatus(v) != 0 { // not up-to-date
|
||||
if v.versionStatus() != 0 { // not up-to-date
|
||||
log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
|
||||
return nil
|
||||
}
|
||||
|
@ -415,11 +433,11 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
|
|||
continue
|
||||
}
|
||||
|
||||
versionStatus := GetVersionStatus(&item)
|
||||
switch versionStatus {
|
||||
vs := item.versionStatus()
|
||||
switch vs {
|
||||
case 0: // latest
|
||||
if err := CollecDepsCheck(&item); err != nil {
|
||||
warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", item.Name, err))
|
||||
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
|
||||
hubIdx[COLLECTIONS][name] = item
|
||||
}
|
||||
case 1: // not up-to-date
|
||||
|
@ -428,7 +446,7 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
|
|||
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
|
||||
}
|
||||
|
||||
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, versionStatus, item.LocalVersion, item.Version, item.Versions)
|
||||
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions)
|
||||
}
|
||||
|
||||
return nil, warnings
|
||||
|
|
|
@ -56,9 +56,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
|
|||
return fmt.Errorf("loading hub: %w", err)
|
||||
}
|
||||
|
||||
if err := cwhub.SetHubBranch(); err != nil {
|
||||
return fmt.Errorf("setting hub branch: %w", err)
|
||||
}
|
||||
cwhub.SetHubBranch()
|
||||
|
||||
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
|
||||
return fmt.Errorf("getting hub index: %w", err)
|
||||
|
|
Loading…
Reference in a new issue