From ffcab0b2bc33316df89e0d55d1aca81b22f2c03d Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Fri, 24 Nov 2023 15:57:32 +0100 Subject: [PATCH] Refactor hub management and cscli commands (#2545) --- cmd/crowdsec-cli/capi.go | 5 +- cmd/crowdsec-cli/collections.go | 176 ------ cmd/crowdsec-cli/config_backup.go | 38 +- cmd/crowdsec-cli/config_restore.go | 54 +- cmd/crowdsec-cli/config_show.go | 1 - cmd/crowdsec-cli/console.go | 5 +- cmd/crowdsec-cli/hub.go | 216 ++++---- cmd/crowdsec-cli/hubtest.go | 66 ++- cmd/crowdsec-cli/hubtest_table.go | 10 +- cmd/crowdsec-cli/item_metrics.go | 236 ++++++++ cmd/crowdsec-cli/item_suggest.go | 85 +++ cmd/crowdsec-cli/itemcommands.go | 606 +++++++++++++++++++++ cmd/crowdsec-cli/items.go | 157 ++++++ cmd/crowdsec-cli/lapi.go | 13 +- cmd/crowdsec-cli/main.go | 33 +- cmd/crowdsec-cli/metrics.go | 35 +- cmd/crowdsec-cli/parsers.go | 194 ------- cmd/crowdsec-cli/postoverflows.go | 191 ------- cmd/crowdsec-cli/require/branch.go | 58 ++ cmd/crowdsec-cli/require/require.go | 44 +- cmd/crowdsec-cli/scenarios.go | 188 ------- cmd/crowdsec-cli/setup.go | 11 +- cmd/crowdsec-cli/simulation.go | 12 +- cmd/crowdsec-cli/support.go | 44 +- cmd/crowdsec-cli/utils.go | 437 --------------- cmd/crowdsec-cli/utils_table.go | 32 +- cmd/crowdsec/crowdsec.go | 19 +- cmd/crowdsec/main.go | 21 +- cmd/crowdsec/metrics.go | 8 - cmd/crowdsec/output.go | 7 +- cmd/crowdsec/serve.go | 19 +- config/config.yaml | 1 - config/config_win.yaml | 1 - config/config_win_no_lapi.yaml | 1 - config/dev.yaml | 1 - config/user.yaml | 1 - docker/config.yaml | 1 - docker/docker_start.sh | 22 +- docker/test/tests/test_hub_collections.py | 8 +- docker/test/tests/test_hub_scenarios.py | 4 +- go.mod | 2 +- go.sum | 4 +- pkg/csconfig/api.go | 4 - pkg/csconfig/api_test.go | 8 +- pkg/csconfig/common.go | 11 +- pkg/csconfig/common_test.go | 83 --- pkg/csconfig/config.go | 38 +- pkg/csconfig/config_paths.go | 2 +- pkg/csconfig/config_test.go | 4 +- pkg/csconfig/crowdsec_service.go | 15 +- pkg/csconfig/crowdsec_service_test.go | 26 +- pkg/csconfig/cscli.go | 20 +- pkg/csconfig/cscli_test.go | 33 +- pkg/csconfig/hub.go | 20 +- pkg/csconfig/hub_test.go | 44 +- pkg/csconfig/prometheus.go | 11 - pkg/csconfig/prometheus_test.go | 42 -- pkg/csconfig/simulation.go | 5 - pkg/csconfig/simulation_test.go | 13 +- pkg/csconfig/testdata/config.yaml | 1 - pkg/cwhub/cwhub.go | 284 +--------- pkg/cwhub/cwhub_test.go | 328 ++--------- pkg/cwhub/dataset.go | 74 ++- pkg/cwhub/dataset_test.go | 17 +- pkg/cwhub/doc.go | 113 ++++ pkg/cwhub/download.go | 324 ----------- pkg/cwhub/download_test.go | 52 -- pkg/cwhub/enable.go | 190 +++++++ pkg/cwhub/enable_test.go | 141 +++++ pkg/cwhub/errors.go | 21 + pkg/cwhub/helpers.go | 501 +++++++++++------ pkg/cwhub/helpers_test.go | 218 ++++---- pkg/cwhub/hub.go | 161 ++++++ pkg/cwhub/hub_test.go | 77 +++ pkg/cwhub/install.go | 214 -------- pkg/cwhub/items.go | 383 +++++++++++++ pkg/cwhub/items_test.go | 71 +++ pkg/cwhub/leakybucket.go | 53 ++ pkg/cwhub/loader.go | 552 ------------------- pkg/cwhub/remote.go | 61 +++ pkg/cwhub/sync.go | 498 +++++++++++++++++ pkg/hubtest/coverage.go | 147 ++--- pkg/hubtest/hubtest.go | 38 +- pkg/hubtest/hubtest_item.go | 118 ++-- pkg/hubtest/parser_assert.go | 170 ++++-- pkg/hubtest/regexp.go | 11 + pkg/hubtest/scenario_assert.go | 57 +- pkg/hubtest/utils.go | 8 +- pkg/hubtest/utils_test.go | 18 +- pkg/leakybucket/buckets_test.go | 39 +- pkg/leakybucket/manager_load.go | 10 +- pkg/leakybucket/tests/hub/index.json | 1 + pkg/parser/unix_parser.go | 21 +- pkg/setup/install.go | 49 +- test/bats/00_wait_for.bats | 71 +++ test/bats/01_crowdsec.bats | 57 +- test/bats/01_cscli.bats | 78 +-- test/bats/02_nolapi.bats | 16 +- test/bats/03_noagent.bats | 17 +- test/bats/04_capi.bats | 4 + test/bats/04_nocapi.bats | 16 +- test/bats/05_config_yaml_local.bats | 17 +- test/bats/07_setup.bats | 45 +- test/bats/08_metrics.bats | 17 +- test/bats/13_capi_whitelists.bats | 36 +- test/bats/20_collections.bats | 145 ----- test/bats/20_hub.bats | 121 ++++ test/bats/20_hub_collections.bats | 381 +++++++++++++ test/bats/20_hub_collections_dep.bats | 126 +++++ test/bats/20_hub_items.bats | 149 +++++ test/bats/20_hub_parsers.bats | 383 +++++++++++++ test/bats/20_hub_postoverflows.bats | 383 +++++++++++++ test/bats/20_hub_scenarios.bats | 382 +++++++++++++ test/bats/30_machines_tls.bats | 10 +- test/bats/40_cold-logs.bats | 6 +- test/bats/40_live-ban.bats | 5 + test/bats/50_simulation.bats | 5 + test/bats/72_plugin_badconfig.bats | 62 ++- test/bats/81_alert_context.bats | 3 + test/bats/testdata/explain/explain-log.txt | 7 +- test/bin/wait-for | 116 ++++ test/lib/config/config-global | 51 +- test/lib/config/config-local | 49 +- test/lib/setup_file.sh | 25 + 124 files changed, 6836 insertions(+), 4414 deletions(-) delete mode 100644 cmd/crowdsec-cli/collections.go create mode 100644 cmd/crowdsec-cli/item_metrics.go create mode 100644 cmd/crowdsec-cli/item_suggest.go create mode 100644 cmd/crowdsec-cli/itemcommands.go create mode 100644 cmd/crowdsec-cli/items.go delete mode 100644 cmd/crowdsec-cli/parsers.go delete mode 100644 cmd/crowdsec-cli/postoverflows.go create mode 100644 cmd/crowdsec-cli/require/branch.go delete mode 100644 cmd/crowdsec-cli/scenarios.go delete mode 100644 pkg/csconfig/common_test.go delete mode 100644 pkg/csconfig/prometheus_test.go create mode 100644 pkg/cwhub/doc.go delete mode 100644 pkg/cwhub/download.go delete mode 100644 pkg/cwhub/download_test.go create mode 100644 pkg/cwhub/enable.go create mode 100644 pkg/cwhub/enable_test.go create mode 100644 pkg/cwhub/errors.go create mode 100644 pkg/cwhub/hub.go create mode 100644 pkg/cwhub/hub_test.go delete mode 100644 pkg/cwhub/install.go create mode 100644 pkg/cwhub/items.go create mode 100644 pkg/cwhub/items_test.go create mode 100644 pkg/cwhub/leakybucket.go delete mode 100644 pkg/cwhub/loader.go create mode 100644 pkg/cwhub/remote.go create mode 100644 pkg/cwhub/sync.go create mode 100644 pkg/hubtest/regexp.go create mode 100644 pkg/leakybucket/tests/hub/index.json create mode 100644 test/bats/00_wait_for.bats delete mode 100644 test/bats/20_collections.bats create mode 100644 test/bats/20_hub.bats create mode 100644 test/bats/20_hub_collections.bats create mode 100644 test/bats/20_hub_collections_dep.bats create mode 100644 test/bats/20_hub_items.bats create mode 100644 test/bats/20_hub_parsers.bats create mode 100644 test/bats/20_hub_postoverflows.bats create mode 100644 test/bats/20_hub_scenarios.bats create mode 100755 test/bin/wait-for diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index 0e0f217aa..0261eab9c 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -151,11 +151,12 @@ func NewCapiStatusCmd() *cobra.Command { return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err) } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig, nil) + if err != nil { return err } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("failed to get scenarios: %w", err) } diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go deleted file mode 100644 index 6806d39a7..000000000 --- a/cmd/crowdsec-cli/collections.go +++ /dev/null @@ -1,176 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewCollectionsCmd() *cobra.Command { - var cmdCollections = &cobra.Command{ - Use: "collections [action]", - Short: "Manage collections from hub", - Long: `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`, - /*TBD fix help*/ - Args: cobra.MinimumNArgs(1), - Aliases: []string{"collection"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - var ignoreError bool - - var cmdCollectionsInstall = &cobra.Command{ - Use: "install collection", - Short: "Install given collection(s)", - Long: `Fetch and install given collection(s) from hub`, - Example: `cscli collections install crowdsec/xxx crowdsec/xyz`, - Args: cobra.MinimumNArgs(1), - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.COLLECTIONS, args, toComplete) - }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.COLLECTIONS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.COLLECTIONS, name) - Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - return nil - }, - } - cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections") - cmdCollections.AddCommand(cmdCollectionsInstall) - - var cmdCollectionsRemove = &cobra.Command{ - Use: "remove collection", - Short: "Remove given collection(s)", - Long: `Remove given collection(s) from hub`, - Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) - }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction) - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one collection to remove or '--all'") - } - - for _, name := range args { - if !forceAction { - item := cwhub.GetItem(cwhub.COLLECTIONS, name) - if item == nil { - return fmt.Errorf("unable to retrieve: %s", name) - } - if len(item.BelongsToCollections) > 0 { - log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections) - log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name) - continue - } - } - cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction) - } - return nil - }, - } - cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections") - cmdCollections.AddCommand(cmdCollectionsRemove) - - var cmdCollectionsUpgrade = &cobra.Command{ - Use: "upgrade collection", - Short: "Upgrade given collection(s)", - Long: `Fetch and upgrade given collection(s) from hub`, - Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) - }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one collection to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction) - } - } - return nil - }, - } - cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections") - cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") - cmdCollections.AddCommand(cmdCollectionsUpgrade) - - var cmdCollectionsInspect = &cobra.Command{ - Use: "inspect collection", - Short: "Inspect given collection", - Long: `Inspect given collection`, - Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) - }, - Run: func(cmd *cobra.Command, args []string) { - for _, name := range args { - InspectItem(name, cwhub.COLLECTIONS) - } - }, - } - cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") - cmdCollections.AddCommand(cmdCollectionsInspect) - - var cmdCollectionsList = &cobra.Command{ - Use: "list collection [-a]", - Short: "List all collections", - Long: `List all collections`, - Example: `cscli collections list`, - Args: cobra.ExactArgs(0), - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all) - }, - } - cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") - cmdCollections.AddCommand(cmdCollectionsList) - - return cmdCollections -} diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 009b35b7c..93772d611 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -14,21 +14,25 @@ import ( ) func backupHub(dirPath string) error { - var err error var itemDirectory string var upstreamParsers []string + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + for _, itemType := range cwhub.ItemTypes { clog := log.WithFields(log.Fields{ "type": itemType, }) - itemMap := cwhub.GetItemMap(itemType) + itemMap := hub.GetItemMap(itemType) if itemMap == nil { clog.Infof("No %s to backup.", itemType) continue } itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType) - if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil { + if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil { return fmt.Errorf("error while creating %s : %s", itemDirectory, err) } upstreamParsers = []string{} @@ -36,30 +40,30 @@ func backupHub(dirPath string) error { clog = clog.WithFields(log.Fields{ "file": v.Name, }) - if !v.Installed { //only backup installed ones + if !v.State.Installed { //only backup installed ones clog.Debugf("[%s] : not installed", k) continue } //for the local/tainted ones, we back up the full file - if v.Tainted || v.Local || !v.UpToDate { - //we need to back up stages for parsers - if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW { + if v.State.Tainted || v.IsLocal() || !v.State.UpToDate { + //we need to backup stages for parsers + if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS { fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) - if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil { + if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil { return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err) } } - clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate) + clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.IsLocal(), v.State.UpToDate) tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) - if err = CopyFile(v.LocalPath, tfile); err != nil { - return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) + if err = CopyFile(v.State.LocalPath, tfile); err != nil { + return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err) } - clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile) + clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile) continue } - clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate) - clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate) + clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate) + clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate) upstreamParsers = append(upstreamParsers, v.Name) } //write the upstream items @@ -100,7 +104,7 @@ func backupConfigToDirectory(dirPath string) error { /*if parent directory doesn't exist, bail out. create final dir with Mkdir*/ parentDir := filepath.Dir(dirPath) - if _, err := os.Stat(parentDir); err != nil { + if _, err = os.Stat(parentDir); err != nil { return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err) } @@ -197,10 +201,6 @@ func backupConfigToDirectory(dirPath string) error { } func runConfigBackup(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { - return err - } - if err := backupConfigToDirectory(args[0]); err != nil { return fmt.Errorf("failed to backup config: %w", err) } diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 395e943bc..56f628281 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -21,45 +21,12 @@ type OldAPICfg struct { Password string `json:"password"` } -// it's a rip of the cli version, but in silent-mode -func silentInstallItem(name string, obtype string) (string, error) { - var item = cwhub.GetItem(obtype, name) - if item == nil { - return "", fmt.Errorf("error retrieving item") - } - if downloadOnly && item.Downloaded && item.UpToDate { - return fmt.Sprintf("%s is already downloaded and up-to-date", item.Name), nil - } - err := cwhub.DownloadLatest(csConfig.Hub, item, forceAction, false) - if err != nil { - return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) - } - if err := cwhub.AddItem(obtype, *item); err != nil { - return "", err - } - - if downloadOnly { - return fmt.Sprintf("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath), nil - } - err = cwhub.EnableItem(csConfig.Hub, item) - if err != nil { - return "", fmt.Errorf("error while enabling %s : %v", item.Name, err) - } - if err := cwhub.AddItem(obtype, *item); err != nil { - return "", err - } - return fmt.Sprintf("Enabled %s", item.Name), nil -} - func restoreHub(dirPath string) error { - var err error - - if err := csConfig.LoadHub(); err != nil { + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + if err != nil { return err } - cwhub.SetHubBranch() - for _, itype := range cwhub.ItemTypes { itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) if _, err = os.Stat(itemDirectory); err != nil { @@ -78,13 +45,14 @@ func restoreHub(dirPath string) error { return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err) } for _, toinstall := range upstreamList { - label, err := silentInstallItem(toinstall, itype) + item := hub.GetItem(itype, toinstall) + if item == nil { + log.Errorf("Item %s/%s not found in hub", itype, toinstall) + continue + } + err := item.Install(false, false) if err != nil { log.Errorf("Error while installing %s : %s", toinstall, err) - } else if label != "" { - log.Infof("Installed %s : %s", toinstall, label) - } else { - log.Printf("Installed %s : ok", toinstall) } } @@ -98,7 +66,7 @@ func restoreHub(dirPath string) error { if file.Name() == fmt.Sprintf("upstream-%s.json", itype) { continue } - if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW { + if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS { //we expect a stage here if !file.IsDir() { continue @@ -302,10 +270,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error { return err } - if err := require.Hub(csConfig); err != nil { - return err - } - if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil { return fmt.Errorf("failed to restore config from %s: %w", args[0], err) } diff --git a/cmd/crowdsec-cli/config_show.go b/cmd/crowdsec-cli/config_show.go index 9f5b11fc1..ca7051195 100644 --- a/cmd/crowdsec-cli/config_show.go +++ b/cmd/crowdsec-cli/config_show.go @@ -82,7 +82,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled cscli: - Output : {{.Cscli.Output}} - Hub Branch : {{.Cscli.HubBranch}} - - Hub Folder : {{.Cscli.HubDir}} {{- end }} {{- if .API }} diff --git a/cmd/crowdsec-cli/console.go b/cmd/crowdsec-cli/console.go index 439a8143b..1caf11752 100644 --- a/cmd/crowdsec-cli/console.go +++ b/cmd/crowdsec-cli/console.go @@ -71,11 +71,12 @@ After running this command your will need to validate the enrollment in the weba return fmt.Errorf("could not parse CAPI URL: %s", err) } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig, nil) + if err != nil { return err } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("failed to get installed scenarios: %s", err) } diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 7bdfd5162..ad3110011 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -1,7 +1,6 @@ package main import ( - "errors" "fmt" "github.com/fatih/color" @@ -13,30 +12,19 @@ import ( ) func NewHubCmd() *cobra.Command { - var cmdHub = &cobra.Command{ + cmdHub := &cobra.Command{ Use: "hub [action]", - Short: "Manage Hub", - Long: ` -Hub management + Short: "Manage hub index", + Long: `Hub management List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net). -The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update. - `, - Example: ` -cscli hub list # List all installed configurations -cscli hub update # Download list of available configurations from the hub - `, +The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`, + Example: `cscli hub list +cscli hub update +cscli hub upgrade`, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if csConfig.Cscli == nil { - return fmt.Errorf("you must configure cli before interacting with hub") - } - - return nil - }, } - cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub") cmdHub.AddCommand(NewHubListCmd()) cmdHub.AddCommand(NewHubUpdateCmd()) @@ -45,116 +33,142 @@ cscli hub update # Download list of available configurations from the hub return cmdHub } +func runHubList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + for _, v := range hub.Warnings { + log.Info(v) + } + + for _, line := range hub.ItemStats() { + log.Info(line) + } + + items := make(map[string][]*cwhub.Item) + + for _, itemType := range cwhub.ItemTypes { + items[itemType], err = selectItems(hub, itemType, nil, !all) + if err != nil { + return err + } + } + + err = listItems(color.Output, cwhub.ItemTypes, items) + if err != nil { + return err + } + + return nil +} + func NewHubListCmd() *cobra.Command { - var cmdHubList = &cobra.Command{ + cmdHubList := &cobra.Command{ Use: "list [-a]", - Short: "List installed configs", + Short: "List all installed configurations", Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { - return err - } - - // use LocalSync to get warnings about tainted / outdated items - warn, _ := cwhub.LocalSync(csConfig.Hub) - for _, v := range warn { - log.Info(v) - } - cwhub.DisplaySummary() - ListItems(color.Output, []string{ - cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW, - }, args, true, false, all) - - return nil - }, + RunE: runHubList, } - cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") + + flags := cmdHubList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") return cmdHubList } +func runHubUpdate(cmd *cobra.Command, args []string) error { + local := csConfig.Hub + remote := require.RemoteHub(csConfig) + + // don't use require.Hub because if there is no index file, it would fail + hub, err := cwhub.NewHub(local, remote, true) + if err != nil { + return fmt.Errorf("failed to update hub: %w", err) + } + + for _, v := range hub.Warnings { + log.Info(v) + } + + return nil +} + func NewHubUpdateCmd() *cobra.Command { - var cmdHubUpdate = &cobra.Command{ + cmdHubUpdate := &cobra.Command{ Use: "update", - Short: "Fetch available configs from hub", + Short: "Download the latest index (catalog of available configurations)", Long: ` -Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs. +Fetches the .index.json file from the hub, containing the list of available configs. `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if csConfig.Cscli == nil { - return fmt.Errorf("you must configure cli before interacting with hub") - } - - cwhub.SetHubBranch() - - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadHub(); err != nil { - return err - } - if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { - if !errors.Is(err, cwhub.ErrIndexNotFound) { - return fmt.Errorf("failed to get Hub index : %w", err) - } - log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch) - cwhub.HubBranch = "master" - if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { - return fmt.Errorf("failed to get Hub index after retry: %w", err) - } - } - // use LocalSync to get warnings about tainted / outdated items - warn, _ := cwhub.LocalSync(csConfig.Hub) - for _, v := range warn { - log.Info(v) - } - - return nil - }, + RunE: runHubUpdate, } return cmdHubUpdate } +func runHubUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + if err != nil { + return err + } + + for _, itemType := range cwhub.ItemTypes { + items, err := hub.GetInstalledItems(itemType) + if err != nil { + return err + } + + updated := 0 + + log.Infof("Upgrading %s", itemType) + for _, item := range items { + didUpdate, err := item.Upgrade(force) + if err != nil { + return err + } + if didUpdate { + updated++ + } + } + log.Infof("Upgraded %d %s", updated, itemType) + } + + return nil +} + func NewHubUpgradeCmd() *cobra.Command { - var cmdHubUpgrade = &cobra.Command{ + cmdHubUpgrade := &cobra.Command{ Use: "upgrade", - Short: "Upgrade all configs installed from hub", + Short: "Upgrade all configurations to their latest version", Long: ` Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available. `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if csConfig.Cscli == nil { - return fmt.Errorf("you must configure cli before interacting with hub") - } - - cwhub.SetHubBranch() - - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { - return err - } - - log.Infof("Upgrading collections") - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction) - log.Infof("Upgrading parsers") - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction) - log.Infof("Upgrading scenarios") - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction) - log.Infof("Upgrading postoverflows") - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction) - - return nil - }, + RunE: runHubUpgrade, } - cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") + + flags := cmdHubUpgrade.Flags() + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") return cmdHubUpgrade } diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index 97bb8c8dd..8b574c3ee 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -18,9 +18,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/hubtest" ) -var ( - HubTest hubtest.HubTest -) +var HubTest hubtest.HubTest func NewHubTestCmd() *cobra.Command { var hubPath string @@ -43,6 +41,7 @@ func NewHubTestCmd() *cobra.Command { return nil }, } + cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder") cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec") cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli") @@ -59,7 +58,6 @@ func NewHubTestCmd() *cobra.Command { return cmdHubTest } - func NewHubTestCreateCmd() *cobra.Command { parsers := []string{} postoverflows := []string{} @@ -138,7 +136,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios } configFilePath := filepath.Join(testPath, "config.yaml") - fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) + fd, err := os.Create(configFilePath) if err != nil { return fmt.Errorf("open: %s", err) } @@ -164,6 +162,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios return nil }, } + cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test") cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test") cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test") @@ -173,7 +172,6 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios return cmdHubTestCreate } - func NewHubTestRunCmd() *cobra.Command { var noClean bool var runAll bool @@ -186,7 +184,7 @@ func NewHubTestRunCmd() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { if !runAll && len(args) == 0 { printHelp(cmd) - return fmt.Errorf("Please provide test to run or --all flag") + return fmt.Errorf("please provide test to run or --all flag") } if runAll { @@ -202,6 +200,9 @@ func NewHubTestRunCmd() *cobra.Command { } } + // set timezone to avoid DST issues + os.Setenv("TZ", "UTC") + for _, test := range HubTest.Tests { if csConfig.Cscli.Output == "human" { log.Infof("Running test '%s'", test.Name) @@ -293,9 +294,11 @@ func NewHubTestRunCmd() *cobra.Command { } } } - if csConfig.Cscli.Output == "human" { + + switch csConfig.Cscli.Output { + case "human": hubTestResultTable(color.Output, testResult) - } else if csConfig.Cscli.Output == "json" { + case "json": jsonResult := make(map[string][]string, 0) jsonResult["success"] = make([]string, 0) jsonResult["fail"] = make([]string, 0) @@ -311,6 +314,8 @@ func NewHubTestRunCmd() *cobra.Command { return fmt.Errorf("unable to json test result: %s", err) } fmt.Println(string(jsonStr)) + default: + return fmt.Errorf("only human/json output modes are supported") } if !success { @@ -320,6 +325,7 @@ func NewHubTestRunCmd() *cobra.Command { return nil }, } + cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed") cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail") cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests") @@ -327,7 +333,6 @@ func NewHubTestRunCmd() *cobra.Command { return cmdHubTestRun } - func NewHubTestCleanCmd() *cobra.Command { var cmdHubTestClean = &cobra.Command{ Use: "clean", @@ -352,7 +357,6 @@ func NewHubTestCleanCmd() *cobra.Command { return cmdHubTestClean } - func NewHubTestInfoCmd() *cobra.Command { var cmdHubTestInfo = &cobra.Command{ Use: "info", @@ -381,7 +385,6 @@ func NewHubTestInfoCmd() *cobra.Command { return cmdHubTestInfo } - func NewHubTestListCmd() *cobra.Command { var cmdHubTestList = &cobra.Command{ Use: "list", @@ -412,7 +415,6 @@ func NewHubTestListCmd() *cobra.Command { return cmdHubTestList } - func NewHubTestCoverageCmd() *cobra.Command { var showParserCov bool var showScenarioCov bool @@ -427,8 +429,8 @@ func NewHubTestCoverageCmd() *cobra.Command { return fmt.Errorf("unable to load all tests: %+v", err) } var err error - scenarioCoverage := []hubtest.ScenarioCoverage{} - parserCoverage := []hubtest.ParserCoverage{} + scenarioCoverage := []hubtest.Coverage{} + parserCoverage := []hubtest.Coverage{} scenarioCoveragePercent := 0 parserCoveragePercent := 0 @@ -443,7 +445,7 @@ func NewHubTestCoverageCmd() *cobra.Command { parserTested := 0 for _, test := range parserCoverage { if test.TestsCount > 0 { - parserTested += 1 + parserTested++ } } parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100))) @@ -454,12 +456,14 @@ func NewHubTestCoverageCmd() *cobra.Command { if err != nil { return fmt.Errorf("while getting scenario coverage: %s", err) } + scenarioTested := 0 for _, test := range scenarioCoverage { if test.TestsCount > 0 { - scenarioTested += 1 + scenarioTested++ } } + scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100))) } @@ -474,7 +478,8 @@ func NewHubTestCoverageCmd() *cobra.Command { os.Exit(0) } - if csConfig.Cscli.Output == "human" { + switch csConfig.Cscli.Output { + case "human": if showParserCov || showAll { hubTestParserCoverageTable(color.Output, parserCoverage) } @@ -489,7 +494,7 @@ func NewHubTestCoverageCmd() *cobra.Command { if showScenarioCov || showAll { fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent) } - } else if csConfig.Cscli.Output == "json" { + case "json": dump, err := json.MarshalIndent(parserCoverage, "", " ") if err != nil { return err @@ -500,13 +505,14 @@ func NewHubTestCoverageCmd() *cobra.Command { return err } fmt.Printf("%s", dump) - } else { + default: return fmt.Errorf("only human/json output modes are supported") } return nil }, } + cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage") @@ -514,7 +520,6 @@ func NewHubTestCoverageCmd() *cobra.Command { return cmdHubTestCoverage } - func NewHubTestEvalCmd() *cobra.Command { var evalExpression string var cmdHubTestEval = &cobra.Command{ @@ -528,26 +533,29 @@ func NewHubTestEvalCmd() *cobra.Command { if err != nil { return fmt.Errorf("can't load test: %+v", err) } + err = test.ParserAssert.LoadTest(test.ParserResultFile) if err != nil { return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err) } + output, err := test.ParserAssert.EvalExpression(evalExpression) if err != nil { return err } + fmt.Print(output) } return nil }, } + cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval") return cmdHubTestEval } - func NewHubTestExplainCmd() *cobra.Command { var cmdHubTestExplain = &cobra.Command{ Use: "explain", @@ -562,24 +570,22 @@ func NewHubTestExplainCmd() *cobra.Command { } err = test.ParserAssert.LoadTest(test.ParserResultFile) if err != nil { - err := test.Run() - if err != nil { + if err = test.Run(); err != nil { return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) } - err = test.ParserAssert.LoadTest(test.ParserResultFile) - if err != nil { + + if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil { return fmt.Errorf("unable to load parser result after run: %s", err) } } err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile) if err != nil { - err := test.Run() - if err != nil { + if err = test.Run(); err != nil { return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) } - err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile) - if err != nil { + + if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil { return fmt.Errorf("unable to load scenario result after run: %s", err) } } diff --git a/cmd/crowdsec-cli/hubtest_table.go b/cmd/crowdsec-cli/hubtest_table.go index 9f28c3699..9b31a79a2 100644 --- a/cmd/crowdsec-cli/hubtest_table.go +++ b/cmd/crowdsec-cli/hubtest_table.go @@ -41,39 +41,41 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) { t.Render() } -func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) { +func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t := newLightTable(out) t.SetHeaders("Parser", "Status", "Number of tests") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) parserTested := 0 + for _, test := range coverage { status := emoji.RedCircle.String() if test.TestsCount > 0 { status = emoji.GreenCircle.String() parserTested++ } - t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) + t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) } t.Render() } -func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) { +func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t := newLightTable(out) t.SetHeaders("Scenario", "Status", "Number of tests") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) parserTested := 0 + for _, test := range coverage { status := emoji.RedCircle.String() if test.TestsCount > 0 { status = emoji.GreenCircle.String() parserTested++ } - t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) + t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) } t.Render() diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go new file mode 100644 index 000000000..51b652abc --- /dev/null +++ b/cmd/crowdsec-cli/item_metrics.go @@ -0,0 +1,236 @@ +package main + +import ( + "fmt" + "math" + "net/http" + "strconv" + "strings" + "time" + + "github.com/fatih/color" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/prom2json" + log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/go-cs-lib/trace" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func ShowMetrics(hubItem *cwhub.Item) error { + switch hubItem.Type { + case cwhub.PARSERS: + metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) + parserMetricsTable(color.Output, hubItem.Name, metrics) + case cwhub.SCENARIOS: + metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) + scenarioMetricsTable(color.Output, hubItem.Name, metrics) + case cwhub.COLLECTIONS: + for _, sub := range hubItem.SubItems() { + if err := ShowMetrics(sub); err != nil { + return err + } + } + default: + // no metrics for this item type + } + return nil +} + +// GetParserMetric is a complete rip from prom2json +func GetParserMetric(url string, itemName string) map[string]map[string]int { + stats := make(map[string]map[string]int) + + result := GetPrometheusMetric(url) + for idx, fam := range result { + if !strings.HasPrefix(fam.Name, "cs_") { + continue + } + log.Tracef("round %d", idx) + for _, m := range fam.Metrics { + metric, ok := m.(prom2json.Metric) + if !ok { + log.Debugf("failed to convert metric to prom2json.Metric") + continue + } + name, ok := metric.Labels["name"] + if !ok { + log.Debugf("no name in Metric %v", metric.Labels) + } + if name != itemName { + continue + } + source, ok := metric.Labels["source"] + if !ok { + log.Debugf("no source in Metric %v", metric.Labels) + } else { + if srctype, ok := metric.Labels["type"]; ok { + source = srctype + ":" + source + } + } + value := m.(prom2json.Metric).Value + fval, err := strconv.ParseFloat(value, 32) + if err != nil { + log.Errorf("Unexpected int value %s : %s", value, err) + continue + } + ival := int(fval) + + switch fam.Name { + case "cs_reader_hits_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + stats[source]["parsed"] = 0 + stats[source]["reads"] = 0 + stats[source]["unparsed"] = 0 + stats[source]["hits"] = 0 + } + stats[source]["reads"] += ival + case "cs_parser_hits_ok_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["parsed"] += ival + case "cs_parser_hits_ko_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["unparsed"] += ival + case "cs_node_hits_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["hits"] += ival + case "cs_node_hits_ok_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["parsed"] += ival + case "cs_node_hits_ko_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["unparsed"] += ival + default: + continue + } + } + } + return stats +} + +func GetScenarioMetric(url string, itemName string) map[string]int { + stats := make(map[string]int) + + stats["instantiation"] = 0 + stats["curr_count"] = 0 + stats["overflow"] = 0 + stats["pour"] = 0 + stats["underflow"] = 0 + + result := GetPrometheusMetric(url) + for idx, fam := range result { + if !strings.HasPrefix(fam.Name, "cs_") { + continue + } + log.Tracef("round %d", idx) + for _, m := range fam.Metrics { + metric, ok := m.(prom2json.Metric) + if !ok { + log.Debugf("failed to convert metric to prom2json.Metric") + continue + } + name, ok := metric.Labels["name"] + if !ok { + log.Debugf("no name in Metric %v", metric.Labels) + } + if name != itemName { + continue + } + value := m.(prom2json.Metric).Value + fval, err := strconv.ParseFloat(value, 32) + if err != nil { + log.Errorf("Unexpected int value %s : %s", value, err) + continue + } + ival := int(fval) + + switch fam.Name { + case "cs_bucket_created_total": + stats["instantiation"] += ival + case "cs_buckets": + stats["curr_count"] += ival + case "cs_bucket_overflowed_total": + stats["overflow"] += ival + case "cs_bucket_poured_total": + stats["pour"] += ival + case "cs_bucket_underflowed_total": + stats["underflow"] += ival + default: + continue + } + } + } + return stats +} + +func GetPrometheusMetric(url string) []*prom2json.Family { + mfChan := make(chan *dto.MetricFamily, 1024) + + // Start with the DefaultTransport for sane defaults. + transport := http.DefaultTransport.(*http.Transport).Clone() + // Conservatively disable HTTP keep-alives as this program will only + // ever need a single HTTP request. + transport.DisableKeepAlives = true + // Timeout early if the server doesn't even return the headers. + transport.ResponseHeaderTimeout = time.Minute + + go func() { + defer trace.CatchPanic("crowdsec/GetPrometheusMetric") + err := prom2json.FetchMetricFamilies(url, mfChan, transport) + if err != nil { + log.Fatalf("failed to fetch prometheus metrics : %v", err) + } + }() + + result := []*prom2json.Family{} + for mf := range mfChan { + result = append(result, prom2json.NewFamily(mf)) + } + log.Debugf("Finished reading prometheus output, %d entries", len(result)) + + return result +} + +type unit struct { + value int64 + symbol string +} + +var ranges = []unit{ + {value: 1e18, symbol: "E"}, + {value: 1e15, symbol: "P"}, + {value: 1e12, symbol: "T"}, + {value: 1e9, symbol: "G"}, + {value: 1e6, symbol: "M"}, + {value: 1e3, symbol: "k"}, + {value: 1, symbol: ""}, +} + +func formatNumber(num int) string { + goodUnit := unit{} + for _, u := range ranges { + if int64(num) >= u.value { + goodUnit = u + break + } + } + + if goodUnit.value == 1 { + return fmt.Sprintf("%d%s", num, goodUnit.symbol) + } + + res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100 + return fmt.Sprintf("%.2f%s", res, goodUnit.symbol) +} diff --git a/cmd/crowdsec-cli/item_suggest.go b/cmd/crowdsec-cli/item_suggest.go new file mode 100644 index 000000000..e9db3b7b9 --- /dev/null +++ b/cmd/crowdsec-cli/item_suggest.go @@ -0,0 +1,85 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/agext/levenshtein" + "github.com/spf13/cobra" + "slices" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +const MaxDistance = 7 + +// SuggestNearestMessage returns a message with the most similar item name, if one is found +func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string { + score := 100 + nearest := "" + + for _, item := range hub.GetItemMap(itemType) { + d := levenshtein.Distance(itemName, item.Name, nil) + if d < score { + score = d + nearest = item.Name + } + } + + msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType) + + if score < MaxDistance { + msg += fmt.Sprintf(", did you mean '%s'?", nearest) + } + + return msg +} + +func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + hub, err := require.Hub(csConfig, nil) + if err != nil { + return nil, cobra.ShellCompDirectiveDefault + } + + comp := make([]string, 0) + + for _, item := range hub.GetItemMap(itemType) { + if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { + comp = append(comp, item.Name) + } + } + + cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) + + return comp, cobra.ShellCompDirectiveNoFileComp +} + +func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + hub, err := require.Hub(csConfig, nil) + if err != nil { + return nil, cobra.ShellCompDirectiveDefault + } + + items, err := hub.GetInstalledItemNames(itemType) + if err != nil { + cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) + return nil, cobra.ShellCompDirectiveDefault + } + + comp := make([]string, 0) + + if toComplete != "" { + for _, item := range items { + if strings.Contains(item, toComplete) { + comp = append(comp, item) + } + } + } else { + comp = items + } + + cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) + + return comp, cobra.ShellCompDirectiveNoFileComp +} diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go new file mode 100644 index 000000000..6b9476d02 --- /dev/null +++ b/cmd/crowdsec-cli/itemcommands.go @@ -0,0 +1,606 @@ +package main + +import ( + "fmt" + + "github.com/fatih/color" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/crowdsecurity/go-cs-lib/coalesce" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +type cmdHelp struct { + // Example is required, the others have a default value + // generated from the item type + use string + short string + long string + example string +} + +type hubItemType struct { + name string // plural, as used in the hub index + singular string + oneOrMore string // parenthetical pluralizaion: "parser(s)" + help cmdHelp + installHelp cmdHelp + removeHelp cmdHelp + upgradeHelp cmdHelp + inspectHelp cmdHelp + listHelp cmdHelp +} + +var hubItemTypes = map[string]hubItemType{ + "parsers": { + name: "parsers", + singular: "parser", + oneOrMore: "parser(s)", + help: cmdHelp{ + example: `cscli parsers list -a +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs +`, + }, + installHelp: cmdHelp{ + example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + removeHelp: cmdHelp{ + example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + upgradeHelp: cmdHelp{ + example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + inspectHelp: cmdHelp{ + example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, + }, + listHelp: cmdHelp{ + example: `cscli parsers list +cscli parsers list -a +cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs + +List only enabled parsers unless "-a" or names are specified.`, + }, + }, + "postoverflows": { + name: "postoverflows", + singular: "postoverflow", + oneOrMore: "postoverflow(s)", + help: cmdHelp{ + example: `cscli postoverflows list -a +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns +`, + }, + installHelp: cmdHelp{ + example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + removeHelp: cmdHelp{ + example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + upgradeHelp: cmdHelp{ + example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + inspectHelp: cmdHelp{ + example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + listHelp: cmdHelp{ + example: `cscli postoverflows list +cscli postoverflows list -a +cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns + +List only enabled postoverflows unless "-a" or names are specified.`, + }, + }, + "scenarios": { + name: "scenarios", + singular: "scenario", + oneOrMore: "scenario(s)", + help: cmdHelp{ + example: `cscli scenarios list -a +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing +`, + }, + installHelp: cmdHelp{ + example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + removeHelp: cmdHelp{ + example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + upgradeHelp: cmdHelp{ + example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + inspectHelp: cmdHelp{ + example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + listHelp: cmdHelp{ + example: `cscli scenarios list +cscli scenarios list -a +cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing + +List only enabled scenarios unless "-a" or names are specified.`, + }, + }, + "collections": { + name: "collections", + singular: "collection", + oneOrMore: "collection(s)", + help: cmdHelp{ + example: `cscli collections list -a +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables +`, + }, + installHelp: cmdHelp{ + example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, + }, + removeHelp: cmdHelp{ + example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, + }, + upgradeHelp: cmdHelp{ + example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, + }, + inspectHelp: cmdHelp{ + example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, + }, + listHelp: cmdHelp{ + example: `cscli collections list +cscli collections list -a +cscli collections list crowdsecurity/http-cve crowdsecurity/iptables + +List only enabled collections unless "-a" or names are specified.`, + }, + }, +} + +func NewItemsCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.help.use, fmt.Sprintf("%s [item]...", it.name)), + Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)), + Long: it.help.long, + Example: it.help.example, + Args: cobra.MinimumNArgs(1), + Aliases: []string{it.singular}, + DisableAutoGenTag: true, + } + + cmd.AddCommand(NewItemsInstallCmd(typeName)) + cmd.AddCommand(NewItemsRemoveCmd(typeName)) + cmd.AddCommand(NewItemsUpgradeCmd(typeName)) + cmd.AddCommand(NewItemsInspectCmd(typeName)) + cmd.AddCommand(NewItemsListCmd(typeName)) + + return cmd +} + +func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + if err != nil { + return err + } + + for _, name := range args { + item := hub.GetItem(it.name, name) + if item == nil { + msg := SuggestNearestMessage(hub, it.name, name) + if !ignoreError { + return fmt.Errorf(msg) + } + + log.Errorf(msg) + + continue + } + + if err := item.Install(force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", item.Name, err) + } + log.Errorf("Error while installing '%s': %s", item.Name, err) + } + } + + log.Infof(ReloadMessage()) + return nil + } + + return run +} + +func NewItemsInstallCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.installHelp.use, "install [item]..."), + Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)), + Long: coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)), + Example: it.installHelp.example, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compAllItems(typeName, args, toComplete) + }, + RunE: itemsInstallRunner(it), + } + + flags := cmd.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") + flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name)) + + return cmd +} + +// return the names of the installed parents of an item, used to check if we can remove it +func istalledParentNames(item *cwhub.Item) []string { + ret := make([]string, 0) + + for _, parent := range item.Ancestors() { + if parent.State.Installed { + ret = append(ret, parent.Name) + } + } + + return ret +} + +func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + if all { + getter := hub.GetInstalledItems + if purge { + getter = hub.GetAllItems + } + + items, err := getter(it.name) + if err != nil { + return err + } + + removed := 0 + + for _, item := range items { + didRemove, err := item.Remove(purge, force) + if err != nil { + return err + } + if didRemove { + removed++ + } + } + + log.Infof("Removed %d %s", removed, it.name) + if removed > 0 { + log.Infof(ReloadMessage()) + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular) + } + + removed := 0 + + for _, itemName := range args { + item := hub.GetItem(it.name, itemName) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", itemName, it.name) + } + + parents := istalledParentNames(item) + + if !force && len(parents) > 0 { + log.Warningf("%s belongs to collections: %s", item.Name, parents) + log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular) + continue + } + + didRemove, err := item.Remove(purge, force) + if err != nil { + return err + } + + if didRemove { + log.Infof("Removed %s", item.Name) + removed++ + } + } + if removed > 0 { + log.Infof(ReloadMessage()) + } + + return nil + } + return run +} + +func NewItemsRemoveCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.removeHelp.use, "remove [item]..."), + Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)), + Long: coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)), + Example: it.removeHelp.example, + Aliases: []string{"delete"}, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: itemsRemoveRunner(it), + } + + flags := cmd.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name)) + + return cmd +} + +func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + if err != nil { + return err + } + + if all { + items, err := hub.GetInstalledItems(it.name) + if err != nil { + return err + } + + updated := 0 + + for _, item := range items { + didUpdate, err := item.Upgrade(force) + if err != nil { + return err + } + if didUpdate { + updated++ + } + } + + log.Infof("Updated %d %s", updated, it.name) + + if updated > 0 { + log.Infof(ReloadMessage()) + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular) + } + + updated := 0 + + for _, itemName := range args { + item := hub.GetItem(it.name, itemName) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", itemName, it.name) + } + + didUpdate, err := item.Upgrade(force) + if err != nil { + return err + } + + if didUpdate { + log.Infof("Updated %s", item.Name) + updated++ + } + } + if updated > 0 { + log.Infof(ReloadMessage()) + } + + return nil + } + + return run +} + +func NewItemsUpgradeCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."), + Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)), + Long: coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)), + Example: it.upgradeHelp.example, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: itemsUpgradeRunner(it), + } + + flags := cmd.Flags() + flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name)) + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") + + return cmd +} + +func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + for _, name := range args { + item := hub.GetItem(it.name, name) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", name, it.name) + } + if err = InspectItem(item, !noMetrics); err != nil { + return err + } + } + + return nil + } + + return run +} + +func NewItemsInspectCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."), + Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)), + Long: coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)), + Example: it.inspectHelp.example, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: itemsInspectRunner(it), + } + + flags := cmd.Flags() + flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") + + return cmd +} + +func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + items := make(map[string][]*cwhub.Item) + + items[it.name], err = selectItems(hub, it.name, args, !all) + if err != nil { + return err + } + + if err = listItems(color.Output, []string{it.name}, items); err != nil { + return err + } + + return nil + } + + return run +} + +func NewItemsListCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.listHelp.use, "list [item... | -a]"), + Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)), + Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)), + Example: it.listHelp.example, + DisableAutoGenTag: true, + RunE: itemsListRunner(it), + } + + flags := cmd.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") + + return cmd +} diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go new file mode 100644 index 000000000..c77ff3f88 --- /dev/null +++ b/cmd/crowdsec-cli/items.go @@ -0,0 +1,157 @@ +package main + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "gopkg.in/yaml.v3" + "slices" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name +func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) { + itemNames := hub.GetItemNames(itemType) + + notExist := []string{} + + if len(args) > 0 { + for _, arg := range args { + if !slices.Contains(itemNames, arg) { + notExist = append(notExist, arg) + } + } + } + + if len(notExist) > 0 { + return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType) + } + + if len(args) > 0 { + itemNames = args + installedOnly = false + } + + items := make([]*cwhub.Item, 0, len(itemNames)) + + for _, itemName := range itemNames { + item := hub.GetItem(itemType, itemName) + if installedOnly && !item.State.Installed { + continue + } + + items = append(items, item) + } + + cwhub.SortItemSlice(items) + + return items, nil +} + +func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item) error { + switch csConfig.Cscli.Output { + case "human": + for _, itemType := range itemTypes { + listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType]) + } + case "json": + type itemHubStatus struct { + Name string `json:"name"` + LocalVersion string `json:"local_version"` + LocalPath string `json:"local_path"` + Description string `json:"description"` + UTF8Status string `json:"utf8_status"` + Status string `json:"status"` + } + + hubStatus := make(map[string][]itemHubStatus) + for _, itemType := range itemTypes { + // empty slice in case there are no items of this type + hubStatus[itemType] = make([]itemHubStatus, len(items[itemType])) + + for i, item := range items[itemType] { + status, emo := item.InstallStatus() + hubStatus[itemType][i] = itemHubStatus{ + Name: item.Name, + LocalVersion: item.State.LocalVersion, + LocalPath: item.State.LocalPath, + Description: item.Description, + Status: status, + UTF8Status: fmt.Sprintf("%v %s", emo, status), + } + } + } + + x, err := json.MarshalIndent(hubStatus, "", " ") + if err != nil { + return fmt.Errorf("failed to unmarshal: %w", err) + } + + out.Write(x) + case "raw": + csvwriter := csv.NewWriter(out) + + header := []string{"name", "status", "version", "description"} + if len(itemTypes) > 1 { + header = append(header, "type") + } + + if err := csvwriter.Write(header); err != nil { + return fmt.Errorf("failed to write header: %s", err) + } + + for _, itemType := range itemTypes { + for _, item := range items[itemType] { + status, _ := item.InstallStatus() + row := []string{ + item.Name, + status, + item.State.LocalVersion, + item.Description, + } + if len(itemTypes) > 1 { + row = append(row, itemType) + } + if err := csvwriter.Write(row); err != nil { + return fmt.Errorf("failed to write raw output: %s", err) + } + } + } + csvwriter.Flush() + default: + return fmt.Errorf("unknown output format '%s'", csConfig.Cscli.Output) + } + + return nil +} + +func InspectItem(item *cwhub.Item, showMetrics bool) error { + switch csConfig.Cscli.Output { + case "human", "raw": + enc := yaml.NewEncoder(os.Stdout) + enc.SetIndent(2) + if err := enc.Encode(item); err != nil { + return fmt.Errorf("unable to encode item: %s", err) + } + case "json": + b, err := json.MarshalIndent(*item, "", " ") + if err != nil { + return fmt.Errorf("unable to marshal item: %s", err) + } + fmt.Print(string(b)) + } + + if csConfig.Cscli.Output == "human" && showMetrics { + fmt.Printf("\nCurrent metrics: \n") + if err := ShowMetrics(item); err != nil { + return err + } + } + + return nil +} diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index 37ee0088c..b2870cb20 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -38,11 +38,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error { log.Fatalf("parsing api url ('%s'): %s", apiurl, err) } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig, nil) + if err != nil { log.Fatal(err) } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) if err != nil { log.Fatalf("failed to get scenarios : %s", err) } @@ -338,12 +339,12 @@ cscli lapi context detect crowdsecurity/sshd-logs log.Fatalf("Failed to init expr helpers : %s", err) } - // Populate cwhub package tools - if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { - log.Fatalf("Failed to load hub index : %s", err) + hub, err := require.Hub(csConfig, nil) + if err != nil { + log.Fatal(err) } - csParsers := parser.NewParsers() + csParsers := parser.NewParsers(hub) if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil { log.Fatalf("unable to load parsers: %s", err) } diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index c3a475f70..1103a21a9 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "path/filepath" - "slices" "strings" "github.com/fatih/color" @@ -12,9 +11,9 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" + "slices" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" @@ -29,15 +28,11 @@ var dbClient *database.Client var OutputFormat string var OutputColor string -var downloadOnly bool -var forceAction bool -var purge bool -var all bool - -var prometheusURL string - var mergedConfig string +// flagBranch overrides the value in csConfig.Cscli.HubBranch +var flagBranch = "" + func initConfig() { var err error if trace_lvl { @@ -58,9 +53,6 @@ func initConfig() { if err != nil { log.Fatal(err) } - if err := csConfig.LoadCSCLI(); err != nil { - log.Fatal(err) - } } else { csConfig = csconfig.NewDefaultConfig() } @@ -71,13 +63,10 @@ func initConfig() { log.Debugf("Enabled feature flags: %s", fflist) } - if csConfig.Cscli == nil { - log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath) + if flagBranch != "" { + csConfig.Cscli.HubBranch = flagBranch } - if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" { - cwhub.HubBranch = csConfig.Cscli.HubBranch - } if OutputFormat != "" { csConfig.Cscli.Output = OutputFormat if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" { @@ -206,7 +195,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error") rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace") - rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github") + rootCmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github") if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil { log.Fatalf("failed to hide flag: %s", err) } @@ -243,10 +232,6 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewSimulationCmds()) rootCmd.AddCommand(NewBouncersCmd()) rootCmd.AddCommand(NewMachinesCmd()) - rootCmd.AddCommand(NewParsersCmd()) - rootCmd.AddCommand(NewScenariosCmd()) - rootCmd.AddCommand(NewCollectionsCmd()) - rootCmd.AddCommand(NewPostOverflowsCmd()) rootCmd.AddCommand(NewCapiCmd()) rootCmd.AddCommand(NewLapiCmd()) rootCmd.AddCommand(NewCompletionCmd()) @@ -255,6 +240,10 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) + rootCmd.AddCommand(NewItemsCmd("collections")) + rootCmd.AddCommand(NewItemsCmd("parsers")) + rootCmd.AddCommand(NewItemsCmd("scenarios")) + rootCmd.AddCommand(NewItemsCmd("postoverflows")) if fflag.CscliSetup.IsEnabled() { rootCmd.AddCommand(NewSetupCmd()) diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index 8ab3f01bd..a03614aae 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -284,8 +284,20 @@ var noUnit bool func runMetrics(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadPrometheus(); err != nil { - return fmt.Errorf("failed to load prometheus config: %w", err) + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noUnit, err = flags.GetBool("no-unit") + if err != nil { + return err } if csConfig.Prometheus == nil { @@ -296,17 +308,8 @@ func runMetrics(cmd *cobra.Command, args []string) error { return fmt.Errorf("prometheus is not enabled, can't show metrics") } - if prometheusURL == "" { - prometheusURL = csConfig.Cscli.PrometheusUrl - } - - if prometheusURL == "" { - return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath) - } - - err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output) - if err != nil { - return fmt.Errorf("could not fetch prometheus metrics: %w", err) + if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil { + return err } return nil } @@ -321,8 +324,10 @@ func NewMetricsCmd() *cobra.Command { DisableAutoGenTag: true, RunE: runMetrics, } - cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://:/metrics)") - cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units") + + flags := cmdMetrics.PersistentFlags() + flags.StringP("url", "u", "", "Prometheus url (http://:/metrics)") + flags.Bool("no-unit", false, "Show the real number instead of formatted with units") return cmdMetrics } diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go deleted file mode 100644 index d97b070db..000000000 --- a/cmd/crowdsec-cli/parsers.go +++ /dev/null @@ -1,194 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewParsersCmd() *cobra.Command { - var cmdParsers = &cobra.Command{ - Use: "parsers [action] [config]", - Short: "Install/Remove/Upgrade/Inspect parser(s) from hub", - Example: `cscli parsers install crowdsecurity/sshd-logs -cscli parsers inspect crowdsecurity/sshd-logs -cscli parsers upgrade crowdsecurity/sshd-logs -cscli parsers list -cscli parsers remove crowdsecurity/sshd-logs -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"parser"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdParsers.AddCommand(NewParsersInstallCmd()) - cmdParsers.AddCommand(NewParsersRemoveCmd()) - cmdParsers.AddCommand(NewParsersUpgradeCmd()) - cmdParsers.AddCommand(NewParsersInspectCmd()) - cmdParsers.AddCommand(NewParsersListCmd()) - - return cmdParsers -} - -func NewParsersInstallCmd() *cobra.Command { - var ignoreError bool - - var cmdParsersInstall = &cobra.Command{ - Use: "install [config]", - Short: "Install given parser(s)", - Long: `Fetch and install given parser(s) from hub`, - Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.PARSERS, args, toComplete) - }, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.PARSERS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.PARSERS, name) - Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - return nil - }, - } - - cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers") - - return cmdParsersInstall -} - -func NewParsersRemoveCmd() *cobra.Command { - cmdParsersRemove := &cobra.Command{ - Use: "remove [config]", - Short: "Remove given parser(s)", - Long: `Remove given parse(s) from hub`, - Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS, args, toComplete) - }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction) - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one parser to remove or '--all'") - } - - for _, name := range args { - cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction) - } - - return nil - }, - } - - cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers") - - return cmdParsersRemove -} - -func NewParsersUpgradeCmd() *cobra.Command { - cmdParsersUpgrade := &cobra.Command{ - Use: "upgrade [config]", - Short: "Upgrade given parser(s)", - Long: `Fetch and upgrade given parser(s) from hub`, - Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS, args, toComplete) - }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one parser to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction) - } - } - return nil - }, - } - - cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers") - cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") - - return cmdParsersUpgrade -} - -func NewParsersInspectCmd() *cobra.Command { - var cmdParsersInspect = &cobra.Command{ - Use: "inspect [name]", - Short: "Inspect given parser", - Long: `Inspect given parser`, - Example: `cscli parsers inspect crowdsec/xxx`, - DisableAutoGenTag: true, - Args: cobra.MinimumNArgs(1), - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS, args, toComplete) - }, - Run: func(cmd *cobra.Command, args []string) { - InspectItem(args[0], cwhub.PARSERS) - }, - } - - cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") - - return cmdParsersInspect -} - -func NewParsersListCmd() *cobra.Command { - var cmdParsersList = &cobra.Command{ - Use: "list [name]", - Short: "List all parsers or given one", - Long: `List all parsers or given one`, - Example: `cscli parsers list -cscli parser list crowdsecurity/xxx`, - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all) - }, - } - - cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") - - return cmdParsersList -} diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go deleted file mode 100644 index f4db0a79e..000000000 --- a/cmd/crowdsec-cli/postoverflows.go +++ /dev/null @@ -1,191 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewPostOverflowsCmd() *cobra.Command { - cmdPostOverflows := &cobra.Command{ - Use: "postoverflows [action] [config]", - Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub", - Example: `cscli postoverflows install crowdsecurity/cdn-whitelist - cscli postoverflows inspect crowdsecurity/cdn-whitelist - cscli postoverflows upgrade crowdsecurity/cdn-whitelist - cscli postoverflows list - cscli postoverflows remove crowdsecurity/cdn-whitelist`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"postoverflow"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsListCmd()) - - return cmdPostOverflows -} - -func NewPostOverflowsInstallCmd() *cobra.Command { - var ignoreError bool - - cmdPostOverflowsInstall := &cobra.Command{ - Use: "install [config]", - Short: "Install given postoverflow(s)", - Long: `Fetch and install given postoverflow(s) from hub`, - Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete) - }, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name) - Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - return nil - }, - } - - cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows") - - return cmdPostOverflowsInstall -} - -func NewPostOverflowsRemoveCmd() *cobra.Command { - cmdPostOverflowsRemove := &cobra.Command{ - Use: "remove [config]", - Short: "Remove given postoverflow(s)", - Long: `remove given postoverflow(s)`, - Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) - }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction) - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one postoverflow to remove or '--all'") - } - - for _, name := range args { - cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction) - } - - return nil - }, - } - - cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows") - - return cmdPostOverflowsRemove -} - -func NewPostOverflowsUpgradeCmd() *cobra.Command { - cmdPostOverflowsUpgrade := &cobra.Command{ - Use: "upgrade [config]", - Short: "Upgrade given postoverflow(s)", - Long: `Fetch and Upgrade given postoverflow(s) from hub`, - Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) - }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction) - } - } - return nil - }, - } - - cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows") - cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") - - return cmdPostOverflowsUpgrade -} - -func NewPostOverflowsInspectCmd() *cobra.Command { - cmdPostOverflowsInspect := &cobra.Command{ - Use: "inspect [config]", - Short: "Inspect given postoverflow", - Long: `Inspect given postoverflow`, - Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`, - DisableAutoGenTag: true, - Args: cobra.MinimumNArgs(1), - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) - }, - Run: func(cmd *cobra.Command, args []string) { - InspectItem(args[0], cwhub.PARSERS_OVFLW) - }, - } - - return cmdPostOverflowsInspect -} - -func NewPostOverflowsListCmd() *cobra.Command { - cmdPostOverflowsList := &cobra.Command{ - Use: "list [config]", - Short: "List all postoverflows or given one", - Long: `List all postoverflows or given one`, - Example: `cscli postoverflows list -cscli postoverflows list crowdsecurity/xxx`, - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all) - }, - } - - cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") - - return cmdPostOverflowsList -} diff --git a/cmd/crowdsec-cli/require/branch.go b/cmd/crowdsec-cli/require/branch.go new file mode 100644 index 000000000..b82f34a27 --- /dev/null +++ b/cmd/crowdsec-cli/require/branch.go @@ -0,0 +1,58 @@ +package require + +// Set the appropriate hub branch according to config settings and crowdsec version + +import ( + log "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" + + "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" +) + +func chooseBranch(cfg *csconfig.Config) string { + // this was set from config.yaml or flag + if cfg.Cscli.HubBranch != "" { + log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch) + return cfg.Cscli.HubBranch + } + + latest, err := cwversion.Latest() + if err != nil { + log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err) + return "master" + } + + csVersion := cwversion.VersionStrip() + if csVersion == latest { + log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion) + return "master" + } + + // if current version is greater than the latest we are in pre-release + if semver.Compare(csVersion, latest) == 1 { + log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion) + return "master" + } + + if csVersion == "" { + log.Warning("Crowdsec version is not set, using hub branch 'master'") + return "master" + } + + log.Warnf("A new CrowdSec release is available (%s). "+ + "Your version is '%s'. Please update it to use new parsers/scenarios/collections.", + latest, csVersion) + return csVersion +} + + +// HubBranch sets the branch (in cscli config) and returns its value +// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag +func HubBranch(cfg *csconfig.Config) string { + branch := chooseBranch(cfg) + + cfg.Cscli.HubBranch = branch + + return branch +} diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index f4129a44f..292862e44 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -23,6 +23,7 @@ func CAPI(c *csconfig.Config) error { if c.API.Server.OnlineClient == nil { return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath) } + return nil } @@ -30,6 +31,7 @@ func PAPI(c *csconfig.Config) error { if c.API.Server.OnlineClient.Credentials.PapiURL == "" { return fmt.Errorf("no PAPI URL in configuration") } + return nil } @@ -45,6 +47,7 @@ func DB(c *csconfig.Config) error { if err := c.LoadDBConfig(); err != nil { return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err) } + return nil } @@ -64,20 +67,33 @@ func Notifications(c *csconfig.Config) error { return nil } -func Hub (c *csconfig.Config) error { - if err := c.LoadHub(); err != nil { - return err +// RemoteHub returns the configuration required to download hub index and items: url, branch, etc. +func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg { + // set branch in config, and log if necessary + branch := HubBranch(c) + remote := &cwhub.RemoteHubCfg { + Branch: branch, + URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s", + // URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s", + IndexPath: ".index.json", } - if c.Hub == nil { - return fmt.Errorf("you must configure cli before interacting with hub") - } - - cwhub.SetHubBranch() - - if err := cwhub.GetHubIdx(c.Hub); err != nil { - return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err) - } - - return nil + return remote +} + +// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items. +// If no remote parameter is provided, the hub can only be used for local operations. +func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) { + local := c.Hub + + if local == nil { + return nil, fmt.Errorf("you must configure cli before interacting with hub") + } + + hub, err := cwhub.NewHub(local, remote, false) + if err != nil { + return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err) + } + + return hub, nil } diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go deleted file mode 100644 index 01e0b02dc..000000000 --- a/cmd/crowdsec-cli/scenarios.go +++ /dev/null @@ -1,188 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewScenariosCmd() *cobra.Command { - var cmdScenarios = &cobra.Command{ - Use: "scenarios [action] [config]", - Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub", - Example: `cscli scenarios list [-a] -cscli scenarios install crowdsecurity/ssh-bf -cscli scenarios inspect crowdsecurity/ssh-bf -cscli scenarios upgrade crowdsecurity/ssh-bf -cscli scenarios remove crowdsecurity/ssh-bf -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"scenario"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdScenarios.AddCommand(NewCmdScenariosInstall()) - cmdScenarios.AddCommand(NewCmdScenariosRemove()) - cmdScenarios.AddCommand(NewCmdScenariosUpgrade()) - cmdScenarios.AddCommand(NewCmdScenariosInspect()) - cmdScenarios.AddCommand(NewCmdScenariosList()) - - return cmdScenarios -} - -func NewCmdScenariosInstall() *cobra.Command { - var ignoreError bool - - var cmdScenariosInstall = &cobra.Command{ - Use: "install [config]", - Short: "Install given scenario(s)", - Long: `Fetch and install given scenario(s) from hub`, - Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`, - Args: cobra.MinimumNArgs(1), - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.SCENARIOS, args, toComplete) - }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.SCENARIOS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.SCENARIOS, name) - Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - return nil - }, - } - cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios") - - return cmdScenariosInstall -} - -func NewCmdScenariosRemove() *cobra.Command { - var cmdScenariosRemove = &cobra.Command{ - Use: "remove [config]", - Short: "Remove given scenario(s)", - Long: `remove given scenario(s)`, - Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`, - Aliases: []string{"delete"}, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.SCENARIOS, args, toComplete) - }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction) - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one scenario to remove or '--all'") - } - - for _, name := range args { - cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction) - } - return nil - }, - } - cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios") - - return cmdScenariosRemove -} - -func NewCmdScenariosUpgrade() *cobra.Command { - var cmdScenariosUpgrade = &cobra.Command{ - Use: "upgrade [config]", - Short: "Upgrade given scenario(s)", - Long: `Fetch and Upgrade given scenario(s) from hub`, - Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.SCENARIOS, args, toComplete) - }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one scenario to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction) - } - } - return nil - }, - } - cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios") - cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") - - return cmdScenariosUpgrade -} - -func NewCmdScenariosInspect() *cobra.Command { - var cmdScenariosInspect = &cobra.Command{ - Use: "inspect [config]", - Short: "Inspect given scenario", - Long: `Inspect given scenario`, - Example: `cscli scenarios inspect crowdsec/xxx`, - Args: cobra.MinimumNArgs(1), - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.SCENARIOS, args, toComplete) - }, - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - InspectItem(args[0], cwhub.SCENARIOS) - }, - } - cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") - - return cmdScenariosInspect -} - -func NewCmdScenariosList() *cobra.Command { - var cmdScenariosList = &cobra.Command{ - Use: "list [config]", - Short: "List all scenario(s) or given one", - Long: `List all scenario(s) or given one`, - Example: `cscli scenarios list -cscli scenarios list crowdsecurity/xxx`, - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all) - }, - } - cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") - - return cmdScenariosList -} diff --git a/cmd/crowdsec-cli/setup.go b/cmd/crowdsec-cli/setup.go index cc0a9a35d..884aa9890 100644 --- a/cmd/crowdsec-cli/setup.go +++ b/cmd/crowdsec-cli/setup.go @@ -6,13 +6,15 @@ import ( "os" "os/exec" + goccyyaml "github.com/goccy/go-yaml" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v3" - goccyyaml "github.com/goccy/go-yaml" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/setup" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" ) // NewSetupCmd defines the "cscli setup" command. @@ -303,7 +305,12 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error { return fmt.Errorf("while reading file %s: %w", fromFile, err) } - if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil { + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + if err != nil { + return err + } + + if err = setup.InstallHubItems(hub, input, dryRun); err != nil { return err } diff --git a/cmd/crowdsec-cli/simulation.go b/cmd/crowdsec-cli/simulation.go index 890785a2d..27aea5831 100644 --- a/cmd/crowdsec-cli/simulation.go +++ b/cmd/crowdsec-cli/simulation.go @@ -3,11 +3,11 @@ package main import ( "fmt" "os" - "slices" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" + "slices" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/cwhub" @@ -112,9 +112,6 @@ cscli simulation disable crowdsecurity/ssh-bf`, if err := csConfig.LoadSimulation(); err != nil { log.Fatal(err) } - if csConfig.Cscli == nil { - return fmt.Errorf("you must configure cli before using simulation") - } if csConfig.Cscli.SimulationConfig == nil { return fmt.Errorf("no simulation configured") } @@ -145,18 +142,19 @@ func NewSimulationEnableCmd() *cobra.Command { Example: `cscli simulation enable`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig, nil) + if err != nil { log.Fatal(err) } if len(args) > 0 { for _, scenario := range args { - var item = cwhub.GetItem(cwhub.SCENARIOS, scenario) + var item = hub.GetItem(cwhub.SCENARIOS, scenario) if item == nil { log.Errorf("'%s' doesn't exist or is not a scenario", scenario) continue } - if !item.Installed { + if !item.State.Installed { log.Warningf("'%s' isn't enabled", scenario) } isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario) diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index e5a4c36ab..1470d37aa 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -58,10 +58,6 @@ func stripAnsiString(str string) string { func collectMetrics() ([]byte, []byte, error) { log.Info("Collecting prometheus metrics") - err := csConfig.LoadPrometheus() - if err != nil { - return nil, nil, err - } if csConfig.Cscli.PrometheusUrl == "" { log.Warn("No Prometheus URL configured, metrics will not be collected") @@ -69,13 +65,13 @@ func collectMetrics() ([]byte, []byte, error) { } humanMetrics := bytes.NewBuffer(nil) - err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human") + err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human") if err != nil { return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err) } - req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil) + req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil) if err != nil { return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err) } @@ -132,10 +128,21 @@ func collectOSInfo() ([]byte, error) { return w.Bytes(), nil } -func collectHubItems(itemType string) []byte { +func collectHubItems(hub *cwhub.Hub, itemType string) []byte { + var err error + out := bytes.NewBuffer(nil) log.Infof("Collecting %s list", itemType) - ListItems(out, []string{itemType}, []string{}, false, true, all) + + items := make(map[string][]*cwhub.Item) + + if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil { + log.Warnf("could not collect %s list: %s", itemType, err) + } + + if err := listItems(out, []string{itemType}, items); err != nil { + log.Warnf("could not collect %s list: %s", itemType, err) + } return out.Bytes() } @@ -157,7 +164,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) { return out.Bytes(), nil } -func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte { +func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte { if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil { return []byte("No agent credentials found, are we LAPI ?") } @@ -167,7 +174,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str if err != nil { return []byte(fmt.Sprintf("cannot parse API URL: %s", err)) } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) if err != nil { return []byte(fmt.Sprintf("could not collect scenarios: %s", err)) } @@ -295,7 +302,8 @@ cscli support dump -f /tmp/crowdsec-support.zip skipAgent = true } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig, nil) + if err != nil { log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected") skipHub = true infos[SUPPORT_PARSERS_PATH] = []byte(err.Error()) @@ -333,10 +341,10 @@ cscli support dump -f /tmp/crowdsec-support.zip infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig() if !skipHub { - infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS) - infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS) - infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW) - infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS) + infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS) + infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS) + infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS) + infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS) } if !skipDB { @@ -358,7 +366,8 @@ cscli support dump -f /tmp/crowdsec-support.zip infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login, csConfig.API.Server.OnlineClient.Credentials.Password, csConfig.API.Server.OnlineClient.Credentials.URL, - CAPIURLPrefix) + CAPIURLPrefix, + hub) } if !skipLAPI { @@ -366,7 +375,8 @@ cscli support dump -f /tmp/crowdsec-support.zip infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login, csConfig.API.Client.Credentials.Password, csConfig.API.Client.Credentials.URL, - LAPIURLPrefix) + LAPIURLPrefix, + hub) infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile() } diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 503653f82..eb7fb51e0 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -1,36 +1,17 @@ package main import ( - "encoding/csv" - "encoding/json" "fmt" - "io" - "math" "net" - "net/http" - "slices" - "strconv" "strings" - "time" - "github.com/fatih/color" - dto "github.com/prometheus/client_model/go" - "github.com/prometheus/prom2json" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/agext/levenshtein" - "gopkg.in/yaml.v2" - "github.com/crowdsecurity/go-cs-lib/trace" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/types" ) -const MaxDistance = 7 - func printHelp(cmd *cobra.Command) { err := cmd.Help() if err != nil { @@ -38,197 +19,6 @@ func printHelp(cmd *cobra.Command) { } } -func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) { - errMsg := "" - if score < MaxDistance { - errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem) - } else { - errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem) - } - if ignoreErr { - log.Error(errMsg) - } else { - log.Fatalf(errMsg) - } -} - -func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { - allItems := make([]string, 0) - nearestScore := 100 - nearestItem := &cwhub.Item{} - hubItems := cwhub.GetHubStatusForItemType(itemType, "", true) - for _, item := range hubItems { - allItems = append(allItems, item.Name) - } - - for _, s := range allItems { - d := levenshtein.Distance(itemName, s, nil) - if d < nearestScore { - nearestScore = d - nearestItem = cwhub.GetItem(itemType, s) - } - } - return nearestItem, nearestScore -} - -func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { - return nil, cobra.ShellCompDirectiveDefault - } - - comp := make([]string, 0) - hubItems := cwhub.GetHubStatusForItemType(itemType, "", true) - for _, item := range hubItems { - if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { - comp = append(comp, item.Name) - } - } - cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) - return comp, cobra.ShellCompDirectiveNoFileComp -} - -func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { - return nil, cobra.ShellCompDirectiveDefault - } - - items, err := cwhub.GetInstalledItemsAsString(itemType) - if err != nil { - cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) - return nil, cobra.ShellCompDirectiveDefault - } - - comp := make([]string, 0) - - if toComplete != "" { - for _, item := range items { - if strings.Contains(item, toComplete) { - comp = append(comp, item) - } - } - } else { - comp = items - } - - cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) - - return comp, cobra.ShellCompDirectiveNoFileComp -} - -func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) { - var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus) - - for _, itemType := range itemTypes { - itemName := "" - if len(args) == 1 { - itemName = args[0] - } - hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all) - } - - if csConfig.Cscli.Output == "human" { - for _, itemType := range itemTypes { - var statuses []cwhub.ItemHubStatus - var ok bool - if statuses, ok = hubStatusByItemType[itemType]; !ok { - log.Errorf("unknown item type: %s", itemType) - continue - } - listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses) - } - } else if csConfig.Cscli.Output == "json" { - x, err := json.MarshalIndent(hubStatusByItemType, "", " ") - if err != nil { - log.Fatalf("failed to unmarshal") - } - out.Write(x) - } else if csConfig.Cscli.Output == "raw" { - csvwriter := csv.NewWriter(out) - if showHeader { - header := []string{"name", "status", "version", "description"} - if showType { - header = append(header, "type") - } - err := csvwriter.Write(header) - if err != nil { - log.Fatalf("failed to write header: %s", err) - } - - } - for _, itemType := range itemTypes { - var statuses []cwhub.ItemHubStatus - var ok bool - if statuses, ok = hubStatusByItemType[itemType]; !ok { - log.Errorf("unknown item type: %s", itemType) - continue - } - for _, status := range statuses { - if status.LocalVersion == "" { - status.LocalVersion = "n/a" - } - row := []string{ - status.Name, - status.Status, - status.LocalVersion, - status.Description, - } - if showType { - row = append(row, itemType) - } - err := csvwriter.Write(row) - if err != nil { - log.Fatalf("failed to write raw output : %s", err) - } - } - } - csvwriter.Flush() - } -} - -func InspectItem(name string, objecitemType string) { - - hubItem := cwhub.GetItem(objecitemType, name) - if hubItem == nil { - log.Fatalf("unable to retrieve item.") - } - var b []byte - var err error - switch csConfig.Cscli.Output { - case "human", "raw": - b, err = yaml.Marshal(*hubItem) - if err != nil { - log.Fatalf("unable to marshal item : %s", err) - } - case "json": - b, err = json.MarshalIndent(*hubItem, "", " ") - if err != nil { - log.Fatalf("unable to marshal item : %s", err) - } - } - fmt.Printf("%s", string(b)) - if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { - return - } - - if prometheusURL == "" { - //This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus - //But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case - listenAddr := csConfig.Prometheus.ListenAddr - if listenAddr == "" { - listenAddr = "127.0.0.1" - } - listenPort := csConfig.Prometheus.ListenPort - if listenPort == 0 { - listenPort = 6060 - } - prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort) - log.Debugf("No prometheus URL provided using: %s", prometheusURL) - } - - fmt.Printf("\nCurrent metrics : \n") - ShowMetrics(hubItem) -} - func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error { /*if a range is provided, change the scope*/ @@ -259,232 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value * return nil } -func ShowMetrics(hubItem *cwhub.Item) { - switch hubItem.Type { - case cwhub.PARSERS: - metrics := GetParserMetric(prometheusURL, hubItem.Name) - parserMetricsTable(color.Output, hubItem.Name, metrics) - case cwhub.SCENARIOS: - metrics := GetScenarioMetric(prometheusURL, hubItem.Name) - scenarioMetricsTable(color.Output, hubItem.Name, metrics) - case cwhub.COLLECTIONS: - for _, item := range hubItem.Parsers { - metrics := GetParserMetric(prometheusURL, item) - parserMetricsTable(color.Output, item, metrics) - } - for _, item := range hubItem.Scenarios { - metrics := GetScenarioMetric(prometheusURL, item) - scenarioMetricsTable(color.Output, item, metrics) - } - for _, item := range hubItem.Collections { - hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item) - if hubItem == nil { - log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name) - } - ShowMetrics(hubItem) - } - default: - log.Errorf("item of type '%s' is unknown", hubItem.Type) - } -} - -// GetParserMetric is a complete rip from prom2json -func GetParserMetric(url string, itemName string) map[string]map[string]int { - stats := make(map[string]map[string]int) - - result := GetPrometheusMetric(url) - for idx, fam := range result { - if !strings.HasPrefix(fam.Name, "cs_") { - continue - } - log.Tracef("round %d", idx) - for _, m := range fam.Metrics { - metric, ok := m.(prom2json.Metric) - if !ok { - log.Debugf("failed to convert metric to prom2json.Metric") - continue - } - name, ok := metric.Labels["name"] - if !ok { - log.Debugf("no name in Metric %v", metric.Labels) - } - if name != itemName { - continue - } - source, ok := metric.Labels["source"] - if !ok { - log.Debugf("no source in Metric %v", metric.Labels) - } else { - if srctype, ok := metric.Labels["type"]; ok { - source = srctype + ":" + source - } - } - value := m.(prom2json.Metric).Value - fval, err := strconv.ParseFloat(value, 32) - if err != nil { - log.Errorf("Unexpected int value %s : %s", value, err) - continue - } - ival := int(fval) - - switch fam.Name { - case "cs_reader_hits_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - stats[source]["parsed"] = 0 - stats[source]["reads"] = 0 - stats[source]["unparsed"] = 0 - stats[source]["hits"] = 0 - } - stats[source]["reads"] += ival - case "cs_parser_hits_ok_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["parsed"] += ival - case "cs_parser_hits_ko_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["unparsed"] += ival - case "cs_node_hits_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["hits"] += ival - case "cs_node_hits_ok_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["parsed"] += ival - case "cs_node_hits_ko_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["unparsed"] += ival - default: - continue - } - } - } - return stats -} - -func GetScenarioMetric(url string, itemName string) map[string]int { - stats := make(map[string]int) - - stats["instantiation"] = 0 - stats["curr_count"] = 0 - stats["overflow"] = 0 - stats["pour"] = 0 - stats["underflow"] = 0 - - result := GetPrometheusMetric(url) - for idx, fam := range result { - if !strings.HasPrefix(fam.Name, "cs_") { - continue - } - log.Tracef("round %d", idx) - for _, m := range fam.Metrics { - metric, ok := m.(prom2json.Metric) - if !ok { - log.Debugf("failed to convert metric to prom2json.Metric") - continue - } - name, ok := metric.Labels["name"] - if !ok { - log.Debugf("no name in Metric %v", metric.Labels) - } - if name != itemName { - continue - } - value := m.(prom2json.Metric).Value - fval, err := strconv.ParseFloat(value, 32) - if err != nil { - log.Errorf("Unexpected int value %s : %s", value, err) - continue - } - ival := int(fval) - - switch fam.Name { - case "cs_bucket_created_total": - stats["instantiation"] += ival - case "cs_buckets": - stats["curr_count"] += ival - case "cs_bucket_overflowed_total": - stats["overflow"] += ival - case "cs_bucket_poured_total": - stats["pour"] += ival - case "cs_bucket_underflowed_total": - stats["underflow"] += ival - default: - continue - } - } - } - return stats -} - -func GetPrometheusMetric(url string) []*prom2json.Family { - mfChan := make(chan *dto.MetricFamily, 1024) - - // Start with the DefaultTransport for sane defaults. - transport := http.DefaultTransport.(*http.Transport).Clone() - // Conservatively disable HTTP keep-alives as this program will only - // ever need a single HTTP request. - transport.DisableKeepAlives = true - // Timeout early if the server doesn't even return the headers. - transport.ResponseHeaderTimeout = time.Minute - - go func() { - defer trace.CatchPanic("crowdsec/GetPrometheusMetric") - err := prom2json.FetchMetricFamilies(url, mfChan, transport) - if err != nil { - log.Fatalf("failed to fetch prometheus metrics : %v", err) - } - }() - - result := []*prom2json.Family{} - for mf := range mfChan { - result = append(result, prom2json.NewFamily(mf)) - } - log.Debugf("Finished reading prometheus output, %d entries", len(result)) - - return result -} - -type unit struct { - value int64 - symbol string -} - -var ranges = []unit{ - {value: 1e18, symbol: "E"}, - {value: 1e15, symbol: "P"}, - {value: 1e12, symbol: "T"}, - {value: 1e9, symbol: "G"}, - {value: 1e6, symbol: "M"}, - {value: 1e3, symbol: "k"}, - {value: 1, symbol: ""}, -} - -func formatNumber(num int) string { - goodUnit := unit{} - for _, u := range ranges { - if int64(num) >= u.value { - goodUnit = u - break - } - } - - if goodUnit.value == 1 { - return fmt.Sprintf("%d%s", num, goodUnit.symbol) - } - - res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100 - return fmt.Sprintf("%.2f%s", res, goodUnit.symbol) -} - func getDBClient() (*database.Client, error) { var err error if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI { @@ -518,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string { } return slice - } diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index 16f42d72a..28b185a01 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -3,6 +3,7 @@ package main import ( "fmt" "io" + "strconv" "github.com/aquasecurity/table" "github.com/enescakir/emoji" @@ -10,14 +11,15 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) { +func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) { t := newLightTable(out) t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) - for _, status := range statuses { - t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath) + for _, item := range items { + status, emo := item.InstallStatus() + t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.State.LocalVersion, item.State.LocalPath) } renderTableTitle(out, title) t.Render() @@ -31,11 +33,11 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired") t.AddRow( - fmt.Sprintf("%d", metrics["curr_count"]), - fmt.Sprintf("%d", metrics["overflow"]), - fmt.Sprintf("%d", metrics["instantiation"]), - fmt.Sprintf("%d", metrics["pour"]), - fmt.Sprintf("%d", metrics["underflow"]), + strconv.Itoa(metrics["curr_count"]), + strconv.Itoa(metrics["overflow"]), + strconv.Itoa(metrics["instantiation"]), + strconv.Itoa(metrics["pour"]), + strconv.Itoa(metrics["underflow"]), ) renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName)) @@ -43,23 +45,25 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int } func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) { - skip := true t := newTable(out) t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed") + // don't show table if no hits + showTable := false + for source, stats := range metrics { if stats["hits"] > 0 { t.AddRow( source, - fmt.Sprintf("%d", stats["hits"]), - fmt.Sprintf("%d", stats["parsed"]), - fmt.Sprintf("%d", stats["unparsed"]), + strconv.Itoa(stats["hits"]), + strconv.Itoa(stats["parsed"]), + strconv.Itoa(stats["unparsed"]), ) - skip = false + showTable = true } } - if !skip { + if showTable { renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName)) t.Render() } diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index c573cd4d4..fc1fdb946 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -20,21 +20,16 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { +func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) { var err error - // Populate cwhub package tools - if err = cwhub.GetHubIdx(cConfig.Hub); err != nil { - return nil, fmt.Errorf("while loading hub index: %w", err) - } - // Start loading configs - csParsers := parser.NewParsers() + csParsers := parser.NewParsers(hub) if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { return nil, fmt.Errorf("while loading parsers: %w", err) } - if err := LoadBuckets(cConfig); err != nil { + if err := LoadBuckets(cConfig, hub); err != nil { return nil, fmt.Errorf("while loading scenarios: %w", err) } @@ -44,7 +39,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { return csParsers, nil } -func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { +func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error { inputEventChan = make(chan types.Event) inputLineChan = make(chan types.Event) @@ -99,7 +94,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ { outputsTomb.Go(func() error { defer trace.CatchPanic("crowdsec/runOutput") - if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil { + if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil { log.Fatalf("starting outputs error : %s", err) return err } @@ -131,7 +126,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { return nil } -func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) { +func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) { crowdsecTomb.Go(func() error { defer trace.CatchPanic("crowdsec/serveCrowdsec") go func() { @@ -139,7 +134,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady // this logs every time, even at config reload log.Debugf("running agent after %s ms", time.Since(crowdsecT0)) agentReady <- true - if err := runCrowdsec(cConfig, parsers); err != nil { + if err := runCrowdsec(cConfig, parsers, hub); err != nil { log.Fatalf("unable to start crowdsec routines: %s", err) } }() diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index c604e670a..8c7fb2991 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -75,20 +75,20 @@ type Flags struct { type labelsMap map[string]string -func LoadBuckets(cConfig *csconfig.Config) error { +func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error { var ( err error files []string ) - for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) { - if hubScenarioItem.Installed { - files = append(files, hubScenarioItem.LocalPath) + for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) { + if hubScenarioItem.State.Installed { + files = append(files, hubScenarioItem.State.LocalPath) } } buckets = leakybucket.NewBuckets() log.Infof("Loading %d scenario files", len(files)) - holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent) + holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent) if err != nil { return fmt.Errorf("scenario loading failed: %v", err) @@ -212,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level { func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) { cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet) if err != nil { - return nil, err - } - - if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) { - return nil, fmt.Errorf("unable to load configuration: common section is empty") + return nil, fmt.Errorf("while loading configuration file: %w", err) } cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags) @@ -228,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo dumpStates = true } - // Configuration paths are dependency to load crowdsec configuration - if err := cConfig.LoadConfigurationPaths(); err != nil { - return nil, err - } - if flags.SingleFileType != "" && flags.OneShotDSN != "" { // if we're in time-machine mode, we don't want to log to file cConfig.Common.LogMedia = "stdout" diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 103becced..6371a6046 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -151,14 +151,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { if !config.Enabled { return } - if config.ListenAddr == "" { - log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'") - config.ListenAddr = "127.0.0.1" - } - if config.ListenPort == 0 { - log.Warning("prometheus is enabled, but the listen port is empty, using '6060'") - config.ListenPort = 6060 - } // Registering prometheus // If in aggregated mode, do not register events associated with a source, to keep the cardinality low diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index 95642bbf3..b04e84981 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -62,7 +62,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error var bucketOverflows []types.Event func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets, - postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error { + postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, + apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error { var err error ticker := time.NewTicker(1 * time.Second) @@ -70,7 +71,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky var cache []types.RuntimeAlert var cacheMutex sync.Mutex - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("loading list of installed hub scenarios: %w", err) } @@ -93,7 +94,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky URL: apiURL, PapiURL: papiURL, VersionPrefix: "v1", - UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)}, + UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemNames(cwhub.SCENARIOS)}, }) if err != nil { return fmt.Errorf("new client api: %w", err) diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index 8513e0046..d51344e6b 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -14,6 +14,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" @@ -76,7 +77,12 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { } if !cConfig.DisableAgent { - csParsers, err := initCrowdsec(cConfig) + hub, err := cwhub.NewHub(cConfig.Hub, nil, false) + if err != nil { + return nil, fmt.Errorf("while loading hub index: %w", err) + } + + csParsers, err := initCrowdsec(cConfig, hub) if err != nil { return nil, fmt.Errorf("unable to init crowdsec: %w", err) } @@ -93,7 +99,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { } agentReady := make(chan bool, 1) - serveCrowdsec(csParsers, cConfig, agentReady) + serveCrowdsec(csParsers, cConfig, hub, agentReady) } log.Printf("Reload is finished") @@ -342,14 +348,19 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e } if !cConfig.DisableAgent { - csParsers, err := initCrowdsec(cConfig) + hub, err := cwhub.NewHub(cConfig.Hub, nil, false) + if err != nil { + return fmt.Errorf("while loading hub index: %w", err) + } + + csParsers, err := initCrowdsec(cConfig, hub) if err != nil { return fmt.Errorf("crowdsec init: %w", err) } // if it's just linting, we're done if !flags.TestMode { - serveCrowdsec(csParsers, cConfig, agentReady) + serveCrowdsec(csParsers, cConfig, hub, agentReady) } } else { agentReady <- true diff --git a/config/config.yaml b/config/config.yaml index 232b0bc43..2b0e4dfca 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -6,7 +6,6 @@ common: log_max_size: 20 compress_logs: true log_max_files: 10 - working_dir: . config_paths: config_dir: /etc/crowdsec/ data_dir: /var/lib/crowdsec/data/ diff --git a/config/config_win.yaml b/config/config_win.yaml index 7863f4fdd..5c34c69a2 100644 --- a/config/config_win.yaml +++ b/config/config_win.yaml @@ -3,7 +3,6 @@ common: log_media: file log_level: info log_dir: C:\ProgramData\CrowdSec\log\ - working_dir: . config_paths: config_dir: C:\ProgramData\CrowdSec\config\ data_dir: C:\ProgramData\CrowdSec\data\ diff --git a/config/config_win_no_lapi.yaml b/config/config_win_no_lapi.yaml index 35c7f2c6f..af240228b 100644 --- a/config/config_win_no_lapi.yaml +++ b/config/config_win_no_lapi.yaml @@ -3,7 +3,6 @@ common: log_media: file log_level: info log_dir: C:\ProgramData\CrowdSec\log\ - working_dir: . config_paths: config_dir: C:\ProgramData\CrowdSec\config\ data_dir: C:\ProgramData\CrowdSec\data\ diff --git a/config/dev.yaml b/config/dev.yaml index 2ff625060..2123dc858 100644 --- a/config/dev.yaml +++ b/config/dev.yaml @@ -2,7 +2,6 @@ common: daemonize: true log_media: stdout log_level: info - working_dir: . config_paths: config_dir: ./config data_dir: ./data/ diff --git a/config/user.yaml b/config/user.yaml index 67bdfa3fc..a1047dcd0 100644 --- a/config/user.yaml +++ b/config/user.yaml @@ -3,7 +3,6 @@ common: log_media: stdout log_level: info log_dir: /var/log/ - working_dir: . config_paths: config_dir: /etc/crowdsec/ data_dir: /var/lib/crowdsec/data diff --git a/docker/config.yaml b/docker/config.yaml index 5259a0fe2..681132909 100644 --- a/docker/config.yaml +++ b/docker/config.yaml @@ -3,7 +3,6 @@ common: log_media: stdout log_level: info log_dir: /var/log/ - working_dir: . config_paths: config_dir: /etc/crowdsec/ data_dir: /var/lib/crowdsec/data/ diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 15308a02e..8a3e55529 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -101,19 +101,23 @@ register_bouncer() { # $2 can be install, remove, upgrade # $3 is a list of object names separated by space cscli_if_clean() { + local itemtype="$1" + local action="$2" + local objs=$3 + shift 3 # loop over all objects - for obj in $3; do - if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then - echo "Object $1/$obj is tainted, skipping" + for obj in $objs; do + if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then + echo "Object $itemtype/$obj is tainted, skipping" else # # Too verbose? Only show errors if not in debug mode # if [ "$DEBUG" != "true" ]; then # error_only=--error # fi error_only="" - echo "Running: cscli $error_only $1 $2 \"$obj\"" + echo "Running: cscli $error_only $itemtype $action \"$obj\" $*" # shellcheck disable=SC2086 - cscli $error_only "$1" "$2" "$obj" + cscli $error_only "$itemtype" "$action" "$obj" "$@" fi done } @@ -327,22 +331,22 @@ fi ## Remove collections, parsers, scenarios & postoverflows if [ "$DISABLE_COLLECTIONS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean collections remove "$DISABLE_COLLECTIONS" + cscli_if_clean collections remove "$DISABLE_COLLECTIONS" --force fi if [ "$DISABLE_PARSERS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean parsers remove "$DISABLE_PARSERS" + cscli_if_clean parsers remove "$DISABLE_PARSERS" --force fi if [ "$DISABLE_SCENARIOS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" + cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" --force fi if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" + cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force fi ## Register bouncers via env diff --git a/docker/test/tests/test_hub_collections.py b/docker/test/tests/test_hub_collections.py index b890bebb9..962f8ff8d 100644 --- a/docker/test/tests/test_hub_collections.py +++ b/docker/test/tests/test_hub_collections.py @@ -30,8 +30,8 @@ def test_install_two_collections(crowdsec, flavor): cs.wait_for_log([ # f'*collections install "{it1}"*' # f'*collections install "{it2}"*' - f'*Enabled collections : {it1}*', - f'*Enabled collections : {it2}*', + f'*Enabled collections: {it1}*', + f'*Enabled collections: {it2}*', ]) @@ -72,7 +72,7 @@ def test_install_and_disable_collection(crowdsec, flavor): assert it not in items logs = cs.log_lines() # check that there was no attempt to install - assert not any(f'Enabled collections : {it}' in line for line in logs) + assert not any(f'Enabled collections: {it}' in line for line in logs) # already done in bats, prividing here as example of a somewhat complex test @@ -91,7 +91,7 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor): # implicit check for tainted=False assert items[coll]['status'] == 'enabled' cs.wait_for_log([ - f'*Enabled collections : {coll}*', + f'*Enabled collections: {coll}*', ]) scenario = 'crowdsecurity/http-crawl-non_statics' diff --git a/docker/test/tests/test_hub_scenarios.py b/docker/test/tests/test_hub_scenarios.py index a60ede667..2a8c3a275 100644 --- a/docker/test/tests/test_hub_scenarios.py +++ b/docker/test/tests/test_hub_scenarios.py @@ -21,8 +21,8 @@ def test_install_two_scenarios(crowdsec, flavor): } with crowdsec(flavor=flavor, environment=env) as cs: cs.wait_for_log([ - f'*scenarios install "{it1}*"', - f'*scenarios install "{it2}*"', + f'*scenarios install "{it1}"*', + f'*scenarios install "{it2}"*', "*Starting processing data*" ]) cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) diff --git a/go.mod b/go.mod index 3d1974ec4..75b0d82c3 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/c-robinson/iplib v1.0.3 github.com/cespare/xxhash/v2 v2.2.0 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.4 + github.com/crowdsecurity/go-cs-lib v0.0.5 github.com/crowdsecurity/grokky v0.2.1 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 23b829d5d..815b66dc7 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims= -github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= +github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= +github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= diff --git a/pkg/csconfig/api.go b/pkg/csconfig/api.go index bbe2e1622..c1577782f 100644 --- a/pkg/csconfig/api.go +++ b/pkg/csconfig/api.go @@ -286,10 +286,6 @@ func (c *Config) LoadAPIServer() error { log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs)) } - if err := c.LoadCommon(); err != nil { - return fmt.Errorf("loading common configuration: %s", err) - } - c.API.Server.LogDir = c.Common.LogDir c.API.Server.LogMedia = c.Common.LogMedia c.API.Server.CompressLogs = c.Common.CompressLogs diff --git a/pkg/csconfig/api_test.go b/pkg/csconfig/api_test.go index 4338de9c1..10128b76b 100644 --- a/pkg/csconfig/api_test.go +++ b/pkg/csconfig/api_test.go @@ -3,7 +3,6 @@ package csconfig import ( "net" "os" - "path/filepath" "strings" "testing" @@ -142,9 +141,6 @@ func TestLoadAPIServer(t *testing.T) { err := tmpLAPI.LoadProfiles() require.NoError(t, err) - LogDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - logLevel := log.InfoLevel config := &Config{} fcontent, err := os.ReadFile("./testdata/config.yaml") @@ -179,7 +175,7 @@ func TestLoadAPIServer(t *testing.T) { DbPath: "./testdata/test.db", }, Common: &CommonCfg{ - LogDir: "./testdata/", + LogDir: "./testdata", LogMedia: "stdout", }, DisableAPI: false, @@ -202,7 +198,7 @@ func TestLoadAPIServer(t *testing.T) { ShareContext: ptr.Of(false), ConsoleManagement: ptr.Of(false), }, - LogDir: LogDirFullPath, + LogDir: "./testdata", LogMedia: "stdout", OnlineClient: &OnlineApiClientCfg{ CredentialsFilePath: "./testdata/online-api-secrets.yaml", diff --git a/pkg/csconfig/common.go b/pkg/csconfig/common.go index 9d80cd95a..7e1ef6e5c 100644 --- a/pkg/csconfig/common.go +++ b/pkg/csconfig/common.go @@ -14,7 +14,7 @@ type CommonCfg struct { LogMedia string `yaml:"log_media"` LogDir string `yaml:"log_dir,omitempty"` //if LogMedia = file LogLevel *log.Level `yaml:"log_level"` - WorkingDir string `yaml:"working_dir,omitempty"` ///var/run + WorkingDir string `yaml:"working_dir,omitempty"` // TODO: This is just for backward compat. Remove this later CompressLogs *bool `yaml:"compress_logs,omitempty"` LogMaxSize int `yaml:"log_max_size,omitempty"` LogMaxAge int `yaml:"log_max_age,omitempty"` @@ -22,15 +22,18 @@ type CommonCfg struct { ForceColorLogs bool `yaml:"force_color_logs,omitempty"` } -func (c *Config) LoadCommon() error { +func (c *Config) loadCommon() error { var err error if c.Common == nil { - return fmt.Errorf("no common block provided in configuration file") + c.Common = &CommonCfg{} + } + + if c.Common.LogMedia == "" { + c.Common.LogMedia = "stdout" } var CommonCleanup = []*string{ &c.Common.LogDir, - &c.Common.WorkingDir, } for _, k := range CommonCleanup { if *k == "" { diff --git a/pkg/csconfig/common_test.go b/pkg/csconfig/common_test.go deleted file mode 100644 index 2c5f798a6..000000000 --- a/pkg/csconfig/common_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package csconfig - -import ( - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/crowdsecurity/go-cs-lib/cstest" -) - -func TestLoadCommon(t *testing.T) { - pidDirPath := "./testdata" - LogDirFullPath, err := filepath.Abs("./testdata/log/") - require.NoError(t, err) - - WorkingDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - tests := []struct { - name string - input *Config - expected *CommonCfg - expectedErr string - }{ - { - name: "basic valid configuration", - input: &Config{ - Common: &CommonCfg{ - Daemonize: true, - PidDir: "./testdata", - LogMedia: "file", - LogDir: "./testdata/log/", - WorkingDir: "./testdata/", - }, - }, - expected: &CommonCfg{ - Daemonize: true, - PidDir: pidDirPath, - LogMedia: "file", - LogDir: LogDirFullPath, - WorkingDir: WorkingDirFullPath, - }, - }, - { - name: "empty working dir", - input: &Config{ - Common: &CommonCfg{ - Daemonize: true, - PidDir: "./testdata", - LogMedia: "file", - LogDir: "./testdata/log/", - }, - }, - expected: &CommonCfg{ - Daemonize: true, - PidDir: pidDirPath, - LogMedia: "file", - LogDir: LogDirFullPath, - }, - }, - { - name: "no common", - input: &Config{}, - expected: nil, - expectedErr: "no common block provided in configuration file", - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadCommon() - cstest.RequireErrorContains(t, err, tc.expectedErr) - if tc.expectedErr != "" { - return - } - - assert.Equal(t, tc.expected, tc.input.Common) - }) - } -} diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index cd9369b17..ccc0a1aaf 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -36,7 +36,7 @@ type Config struct { PluginConfig *PluginCfg `yaml:"plugin_config,omitempty"` DisableAPI bool `yaml:"-"` DisableAgent bool `yaml:"-"` - Hub *Hub `yaml:"-"` + Hub *LocalHubCfg `yaml:"-"` } func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) { @@ -58,6 +58,37 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool // this is actually the "merged" yaml return nil, "", fmt.Errorf("%s: %w", configFile, err) } + + if cfg.Prometheus == nil { + cfg.Prometheus = &PrometheusCfg{} + } + + if cfg.Prometheus.ListenAddr == "" { + cfg.Prometheus.ListenAddr = "127.0.0.1" + log.Debugf("prometheus.listen_addr is empty, defaulting to %s", cfg.Prometheus.ListenAddr) + } + + if cfg.Prometheus.ListenPort == 0 { + cfg.Prometheus.ListenPort = 6060 + log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort) + } + + if err = cfg.loadCommon(); err != nil { + return nil, "", err + } + + if err = cfg.loadConfigurationPaths(); err != nil { + return nil, "", err + } + + if err = cfg.loadHub(); err != nil { + return nil, "", err + } + + if err = cfg.loadCSCLI(); err != nil { + return nil, "", err + } + return &cfg, configData, nil } @@ -65,11 +96,8 @@ func NewDefaultConfig() *Config { logLevel := log.InfoLevel commonCfg := CommonCfg{ Daemonize: false, - PidDir: "/tmp/", LogMedia: "stdout", - //LogDir unneeded - LogLevel: &logLevel, - WorkingDir: ".", + LogLevel: &logLevel, } prometheus := PrometheusCfg{ Enabled: true, diff --git a/pkg/csconfig/config_paths.go b/pkg/csconfig/config_paths.go index 24ff454b7..71e3bacda 100644 --- a/pkg/csconfig/config_paths.go +++ b/pkg/csconfig/config_paths.go @@ -15,7 +15,7 @@ type ConfigurationPaths struct { NotificationDir string `yaml:"notification_dir,omitempty"` } -func (c *Config) LoadConfigurationPaths() error { +func (c *Config) loadConfigurationPaths() error { var err error if c.ConfigPaths == nil { return fmt.Errorf("no configuration paths provided") diff --git a/pkg/csconfig/config_test.go b/pkg/csconfig/config_test.go index 9bdf2da6d..4843c2f70 100644 --- a/pkg/csconfig/config_test.go +++ b/pkg/csconfig/config_test.go @@ -15,10 +15,10 @@ func TestNormalLoad(t *testing.T) { require.NoError(t, err) _, _, err = NewConfig("./testdata/xxx.yaml", false, false, false) - assert.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage) + require.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage) _, _, err = NewConfig("./testdata/simulation.yaml", false, false, false) - assert.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config") + require.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config") } func TestNewCrowdSecConfig(t *testing.T) { diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index 28d6e77f0..dc226cfd6 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -28,10 +28,6 @@ type CrowdsecServiceCfg struct { BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode - HubDir string `yaml:"-"` - DataDir string `yaml:"-"` - ConfigDir string `yaml:"-"` - HubIndexFile string `yaml:"-"` SimulationFilePath string `yaml:"-"` ContextToSend map[string][]string `yaml:"-"` } @@ -101,11 +97,6 @@ func (c *Config) LoadCrowdsec() error { return fmt.Errorf("load error (simulation): %w", err) } - c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir - c.Crowdsec.DataDir = c.ConfigPaths.DataDir - c.Crowdsec.HubDir = c.ConfigPaths.HubDir - c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile - if c.Crowdsec.ParserRoutinesCount <= 0 { c.Crowdsec.ParserRoutinesCount = 1 } @@ -145,15 +136,11 @@ func (c *Config) LoadCrowdsec() error { return fmt.Errorf("loading api client: %s", err) } - if err := c.LoadHub(); err != nil { - return fmt.Errorf("while loading hub: %w", err) - } - c.Crowdsec.ContextToSend = make(map[string][]string, 0) fallback := false if c.Crowdsec.ConsoleContextPath == "" { // fallback to default config file - c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml") + c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml") fallback = true } diff --git a/pkg/csconfig/crowdsec_service_test.go b/pkg/csconfig/crowdsec_service_test.go index aa1d341f5..e9d7e8de3 100644 --- a/pkg/csconfig/crowdsec_service_test.go +++ b/pkg/csconfig/crowdsec_service_test.go @@ -20,18 +20,6 @@ func TestLoadCrowdsec(t *testing.T) { acquisDirFullPath, err := filepath.Abs("./testdata/acquis") require.NoError(t, err) - hubFullPath, err := filepath.Abs("./hub") - require.NoError(t, err) - - dataFullPath, err := filepath.Abs("./data") - require.NoError(t, err) - - configDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json") - require.NoError(t, err) - contextFileFullPath, err := filepath.Abs("./testdata/context.yaml") require.NoError(t, err) @@ -66,10 +54,6 @@ func TestLoadCrowdsec(t *testing.T) { AcquisitionDirPath: "", ConsoleContextPath: contextFileFullPath, AcquisitionFilePath: acquisFullPath, - ConfigDir: configDirFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, - HubIndexFile: hubIndexFileFullPath, BucketsRoutinesCount: 1, ParserRoutinesCount: 1, OutputRoutinesCount: 1, @@ -109,10 +93,6 @@ func TestLoadCrowdsec(t *testing.T) { AcquisitionDirPath: acquisDirFullPath, AcquisitionFilePath: acquisFullPath, ConsoleContextPath: contextFileFullPath, - ConfigDir: configDirFullPath, - HubIndexFile: hubIndexFileFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, BucketsRoutinesCount: 1, ParserRoutinesCount: 1, OutputRoutinesCount: 1, @@ -141,7 +121,7 @@ func TestLoadCrowdsec(t *testing.T) { }, }, Crowdsec: &CrowdsecServiceCfg{ - ConsoleContextPath: contextFileFullPath, + ConsoleContextPath: "./testdata/context.yaml", ConsoleContextValueLength: 10, }, }, @@ -149,10 +129,6 @@ func TestLoadCrowdsec(t *testing.T) { Enable: ptr.Of(true), AcquisitionDirPath: "", AcquisitionFilePath: "", - ConfigDir: configDirFullPath, - HubIndexFile: hubIndexFileFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, ConsoleContextPath: contextFileFullPath, BucketsRoutinesCount: 1, ParserRoutinesCount: 1, diff --git a/pkg/csconfig/cscli.go b/pkg/csconfig/cscli.go index 6b0bf5ae4..2a3fa7df3 100644 --- a/pkg/csconfig/cscli.go +++ b/pkg/csconfig/cscli.go @@ -1,5 +1,9 @@ package csconfig +import ( + "fmt" +) + /*cscli specific config, such as hub directory*/ type CscliCfg struct { Output string `yaml:"output,omitempty"` @@ -7,25 +11,19 @@ type CscliCfg struct { HubBranch string `yaml:"hub_branch"` SimulationConfig *SimulationConfig `yaml:"-"` DbConfig *DatabaseCfg `yaml:"-"` - HubDir string `yaml:"-"` - DataDir string `yaml:"-"` - ConfigDir string `yaml:"-"` - HubIndexFile string `yaml:"-"` + SimulationFilePath string `yaml:"-"` PrometheusUrl string `yaml:"prometheus_uri"` } -func (c *Config) LoadCSCLI() error { +func (c *Config) loadCSCLI() error { if c.Cscli == nil { c.Cscli = &CscliCfg{} } - if err := c.LoadConfigurationPaths(); err != nil { - return err + + if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 { + c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) } - c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir - c.Cscli.DataDir = c.ConfigPaths.DataDir - c.Cscli.HubDir = c.ConfigPaths.HubDir - c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile return nil } diff --git a/pkg/csconfig/cscli_test.go b/pkg/csconfig/cscli_test.go index b3d0abc6b..b814fda88 100644 --- a/pkg/csconfig/cscli_test.go +++ b/pkg/csconfig/cscli_test.go @@ -1,28 +1,14 @@ package csconfig import ( - "path/filepath" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" ) func TestLoadCSCLI(t *testing.T) { - hubFullPath, err := filepath.Abs("./hub") - require.NoError(t, err) - - dataFullPath, err := filepath.Abs("./data") - require.NoError(t, err) - - configDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json") - require.NoError(t, err) - tests := []struct { name string input *Config @@ -38,26 +24,23 @@ func TestLoadCSCLI(t *testing.T) { HubDir: "./hub", HubIndexFile: "./hub/.index.json", }, + Prometheus: &PrometheusCfg{ + Enabled: true, + Level: "full", + ListenAddr: "127.0.0.1", + ListenPort: 6060, + }, }, expected: &CscliCfg{ - ConfigDir: configDirFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, - HubIndexFile: hubIndexFileFullPath, + PrometheusUrl: "http://127.0.0.1:6060/metrics", }, }, - { - name: "no configuration path", - input: &Config{}, - expected: &CscliCfg{}, - expectedErr: "no configuration paths provided", - }, } for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadCSCLI() + err := tc.input.loadCSCLI() cstest.RequireErrorContains(t, err, tc.expectedErr) if tc.expectedErr != "" { return diff --git a/pkg/csconfig/hub.go b/pkg/csconfig/hub.go index 4c3c610aa..ca3750e58 100644 --- a/pkg/csconfig/hub.go +++ b/pkg/csconfig/hub.go @@ -1,19 +1,15 @@ package csconfig -/*cscli specific config, such as hub directory*/ -type Hub struct { - HubIndexFile string - HubDir string - InstallDir string - InstallDataDir string +// LocalHubCfg holds the configuration for a local hub: where to download etc. +type LocalHubCfg struct { + HubIndexFile string // Path to the local index file + HubDir string // Where the hub items are downloaded + InstallDir string // Where to install items + InstallDataDir string // Where to install data } -func (c *Config) LoadHub() error { - if err := c.LoadConfigurationPaths(); err != nil { - return err - } - - c.Hub = &Hub{ +func (c *Config) loadHub() error { + c.Hub = &LocalHubCfg{ HubIndexFile: c.ConfigPaths.HubIndexFile, HubDir: c.ConfigPaths.HubDir, InstallDir: c.ConfigPaths.ConfigDir, diff --git a/pkg/csconfig/hub_test.go b/pkg/csconfig/hub_test.go index d573e4690..2f9528c60 100644 --- a/pkg/csconfig/hub_test.go +++ b/pkg/csconfig/hub_test.go @@ -1,32 +1,18 @@ package csconfig import ( - "path/filepath" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" ) func TestLoadHub(t *testing.T) { - hubFullPath, err := filepath.Abs("./hub") - require.NoError(t, err) - - dataFullPath, err := filepath.Abs("./data") - require.NoError(t, err) - - configDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json") - require.NoError(t, err) - tests := []struct { name string input *Config - expected *Hub + expected *LocalHubCfg expectedErr string }{ { @@ -39,35 +25,19 @@ func TestLoadHub(t *testing.T) { HubIndexFile: "./hub/.index.json", }, }, - expected: &Hub{ - HubDir: hubFullPath, - HubIndexFile: hubIndexFileFullPath, - InstallDir: configDirFullPath, - InstallDataDir: dataFullPath, + expected: &LocalHubCfg{ + HubDir: "./hub", + HubIndexFile: "./hub/.index.json", + InstallDir: "./testdata", + InstallDataDir: "./data", }, }, - { - name: "no data dir", - input: &Config{ - ConfigPaths: &ConfigurationPaths{ - ConfigDir: "./testdata", - HubDir: "./hub", - HubIndexFile: "./hub/.index.json", - }, - }, - expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section", - }, - { - name: "no configuration path", - input: &Config{}, - expectedErr: "no configuration paths provided", - }, } for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadHub() + err := tc.input.loadHub() cstest.RequireErrorContains(t, err, tc.expectedErr) if tc.expectedErr != "" { return diff --git a/pkg/csconfig/prometheus.go b/pkg/csconfig/prometheus.go index eea768ab7..9b80fe398 100644 --- a/pkg/csconfig/prometheus.go +++ b/pkg/csconfig/prometheus.go @@ -1,19 +1,8 @@ package csconfig -import "fmt" - type PrometheusCfg struct { Enabled bool `yaml:"enabled"` Level string `yaml:"level"` //aggregated|full ListenAddr string `yaml:"listen_addr"` ListenPort int `yaml:"listen_port"` } - -func (c *Config) LoadPrometheus() error { - if c.Cscli != nil && c.Cscli.PrometheusUrl == "" && c.Prometheus != nil { - if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 { - c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) - } - } - return nil -} diff --git a/pkg/csconfig/prometheus_test.go b/pkg/csconfig/prometheus_test.go deleted file mode 100644 index 79c9ec58f..000000000 --- a/pkg/csconfig/prometheus_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package csconfig - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/crowdsecurity/go-cs-lib/cstest" -) - -func TestLoadPrometheus(t *testing.T) { - tests := []struct { - name string - input *Config - expectedURL string - expectedErr string - }{ - { - name: "basic valid configuration", - input: &Config{ - Prometheus: &PrometheusCfg{ - Enabled: true, - Level: "full", - ListenAddr: "127.0.0.1", - ListenPort: 6060, - }, - Cscli: &CscliCfg{}, - }, - expectedURL: "http://127.0.0.1:6060", - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadPrometheus() - cstest.RequireErrorContains(t, err, tc.expectedErr) - - require.Equal(t, tc.expectedURL, tc.input.Cscli.PrometheusUrl) - }) - } -} diff --git a/pkg/csconfig/simulation.go b/pkg/csconfig/simulation.go index 184708f0d..0d09aa478 100644 --- a/pkg/csconfig/simulation.go +++ b/pkg/csconfig/simulation.go @@ -30,11 +30,6 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool { } func (c *Config) LoadSimulation() error { - - if err := c.LoadConfigurationPaths(); err != nil { - return err - } - simCfg := SimulationConfig{} if c.ConfigPaths.SimulationFilePath == "" { c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml") diff --git a/pkg/csconfig/simulation_test.go b/pkg/csconfig/simulation_test.go index 44b8909a2..01f05e397 100644 --- a/pkg/csconfig/simulation_test.go +++ b/pkg/csconfig/simulation_test.go @@ -2,7 +2,6 @@ package csconfig import ( "fmt" - "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -12,12 +11,6 @@ import ( ) func TestSimulationLoading(t *testing.T) { - testXXFullPath, err := filepath.Abs("./testdata/xxx.yaml") - require.NoError(t, err) - - badYamlFullPath, err := filepath.Abs("./testdata/config.yaml") - require.NoError(t, err) - tests := []struct { name string input *Config @@ -56,7 +49,7 @@ func TestSimulationLoading(t *testing.T) { }, Crowdsec: &CrowdsecServiceCfg{}, }, - expectedErr: fmt.Sprintf("while reading yaml file: open %s: %s", testXXFullPath, cstest.FileNotFoundMessage), + expectedErr: fmt.Sprintf("while reading yaml file: open ./testdata/xxx.yaml: %s", cstest.FileNotFoundMessage), }, { name: "basic bad file content", @@ -67,7 +60,7 @@ func TestSimulationLoading(t *testing.T) { }, Crowdsec: &CrowdsecServiceCfg{}, }, - expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath), + expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors", }, { name: "basic bad file content", @@ -78,7 +71,7 @@ func TestSimulationLoading(t *testing.T) { }, Crowdsec: &CrowdsecServiceCfg{}, }, - expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath), + expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors", }, } diff --git a/pkg/csconfig/testdata/config.yaml b/pkg/csconfig/testdata/config.yaml index 288c09b84..17975b105 100644 --- a/pkg/csconfig/testdata/config.yaml +++ b/pkg/csconfig/testdata/config.yaml @@ -2,7 +2,6 @@ common: daemonize: false log_media: stdout log_level: info - working_dir: . prometheus: enabled: true level: full diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 2fe45bcda..ff34bed59 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -2,287 +2,31 @@ package cwhub import ( "fmt" - "os" + "net/http" "path/filepath" - "sort" "strings" - - "github.com/enescakir/emoji" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "golang.org/x/mod/semver" + "time" ) -const ( - HubIndexFile = ".index.json" - - // managed item types - PARSERS = "parsers" - PARSERS_OVFLW = "postoverflows" - SCENARIOS = "scenarios" - COLLECTIONS = "collections" -) - -var ( - ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS} - - ErrMissingReference = errors.New("Reference(s) missing in collection") - - // XXX: can we remove these globals? - skippedLocal = 0 - skippedTainted = 0 - RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" - HubBranch = "master" - hubIdx map[string]map[string]Item -) - -type ItemVersion struct { - Digest string `json:"digest,omitempty"` // meow - Deprecated bool `json:"deprecated,omitempty"` +var hubClient = &http.Client{ + Timeout: 120 * time.Second, } -type ItemHubStatus struct { - Name string `json:"name"` - LocalVersion string `json:"local_version"` - LocalPath string `json:"local_path"` - Description string `json:"description"` - UTF8Status string `json:"utf8_status"` - Status string `json:"status"` -} - -// Item can be: parser, scenario, collection.. -type Item struct { - // descriptive info - Type string `json:"type,omitempty" yaml:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich) - Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... - Name string `json:"name,omitempty"` // as seen in .config.json, usually "author/name" - FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml - Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .config.json - Author string `json:"author,omitempty"` // as seen in .config.json - References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .config.json - BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any - - // remote (hub) info - RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml - Version string `json:"version,omitempty"` // the last version - Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions - - // local (deployed) info - LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} - LocalVersion string `json:"local_version,omitempty"` - LocalHash string `json:"local_hash,omitempty"` // the local meow - Installed bool `json:"installed,omitempty"` - Downloaded bool `json:"downloaded,omitempty"` - UpToDate bool `json:"up_to_date,omitempty"` - Tainted bool `json:"tainted,omitempty"` // has it been locally modified - Local bool `json:"local,omitempty"` // if it's a non versioned control one - - // if it's a collection, it's not a single file - Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` - PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` - Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` - Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` -} - -func (i *Item) status() (string, emoji.Emoji) { - status := "disabled" - ok := false - - if i.Installed { - ok = true - status = "enabled" - } - - managed := true - if i.Local { - managed = false - status += ",local" - } - - warning := false - if i.Tainted { - warning = true - status += ",tainted" - } else if !i.UpToDate && !i.Local { - warning = true - status += ",update-available" - } - - emo := emoji.QuestionMark - - switch { - case !managed: - emo = emoji.House - case !i.Installed: - emo = emoji.Prohibited - case warning: - emo = emoji.Warning - case ok: - emo = emoji.CheckMark - } - - return status, emo -} - -func (i *Item) hubStatus() ItemHubStatus { - status, emo := i.status() - - return ItemHubStatus{ - Name: i.Name, - LocalVersion: i.LocalVersion, - LocalPath: i.LocalPath, - Description: i.Description, - Status: status, - UTF8Status: fmt.Sprintf("%v %s", emo, status), - } -} - -// versionStatus: semver requires 'v' prefix -func (i *Item) versionStatus() int { - return semver.Compare("v"+i.Version, "v"+i.LocalVersion) -} - -func GetItemMap(itemType string) map[string]Item { - m, ok := hubIdx[itemType] - if !ok { - return nil - } - - return m -} - -// Given a FileInfo, extract the map key. Follow a symlink if necessary -func itemKey(itemPath string) (string, error) { - f, err := os.Lstat(itemPath) +// safePath returns a joined path and ensures that it does not escape the base directory. +func safePath(dir, filePath string) (string, error) { + absBaseDir, err := filepath.Abs(filepath.Clean(dir)) if err != nil { - return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err) + return "", err } - if f.Mode()&os.ModeSymlink == 0 { - // it's not a symlink, so the filename itsef should be the key - return filepath.Base(itemPath), nil - } - - // resolve the symlink to hub file - pathInHub, err := os.Readlink(itemPath) + absFilePath, err := filepath.Abs(filepath.Join(dir, filePath)) if err != nil { - return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err) + return "", err } - author := filepath.Base(filepath.Dir(pathInHub)) + if !strings.HasPrefix(absFilePath, absBaseDir) { + return "", fmt.Errorf("path %s escapes base directory %s", filePath, dir) + } - fname := filepath.Base(pathInHub) - fname = strings.TrimSuffix(fname, ".yaml") - fname = strings.TrimSuffix(fname, ".yml") - - return fmt.Sprintf("%s/%s", author, fname), nil -} - -// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. -func GetItemByPath(itemType string, itemPath string) (*Item, error) { - itemKey, err := itemKey(itemPath) - if err != nil { - return nil, err - } - - m := GetItemMap(itemType) - if m == nil { - return nil, fmt.Errorf("item type %s doesn't exist", itemType) - } - - v, ok := m[itemKey] - if !ok { - return nil, fmt.Errorf("%s not found in %s", itemKey, itemType) - } - - return &v, nil -} - -func GetItem(itemType string, itemName string) *Item { - if m, ok := GetItemMap(itemType)[itemName]; ok { - return &m - } - - return nil -} - -func AddItem(itemType string, item Item) error { - for _, itype := range ItemTypes { - if itype == itemType { - hubIdx[itemType][item.Name] = item - return nil - } - } - - return fmt.Errorf("ItemType %s is unknown", itemType) -} - -func DisplaySummary() { - log.Infof("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]), - len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW])) - - if skippedLocal > 0 || skippedTainted > 0 { - log.Infof("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted) - } -} - -func GetInstalledItems(itemType string) ([]Item, error) { - items, ok := hubIdx[itemType] - if !ok { - return nil, fmt.Errorf("no %s in hubIdx", itemType) - } - - retItems := make([]Item, 0) - - for _, item := range items { - if item.Installed { - retItems = append(retItems, item) - } - } - - return retItems, nil -} - -func GetInstalledItemsAsString(itemType string) ([]string, error) { - items, err := GetInstalledItems(itemType) - if err != nil { - return nil, err - } - - retStr := make([]string, len(items)) - - for i, it := range items { - retStr[i] = it.Name - } - - return retStr, nil -} - -// Returns a slice of entries for packages: name, status, local_path, local_version, utf8_status (fancy) -func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus { - if _, ok := hubIdx[itemType]; !ok { - log.Errorf("type %s doesn't exist", itemType) - - return nil - } - - ret := make([]ItemHubStatus, 0) - - // remember, you do it for the user :) - for _, item := range hubIdx[itemType] { - if name != "" && name != item.Name { - // user has requested a specific name - continue - } - // Only enabled items ? - if !all && !item.Installed { - continue - } - // Check the item status - ret = append(ret, item.hubStatus()) - } - - sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) - - return ret + return absFilePath, nil } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 2fef828c2..270f003c3 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -9,14 +9,13 @@ import ( "testing" log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) +const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" + /* To test : - Download 'first' hub index @@ -28,294 +27,63 @@ import ( var responseByPath map[string]string -func TestItemStatus(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) +// testHub initializes a temporary hub with an empty json file, optionally updating it. +func testHub(t *testing.T, update bool) *Hub { + tmpDir, err := os.MkdirTemp("", "testhub") + require.NoError(t, err) - // DownloadHubIdx() - err := UpdateHubIdx(cfg.Hub) - require.NoError(t, err, "failed to download index") - - err = GetHubIdx(cfg.Hub) - require.NoError(t, err, "failed to load hub index") - - // get existing map - x := GetItemMap(COLLECTIONS) - require.NotEmpty(t, x) - - // Get item : good and bad - for k := range x { - item := GetItem(COLLECTIONS, k) - require.NotNil(t, item) - - item.Installed = true - item.UpToDate = false - item.Local = false - item.Tainted = false - - txt, _ := item.status() - require.Equal(t, "enabled,update-available", txt) - - item.Installed = false - item.UpToDate = false - item.Local = true - item.Tainted = false - - txt, _ = item.status() - require.Equal(t, "disabled,local", txt) + local := &csconfig.LocalHubCfg{ + HubDir: filepath.Join(tmpDir, "crowdsec", "hub"), + HubIndexFile: filepath.Join(tmpDir, "crowdsec", "hub", ".index.json"), + InstallDir: filepath.Join(tmpDir, "crowdsec"), + InstallDataDir: filepath.Join(tmpDir, "installed-data"), } - DisplaySummary() -} + err = os.MkdirAll(local.HubDir, 0o700) + require.NoError(t, err) -func TestGetters(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) + err = os.MkdirAll(local.InstallDir, 0o700) + require.NoError(t, err) - // DownloadHubIdx() - err := UpdateHubIdx(cfg.Hub) - require.NoError(t, err, "failed to download index") + err = os.MkdirAll(local.InstallDataDir, 0o700) + require.NoError(t, err) - err = GetHubIdx(cfg.Hub) - require.NoError(t, err, "failed to load hub index") - - // get non existing map - empty := GetItemMap("ratata") - require.Nil(t, empty) - - // get existing map - x := GetItemMap(COLLECTIONS) - require.NotEmpty(t, x) - - // Get item : good and bad - for k := range x { - empty := GetItem(COLLECTIONS, k+"nope") - require.Nil(t, empty) - - item := GetItem(COLLECTIONS, k) - require.NotNil(t, item) - - // Add item and get it - item.Name += "nope" - err := AddItem(COLLECTIONS, *item) - require.NoError(t, err) - - newitem := GetItem(COLLECTIONS, item.Name) - require.NotNil(t, newitem) - - err = AddItem("ratata", *item) - cstest.RequireErrorContains(t, err, "ItemType ratata is unknown") - } -} - -func TestIndexDownload(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) - - // DownloadHubIdx() - err := UpdateHubIdx(cfg.Hub) - require.NoError(t, err, "failed to download index") - - err = GetHubIdx(cfg.Hub) - require.NoError(t, err, "failed to load hub index") -} - -func getTestCfg() *csconfig.Config { - cfg := &csconfig.Config{Hub: &csconfig.Hub{}} - cfg.Hub.InstallDir, _ = filepath.Abs("./install") - cfg.Hub.HubDir, _ = filepath.Abs("./hubdir") - cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json") - - return cfg -} - -func envSetup(t *testing.T) *csconfig.Config { - resetResponseByPath() - log.SetLevel(log.DebugLevel) - - cfg := getTestCfg() - - defaultTransport := http.DefaultClient.Transport + err = os.WriteFile(local.HubIndexFile, []byte("{}"), 0o644) + require.NoError(t, err) t.Cleanup(func() { - http.DefaultClient.Transport = defaultTransport + os.RemoveAll(tmpDir) + }) + + remote := &RemoteHubCfg{ + Branch: "master", + URLTemplate: mockURLTemplate, + IndexPath: ".index.json", + } + + hub, err := NewHub(local, remote, update) + require.NoError(t, err) + + return hub +} + +// envSetup initializes the temporary hub and mocks the http client. +func envSetup(t *testing.T) *Hub { + setResponseByPath() + log.SetLevel(log.DebugLevel) + + defaultTransport := hubClient.Transport + + t.Cleanup(func() { + hubClient.Transport = defaultTransport }) // Mock the http client - http.DefaultClient.Transport = newMockTransport() + hubClient.Transport = newMockTransport() - err := os.MkdirAll(cfg.Hub.InstallDir, 0700) - require.NoError(t, err) + hub := testHub(t, true) - err = os.MkdirAll(cfg.Hub.HubDir, 0700) - require.NoError(t, err) - - err = UpdateHubIdx(cfg.Hub) - require.NoError(t, err) - - // if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil { - // log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err) - // } - // if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil { - // log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err) - // } - return cfg -} - -func envTearDown(cfg *csconfig.Config) { - if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil { - log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err) - } - - if err := os.RemoveAll(cfg.Hub.HubDir); err != nil { - log.Fatalf("failed to remove %s : %s", cfg.Hub.HubDir, err) - } -} - -func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { - // Install the parser - err := DownloadLatest(cfg, &item, false, false) - require.NoError(t, err, "failed to download %s", item.Name) - - _, err = LocalSync(cfg) - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) - - err = EnableItem(cfg, &item) - require.NoError(t, err, "failed to enable %s", item.Name) - - _, err = LocalSync(cfg) - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name) -} - -func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) { - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) - - f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) - require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name) - - defer f.Close() - - _, err = f.WriteString("tainted") - require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name) - - // Local sync and check status - _, err = LocalSync(cfg) - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hubIdx[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) -} - -func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) { - assert.False(t, hubIdx[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) - - // Update it + check status - err := DownloadLatest(cfg, &item, true, true) - require.NoError(t, err, "failed to update %s", item.Name) - - // Local sync and check status - _, err = LocalSync(cfg) - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) -} - -func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) { - assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name) - - // Remove - err := DisableItem(cfg, &item, false, false) - require.NoError(t, err, "failed to disable %s", item.Name) - - // Local sync and check status - warns, err := LocalSync(cfg) - require.NoError(t, err, "failed to run localSync") - require.Empty(t, warns, "unexpected warnings : %+v", warns) - - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.True(t, hubIdx[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) - - // Purge - err = DisableItem(cfg, &item, true, false) - require.NoError(t, err, "failed to purge %s", item.Name) - - // Local sync and check status - warns, err = LocalSync(cfg) - require.NoError(t, err, "failed to run localSync") - require.Empty(t, warns, "unexpected warnings : %+v", warns) - - assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) -} - -func TestInstallParser(t *testing.T) { - /* - - install a random parser - - check its status - - taint it - - check its status - - force update it - - check its status - - remove it - */ - cfg := envSetup(t) - defer envTearDown(cfg) - - getHubIdxOrFail(t) - // map iteration is random by itself - for _, it := range hubIdx[PARSERS] { - testInstallItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] - _ = GetHubStatusForItemType(PARSERS, it.Name, false) - testTaintItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] - _ = GetHubStatusForItemType(PARSERS, it.Name, false) - testUpdateItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] - testDisableItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] - - break - } -} - -func TestInstallCollection(t *testing.T) { - /* - - install a random parser - - check its status - - taint it - - check its status - - force update it - - check its status - - remove it - */ - cfg := envSetup(t) - defer envTearDown(cfg) - - getHubIdxOrFail(t) - // map iteration is random by itself - for _, it := range hubIdx[COLLECTIONS] { - testInstallItem(cfg.Hub, t, it) - it = hubIdx[COLLECTIONS][it.Name] - testTaintItem(cfg.Hub, t, it) - it = hubIdx[COLLECTIONS][it.Name] - testUpdateItem(cfg.Hub, t, it) - it = hubIdx[COLLECTIONS][it.Name] - testDisableItem(cfg.Hub, t, it) - - it = hubIdx[COLLECTIONS][it.Name] - x := GetHubStatusForItemType(COLLECTIONS, it.Name, false) - log.Infof("%+v", x) - - break - } + return hub } type mockTransport struct{} @@ -324,7 +92,7 @@ func newMockTransport() http.RoundTripper { return &mockTransport{} } -// Implement http.RoundTripper +// Implement http.RoundTripper. func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { // Create mocked http.Response response := &http.Response{ @@ -362,7 +130,7 @@ func fileToStringX(path string) string { return strings.ReplaceAll(string(data), "\r\n", "\n") } -func resetResponseByPath() { +func setResponseByPath() { responseByPath = map[string]string{ "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml": fileToStringX("./testdata/foobar_parser.yaml"), "/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml": fileToStringX("./testdata/foobar_parser.yaml"), diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index 2255d40a7..e624436c8 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -1,70 +1,84 @@ package cwhub import ( + "errors" "fmt" "io" "net/http" "os" - "path/filepath" log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/types" ) +// The DataSet is a list of data sources required by an item (built from the data: section in the yaml). type DataSet struct { - Data []*types.DataSource `yaml:"data,omitempty"` + Data []types.DataSource `yaml:"data,omitempty"` } +// downloadFile downloads a file and writes it to disk, with no hash verification. func downloadFile(url string, destPath string) error { log.Debugf("downloading %s in %s", url, destPath) - req, err := http.NewRequest(http.MethodGet, url, nil) + resp, err := hubClient.Get(url) if err != nil { - return err - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err + return fmt.Errorf("while downloading %s: %w", url, err) } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body)) + return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) } - file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + file, err := os.Create(destPath) + if err != nil { + return err + } + defer file.Close() + + // avoid reading the whole file in memory + _, err = io.Copy(file, resp.Body) if err != nil { return err } - _, err = file.Write(body) - if err != nil { - return err - } - - err = file.Sync() - if err != nil { + if err = file.Sync(); err != nil { return err } return nil } -func GetData(data []*types.DataSource, dataDir string) error { - for _, dataS := range data { - destPath := filepath.Join(dataDir, dataS.DestPath) - log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) +// downloadDataSet downloads all the data files for an item. +func downloadDataSet(dataFolder string, force bool, reader io.Reader) error { + dec := yaml.NewDecoder(reader) - err := downloadFile(dataS.SourceURL, destPath) - if err != nil { - return err + for { + data := &DataSet{} + + if err := dec.Decode(data); err != nil { + if errors.Is(err, io.EOF) { + break + } + + return fmt.Errorf("while reading file: %w", err) + } + + for _, dataS := range data.Data { + destPath, err := safePath(dataFolder, dataS.DestPath) + if err != nil { + return err + } + + if _, err := os.Stat(destPath); os.IsNotExist(err) || force { + log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) + + if err := downloadFile(dataS.SourceURL, destPath); err != nil { + return fmt.Errorf("while getting data: %w", err) + } + } } } diff --git a/pkg/cwhub/dataset_test.go b/pkg/cwhub/dataset_test.go index 40f6ba847..f23f48782 100644 --- a/pkg/cwhub/dataset_test.go +++ b/pkg/cwhub/dataset_test.go @@ -6,6 +6,7 @@ import ( "github.com/jarcoal/httpmock" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDownloadFile(t *testing.T) { @@ -14,12 +15,14 @@ func TestDownloadFile(t *testing.T) { httpmock.Activate() defer httpmock.DeactivateAndReset() + //OK httpmock.RegisterResponder( "GET", "https://example.com/xx", httpmock.NewStringResponder(200, "example content oneoneone"), ) + httpmock.RegisterResponder( "GET", "https://example.com/x", @@ -27,17 +30,21 @@ func TestDownloadFile(t *testing.T) { ) err := downloadFile("https://example.com/xx", examplePath) - assert.NoError(t, err) + require.NoError(t, err) + content, err := os.ReadFile(examplePath) assert.Equal(t, "example content oneoneone", string(content)) - assert.NoError(t, err) + require.NoError(t, err) + //bad uri err = downloadFile("https://zz.com", examplePath) - assert.Error(t, err) + require.Error(t, err) + //404 err = downloadFile("https://example.com/x", examplePath) - assert.Error(t, err) + require.Error(t, err) + //bad target err = downloadFile("https://example.com/xx", "") - assert.Error(t, err) + require.Error(t, err) } diff --git a/pkg/cwhub/doc.go b/pkg/cwhub/doc.go new file mode 100644 index 000000000..857672650 --- /dev/null +++ b/pkg/cwhub/doc.go @@ -0,0 +1,113 @@ +// Package cwhub is responsible for installing and upgrading the local hub files for CrowdSec. +// +// # Definitions +// +// - A hub ITEM is a file that defines a parser, a scenario, a collection... in the case of a collection, it has dependencies on other hub items. +// - The hub INDEX is a JSON file that contains a tree of available hub items. +// - A REMOTE HUB is an HTTP server that hosts the hub index and the hub items. It can serve from several branches, usually linked to the CrowdSec version. +// - A LOCAL HUB is a directory that contains a copy of the hub index and the downloaded hub items. +// +// Once downloaded, hub items can be installed by linking to them from the configuration directory. +// If an item is present in the configuration directory but it's not a link to the local hub, it is +// considered as a LOCAL ITEM and won't be removed or upgraded. +// +// # Directory Structure +// +// A typical directory layout is the following: +// +// For the local hub (HubDir = /etc/crowdsec/hub): +// +// - /etc/crowdsec/hub/.index.json +// - /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml +// - /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml +// +// For the configuration directory (InstallDir = /etc/crowdsec): +// +// - /etc/crowdsec/parsers/{stage}/{parser-name.yaml} -> /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml +// - /etc/crowdsec/scenarios/{scenario-name.yaml} -> /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml +// - /etc/crowdsec/scenarios/local-scenario.yaml +// +// Note that installed items are not grouped by author, this may change in the future if we want to +// support items with the same name from different authors. +// +// Only parsers and postoverflows have the concept of stage. +// +// Additionally, an item can reference a DATA SET that is installed in a different location than +// the item itself. These files are stored in the data directory (InstallDataDir = /var/lib/crowdsec/data). +// +// - /var/lib/crowdsec/data/http_path_traversal.txt +// - /var/lib/crowdsec/data/jira_cve_2021-26086.txt +// - /var/lib/crowdsec/data/log4j2_cve_2021_44228.txt +// - /var/lib/crowdsec/data/sensitive_data.txt +// +// +// # Using the package +// +// The main entry point is the Hub struct. You can create a new instance with NewHub(). +// This constructor takes three parameters, but only the LOCAL HUB configuration is required: +// +// import ( +// "fmt" +// "github.com/crowdsecurity/crowdsec/pkg/csconfig" +// "github.com/crowdsecurity/crowdsec/pkg/cwhub" +// ) +// +// localHub := csconfig.LocalHubCfg{ +// HubIndexFile: "/etc/crowdsec/hub/.index.json", +// HubDir: "/etc/crowdsec/hub", +// InstallDir: "/etc/crowdsec", +// InstallDataDir: "/var/lib/crowdsec/data", +// } +// hub, err := cwhub.NewHub(localHub, nil, false) +// if err != nil { +// return fmt.Errorf("unable to initialize hub: %w", err) +// } +// +// Now you can use the hub to access the existing items: +// +// // list all the parsers +// for _, parser := range hub.GetItemMap(cwhub.PARSERS) { +// fmt.Printf("parser: %s\n", parser.Name) +// } +// +// // retrieve a specific collection +// coll := hub.GetItem(cwhub.COLLECTIONS, "crowdsecurity/linux") +// if coll == nil { +// return fmt.Errorf("collection not found") +// } +// +// You can also install items if they have already been downloaded: +// +// // install a parser +// force := false +// downloadOnly := false +// err := parser.Install(force, downloadOnly) +// if err != nil { +// return fmt.Errorf("unable to install parser: %w", err) +// } +// +// As soon as you try to install an item that is not downloaded or is not up-to-date (meaning its computed hash +// does not correspond to the latest version available in the index), a download will be attempted and you'll +// get the error "remote hub configuration is not provided". +// +// To provide the remote hub configuration, use the second parameter of NewHub(): +// +// remoteHub := cwhub.RemoteHubCfg{ +// URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s", +// Branch: "master", +// IndexPath: ".index.json", +// } +// updateIndex := false +// hub, err := cwhub.NewHub(localHub, remoteHub, updateIndex) +// if err != nil { +// return fmt.Errorf("unable to initialize hub: %w", err) +// } +// +// The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two +// placeholders: the branch and the file path (it will be an index or an item). +// +// Setting the third parameter to true will download the latest version of the index, if available on the +// specified branch. +// There is no exported method to update the index once the hub struct is created. +// +package cwhub diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go deleted file mode 100644 index ef111ba62..000000000 --- a/pkg/cwhub/download.go +++ /dev/null @@ -1,324 +0,0 @@ -package cwhub - -import ( - "bytes" - "crypto/sha256" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" -) - -var ErrIndexNotFound = fmt.Errorf("index not found") - -func UpdateHubIdx(hub *csconfig.Hub) error { - bidx, err := DownloadHubIdx(hub) - if err != nil { - return fmt.Errorf("failed to download index: %w", err) - } - - ret, err := LoadPkgIndex(bidx) - if err != nil { - if !errors.Is(err, ErrMissingReference) { - return fmt.Errorf("failed to read index: %w", err) - } - } - - hubIdx = ret - - if _, err := LocalSync(hub); err != nil { - return fmt.Errorf("failed to sync: %w", err) - } - - return nil -} - -func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) { - log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) - if err != nil { - return nil, fmt.Errorf("failed to build request for hub index: %w", err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("failed http request for hub index: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - if resp.StatusCode == http.StatusNotFound { - return nil, ErrIndexNotFound - } - - return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String()) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) - } - - oldContent, err := os.ReadFile(hub.HubIndexFile) - if err != nil { - if !os.IsNotExist(err) { - log.Warningf("failed to read hub index: %s", err) - } - } else if bytes.Equal(body, oldContent) { - log.Info("hub index is up to date") - // write it anyway, can't hurt - } - - file, err := os.OpenFile(hub.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - - if err != nil { - return nil, fmt.Errorf("while opening hub index file: %w", err) - } - defer file.Close() - - wsize, err := file.Write(body) - if err != nil { - return nil, fmt.Errorf("while writing hub index file: %w", err) - } - - log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile) - - return body, nil -} - -// DownloadLatest will download the latest version of Item to the tdir directory -func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly bool) error { - var err error - - log.Debugf("Downloading %s %s", target.Type, target.Name) - - if target.Type != COLLECTIONS { - if !target.Installed && updateOnly && target.Downloaded { - log.Debugf("skipping upgrade of %s : not installed", target.Name) - return nil - } - - return DownloadItem(hub, target, overwrite) - } - - // collection - var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} - for idx, ptr := range tmp { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - val, ok := hubIdx[ptrtype][p] - if !ok { - return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) - } - - if !val.Installed && updateOnly && val.Downloaded { - log.Debugf("skipping upgrade of %s : not installed", target.Name) - continue - } - - log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly) - //recurse as it's a collection - if ptrtype == COLLECTIONS { - log.Tracef("collection, recurse") - - err = DownloadLatest(hub, &val, overwrite, updateOnly) - if err != nil { - return fmt.Errorf("while downloading %s: %w", val.Name, err) - } - } - - downloaded := val.Downloaded - - err = DownloadItem(hub, &val, overwrite) - if err != nil { - return fmt.Errorf("while downloading %s: %w", val.Name, err) - } - - // We need to enable an item when it has been added to a collection since latest release of the collection. - // We check if val.Downloaded is false because maybe the item has been disabled by the user. - if !val.Installed && !downloaded { - if err = EnableItem(hub, &val); err != nil { - return fmt.Errorf("enabling '%s': %w", val.Name, err) - } - } - - hubIdx[ptrtype][p] = val - } - } - - err = DownloadItem(hub, target, overwrite) - if err != nil { - return fmt.Errorf("failed to download item: %w", err) - } - - return nil -} - -func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error { - tdir := hub.HubDir - - // if user didn't --force, don't overwrite local, tainted, up-to-date files - if !overwrite { - if target.Tainted { - log.Debugf("%s : tainted, not updated", target.Name) - return nil - } - - if target.UpToDate { - // We still have to check if data files are present - log.Debugf("%s : up-to-date, not updated", target.Name) - } - } - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) - if err != nil { - return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) - } - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) - } - - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("while reading %s: %w", req.URL.String(), err) - } - - h := sha256.New() - if _, err = h.Write(body); err != nil { - return fmt.Errorf("while hashing %s: %w", target.Name, err) - } - - meow := fmt.Sprintf("%x", h.Sum(nil)) - if meow != target.Versions[target.Version].Digest { - log.Errorf("Downloaded version doesn't match index, please 'hub update'") - log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) - - return fmt.Errorf("invalid download hash for %s", target.Name) - } - - //all good, install - //check if parent dir exists - tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") - parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") - - // ensure that target file is within target dir - finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath) - if err != nil { - return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err) - } - - if !strings.HasPrefix(finalPath, tdir) { - return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir) - } - - // check dir - if _, err = os.Stat(parentDir); os.IsNotExist(err) { - log.Debugf("%s doesn't exist, create", parentDir) - - if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { - return fmt.Errorf("while creating parent directories: %w", err) - } - } - - // check actual file - if _, err = os.Stat(finalPath); !os.IsNotExist(err) { - log.Warningf("%s : overwrite", target.Name) - log.Debugf("target: %s/%s", tdir, target.RemotePath) - } else { - log.Infof("%s : OK", target.Name) - } - - f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return fmt.Errorf("while opening file: %w", err) - } - - defer f.Close() - - _, err = f.Write(body) - if err != nil { - return fmt.Errorf("while writing file: %w", err) - } - - target.Downloaded = true - target.Tainted = false - target.UpToDate = true - - if err = downloadData(hub.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { - return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) - } - - hubIdx[target.Type][target.Name] = *target - - return nil -} - -func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error { - itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName) - - itemFile, err := os.Open(itemFilePath) - if err != nil { - return fmt.Errorf("while opening %s: %w", itemFilePath, err) - } - - defer itemFile.Close() - - if err = downloadData(hub.InstallDataDir, force, itemFile); err != nil { - return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) - } - - return nil -} - -func downloadData(dataFolder string, force bool, reader io.Reader) error { - var err error - - dec := yaml.NewDecoder(reader) - - for { - data := &DataSet{} - - err = dec.Decode(data) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - - return fmt.Errorf("while reading file: %w", err) - } - - download := false - - for _, dataS := range data.Data { - if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { - download = true - } - } - - if download || force { - err = GetData(data.Data, dataFolder) - if err != nil { - return fmt.Errorf("while getting data: %w", err) - } - } - } - - return nil -} diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go deleted file mode 100644 index 351b08f8e..000000000 --- a/pkg/cwhub/download_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package cwhub - -import ( - "fmt" - "strings" - "testing" - - log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" -) - -func TestDownloadHubIdx(t *testing.T) { - back := RawFileURLTemplate - // bad url template - fmt.Println("Test 'bad URL'") - - RawFileURLTemplate = "x" - - ret, err := DownloadHubIdx(&csconfig.Hub{}) - if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") { - log.Errorf("unexpected error %s", err) - } - - fmt.Printf("->%+v", ret) - - // bad domain - fmt.Println("Test 'bad domain'") - - RawFileURLTemplate = "https://baddomain/%s/%s" - - ret, err = DownloadHubIdx(&csconfig.Hub{}) - if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") { - log.Errorf("unexpected error %s", err) - } - - fmt.Printf("->%+v", ret) - - // bad target path - fmt.Println("Test 'bad target path'") - - RawFileURLTemplate = back - - ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"}) - if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") { - log.Errorf("unexpected error %s", err) - } - - RawFileURLTemplate = back - - fmt.Printf("->%+v", ret) -} diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go new file mode 100644 index 000000000..a8f46c6b3 --- /dev/null +++ b/pkg/cwhub/enable.go @@ -0,0 +1,190 @@ +package cwhub + +// Enable/disable items already downloaded + +import ( + "fmt" + "os" + "path/filepath" + + log "github.com/sirupsen/logrus" +) + +// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local +// (eg. /etc/crowdsec/collections/xyz.yaml). +// Raises an error if the path goes outside of the install dir. +func (i *Item) installPath() (string, error) { + p := i.Type + if i.Stage != "" { + p = filepath.Join(p, i.Stage) + } + + return safePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName)) +} + +// downloadPath returns the location of the actual config file in the hub +// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml). +// Raises an error if the path goes outside of the hub dir. +func (i *Item) downloadPath() (string, error) { + ret, err := safePath(i.hub.local.HubDir, i.RemotePath) + if err != nil { + return "", err + } + + return ret, nil +} + +// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir. +func (i *Item) createInstallLink() error { + dest, err := i.installPath() + if err != nil { + return err + } + + destDir := filepath.Dir(dest) + if err = os.MkdirAll(destDir, os.ModePerm); err != nil { + return fmt.Errorf("while creating %s: %w", destDir, err) + } + + if _, err = os.Lstat(dest); !os.IsNotExist(err) { + log.Infof("%s already exists.", dest) + return nil + } + + src, err := i.downloadPath() + if err != nil { + return err + } + + if err = os.Symlink(src, dest); err != nil { + return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err) + } + + return nil +} + +// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items. +func (i *Item) enable() error { + if i.State.Installed { + if i.State.Tainted { + return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name) + } + + if i.IsLocal() { + return fmt.Errorf("%s is local, won't enable", i.Name) + } + + // if it's a collection, check sub-items even if the collection file itself is up-to-date + if i.State.UpToDate && !i.HasSubItems() { + log.Tracef("%s is installed and up-to-date, skip.", i.Name) + return nil + } + } + + for _, sub := range i.SubItems() { + if err := sub.enable(); err != nil { + return fmt.Errorf("while installing %s: %w", sub.Name, err) + } + } + + if err := i.createInstallLink(); err != nil { + return err + } + + log.Infof("Enabled %s: %s", i.Type, i.Name) + i.State.Installed = true + + return nil +} + +// purge removes the actual config file that was downloaded. +func (i *Item) purge() error { + if !i.State.Downloaded { + log.Infof("removing %s: not downloaded -- no need to remove", i.Name) + return nil + } + + src, err := i.downloadPath() + if err != nil { + return err + } + + if err := os.Remove(src); err != nil { + if os.IsNotExist(err) { + log.Debugf("%s doesn't exist, no need to remove", src) + return nil + } + + return fmt.Errorf("while removing file: %w", err) + } + + i.State.Downloaded = false + log.Infof("Removed source file [%s]: %s", i.Name, src) + + return nil +} + +// removeInstallLink removes the symlink to the downloaded content. +func (i *Item) removeInstallLink() error { + syml, err := i.installPath() + if err != nil { + return err + } + + stat, err := os.Lstat(syml) + if err != nil { + return err + } + + // if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ... + if stat.Mode()&os.ModeSymlink == 0 { + log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml) + return fmt.Errorf("%s isn't managed by hub", i.Name) + } + + hubpath, err := os.Readlink(syml) + if err != nil { + return fmt.Errorf("while reading symlink: %w", err) + } + + src, err := i.downloadPath() + if err != nil { + return err + } + + if hubpath != src { + log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src) + return fmt.Errorf("%s isn't managed by hub", i.Name) + } + + if err := os.Remove(syml); err != nil { + return fmt.Errorf("while removing symlink: %w", err) + } + + log.Infof("Removed symlink [%s]: %s", i.Name, syml) + + return nil +} + +// disable removes the install link, and optionally the downloaded content. +func (i *Item) disable(purge bool, force bool) error { + err := i.removeInstallLink() + if os.IsNotExist(err) { + if !purge && !force { + link, _ := i.installPath() + return fmt.Errorf("link %s does not exist (override with --force or --purge)", link) + } + } else if err != nil { + return err + } + + i.State.Installed = false + + if purge { + if err := i.purge(); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go new file mode 100644 index 000000000..35e56915b --- /dev/null +++ b/pkg/cwhub/enable_test.go @@ -0,0 +1,141 @@ +package cwhub + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testInstall(hub *Hub, t *testing.T, item *Item) { + // Install the parser + _, err := item.downloadLatest(false, false) + require.NoError(t, err, "failed to download %s", item.Name) + + err = hub.localSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name) + + err = item.enable() + require.NoError(t, err, "failed to enable %s", item.Name) + + err = hub.localSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name) +} + +func testTaint(hub *Hub, t *testing.T, item *Item) { + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name) + + // truncate the file + f, err := os.Create(item.State.LocalPath) + require.NoError(t, err) + f.Close() + + // Local sync and check status + err = hub.localSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should be tainted", item.Name) +} + +func testUpdate(hub *Hub, t *testing.T, item *Item) { + assert.False(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should not be up-to-date", item.Name) + + // Update it + check status + _, err := item.downloadLatest(true, true) + require.NoError(t, err, "failed to update %s", item.Name) + + // Local sync and check status + err = hub.localSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name) +} + +func testDisable(hub *Hub, t *testing.T, item *Item) { + assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name) + + // Remove + err := item.disable(false, false) + require.NoError(t, err, "failed to disable %s", item.Name) + + // Local sync and check status + err = hub.localSync() + require.NoError(t, err, "failed to run localSync") + require.Empty(t, hub.Warnings) + + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should still be downloaded", item.Name) + + // Purge + err = item.disable(true, false) + require.NoError(t, err, "failed to purge %s", item.Name) + + // Local sync and check status + err = hub.localSync() + require.NoError(t, err, "failed to run localSync") + require.Empty(t, hub.Warnings) + + assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should not be downloaded", item.Name) +} + +func TestInstallParser(t *testing.T) { + /* + - install a random parser + - check its status + - taint it + - check its status + - force update it + - check its status + - remove it + */ + hub := envSetup(t) + + // map iteration is random by itself + for _, it := range hub.Items[PARSERS] { + testInstall(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testTaint(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testUpdate(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testDisable(hub, t, it) + + break + } +} + +func TestInstallCollection(t *testing.T) { + /* + - install a random parser + - check its status + - taint it + - check its status + - force update it + - check its status + - remove it + */ + hub := envSetup(t) + + // map iteration is random by itself + for _, it := range hub.Items[COLLECTIONS] { + testInstall(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testTaint(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testUpdate(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testDisable(hub, t, it) + + break + } +} diff --git a/pkg/cwhub/errors.go b/pkg/cwhub/errors.go new file mode 100644 index 000000000..789c2eced --- /dev/null +++ b/pkg/cwhub/errors.go @@ -0,0 +1,21 @@ +package cwhub + +import ( + "errors" + "fmt" +) + +var ( + // ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor. + ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers") +) + +// IndexNotFoundError is returned when the remote hub index is not found. +type IndexNotFoundError struct { + URL string + Branch string +} + +func (e IndexNotFoundError) Error() string { + return fmt.Sprintf("index not found at %s, branch '%s'. Please check the .cscli.hub_branch value if you specified it in config.yaml, or use 'master' if not sure", e.URL, e.Branch) +} diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index c17e6758d..fc8a98d43 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -1,222 +1,373 @@ package cwhub +// Install, upgrade and remove items from the hub to the local configuration + import ( + "bytes" + "crypto/sha256" + "encoding/hex" "fmt" + "io" + "net/http" + "os" "path/filepath" "github.com/enescakir/emoji" log "github.com/sirupsen/logrus" - "golang.org/x/mod/semver" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "slices" ) -// pick a hub branch corresponding to the current crowdsec version. -func chooseHubBranch() string { - latest, err := cwversion.Latest() - if err != nil { - log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err) - //lint:ignore nilerr - return "master" - } - - csVersion := cwversion.VersionStrip() - if csVersion == latest { - log.Debugf("current version is equal to latest (%s)", csVersion) - return "master" - } - - // if current version is greater than the latest we are in pre-release - if semver.Compare(csVersion, latest) == 1 { - log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion) - return "master" - } - - if csVersion == "" { - log.Warning("Crowdsec version is not set, using master branch for the hub") - return "master" - } - - log.Warnf("Crowdsec is not the latest version. "+ - "Current version is '%s' and the latest stable version is '%s'. Please update it!", - csVersion, latest) - - log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+ - "added to Crowdsec Hub after CrowdSec %s", latest) - - return csVersion -} - -// SetHubBranch sets the package variable that points to the hub branch. -func SetHubBranch() { - // a branch is already set, or specified from the flags - if HubBranch != "" { - return - } - - // use the branch corresponding to the crowdsec version - HubBranch = chooseHubBranch() - - log.Debugf("Using branch '%s' for the hub", HubBranch) -} - -func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error { - item := GetItem(obtype, name) - if item == nil { - return fmt.Errorf("unable to retrieve item: %s", name) - } - - if downloadOnly && item.Downloaded && item.UpToDate { - log.Warningf("%s is already downloaded and up-to-date", item.Name) +// Install installs the item from the hub, downloading it if needed. +func (i *Item) Install(force bool, downloadOnly bool) error { + if downloadOnly && i.State.Downloaded && i.State.UpToDate { + log.Infof("%s is already downloaded and up-to-date", i.Name) if !force { return nil } } - err := DownloadLatest(csConfig.Hub, item, force, true) + filePath, err := i.downloadLatest(force, true) if err != nil { - return fmt.Errorf("while downloading %s: %w", item.Name, err) - } - - if err = AddItem(obtype, *item); err != nil { - return fmt.Errorf("while adding %s: %w", item.Name, err) + return fmt.Errorf("while downloading %s: %w", i.Name, err) } if downloadOnly { - log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath)) + log.Infof("Downloaded %s to %s", i.Name, filePath) return nil } - err = EnableItem(csConfig.Hub, item) - if err != nil { - return fmt.Errorf("while enabling %s: %w", item.Name, err) + if err := i.enable(); err != nil { + return fmt.Errorf("while enabling %s: %w", i.Name, err) } - if err := AddItem(obtype, *item); err != nil { - return fmt.Errorf("while adding %s: %w", item.Name, err) - } - - log.Infof("Enabled %s", item.Name) + log.Infof("Enabled %s", i.Name) return nil } -// XXX this must return errors instead of log.Fatal -func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) { - if name != "" { - item := GetItem(itemType, name) +// descendants returns a list of all (direct or indirect) dependencies of the item. +func (i *Item) descendants() ([]*Item, error) { + var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error + + collectSubItems = func(item *Item, visited map[*Item]bool, result *[]*Item) error { if item == nil { - log.Fatalf("unable to retrieve: %s", name) + return nil } - err := DisableItem(csConfig.Hub, item, purge, forceAction) - - if err != nil { - log.Fatalf("unable to disable %s : %v", item.Name, err) + if visited[item] { + return nil } - if err = AddItem(itemType, *item); err != nil { - log.Fatalf("unable to add %s: %v", item.Name, err) - } + visited[item] = true - return - } - - if !all { - log.Fatal("removing item: no item specified") - } - - disabled := 0 - - // remove all - for _, v := range GetItemMap(itemType) { - if !v.Installed { - continue - } - - err := DisableItem(csConfig.Hub, &v, purge, forceAction) - if err != nil { - log.Fatalf("unable to disable %s : %v", v.Name, err) - } - - if err := AddItem(itemType, v); err != nil { - log.Fatalf("unable to add %s: %v", v.Name, err) - } - disabled++ - } - - log.Infof("Disabled %d items", disabled) -} - -func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) { - updated := 0 - found := false - - for _, v := range GetItemMap(itemType) { - if name != "" && name != v.Name { - continue - } - - if !v.Installed { - log.Tracef("skip %s, not installed", v.Name) - continue - } - - if !v.Downloaded { - log.Warningf("%s : not downloaded, please install.", v.Name) - continue - } - - found = true - - if v.UpToDate { - log.Infof("%s : up-to-date", v.Name) - - if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil { - log.Fatalf("%s : download failed : %v", v.Name, err) + for _, subItem := range item.SubItems() { + if subItem == i { + return fmt.Errorf("circular dependency detected: %s depends on %s", item.Name, i.Name) } - if !force { + *result = append(*result, subItem) + + err := collectSubItems(subItem, visited, result) + if err != nil { + return err + } + } + + return nil + } + + ret := []*Item{} + visited := map[*Item]bool{} + + err := collectSubItems(i, visited, &ret) + if err != nil { + return nil, err + } + + return ret, nil +} + +// Remove disables the item, optionally removing the downloaded content. +func (i *Item) Remove(purge bool, force bool) (bool, error) { + if i.IsLocal() { + return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name) + } + + if i.State.Tainted && !force { + return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name) + } + + if !i.State.Installed && !purge { + log.Infof("removing %s: not installed -- no need to remove", i.Name) + return false, nil + } + + removed := false + + descendants, err := i.descendants() + if err != nil { + return false, err + } + + ancestors := i.Ancestors() + + for _, sub := range i.SubItems() { + if !sub.State.Installed { + continue + } + + // if the sub depends on a collection that is not a direct or indirect dependency + // of the current item, it is not removed + for _, subParent := range sub.Ancestors() { + if !purge && !subParent.State.Installed { + continue + } + + // the ancestor that would block the removal of the sub item is also an ancestor + // of the item we are removing, so we don't want false warnings + // (e.g. crowdsecurity/sshd-logs was not removed because it also belongs to crowdsecurity/linux, + // while we are removing crowdsecurity/sshd) + if slices.Contains(ancestors, subParent) { + continue + } + + // the sub-item belongs to the item we are removing, but we already knew that + if subParent == i { + continue + } + + if !slices.Contains(descendants, subParent) { + log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name) continue } } - if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil { - log.Fatalf("%s : download failed : %v", v.Name, err) + subRemoved, err := sub.Remove(purge, force) + if err != nil { + return false, fmt.Errorf("unable to disable %s: %w", i.Name, err) } - if !v.UpToDate { - if v.Tainted { - log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name) - } else if v.Local { - log.Infof("%v %s is local", emoji.Prohibited, v.Name) - } - } else { - // this is used while scripting to know if the hub has been upgraded - // and a configuration reload is required - fmt.Printf("updated %s\n", v.Name) - log.Infof("%v %s : updated", emoji.Package, v.Name) - updated++ - } - - if err := AddItem(itemType, v); err != nil { - log.Fatalf("unable to add %s: %v", v.Name, err) - } + removed = removed || subRemoved } - if !found && name == "" { - log.Infof("No %s installed, nothing to upgrade", itemType) - } else if !found { - log.Errorf("Item '%s' not found in hub", name) - } else if updated == 0 && found { - if name == "" { - log.Infof("All %s are already up-to-date", itemType) - } else { - log.Infof("Item '%s' is up-to-date", name) - } - } else if updated != 0 { - log.Infof("Upgraded %d items", updated) + if err = i.disable(purge, force); err != nil { + return false, fmt.Errorf("while removing %s: %w", i.Name, err) } + + removed = true + + return removed, nil +} + +// Upgrade downloads and applies the last version of the item from the hub. +func (i *Item) Upgrade(force bool) (bool, error) { + updated := false + + if !i.State.Downloaded { + return false, fmt.Errorf("can't upgrade %s: not installed", i.Name) + } + + if !i.State.Installed { + return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", i.Name) + } + + if i.State.UpToDate { + log.Infof("%s: up-to-date", i.Name) + + if err := i.DownloadDataIfNeeded(force); err != nil { + return false, fmt.Errorf("%s: download failed: %w", i.Name, err) + } + + if !force { + // no upgrade needed + return false, nil + } + } + + if _, err := i.downloadLatest(force, true); err != nil { + return false, fmt.Errorf("%s: download failed: %w", i.Name, err) + } + + if !i.State.UpToDate { + if i.State.Tainted { + log.Warningf("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name) + } else if i.IsLocal() { + log.Infof("%v %s is local", emoji.Prohibited, i.Name) + } + } else { + // a check on stdout is used while scripting to know if the hub has been upgraded + // and a configuration reload is required + // TODO: use a better way to communicate this + fmt.Printf("updated %s\n", i.Name) + log.Infof("%v %s: updated", emoji.Package, i.Name) + updated = true + } + + return updated, nil +} + +// downloadLatest downloads the latest version of the item to the hub directory. +func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) { + log.Debugf("Downloading %s %s", i.Type, i.Name) + + for _, sub := range i.SubItems() { + if !sub.State.Installed && updateOnly && sub.State.Downloaded { + log.Debugf("skipping upgrade of %s: not installed", i.Name) + continue + } + + log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.State.Installed, updateOnly) + + // recurse as it's a collection + if sub.HasSubItems() { + log.Tracef("collection, recurse") + + if _, err := sub.downloadLatest(overwrite, updateOnly); err != nil { + return "", fmt.Errorf("while downloading %s: %w", sub.Name, err) + } + } + + downloaded := sub.State.Downloaded + + if _, err := sub.download(overwrite); err != nil { + return "", fmt.Errorf("while downloading %s: %w", sub.Name, err) + } + + // We need to enable an item when it has been added to a collection since latest release of the collection. + // We check if sub.Downloaded is false because maybe the item has been disabled by the user. + if !sub.State.Installed && !downloaded { + if err := sub.enable(); err != nil { + return "", fmt.Errorf("enabling '%s': %w", sub.Name, err) + } + } + } + + if !i.State.Installed && updateOnly && i.State.Downloaded { + log.Debugf("skipping upgrade of %s: not installed", i.Name) + return "", nil + } + + ret, err := i.download(overwrite) + if err != nil { + return "", fmt.Errorf("failed to download item: %w", err) + } + + return ret, nil +} + +// fetch downloads the item from the hub, verifies the hash and returns the content. +func (i *Item) fetch() ([]byte, error) { + url, err := i.hub.remote.urlTo(i.RemotePath) + if err != nil { + return nil, fmt.Errorf("failed to build hub item request: %w", err) + } + + resp, err := hubClient.Get(url) + if err != nil { + return nil, fmt.Errorf("while downloading %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("while downloading %s: %w", url, err) + } + + hash := sha256.New() + if _, err = hash.Write(body); err != nil { + return nil, fmt.Errorf("while hashing %s: %w", i.Name, err) + } + + meow := hex.EncodeToString(hash.Sum(nil)) + if meow != i.Versions[i.Version].Digest { + log.Errorf("Downloaded version doesn't match index, please 'hub update'") + log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest) + + return nil, fmt.Errorf("invalid download hash for %s", i.Name) + } + + return body, nil +} + +// download downloads the item from the hub and writes it to the hub directory. +func (i *Item) download(overwrite bool) (string, error) { + // if user didn't --force, don't overwrite local, tainted, up-to-date files + if !overwrite { + if i.State.Tainted { + log.Debugf("%s: tainted, not updated", i.Name) + return "", nil + } + + if i.State.UpToDate { + // We still have to check if data files are present + log.Debugf("%s: up-to-date, not updated", i.Name) + } + } + + body, err := i.fetch() + if err != nil { + return "", err + } + + // all good, install + + // ensure that target file is within target dir + finalPath, err := i.downloadPath() + if err != nil { + return "", err + } + + parentDir := filepath.Dir(finalPath) + + if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { + return "", fmt.Errorf("while creating %s: %w", parentDir, err) + } + + // check actual file + if _, err = os.Stat(finalPath); !os.IsNotExist(err) { + log.Warningf("%s: overwrite", i.Name) + log.Debugf("target: %s", finalPath) + } else { + log.Infof("%s: OK", i.Name) + } + + if err = os.WriteFile(finalPath, body, 0o644); err != nil { + return "", fmt.Errorf("while writing %s: %w", finalPath, err) + } + + i.State.Downloaded = true + i.State.Tainted = false + i.State.UpToDate = true + + if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { + return "", fmt.Errorf("while downloading data for %s: %w", i.FileName, err) + } + + return finalPath, nil +} + +// DownloadDataIfNeeded downloads the data set for the item. +func (i *Item) DownloadDataIfNeeded(force bool) error { + itemFilePath, err := i.installPath() + if err != nil { + return err + } + + itemFile, err := os.Open(itemFilePath) + if err != nil { + return fmt.Errorf("while opening %s: %w", itemFilePath, err) + } + + defer itemFile.Close() + + if err = downloadDataSet(i.hub.local.InstallDataDir, force, itemFile); err != nil { + return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) + } + + return nil } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index c8bb28c36..60b70f491 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -4,157 +4,187 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection -// We expect the new scenario to be installed -func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) +// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection. +// We expect the new scenario to be installed. +func TestUpgradeItemNewScenarioInCollection(t *testing.T) { + hub := envSetup(t) // fresh install of collection - getHubIdxOrFail(t) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + require.NoError(t, item.Install(false, false)) - require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) - - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) // This is the scenario that gets added in next version of collection - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"]) - assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") + assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" pushUpdateToCollectionInHub() - if err := UpdateHubIdx(cfg.Hub); err != nil { - t.Fatalf("failed to download index : %s", err) + remote := &RemoteHubCfg{ + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", } - getHubIdxOrFail(t) + hub, err := NewHub(hub.local, remote, true) + require.NoError(t, err, "failed to download index: %s", err) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + hub = getHubOrFail(t, hub.local, remote) - UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) - assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + didUpdate, err := item.Upgrade(false) + require.NoError(t, err) + require.True(t, didUpdate) + assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") + + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Downloaded) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed) } // Install a collection, disable a scenario. // Upgrade should install should not enable/download the disabled scenario. -func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) +func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { + hub := envSetup(t) // fresh install of collection - getHubIdxOrFail(t) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + require.NoError(t, item.Install(false, false)) - require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") + item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") + didRemove, err := item.Remove(false, false) + require.NoError(t, err) + require.True(t, didRemove) - RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) - getHubIdxOrFail(t) - // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - - if err := UpdateHubIdx(cfg.Hub); err != nil { - t.Fatalf("failed to download index : %s", err) + remote := &RemoteHubCfg{ + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", } - UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + hub = getHubOrFail(t, hub.local, remote) + // scenario referenced by collection was deleted hence, collection should be tainted + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) - getHubIdxOrFail(t) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + hub, err = NewHub(hub.local, remote, true) + require.NoError(t, err, "failed to download index: %s", err) + + item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + didUpdate, err := item.Upgrade(false) + require.NoError(t, err) + require.False(t, didUpdate) + + hub = getHubOrFail(t, hub.local, remote) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) } -func getHubIdxOrFail(t *testing.T) { - if err := GetHubIdx(getTestCfg().Hub); err != nil { - t.Fatalf("failed to load hub index") - } +// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test. +func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub { + hub, err := NewHub(local, remote, false) + require.NoError(t, err, "failed to load hub index") + + return hub } // Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario // Upgrade should not enable/download the disabled scenario. // Upgrade should install and enable the newly added scenario. -func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) +func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { + hub := envSetup(t) // fresh install of collection - getHubIdxOrFail(t) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + require.NoError(t, item.Install(false, false)) - require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") + item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") + didRemove, err := item.Remove(false, false) + require.NoError(t, err) + require.True(t, didRemove) - RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) - getHubIdxOrFail(t) + remote := &RemoteHubCfg{ + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", + } + + hub = getHubOrFail(t, hub.local, remote) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Downloaded) // this fails + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" // we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario // we just removed. Nor should it install the newly added scenario pushUpdateToCollectionInHub() - if err := UpdateHubIdx(cfg.Hub); err != nil { - t.Fatalf("failed to download index : %s", err) - } + hub, err = NewHub(hub.local, remote, true) + require.NoError(t, err, "failed to download index: %s", err) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - getHubIdxOrFail(t) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + hub = getHubOrFail(t, hub.local, remote) - UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) - getHubIdxOrFail(t) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + didUpdate, err := item.Upgrade(false) + require.NoError(t, err) + require.True(t, didUpdate) + + hub = getHubOrFail(t, hub.local, remote) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed) } -func assertCollectionDepsInstalled(t *testing.T, collection string) { +func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) { t.Helper() - c := hubIdx[COLLECTIONS][collection] - require.NoError(t, CollecDepsCheck(&c)) + c := hub.Items[COLLECTIONS][collection] + require.NoError(t, c.checkSubItemVersions()) } func pushUpdateToCollectionInHub() { diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go new file mode 100644 index 000000000..ff1c3cf15 --- /dev/null +++ b/pkg/cwhub/hub.go @@ -0,0 +1,161 @@ +package cwhub + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path" + "strings" + + log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" +) + +// Hub is the main structure for the package. +type Hub struct { + Items HubItems // Items read from HubDir and InstallDir + local *csconfig.LocalHubCfg + remote *RemoteHubCfg + Warnings []string // Warnings encountered during sync +} + +// GetDataDir returns the data directory, where data sets are installed. +func (h *Hub) GetDataDir() string { + return h.local.InstallDataDir +} + +// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state. +// If updateIndex is true, the local index file is updated from the remote before reading the state of the items. +// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set. +func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) { + if local == nil { + return nil, fmt.Errorf("no hub configuration found") + } + + hub := &Hub{ + local: local, + remote: remote, + } + + if updateIndex { + if err := hub.updateIndex(); err != nil { + return nil, err + } + } + + log.Debugf("loading hub idx %s", local.HubIndexFile) + + if err := hub.parseIndex(); err != nil { + return nil, fmt.Errorf("failed to load index: %w", err) + } + + if err := hub.localSync(); err != nil { + return nil, fmt.Errorf("failed to sync items: %w", err) + } + + return hub, nil +} + +// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections. +func (h *Hub) parseIndex() error { + bidx, err := os.ReadFile(h.local.HubIndexFile) + if err != nil { + return fmt.Errorf("unable to read index file: %w", err) + } + + if err := json.Unmarshal(bidx, &h.Items); err != nil { + return fmt.Errorf("failed to unmarshal index: %w", err) + } + + log.Debugf("%d item types in hub index", len(ItemTypes)) + + // Iterate over the different types to complete the struct + for _, itemType := range ItemTypes { + log.Tracef("%s: %d items", itemType, len(h.Items[itemType])) + + for name, item := range h.Items[itemType] { + item.hub = h + item.Name = name + + // if the item has no (redundant) author, take it from the json key + if item.Author == "" && strings.Contains(name, "/") { + item.Author = strings.Split(name, "/")[0] + } + + item.Type = itemType + item.FileName = path.Base(item.RemotePath) + + item.logMissingSubItems() + } + } + + return nil +} + +// ItemStats returns total counts of the hub items, including local and tainted. +func (h *Hub) ItemStats() []string { + loaded := "" + local := 0 + tainted := 0 + + for _, itemType := range ItemTypes { + if len(h.Items[itemType]) == 0 { + continue + } + + loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) + + for _, item := range h.Items[itemType] { + if item.IsLocal() { + local++ + } + + if item.State.Tainted { + tainted++ + } + } + } + + loaded = strings.Trim(loaded, ", ") + if loaded == "" { + loaded = "0 items" + } + + ret := []string{ + fmt.Sprintf("Loaded: %s", loaded), + } + + if local > 0 || tainted > 0 { + ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", local, tainted)) + } + + return ret +} + +// updateIndex downloads the latest version of the index and writes it to disk if it changed. +func (h *Hub) updateIndex() error { + body, err := h.remote.fetchIndex() + if err != nil { + return err + } + + oldContent, err := os.ReadFile(h.local.HubIndexFile) + if err != nil { + if !os.IsNotExist(err) { + log.Warningf("failed to read hub index: %s", err) + } + } else if bytes.Equal(body, oldContent) { + log.Info("hub index is up to date") + return nil + } + + if err = os.WriteFile(h.local.HubIndexFile, body, 0o644); err != nil { + return fmt.Errorf("failed to write hub index: %w", err) + } + + log.Infof("Wrote index to %s, %d bytes", h.local.HubIndexFile, len(body)) + + return nil +} diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go new file mode 100644 index 000000000..670f8d843 --- /dev/null +++ b/pkg/cwhub/hub_test.go @@ -0,0 +1,77 @@ +package cwhub + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/cstest" +) + +func TestInitHubUpdate(t *testing.T) { + hub := envSetup(t) + + remote := &RemoteHubCfg{ + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", + } + + _, err := NewHub(hub.local, remote, true) + require.NoError(t, err) +} + +func TestUpdateIndex(t *testing.T) { + // bad url template + fmt.Println("Test 'bad URL'") + + tmpIndex, err := os.CreateTemp("", "index.json") + require.NoError(t, err) + + t.Cleanup(func() { + os.Remove(tmpIndex.Name()) + }) + + hub := envSetup(t) + + hub.remote = &RemoteHubCfg{ + URLTemplate: "x", + Branch: "", + IndexPath: "", + } + + hub.local.HubIndexFile = tmpIndex.Name() + + err = hub.updateIndex() + cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") + + // bad domain + fmt.Println("Test 'bad domain'") + + hub.remote = &RemoteHubCfg{ + URLTemplate: "https://baddomain/%s/%s", + Branch: "master", + IndexPath: ".index.json", + } + + err = hub.updateIndex() + require.NoError(t, err) + // XXX: this is not failing + // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") + + // bad target path + fmt.Println("Test 'bad target path'") + + hub.remote = &RemoteHubCfg{ + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", + } + + hub.local.HubIndexFile = "/does/not/exist/index.json" + + err = hub.updateIndex() + cstest.RequireErrorContains(t, err, "failed to write hub index: open /does/not/exist/index.json:") +} diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go deleted file mode 100644 index 45e2ba419..000000000 --- a/pkg/cwhub/install.go +++ /dev/null @@ -1,214 +0,0 @@ -package cwhub - -import ( - "fmt" - "os" - "path/filepath" - - log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" -) - -func purgeItem(hub *csconfig.Hub, target Item) (Item, error) { - itempath := hub.HubDir + "/" + target.RemotePath - - // disable hub file - if err := os.Remove(itempath); err != nil { - return target, fmt.Errorf("while removing file: %w", err) - } - - target.Downloaded = false - log.Infof("Removed source file [%s]: %s", target.Name, itempath) - hubIdx[target.Type][target.Name] = target - - return target, nil -} - -// DisableItem to disable an item managed by the hub, removes the symlink if purge is true -func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error { - var err error - - // already disabled, noop unless purge - if !target.Installed { - if purge { - *target, err = purgeItem(hub, *target) - if err != nil { - return err - } - } - - return nil - } - - if target.Local { - return fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name) - } - - if target.Tainted && !force { - return fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name) - } - - // for a COLLECTIONS, disable sub-items - if target.Type == COLLECTIONS { - for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if val, ok := hubIdx[ptrtype][p]; ok { - // check if the item doesn't belong to another collection before removing it - toRemove := true - - for _, collection := range val.BelongsToCollections { - if collection != target.Name { - toRemove = false - break - } - } - - if toRemove { - err = DisableItem(hub, &val, purge, force) - if err != nil { - return fmt.Errorf("while disabling %s: %w", p, err) - } - } else { - log.Infof("%s was not removed because it belongs to another collection", val.Name) - } - } else { - log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name) - } - } - } - } - - syml, err := filepath.Abs(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName) - if err != nil { - return err - } - - stat, err := os.Lstat(syml) - if os.IsNotExist(err) { - // we only accept to "delete" non existing items if it's a forced purge - if !purge && !force { - return fmt.Errorf("can't delete %s : %s doesn't exist", target.Name, syml) - } - } else { - // if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ... - if stat.Mode()&os.ModeSymlink == 0 { - log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml) - return fmt.Errorf("%s isn't managed by hub", target.Name) - } - - hubpath, err := os.Readlink(syml) - if err != nil { - return fmt.Errorf("while reading symlink: %w", err) - } - - absPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath) - if err != nil { - return fmt.Errorf("while abs path: %w", err) - } - - if hubpath != absPath { - log.Warningf("%s (%s) isn't a symlink to %s", target.Name, syml, absPath) - return fmt.Errorf("%s isn't managed by hub", target.Name) - } - - // remove the symlink - if err = os.Remove(syml); err != nil { - return fmt.Errorf("while removing symlink: %w", err) - } - - log.Infof("Removed symlink [%s] : %s", target.Name, syml) - } - - target.Installed = false - - if purge { - *target, err = purgeItem(hub, *target) - if err != nil { - return err - } - } - - hubIdx[target.Type][target.Name] = *target - - return nil -} - -// creates symlink between actual config file at hub.HubDir and hub.ConfigDir -// Handles collections recursively -func EnableItem(hub *csconfig.Hub, target *Item) error { - var err error - - parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/") - - // create directories if needed - if target.Installed { - if target.Tainted { - return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) - } - - if target.Local { - return fmt.Errorf("%s is local, won't enable", target.Name) - } - - // if it's a collection, check sub-items even if the collection file itself is up-to-date - if target.UpToDate && target.Type != COLLECTIONS { - log.Tracef("%s is installed and up-to-date, skip.", target.Name) - return nil - } - } - - if _, err = os.Stat(parentDir); os.IsNotExist(err) { - log.Infof("%s doesn't exist, create", parentDir) - - if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { - return fmt.Errorf("while creating directory: %w", err) - } - } - - // install sub-items if it's a collection - if target.Type == COLLECTIONS { - for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - val, ok := hubIdx[ptrtype][p] - if !ok { - return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) - } - - err = EnableItem(hub, &val) - if err != nil { - return fmt.Errorf("while installing %s: %w", p, err) - } - } - } - } - - // check if file already exists where it should in configdir (eg /etc/crowdsec/collections/) - if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) { - log.Infof("%s already exists.", parentDir+"/"+target.FileName) - return nil - } - - // hub.ConfigDir + target.RemotePath - srcPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath) - if err != nil { - return fmt.Errorf("while getting source path: %w", err) - } - - dstPath, err := filepath.Abs(parentDir + "/" + target.FileName) - if err != nil { - return fmt.Errorf("while getting destination path: %w", err) - } - - if err = os.Symlink(srcPath, dstPath); err != nil { - return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err) - } - - log.Infof("Enabled %s : %s", target.Type, target.Name) - target.Installed = true - hubIdx[target.Type][target.Name] = *target - - return nil -} diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go new file mode 100644 index 000000000..9fc528590 --- /dev/null +++ b/pkg/cwhub/items.go @@ -0,0 +1,383 @@ +package cwhub + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/enescakir/emoji" + log "github.com/sirupsen/logrus" +) + +const ( + // managed item types. + COLLECTIONS = "collections" + PARSERS = "parsers" + POSTOVERFLOWS = "postoverflows" + SCENARIOS = "scenarios" +) + +const ( + versionUpToDate = iota // the latest version from index is installed + versionUpdateAvailable // not installed, or lower than latest + versionUnknown // local file with no version, or invalid version number + versionFuture // local version is higher latest, but is included in the index: should not happen +) + +var ( + // The order is important, as it is used to range over sub-items in collections. + ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} +) + +type HubItems map[string]map[string]*Item + +// ItemVersion is used to detect the version of a given item +// by comparing the hash of each version to the local file. +// If the item does not match any known version, it is considered tainted (modified). +type ItemVersion struct { + Digest string `json:"digest,omitempty" yaml:"digest,omitempty"` + Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"` +} + +// ItemState is used to keep the local state (i.e. at runtime) of an item. +// This data is not stored in the index, but is displayed with "cscli ... inspect". +type ItemState struct { + LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` + LocalVersion string `json:"local_version,omitempty" yaml:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty" yaml:"local_hash,omitempty"` + Installed bool `json:"installed"` + Downloaded bool `json:"downloaded"` + UpToDate bool `json:"up_to_date"` + Tainted bool `json:"tainted"` + BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` +} + +// Item is created from an index file and enriched with local info. +type Item struct { + hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods + + State ItemState `json:"-" yaml:"-"` // local state, not stored in the index + + Type string `json:"type,omitempty" yaml:"type,omitempty"` // one of the ItemTypes + Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... + Name string `json:"name,omitempty" yaml:"name,omitempty"` // usually "author/name" + FileName string `json:"file_name,omitempty" yaml:"file_name,omitempty"` // eg. apache2-logs.yaml + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Author string `json:"author,omitempty" yaml:"author,omitempty"` + References []string `json:"references,omitempty" yaml:"references,omitempty"` + + RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml + Version string `json:"version,omitempty" yaml:"version,omitempty"` // the last available version + Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // all the known versions + + // if it's a collection, it can have sub items + Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` + PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` + Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` + Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` +} + +// HasSubItems returns true if items of this type can have sub-items. Currently only collections. +func (i *Item) HasSubItems() bool { + return i.Type == COLLECTIONS +} + +// IsLocal returns true if the item has been create by a user (not downloaded from the hub). +func (i *Item) IsLocal() bool { + return i.State.Installed && !i.State.Downloaded +} + +// MarshalJSON is used to prepare the output for "cscli ... inspect -o json". +// It must not use a pointer receiver. +func (i Item) MarshalJSON() ([]byte, error) { + type Alias Item + + return json.Marshal(&struct { + Alias + // we have to repeat the fields here, json will have inline support in v2 + LocalPath string `json:"local_path,omitempty"` + LocalVersion string `json:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty"` + Installed bool `json:"installed"` + Downloaded bool `json:"downloaded"` + UpToDate bool `json:"up_to_date"` + Tainted bool `json:"tainted"` + Local bool `json:"local"` + BelongsToCollections []string `json:"belongs_to_collections,omitempty"` + }{ + Alias: Alias(i), + LocalPath: i.State.LocalPath, + LocalVersion: i.State.LocalVersion, + LocalHash: i.State.LocalHash, + Installed: i.State.Installed, + Downloaded: i.State.Downloaded, + UpToDate: i.State.UpToDate, + Tainted: i.State.Tainted, + BelongsToCollections: i.State.BelongsToCollections, + Local: i.IsLocal(), + }) +} + +// MarshalYAML is used to prepare the output for "cscli ... inspect -o raw". +// It must not use a pointer receiver. +func (i Item) MarshalYAML() (interface{}, error) { + type Alias Item + + return &struct { + Alias `yaml:",inline"` + State ItemState `yaml:",inline"` + Local bool `yaml:"local"` + }{ + Alias: Alias(i), + State: i.State, + Local: i.IsLocal(), + }, nil +} + +// SubItems returns a slice of sub-items, excluding the ones that were not found. +func (i *Item) SubItems() []*Item { + sub := make([]*Item, 0) + + for _, name := range i.Parsers { + s := i.hub.GetItem(PARSERS, name) + if s == nil { + continue + } + + sub = append(sub, s) + } + + for _, name := range i.PostOverflows { + s := i.hub.GetItem(POSTOVERFLOWS, name) + if s == nil { + continue + } + + sub = append(sub, s) + } + + for _, name := range i.Scenarios { + s := i.hub.GetItem(SCENARIOS, name) + if s == nil { + continue + } + + sub = append(sub, s) + } + + for _, name := range i.Collections { + s := i.hub.GetItem(COLLECTIONS, name) + if s == nil { + continue + } + + sub = append(sub, s) + } + + return sub +} + +func (i *Item) logMissingSubItems() { + if !i.HasSubItems() { + return + } + + for _, subName := range i.Parsers { + if i.hub.GetItem(PARSERS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name) + } + } + + for _, subName := range i.Scenarios { + if i.hub.GetItem(SCENARIOS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name) + } + } + + for _, subName := range i.PostOverflows { + if i.hub.GetItem(POSTOVERFLOWS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name) + } + } + + for _, subName := range i.Collections { + if i.hub.GetItem(COLLECTIONS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name) + } + } +} + +// Ancestors returns a slice of items (typically collections) that have this item as a direct or indirect dependency. +func (i *Item) Ancestors() []*Item { + ret := make([]*Item, 0) + + for _, parentName := range i.State.BelongsToCollections { + parent := i.hub.GetItem(COLLECTIONS, parentName) + if parent == nil { + continue + } + + ret = append(ret, parent) + } + + return ret +} + +// InstallStatus returns the status of the item as a string and an emoji +// (eg. "enabled,update-available" and emoji.Warning). +func (i *Item) InstallStatus() (string, emoji.Emoji) { + status := "disabled" + ok := false + + if i.State.Installed { + ok = true + status = "enabled" + } + + managed := true + if i.IsLocal() { + managed = false + status += ",local" + } + + warning := false + if i.State.Tainted { + warning = true + status += ",tainted" + } else if !i.State.UpToDate && !i.IsLocal() { + warning = true + status += ",update-available" + } + + emo := emoji.QuestionMark + + switch { + case !managed: + emo = emoji.House + case !i.State.Installed: + emo = emoji.Prohibited + case warning: + emo = emoji.Warning + case ok: + emo = emoji.CheckMark + } + + return status, emo +} + +// versionStatus returns the status of the item version compared to the hub version. +// semver requires the 'v' prefix. +func (i *Item) versionStatus() int { + local, err := semver.NewVersion(i.State.LocalVersion) + if err != nil { + return versionUnknown + } + + // hub versions are already validated while syncing, ignore errors + latest, _ := semver.NewVersion(i.Version) + + if local.LessThan(latest) { + return versionUpdateAvailable + } + + if local.Equal(latest) { + return versionUpToDate + } + + return versionFuture +} + +// validPath returns true if the (relative) path is allowed for the item. +// dirNname: the directory name (ie. crowdsecurity). +// fileName: the filename (ie. apache2-logs.yaml). +func (i *Item) validPath(dirName, fileName string) bool { + return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml") +} + +// GetItemMap returns the map of items for a given type. +func (h *Hub) GetItemMap(itemType string) map[string]*Item { + return h.Items[itemType] +} + +// GetItem returns an item from hub based on its type and full name (author/name). +func (h *Hub) GetItem(itemType string, itemName string) *Item { + return h.GetItemMap(itemType)[itemName] +} + +// GetItemNames returns a slice of (full) item names for a given type +// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx). +func (h *Hub) GetItemNames(itemType string) []string { + m := h.GetItemMap(itemType) + if m == nil { + return nil + } + + names := make([]string, 0, len(m)) + for k := range m { + names = append(names, k) + } + + return names +} + +// GetAllItems returns a slice of all the items of a given type, installed or not. +func (h *Hub) GetAllItems(itemType string) ([]*Item, error) { + items, ok := h.Items[itemType] + if !ok { + return nil, fmt.Errorf("no %s in the hub index", itemType) + } + + ret := make([]*Item, len(items)) + + idx := 0 + + for _, item := range items { + ret[idx] = item + idx++ + } + + return ret, nil +} + +// GetInstalledItems returns a slice of the installed items of a given type. +func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { + items, ok := h.Items[itemType] + if !ok { + return nil, fmt.Errorf("no %s in the hub index", itemType) + } + + retItems := make([]*Item, 0) + + for _, item := range items { + if item.State.Installed { + retItems = append(retItems, item) + } + } + + return retItems, nil +} + +// GetInstalledItemNames returns the names of the installed items of a given type. +func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) { + items, err := h.GetInstalledItems(itemType) + if err != nil { + return nil, err + } + + retStr := make([]string, len(items)) + + for idx, it := range items { + retStr[idx] = it.Name + } + + return retStr, nil +} + +// SortItemSlice sorts a slice of items by name, case insensitive. +func SortItemSlice(items []*Item) { + sort.Slice(items, func(i, j int) bool { + return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) + }) +} diff --git a/pkg/cwhub/items_test.go b/pkg/cwhub/items_test.go new file mode 100644 index 000000000..a4c2bc0a0 --- /dev/null +++ b/pkg/cwhub/items_test.go @@ -0,0 +1,71 @@ +package cwhub + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestItemStatus(t *testing.T) { + hub := envSetup(t) + + // get existing map + x := hub.GetItemMap(COLLECTIONS) + require.NotEmpty(t, x) + + // Get item: good and bad + for k := range x { + item := hub.GetItem(COLLECTIONS, k) + require.NotNil(t, item) + + item.State.Installed = true + item.State.UpToDate = false + item.State.Tainted = false + item.State.Downloaded = true + + txt, _ := item.InstallStatus() + require.Equal(t, "enabled,update-available", txt) + + item.State.Installed = true + item.State.UpToDate = false + item.State.Tainted = false + item.State.Downloaded = false + + txt, _ = item.InstallStatus() + require.Equal(t, "enabled,local", txt) + } + + stats := hub.ItemStats() + require.Equal(t, []string{ + "Loaded: 2 parsers, 1 scenarios, 3 collections", + "Unmanaged items: 3 local, 0 tainted", + }, stats) +} + +func TestGetters(t *testing.T) { + hub := envSetup(t) + + // get non existing map + empty := hub.GetItemMap("ratata") + require.Nil(t, empty) + + // get existing map + x := hub.GetItemMap(COLLECTIONS) + require.NotEmpty(t, x) + + // Get item: good and bad + for k := range x { + empty := hub.GetItem(COLLECTIONS, k+"nope") + require.Nil(t, empty) + + item := hub.GetItem(COLLECTIONS, k) + require.NotNil(t, item) + + // Add item and get it + item.Name += "nope" + hub.Items[item.Type][item.Name] = item + + newitem := hub.GetItem(COLLECTIONS, item.Name) + require.NotNil(t, newitem) + } +} diff --git a/pkg/cwhub/leakybucket.go b/pkg/cwhub/leakybucket.go new file mode 100644 index 000000000..8143e9433 --- /dev/null +++ b/pkg/cwhub/leakybucket.go @@ -0,0 +1,53 @@ +package cwhub + +// Resolve a symlink to find the hub item it points to. +// This file is used only by pkg/leakybucket + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary. +func itemKey(itemPath string) (string, error) { + f, err := os.Lstat(itemPath) + if err != nil { + return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err) + } + + if f.Mode()&os.ModeSymlink == 0 { + // it's not a symlink, so the filename itsef should be the key + return filepath.Base(itemPath), nil + } + + // resolve the symlink to hub file + pathInHub, err := os.Readlink(itemPath) + if err != nil { + return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err) + } + + author := filepath.Base(filepath.Dir(pathInHub)) + + fname := filepath.Base(pathInHub) + fname = strings.TrimSuffix(fname, ".yaml") + fname = strings.TrimSuffix(fname, ".yml") + + return fmt.Sprintf("%s/%s", author, fname), nil +} + +// GetItemByPath retrieves an item from the hub index based on its local path. +func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) { + itemKey, err := itemKey(itemPath) + if err != nil { + return nil, err + } + + item := h.GetItem(itemType, itemKey) + if item == nil { + return nil, fmt.Errorf("%s not found in %s", itemKey, itemType) + } + + return item, nil +} diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go deleted file mode 100644 index 3c4566493..000000000 --- a/pkg/cwhub/loader.go +++ /dev/null @@ -1,552 +0,0 @@ -package cwhub - -import ( - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "sort" - "strings" - - log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" -) - -func isYAMLFileName(path string) bool { - return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") -} - -func validItemFileName(vname string, fauthor string, fname string) bool { - return (fauthor+"/"+fname == vname+".yaml") || (fauthor+"/"+fname == vname+".yml") -} - -func handleSymlink(path string) (string, error) { - hubpath, err := os.Readlink(path) - if err != nil { - return "", fmt.Errorf("unable to read symlink of %s", path) - } - // the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml - _, err = os.Lstat(hubpath) - if os.IsNotExist(err) { - log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath) - // remove the symlink - if err = os.Remove(path); err != nil { - return "", fmt.Errorf("failed to unlink %s: %w", path, err) - } - - // XXX: is this correct? - return "", nil - } - - return hubpath, nil -} - -func getSHA256(filepath string) (string, error) { - f, err := os.Open(filepath) - if err != nil { - return "", fmt.Errorf("unable to open '%s': %w", filepath, err) - } - - defer f.Close() - - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err) - } - - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - -type Walker struct { - // the walk/parserVisit function can't receive extra args - hubdir string - installdir string -} - -func NewWalker(hub *csconfig.Hub) Walker { - return Walker{ - hubdir: hub.HubDir, - installdir: hub.InstallDir, - } -} - -type itemFileInfo struct { - fname string - stage string - ftype string - fauthor string -} - -func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { - ret := itemFileInfo{} - inhub := false - - subs := strings.Split(path, string(os.PathSeparator)) - - log.Tracef("path:%s, hubdir:%s, installdir:%s", path, w.hubdir, w.installdir) - log.Tracef("subs:%v", subs) - // we're in hub (~/.hub/hub/) - if strings.HasPrefix(path, w.hubdir) { - log.Tracef("in hub dir") - - inhub = true - //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml - //.../hub/scenarios/crowdsec/ssh_bf.yaml - //.../hub/profiles/crowdsec/linux.yaml - if len(subs) < 4 { - log.Fatalf("path is too short : %s (%d)", path, len(subs)) - } - - ret.fname = subs[len(subs)-1] - ret.fauthor = subs[len(subs)-2] - ret.stage = subs[len(subs)-3] - ret.ftype = subs[len(subs)-4] - } else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec//... - log.Tracef("in install dir") - if len(subs) < 3 { - log.Fatalf("path is too short : %s (%d)", path, len(subs)) - } - ///.../config/parser/stage/file.yaml - ///.../config/postoverflow/stage/file.yaml - ///.../config/scenarios/scenar.yaml - ///.../config/collections/linux.yaml //file is empty - ret.fname = subs[len(subs)-1] - ret.stage = subs[len(subs)-2] - ret.ftype = subs[len(subs)-3] - ret.fauthor = "" - } else { - return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, w.hubdir, w.installdir) - } - - log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) - // log.Infof("%s -> name:%s stage:%s", path, fname, stage) - - if ret.stage == SCENARIOS { - ret.ftype = SCENARIOS - ret.stage = "" - } else if ret.stage == COLLECTIONS { - ret.ftype = COLLECTIONS - ret.stage = "" - } else if ret.ftype != PARSERS && ret.ftype != PARSERS_OVFLW { - // its a PARSER / PARSER_OVFLW with a stage - return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path) - } - - log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype) - - return ret, inhub, nil -} - -func (w Walker) parserVisit(path string, f os.DirEntry, err error) error { - var ( - local bool - hubpath string - ) - - if err != nil { - log.Debugf("while syncing hub dir: %s", err) - // there is a path error, we ignore the file - return nil - } - - path, err = filepath.Abs(path) - if err != nil { - return err - } - - // we only care about files - if f == nil || f.IsDir() { - return nil - } - - if !isYAMLFileName(f.Name()) { - return nil - } - - info, inhub, err := w.getItemInfo(path) - if err != nil { - return err - } - - /* - we can encounter 'collections' in the form of a symlink : - /etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml - when the collection is installed, both files are created - */ - // non symlinks are local user files or hub files - if f.Type()&os.ModeSymlink == 0 { - local = true - - log.Tracef("%s isn't a symlink", path) - } else { - hubpath, err = handleSymlink(path) - if err != nil { - return err - } - log.Tracef("%s points to %s", path, hubpath) - - if hubpath == "" { - // XXX: is this correct? - return nil - } - } - - // if it's not a symlink and not in hub, it's a local file, don't bother - if local && !inhub { - log.Tracef("%s is a local file, skip", path) - skippedLocal++ - // log.Infof("local scenario, skip.") - - _, fileName := filepath.Split(path) - - hubIdx[info.ftype][info.fname] = Item{ - Name: info.fname, - Stage: info.stage, - Installed: true, - Type: info.ftype, - Local: true, - LocalPath: path, - UpToDate: true, - FileName: fileName, - } - - return nil - } - - // try to find which configuration item it is - log.Tracef("check [%s] of %s", info.fname, info.ftype) - - match := false - - for name, item := range hubIdx[info.ftype] { - log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml") - - if info.fname != item.FileName { - log.Tracef("%s != %s (filename)", info.fname, item.FileName) - continue - } - - // wrong stage - if item.Stage != info.stage { - continue - } - - // if we are walking hub dir, just mark present files as downloaded - if inhub { - // wrong author - if info.fauthor != item.Author { - continue - } - - // wrong file - if !validItemFileName(item.Name, info.fauthor, info.fname) { - continue - } - - if path == w.hubdir+"/"+item.RemotePath { - log.Tracef("marking %s as downloaded", item.Name) - item.Downloaded = true - } - } else if !hasPathSuffix(hubpath, item.RemotePath) { - // wrong file - // ///.yaml - continue - } - - sha, err := getSHA256(path) - if err != nil { - log.Fatalf("Failed to get sha of %s : %v", path, err) - } - - // let's reverse sort the versions to deal with hash collisions (#154) - versions := make([]string, 0, len(item.Versions)) - for k := range item.Versions { - versions = append(versions, k) - } - - sort.Sort(sort.Reverse(sort.StringSlice(versions))) - - for _, version := range versions { - val := item.Versions[version] - if sha != val.Digest { - // log.Infof("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v)) - continue - } - - // we got an exact match, update struct - - item.Downloaded = true - item.LocalHash = sha - - if !inhub { - log.Tracef("found exact match for %s, version is %s, latest is %s", item.Name, version, item.Version) - item.LocalPath = path - item.LocalVersion = version - item.Tainted = false - // if we're walking the hub, present file doesn't means installed file - item.Installed = true - } - - if version == item.Version { - log.Tracef("%s is up-to-date", item.Name) - item.UpToDate = true - } - - match = true - - break - } - - if !match { - log.Tracef("got tainted match for %s: %s", item.Name, path) - - skippedTainted++ - // the file and the stage is right, but the hash is wrong, it has been tainted by user - if !inhub { - item.LocalPath = path - item.Installed = true - } - - item.UpToDate = false - item.LocalVersion = "?" - item.Tainted = true - item.LocalHash = sha - } - - // update the entry if appropriate - // if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D { - // fmt.Printf("Updating %s", k) - // hubIdx[ftype][k] = v - // } else if !inhub { - - // } else if - hubIdx[info.ftype][name] = item - - return nil - } - - log.Infof("Ignoring file %s of type %s", path, info.ftype) - - return nil -} - -func CollecDepsCheck(v *Item) error { - if v.versionStatus() != 0 { // not up-to-date - log.Debugf("%s dependencies not checked : not up-to-date", v.Name) - return nil - } - - if v.Type != COLLECTIONS { - return nil - } - - // if it's a collection, ensure all the items are installed, or tag it as tainted - log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) - - for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} { - sliceType := ItemTypes[idx] - for _, subName := range itemSlice { - subItem, ok := hubIdx[sliceType][subName] - if !ok { - log.Fatalf("Referred %s %s in collection %s doesn't exist.", sliceType, subName, v.Name) - } - - log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed) - - if !v.Installed { - continue - } - - if subItem.Type == COLLECTIONS { - log.Tracef("collec, recurse.") - - if err := CollecDepsCheck(&subItem); err != nil { - if subItem.Tainted { - v.Tainted = true - } - - return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) - } - - hubIdx[sliceType][subName] = subItem - } - - // propagate the state of sub-items to set - if subItem.Tainted { - v.Tainted = true - return fmt.Errorf("tainted %s %s, tainted", sliceType, subName) - } - - if !subItem.Installed && v.Installed { - v.Tainted = true - return fmt.Errorf("missing %s %s, tainted", sliceType, subName) - } - - if !subItem.UpToDate { - v.UpToDate = false - return fmt.Errorf("outdated %s %s", sliceType, subName) - } - - skip := false - - for idx := range subItem.BelongsToCollections { - if subItem.BelongsToCollections[idx] == v.Name { - skip = true - } - } - - if !skip { - subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) - } - - hubIdx[sliceType][subName] = subItem - - log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate) - } - } - - return nil -} - -func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) { - warnings := []string{} - - // For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last - for _, scan := range ItemTypes { - cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) - if err != nil { - log.Errorf("failed %s : %s", cpath, err) - } - - err = filepath.WalkDir(cpath, NewWalker(hub).parserVisit) - if err != nil { - return warnings, err - } - } - - for name, item := range hubIdx[COLLECTIONS] { - if !item.Installed { - continue - } - - vs := item.versionStatus() - switch vs { - case 0: // latest - if err := CollecDepsCheck(&item); err != nil { - warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) - hubIdx[COLLECTIONS][name] = item - } - case 1: // not up-to-date - warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) - default: // version is higher than the highest available from hub? - warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) - } - - log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions) - } - - return warnings, nil -} - -// Updates the info from HubInit() with the local state -func LocalSync(hub *csconfig.Hub) ([]string, error) { - skippedLocal = 0 - skippedTainted = 0 - - warnings, err := SyncDir(hub, hub.InstallDir) - if err != nil { - return warnings, fmt.Errorf("failed to scan %s: %w", hub.InstallDir, err) - } - - _, err = SyncDir(hub, hub.HubDir) - if err != nil { - return warnings, fmt.Errorf("failed to scan %s: %w", hub.HubDir, err) - } - - return warnings, nil -} - -func GetHubIdx(hub *csconfig.Hub) error { - if hub == nil { - return fmt.Errorf("no configuration found for hub") - } - - log.Debugf("loading hub idx %s", hub.HubIndexFile) - - bidx, err := os.ReadFile(hub.HubIndexFile) - if err != nil { - return fmt.Errorf("unable to read index file: %w", err) - } - - ret, err := LoadPkgIndex(bidx) - if err != nil { - if !errors.Is(err, ErrMissingReference) { - return fmt.Errorf("unable to load existing index: %w", err) - } - - // XXX: why the error check if we bail out anyway? - return err - } - - hubIdx = ret - - _, err = LocalSync(hub) - if err != nil { - return fmt.Errorf("failed to sync Hub index with local deployment : %w", err) - } - - return nil -} - -// LoadPkgIndex loads a local .index.json file and returns the map of associated parsers/scenarios/collections -func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) { - var ( - RawIndex map[string]map[string]Item - missingItems []string - ) - - if err := json.Unmarshal(buff, &RawIndex); err != nil { - return nil, fmt.Errorf("failed to unmarshal index: %w", err) - } - - log.Debugf("%d item types in hub index", len(ItemTypes)) - - // Iterate over the different types to complete the struct - for _, itemType := range ItemTypes { - log.Tracef("%d item", len(RawIndex[itemType])) - - for name, item := range RawIndex[itemType] { - item.Name = name - item.Type = itemType - x := strings.Split(item.RemotePath, "/") - item.FileName = x[len(x)-1] - RawIndex[itemType][name] = item - - if itemType != COLLECTIONS { - continue - } - - // if it's a collection, check its sub-items are present - // XXX should be done later - for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if _, ok := RawIndex[ptrtype][p]; !ok { - log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name) - missingItems = append(missingItems, p) - } - } - } - } - } - - if len(missingItems) > 0 { - return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference) - } - - return RawIndex, nil -} diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go new file mode 100644 index 000000000..c1eb5a708 --- /dev/null +++ b/pkg/cwhub/remote.go @@ -0,0 +1,61 @@ +package cwhub + +import ( + "fmt" + "io" + "net/http" +) + +// RemoteHubCfg is used to retrieve index and items from the remote hub. +type RemoteHubCfg struct { + Branch string + URLTemplate string + IndexPath string +} + +// urlTo builds the URL to download a file from the remote hub. +func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { + if r == nil { + return "", ErrNilRemoteHub + } + + // the template must contain two string placeholders + if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate { + return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate) + } + + return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil +} + +// fetchIndex downloads the index from the hub and returns the content. +func (r *RemoteHubCfg) fetchIndex() ([]byte, error) { + if r == nil { + return nil, ErrNilRemoteHub + } + + url, err := r.urlTo(r.IndexPath) + if err != nil { + return nil, fmt.Errorf("failed to build hub index request: %w", err) + } + + resp, err := hubClient.Get(url) + if err != nil { + return nil, fmt.Errorf("failed http request for hub index: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return nil, IndexNotFoundError{url, r.Branch} + } + + return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) + } + + return body, nil +} diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go new file mode 100644 index 000000000..bc64cd01c --- /dev/null +++ b/pkg/cwhub/sync.go @@ -0,0 +1,498 @@ +package cwhub + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/Masterminds/semver/v3" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" +) + +func isYAMLFileName(path string) bool { + return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") +} + +// linkTarget returns the target of a symlink, or empty string if it's dangling. +func linkTarget(path string) (string, error) { + hubpath, err := os.Readlink(path) + if err != nil { + return "", fmt.Errorf("unable to read symlink: %s", path) + } + + log.Tracef("symlink %s -> %s", path, hubpath) + + _, err = os.Lstat(hubpath) + if os.IsNotExist(err) { + log.Infof("link target does not exist: %s -> %s", path, hubpath) + return "", nil + } + + return hubpath, nil +} + +func getSHA256(filepath string) (string, error) { + f, err := os.Open(filepath) + if err != nil { + return "", fmt.Errorf("unable to open '%s': %w", filepath, err) + } + + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err) + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + +// information used to create a new Item, from a file path. +type itemFileInfo struct { + inhub bool + fname string + stage string + ftype string + fauthor string +} + +func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) { + var ret *itemFileInfo + + hubDir := h.local.HubDir + installDir := h.local.InstallDir + + subs := strings.Split(path, string(os.PathSeparator)) + + log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir) + log.Tracef("subs:%v", subs) + // we're in hub (~/.hub/hub/) + if strings.HasPrefix(path, hubDir) { + log.Tracef("in hub dir") + + //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml + //.../hub/scenarios/crowdsec/ssh_bf.yaml + //.../hub/profiles/crowdsec/linux.yaml + if len(subs) < 4 { + return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) + } + + ret = &itemFileInfo{ + inhub: true, + fname: subs[len(subs)-1], + fauthor: subs[len(subs)-2], + stage: subs[len(subs)-3], + ftype: subs[len(subs)-4], + } + } else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec//... + log.Tracef("in install dir") + if len(subs) < 3 { + return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) + } + ///.../config/parser/stage/file.yaml + ///.../config/postoverflow/stage/file.yaml + ///.../config/scenarios/scenar.yaml + ///.../config/collections/linux.yaml //file is empty + ret = &itemFileInfo{ + inhub: false, + fname: subs[len(subs)-1], + stage: subs[len(subs)-2], + ftype: subs[len(subs)-3], + fauthor: "", + } + } else { + return nil, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir) + } + + log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) + + if ret.stage == SCENARIOS { + ret.ftype = SCENARIOS + ret.stage = "" + } else if ret.stage == COLLECTIONS { + ret.ftype = COLLECTIONS + ret.stage = "" + } else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { + // it's a PARSER / POSTOVERFLOW with a stage + return nil, fmt.Errorf("unknown configuration type for file '%s'", path) + } + + log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype) + + return ret, nil +} + +// sortedVersions returns the input data, sorted in reverse order (new, old) by semver. +func sortedVersions(raw []string) ([]string, error) { + vs := make([]*semver.Version, len(raw)) + + for idx, r := range raw { + v, err := semver.NewVersion(r) + if err != nil { + return nil, fmt.Errorf("%s: %w", r, err) + } + + vs[idx] = v + } + + sort.Sort(sort.Reverse(semver.Collection(vs))) + + ret := make([]string, len(vs)) + for idx, v := range vs { + ret[idx] = v.Original() + } + + return ret, nil +} + +func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) { + type localItemName struct { + Name string `yaml:"name"` + } + + _, fileName := filepath.Split(path) + + item := &Item{ + hub: h, + Name: info.fname, + Stage: info.stage, + Type: info.ftype, + FileName: fileName, + State: ItemState{ + LocalPath: path, + Installed: true, + UpToDate: true, + }, + } + + // try to read the name from the file + itemName := localItemName{} + + itemContent, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", path, err) + } + + err = yaml.Unmarshal(itemContent, &itemName) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal %s: %w", path, err) + } + + if itemName.Name != "" { + item.Name = itemName.Name + } + + return item, nil +} + +func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { + hubpath := "" + + if err != nil { + log.Debugf("while syncing hub dir: %s", err) + // there is a path error, we ignore the file + return nil + } + + // only happens if the current working directory was removed (!) + path, err = filepath.Abs(path) + if err != nil { + return err + } + + // we only care about YAML files + if f == nil || f.IsDir() || !isYAMLFileName(f.Name()) { + return nil + } + + info, err := h.getItemFileInfo(path) + if err != nil { + return err + } + + // non symlinks are local user files or hub files + if f.Type()&os.ModeSymlink == 0 { + log.Tracef("%s is not a symlink", path) + + if !info.inhub { + log.Tracef("%s is a local file, skip", path) + item, err := newLocalItem(h, path, info) + if err != nil { + return err + } + h.Items[info.ftype][item.Name] = item + + return nil + } + } else { + hubpath, err = linkTarget(path) + if err != nil { + return err + } + + if hubpath == "" { + // target does not exist, the user might have removed the file + // or switched to a hub branch without it + return nil + } + } + + // try to find which configuration item it is + log.Tracef("check [%s] of %s", info.fname, info.ftype) + + for name, item := range h.Items[info.ftype] { + if info.fname != item.FileName { + continue + } + + if item.Stage != info.stage { + continue + } + + // if we are walking hub dir, just mark present files as downloaded + if info.inhub { + // wrong author + if info.fauthor != item.Author { + continue + } + + // not the item we're looking for + if !item.validPath(info.fauthor, info.fname) { + continue + } + + src, err := item.downloadPath() + if err != nil { + return err + } + + if path == src { + log.Tracef("marking %s as downloaded", item.Name) + item.State.Downloaded = true + } + } else if !hasPathSuffix(hubpath, item.RemotePath) { + // wrong file + // ///.yaml + continue + } + + err := item.setVersionState(path, info.inhub) + if err != nil { + return err + } + + h.Items[info.ftype][name] = item + + return nil + } + + log.Infof("Ignoring file %s of type %s", path, info.ftype) + + return nil +} + +// checkSubItemVersions checks for the presence, taint and version state of sub-items. +func (i *Item) checkSubItemVersions() error { + if !i.HasSubItems() { + return nil + } + + if i.versionStatus() != versionUpToDate { + log.Debugf("%s dependencies not checked: not up-to-date", i.Name) + return nil + } + + // ensure all the sub-items are installed, or tag the parent as tainted + log.Tracef("checking submembers of %s installed:%t", i.Name, i.State.Installed) + + for _, sub := range i.SubItems() { + log.Tracef("check %s installed:%t", sub.Name, sub.State.Installed) + + if !i.State.Installed { + continue + } + + if err := sub.checkSubItemVersions(); err != nil { + if sub.State.Tainted { + i.State.Tainted = true + } + + return fmt.Errorf("dependency of %s: sub collection %s is broken: %w", i.Name, sub.Name, err) + } + + if sub.State.Tainted { + i.State.Tainted = true + return fmt.Errorf("%s is tainted because %s:%s is tainted", i.Name, sub.Type, sub.Name) + } + + if !sub.State.Installed && i.State.Installed { + i.State.Tainted = true + return fmt.Errorf("%s is tainted because %s:%s is missing", i.Name, sub.Type, sub.Name) + } + + if !sub.State.UpToDate { + i.State.UpToDate = false + return fmt.Errorf("dependency of %s: outdated %s:%s", i.Name, sub.Type, sub.Name) + } + + log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, i.State.Tainted, i.State.UpToDate) + } + + return nil +} + +// syncDir scans a directory for items, and updates the Hub state accordingly. +func (h *Hub) syncDir(dir string) error { + // For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last + for _, scan := range ItemTypes { + // cpath: top-level item directory, either downloaded or installed items. + // i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ... + cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) + if err != nil { + log.Errorf("failed %s: %s", cpath, err) + continue + } + + // explicit check for non existing directory, avoid spamming log.Debug + if _, err = os.Stat(cpath); os.IsNotExist(err) { + log.Tracef("directory %s doesn't exist, skipping", cpath) + continue + } + + if err = filepath.WalkDir(cpath, h.itemVisit); err != nil { + return err + } + } + + return nil +} + +// insert a string in a sorted slice, case insensitive, and return the new slice. +func insertInOrderNoCase(sl []string, value string) []string { + i := sort.Search(len(sl), func(i int) bool { + return strings.ToLower(sl[i]) >= strings.ToLower(value) + }) + + return append(sl[:i], append([]string{value}, sl[i:]...)...) +} + +// localSync updates the hub state with downloaded, installed and local items. +func (h *Hub) localSync() error { + err := h.syncDir(h.local.InstallDir) + if err != nil { + return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err) + } + + if err = h.syncDir(h.local.HubDir); err != nil { + return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err) + } + + warnings := make([]string, 0) + + for _, item := range h.Items[COLLECTIONS] { + // check for cyclic dependencies + subs, err := item.descendants() + if err != nil { + return err + } + + // populate the sub- and sub-sub-items with the collections they belong to + for _, sub := range subs { + sub.State.BelongsToCollections = insertInOrderNoCase(sub.State.BelongsToCollections, item.Name) + } + + if !item.State.Installed { + continue + } + + vs := item.versionStatus() + switch vs { + case versionUpToDate: // latest + if err := item.checkSubItemVersions(); err != nil { + warnings = append(warnings, err.Error()) + } + case versionUpdateAvailable: // not up-to-date + warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version)) + case versionFuture: + warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version)) + case versionUnknown: + if !item.IsLocal() { + warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version)) + } + } + + log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions) + } + + h.Warnings = warnings + + return nil +} + +func (i *Item) setVersionState(path string, inhub bool) error { + var err error + + i.State.LocalHash, err = getSHA256(path) + if err != nil { + return fmt.Errorf("failed to get sha256 of %s: %w", path, err) + } + + // let's reverse sort the versions to deal with hash collisions (#154) + versions := make([]string, 0, len(i.Versions)) + for k := range i.Versions { + versions = append(versions, k) + } + + versions, err = sortedVersions(versions) + if err != nil { + return fmt.Errorf("while syncing %s %s: %w", i.Type, i.FileName, err) + } + + i.State.LocalVersion = "?" + + for _, version := range versions { + if i.Versions[version].Digest == i.State.LocalHash { + i.State.LocalVersion = version + break + } + } + + if i.State.LocalVersion == "?" { + log.Tracef("got tainted match for %s: %s", i.Name, path) + + if !inhub { + i.State.LocalPath = path + i.State.Installed = true + } + + i.State.UpToDate = false + i.State.Tainted = true + + return nil + } + + // we got an exact match, update struct + + i.State.Downloaded = true + + if !inhub { + log.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.State.LocalVersion, i.Version) + i.State.LocalPath = path + i.State.Tainted = false + // if we're walking the hub, present file doesn't means installed file + i.State.Installed = true + } + + if i.State.LocalVersion == i.Version { + log.Tracef("%s is up-to-date", i.Name) + i.State.UpToDate = true + } + + return nil +} diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index eeff24b57..5d3b79fe1 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -5,173 +5,194 @@ import ( "fmt" "os" "path/filepath" - "regexp" - "sort" "strings" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -type ParserCoverage struct { - Parser string +type Coverage struct { + Name string TestsCount int PresentIn map[string]bool //poorman's set } -type ScenarioCoverage struct { - Scenario string - TestsCount int - PresentIn map[string]bool -} +func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { + if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok { + return nil, fmt.Errorf("no parsers in hub index") + } -func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) { - var coverage []ParserCoverage - if _, ok := h.HubIndex.Data[cwhub.PARSERS]; !ok { - return coverage, fmt.Errorf("no parsers in hub index") - } - //populate from hub, iterate in alphabetical order - var pkeys []string - for pname := range h.HubIndex.Data[cwhub.PARSERS] { - pkeys = append(pkeys, pname) - } - sort.Strings(pkeys) - for _, pname := range pkeys { - coverage = append(coverage, ParserCoverage{ - Parser: pname, + // populate from hub, iterate in alphabetical order + pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.PARSERS]) + coverage := make([]Coverage, len(pkeys)) + + for i, name := range pkeys { + coverage[i] = Coverage{ + Name: name, TestsCount: 0, PresentIn: make(map[string]bool), - }) + } } - //parser the expressions a-la-oneagain + // parser the expressions a-la-oneagain passerts, err := filepath.Glob(".tests/*/parser.assert") if err != nil { - return coverage, fmt.Errorf("while find parser asserts : %s", err) + return nil, fmt.Errorf("while find parser asserts : %s", err) } + for _, assert := range passerts { file, err := os.Open(assert) if err != nil { - return coverage, fmt.Errorf("while reading %s : %s", assert, err) + return nil, fmt.Errorf("while reading %s : %s", assert, err) } + scanner := bufio.NewScanner(file) for scanner.Scan() { - assertLine := regexp.MustCompile(`^results\["[^"]+"\]\["(?P[^"]+)"\]\[[0-9]+\]\.Evt\..*`) line := scanner.Text() log.Debugf("assert line : %s", line) - match := assertLine.FindStringSubmatch(line) + + match := parserResultRE.FindStringSubmatch(line) if len(match) == 0 { log.Debugf("%s doesn't match", line) continue } - sidx := assertLine.SubexpIndex("parser") + + sidx := parserResultRE.SubexpIndex("parser") capturedParser := match[sidx] + for idx, pcover := range coverage { - if pcover.Parser == capturedParser { + if pcover.Name == capturedParser { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - parserNameSplit := strings.Split(pcover.Parser, "/") + + parserNameSplit := strings.Split(pcover.Name, "/") parserNameOnly := parserNameSplit[len(parserNameSplit)-1] + if parserNameOnly == capturedParser { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } + capturedParserSplit := strings.Split(capturedParser, "/") capturedParserName := capturedParserSplit[len(capturedParserSplit)-1] + if capturedParserName == parserNameOnly { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } + if capturedParserName == parserNameOnly+"-logs" { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } } } + file.Close() } + return coverage, nil } -func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) { - var coverage []ScenarioCoverage - if _, ok := h.HubIndex.Data[cwhub.SCENARIOS]; !ok { - return coverage, fmt.Errorf("no scenarios in hub index") - } - //populate from hub, iterate in alphabetical order - var pkeys []string - for scenarioName := range h.HubIndex.Data[cwhub.SCENARIOS] { - pkeys = append(pkeys, scenarioName) - } - sort.Strings(pkeys) - for _, scenarioName := range pkeys { - coverage = append(coverage, ScenarioCoverage{ - Scenario: scenarioName, - TestsCount: 0, - PresentIn: make(map[string]bool), - }) +func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { + if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok { + return nil, fmt.Errorf("no scenarios in hub index") } - //parser the expressions a-la-oneagain + // populate from hub, iterate in alphabetical order + pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.SCENARIOS]) + coverage := make([]Coverage, len(pkeys)) + + for i, name := range pkeys { + coverage[i] = Coverage{ + Name: name, + TestsCount: 0, + PresentIn: make(map[string]bool), + } + } + + // parser the expressions a-la-oneagain passerts, err := filepath.Glob(".tests/*/scenario.assert") if err != nil { - return coverage, fmt.Errorf("while find scenario asserts : %s", err) + return nil, fmt.Errorf("while find scenario asserts : %s", err) } + + for _, assert := range passerts { file, err := os.Open(assert) if err != nil { - return coverage, fmt.Errorf("while reading %s : %s", assert, err) + return nil, fmt.Errorf("while reading %s : %s", assert, err) } + scanner := bufio.NewScanner(file) for scanner.Scan() { - assertLine := regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P[^"]+)"`) line := scanner.Text() log.Debugf("assert line : %s", line) - match := assertLine.FindStringSubmatch(line) + match := scenarioResultRE.FindStringSubmatch(line) + if len(match) == 0 { log.Debugf("%s doesn't match", line) continue } - sidx := assertLine.SubexpIndex("scenario") - scanner_name := match[sidx] + + sidx := scenarioResultRE.SubexpIndex("scenario") + scannerName := match[sidx] + for idx, pcover := range coverage { - if pcover.Scenario == scanner_name { + if pcover.Name == scannerName { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - scenarioNameSplit := strings.Split(pcover.Scenario, "/") + + scenarioNameSplit := strings.Split(pcover.Name, "/") scenarioNameOnly := scenarioNameSplit[len(scenarioNameSplit)-1] - if scenarioNameOnly == scanner_name { + + if scenarioNameOnly == scannerName { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - fixedProbingWord := strings.ReplaceAll(pcover.Scenario, "probbing", "probing") - fixedProbingAssert := strings.ReplaceAll(scanner_name, "probbing", "probing") + + fixedProbingWord := strings.ReplaceAll(pcover.Name, "probbing", "probing") + fixedProbingAssert := strings.ReplaceAll(scannerName, "probbing", "probing") + if fixedProbingWord == fixedProbingAssert { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - if fmt.Sprintf("%s-detection", pcover.Scenario) == scanner_name { + + if fmt.Sprintf("%s-detection", pcover.Name) == scannerName { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } + if fmt.Sprintf("%s-detection", fixedProbingWord) == fixedProbingAssert { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } } } file.Close() } + return coverage, nil } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index c1aa4251c..ec1f6ee5e 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -18,7 +19,7 @@ type HubTest struct { TemplateConfigPath string TemplateProfilePath string TemplateSimulationPath string - HubIndex *HubIndex + HubIndex *cwhub.Hub Tests []*HubTestItem } @@ -29,42 +30,44 @@ const ( ) func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) { - var err error - - hubPath, err = filepath.Abs(hubPath) + hubPath, err := filepath.Abs(hubPath) if err != nil { return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err) } + // we can't use hubtest without the hub - if _, err := os.Stat(hubPath); os.IsNotExist(err) { + if _, err = os.Stat(hubPath); os.IsNotExist(err) { return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath) } + HubTestPath := filepath.Join(hubPath, "./.tests/") // we can't use hubtest without crowdsec binary - if _, err := exec.LookPath(crowdsecPath); err != nil { - if _, err := os.Stat(crowdsecPath); os.IsNotExist(err) { + if _, err = exec.LookPath(crowdsecPath); err != nil { + if _, err = os.Stat(crowdsecPath); os.IsNotExist(err) { return HubTest{}, fmt.Errorf("path to crowdsec binary '%s' doesn't exist or is not in $PATH, can't run", crowdsecPath) } } // we can't use hubtest without cscli binary - if _, err := exec.LookPath(cscliPath); err != nil { - if _, err := os.Stat(cscliPath); os.IsNotExist(err) { + if _, err = exec.LookPath(cscliPath); err != nil { + if _, err = os.Stat(cscliPath); os.IsNotExist(err) { return HubTest{}, fmt.Errorf("path to cscli binary '%s' doesn't exist or is not in $PATH, can't run", cscliPath) } } hubIndexFile := filepath.Join(hubPath, ".index.json") - bidx, err := os.ReadFile(hubIndexFile) - if err != nil { - return HubTest{}, fmt.Errorf("unable to read index file: %s", err) + + local := &csconfig.LocalHubCfg{ + HubDir: hubPath, + HubIndexFile: hubIndexFile, + InstallDir: HubTestPath, + InstallDataDir: HubTestPath, } - // load hub index - hubIndex, err := cwhub.LoadPkgIndex(bidx) + hub, err := cwhub.NewHub(local, nil, false) if err != nil { - return HubTest{}, fmt.Errorf("unable to load hub index file: %s", err) + return HubTest{}, fmt.Errorf("unable to load hub: %s", err) } templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile) @@ -80,16 +83,18 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, TemplateConfigPath: templateConfigFilePath, TemplateProfilePath: templateProfilePath, TemplateSimulationPath: templateSimulationPath, - HubIndex: &HubIndex{Data: hubIndex}, + HubIndex: hub, }, nil } func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) { HubTestItem := &HubTestItem{} + testItem, err := NewTest(name, h) if err != nil { return HubTestItem, err } + h.Tests = append(h.Tests, testItem) return testItem, nil @@ -108,5 +113,6 @@ func (h *HubTest) LoadAllTests() error { } } } + return nil } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 47a151220..717c876ed 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -7,11 +7,12 @@ import ( "path/filepath" "strings" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/parser" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" ) type HubTestItemConfig struct { @@ -25,10 +26,6 @@ type HubTestItemConfig struct { OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 } -type HubIndex struct { - Data map[string]map[string]cwhub.Item -} - type HubTestItem struct { Name string Path string @@ -43,7 +40,7 @@ type HubTestItem struct { RuntimeConfigFilePath string RuntimeProfileFilePath string RuntimeSimulationFilePath string - RuntimeHubConfig *csconfig.Hub + RuntimeHubConfig *csconfig.LocalHubCfg ResultsPath string ParserResultFile string @@ -56,7 +53,7 @@ type HubTestItem struct { TemplateConfigPath string TemplateProfilePath string TemplateSimulationPath string - HubIndex *HubIndex + HubIndex *cwhub.Hub Config *HubTestItemConfig @@ -80,8 +77,6 @@ const ( BucketPourResultFileName = "bucketpour-dump.yaml" ) -var crowdsecPatternsFolder = csconfig.DefaultConfigPath("patterns") - func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { testPath := filepath.Join(hubTest.HubTestPath, name) runtimeFolder := filepath.Join(testPath, "runtime") @@ -91,10 +86,12 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { // read test configuration file configFileData := &HubTestItemConfig{} + yamlFile, err := os.ReadFile(configFilePath) if err != nil { log.Printf("no config file found in '%s': %v", testPath, err) } + err = yaml.Unmarshal(yamlFile, configFileData) if err != nil { return nil, fmt.Errorf("unmarshal: %v", err) @@ -105,6 +102,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName) ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath) + return &HubTestItem{ Name: name, Path: testPath, @@ -121,7 +119,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { ParserResultFile: filepath.Join(resultPath, ParserResultFileName), ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName), BucketPourResultFile: filepath.Join(resultPath, BucketPourResultFileName), - RuntimeHubConfig: &csconfig.Hub{ + RuntimeHubConfig: &csconfig.LocalHubCfg{ HubDir: runtimeHubFolder, HubIndexFile: hubTest.HubIndexFile, InstallDir: runtimeFolder, @@ -147,23 +145,25 @@ func (t *HubTestItem) InstallHub() error { if parser == "" { continue } - var parserDirDest string - if hubParser, ok := t.HubIndex.Data[cwhub.PARSERS][parser]; ok { + + if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok { parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err) } + parserFileName := filepath.Base(parserSource) // runtime/hub/parsers/s00-raw/crowdsecurity/ hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubParser.RemotePath)) // runtime/parsers/s00-raw/ - parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage) + parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage) if err := os.MkdirAll(hubDirParserDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", hubDirParserDest, err) } + if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err) } @@ -204,7 +204,7 @@ func (t *HubTestItem) InstallHub() error { //return fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath) } - parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage) + parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage) if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil { continue //return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err) @@ -231,23 +231,25 @@ func (t *HubTestItem) InstallHub() error { if scenario == "" { continue } - var scenarioDirDest string - if hubScenario, ok := t.HubIndex.Data[cwhub.SCENARIOS][scenario]; ok { + + if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok { scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path to: %s", scenarioSource) } + scenarioFileName := filepath.Base(scenarioSource) // runtime/hub/scenarios/crowdsecurity/ hubDirScenarioDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubScenario.RemotePath)) // runtime/parsers/scenarios/ - scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath) + scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath) if err := os.MkdirAll(hubDirScenarioDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", hubDirScenarioDest, err) } + if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err) } @@ -275,7 +277,7 @@ func (t *HubTestItem) InstallHub() error { //return fmt.Errorf("scenarios '%s' doesn't exist in the hub and doesn't appear to be a custom one.", scenario) } - scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath) + scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath) if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err) } @@ -300,23 +302,25 @@ func (t *HubTestItem) InstallHub() error { if postoverflow == "" { continue } - var postoverflowDirDest string - if hubPostOverflow, ok := t.HubIndex.Data[cwhub.PARSERS_OVFLW][postoverflow]; ok { + + if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok { postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err) } + postoverflowFileName := filepath.Base(postoverflowSource) // runtime/hub/postoverflows/s00-enrich/crowdsecurity/ hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubPostOverflow.RemotePath)) // runtime/postoverflows/s00-enrich - postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage) + postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage) if err := os.MkdirAll(hubDirPostoverflowDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", hubDirPostoverflowDest, err) } + if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err) } @@ -357,7 +361,7 @@ func (t *HubTestItem) InstallHub() error { //return fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath) } - postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage) + postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage) if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil { continue //return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err) @@ -384,10 +388,12 @@ func (t *HubTestItem) InstallHub() error { Filter: "1==1", Statics: t.Config.OverrideStatics, } + b, err := yaml.Marshal(n) if err != nil { return fmt.Errorf("unable to marshal overrides: %s", err) } + tgtFilename := fmt.Sprintf("%s/parsers/s00-raw/00_overrides.yaml", t.RuntimePath) if err := os.WriteFile(tgtFilename, b, os.ModePerm); err != nil { return fmt.Errorf("unable to write overrides to '%s': %s", tgtFilename, err) @@ -395,40 +401,43 @@ func (t *HubTestItem) InstallHub() error { } // load installed hub - err := cwhub.GetHubIdx(t.RuntimeHubConfig) + hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, false) if err != nil { - log.Fatalf("can't local sync the hub: %+v", err) + log.Fatal(err) } // install data for parsers if needed - ret := cwhub.GetItemMap(cwhub.PARSERS) + ret := hub.GetItemMap(cwhub.PARSERS) for parserName, item := range ret { - if item.Installed { - if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { + if item.State.Installed { + if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err) } + log.Debugf("parser '%s' installed successfully in runtime environment", parserName) } } // install data for scenarios if needed - ret = cwhub.GetItemMap(cwhub.SCENARIOS) + ret = hub.GetItemMap(cwhub.SCENARIOS) for scenarioName, item := range ret { - if item.Installed { - if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { + if item.State.Installed { + if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err) } + log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName) } } // install data for postoverflows if needed - ret = cwhub.GetItemMap(cwhub.PARSERS_OVFLW) + ret = hub.GetItemMap(cwhub.POSTOVERFLOWS) for postoverflowName, item := range ret { - if item.Installed { - if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { + if item.State.Installed { + if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err) } + log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName) } } @@ -455,51 +464,53 @@ func (t *HubTestItem) Run() error { } // create runtime folder - if err := os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil { + if err = os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err) } // create runtime data folder - if err := os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil { + if err = os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err) } // create runtime hub folder - if err := os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil { + if err = os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err) } - if err := Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil { + if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil { return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err) } // create results folder - if err := os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil { + if err = os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err) } // copy template config file to runtime folder - if err := Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil { + if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err) } // copy template profile file to runtime folder - if err := Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil { + if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err) } // copy template simulation file to runtime folder - if err := Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil { + if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err) } + crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns") + // copy template patterns folder to runtime folder - if err := CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { + if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err) } // install the hub in the runtime folder - if err := t.InstallHub(); err != nil { + if err = t.InstallHub(); err != nil { return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err) } @@ -507,7 +518,7 @@ func (t *HubTestItem) Run() error { logType := t.Config.LogType dsn := fmt.Sprintf("file://%s", logFile) - if err := os.Chdir(testPath); err != nil { + if err = os.Chdir(testPath); err != nil { return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err) } @@ -515,6 +526,7 @@ func (t *HubTestItem) Run() error { if err != nil { return fmt.Errorf("unable to stat log file '%s': %s", logFile, err) } + if logFileStat.Size() == 0 { return fmt.Errorf("log file '%s' is empty, please fill it with log", logFile) } @@ -522,6 +534,7 @@ func (t *HubTestItem) Run() error { cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"} cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) log.Debugf("%s", cscliRegisterCmd.String()) + output, err := cscliRegisterCmd.CombinedOutput() if err != nil { if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") { @@ -531,16 +544,20 @@ func (t *HubTestItem) Run() error { } cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", t.ResultsPath, "-order-event"} + for labelKey, labelValue := range t.Config.Labels { arg := fmt.Sprintf("%s:%s", labelKey, labelValue) cmdArgs = append(cmdArgs, "-label", arg) } + crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...) log.Debugf("%s", crowdsecCmd.String()) output, err = crowdsecCmd.CombinedOutput() + if log.GetLevel() >= log.DebugLevel || err != nil { fmt.Println(string(output)) } + if err != nil { return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err) } @@ -557,8 +574,10 @@ func (t *HubTestItem) Run() error { if err != nil { return err } + parserAssertFile.Close() } + assertFileStat, err := os.Stat(t.ParserAssert.File) if err != nil { return fmt.Errorf("error while stats '%s': %s", t.ParserAssert.File, err) @@ -569,6 +588,7 @@ func (t *HubTestItem) Run() error { if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + t.ParserAssert.AutoGenAssertData = assertData t.ParserAssert.AutoGenAssert = true } else { @@ -580,12 +600,15 @@ func (t *HubTestItem) Run() error { // assert scenarios nbScenario := 0 + for _, scenario := range t.Config.Scenarios { if scenario == "" { continue } - nbScenario += 1 + + nbScenario++ } + if nbScenario > 0 { _, err := os.Stat(t.ScenarioAssert.File) if os.IsNotExist(err) { @@ -593,8 +616,10 @@ func (t *HubTestItem) Run() error { if err != nil { return err } + scenarioAssertFile.Close() } + assertFileStat, err := os.Stat(t.ScenarioAssert.File) if err != nil { return fmt.Errorf("error while stats '%s': %s", t.ScenarioAssert.File, err) @@ -605,6 +630,7 @@ func (t *HubTestItem) Run() error { if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + t.ScenarioAssert.AutoGenAssertData = assertData t.ScenarioAssert.AutoGenAssert = true } else { diff --git a/pkg/hubtest/parser_assert.go b/pkg/hubtest/parser_assert.go index c9a183336..aadf16af7 100644 --- a/pkg/hubtest/parser_assert.go +++ b/pkg/hubtest/parser_assert.go @@ -5,13 +5,11 @@ import ( "fmt" "io" "os" - "regexp" "sort" "strings" "time" "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" "github.com/enescakir/emoji" "github.com/fatih/color" diff "github.com/r3labs/diff/v2" @@ -43,10 +41,10 @@ type ParserResult struct { Evt types.Event Success bool } + type ParserResults map[string]map[string][]ParserResult func NewParserAssert(file string) *ParserAssert { - ParserAssert := &ParserAssert{ File: file, NbAssert: 0, @@ -55,6 +53,7 @@ func NewParserAssert(file string) *ParserAssert { AutoGenAssert: false, TestData: &ParserResults{}, } + return ParserAssert } @@ -63,22 +62,24 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) { if err != nil { return "", err } + ret := p.AutoGenParserAssert() + return ret, nil } func (p *ParserAssert) LoadTest(filename string) error { - var err error parserDump, err := LoadParserDump(filename) if err != nil { return fmt.Errorf("loading parser dump file: %+v", err) } + p.TestData = parserDump + return nil } func (p *ParserAssert) AssertFile(testFile string) error { - file, err := os.Open(p.File) if err != nil { @@ -88,19 +89,26 @@ func (p *ParserAssert) AssertFile(testFile string) error { if err := p.LoadTest(testFile); err != nil { return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err) } + scanner := bufio.NewScanner(file) scanner.Split(bufio.ScanLines) + nbLine := 0 + for scanner.Scan() { - nbLine += 1 + nbLine++ + if scanner.Text() == "" { continue } + ok, err := p.Run(scanner.Text()) if err != nil { return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) } - p.NbAssert += 1 + + p.NbAssert++ + if !ok { log.Debugf("%s is FALSE", scanner.Text()) failedAssert := &AssertFail{ @@ -109,37 +117,43 @@ func (p *ParserAssert) AssertFile(testFile string) error { Expression: scanner.Text(), Debug: make(map[string]string), } - variableRE := regexp.MustCompile(`(?P[^ =]+) == .*`) + match := variableRE.FindStringSubmatch(scanner.Text()) variable := "" + if len(match) == 0 { log.Infof("Couldn't get variable of line '%s'", scanner.Text()) variable = scanner.Text() } else { variable = match[1] } + result, err := p.EvalExpression(variable) if err != nil { log.Errorf("unable to evaluate variable '%s': %s", variable, err) continue } + failedAssert.Debug[variable] = result p.Fails = append(p.Fails, *failedAssert) continue } //fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text()) - } + file.Close() + if p.NbAssert == 0 { assertData, err := p.AutoGenFromFile(testFile) if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + p.AutoGenAssertData = assertData p.AutoGenAssert = true } + if len(p.Fails) == 0 { p.Success = true } @@ -148,15 +162,14 @@ func (p *ParserAssert) AssertFile(testFile string) error { } func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { - var err error //debug doesn't make much sense with the ability to evaluate "on the fly" //var debugFilter *exprhelpers.ExprDebugger - var runtimeFilter *vm.Program var output interface{} env := map[string]interface{}{"results": *p.TestData} - if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { + runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...) + if err != nil { log.Errorf("failed to compile '%s' : %s", expression, err) return output, err } @@ -168,8 +181,10 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) + return output, fmt.Errorf("while running expression %s: %w", expression, err) } + return output, nil } @@ -178,10 +193,13 @@ func (p *ParserAssert) EvalExpression(expression string) (string, error) { if err != nil { return "", err } + ret, err := yaml.Marshal(output) + if err != nil { return "", err } + return string(ret), nil } @@ -190,6 +208,7 @@ func (p *ParserAssert) Run(assert string) (bool, error) { if err != nil { return false, err } + switch out := output.(type) { case bool: return out, nil @@ -201,80 +220,89 @@ func (p *ParserAssert) Run(assert string) (bool, error) { func Escape(val string) string { val = strings.ReplaceAll(val, `\`, `\\`) val = strings.ReplaceAll(val, `"`, `\"`) + return val } func (p *ParserAssert) AutoGenParserAssert() string { //attempt to autogen parser asserts - var ret string + ret := fmt.Sprintf("len(results) == %d\n", len(*p.TestData)) + + //sort map keys for consistent order + stages := sortedMapKeys(*p.TestData) - //sort map keys for consistent ordre - var stages []string - for stage := range *p.TestData { - stages = append(stages, stage) - } - sort.Strings(stages) - ret += fmt.Sprintf("len(results) == %d\n", len(*p.TestData)) for _, stage := range stages { parsers := (*p.TestData)[stage] - //sort map keys for consistent ordre - var pnames []string - for pname := range parsers { - pnames = append(pnames, pname) - } - sort.Strings(pnames) + + //sort map keys for consistent order + pnames := sortedMapKeys(parsers) + for _, parser := range pnames { presults := parsers[parser] ret += fmt.Sprintf(`len(results["%s"]["%s"]) == %d`+"\n", stage, parser, len(presults)) + for pidx, result := range presults { ret += fmt.Sprintf(`results["%s"]["%s"][%d].Success == %t`+"\n", stage, parser, pidx, result.Success) if !result.Success { continue } + for _, pkey := range sortedMapKeys(result.Evt.Parsed) { pval := result.Evt.Parsed[pkey] if pval == "" { continue } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval)) } + for _, mkey := range sortedMapKeys(result.Evt.Meta) { mval := result.Evt.Meta[mkey] if mval == "" { continue } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval)) } + for _, ekey := range sortedMapKeys(result.Evt.Enriched) { eval := result.Evt.Enriched[ekey] if eval == "" { continue } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval)) } + for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) { uval := result.Evt.Unmarshaled[ukey] if uval == "" { continue } + base := fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Unmarshaled["%s"]`, stage, parser, pidx, ukey) + for _, line := range p.buildUnmarshaledAssert(base, uval) { ret += line } } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Whitelisted == %t`+"\n", stage, parser, pidx, result.Evt.Whitelisted) + if result.Evt.WhitelistReason != "" { ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.WhitelistReason == "%s"`+"\n", stage, parser, pidx, Escape(result.Evt.WhitelistReason)) } } } } + return ret } func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []string { ret := make([]string, 0) + switch val := eval.(type) { case map[string]interface{}: for k, v := range val { @@ -297,12 +325,11 @@ func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []s default: log.Warningf("unknown type '%T' for key '%s'", val, ekey) } + return ret } func LoadParserDump(filepath string) (*ParserResults, error) { - var pdump ParserResults - dumpData, err := os.Open(filepath) if err != nil { return nil, err @@ -314,18 +341,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) { return nil, err } + pdump := ParserResults{} + if err := yaml.Unmarshal(results, &pdump); err != nil { return nil, err } /* we know that some variables should always be set, let's check if they're present in last parser output of last stage */ - stages := make([]string, 0, len(pdump)) - for k := range pdump { - stages = append(stages, k) - } - sort.Strings(stages) + + stages := sortedMapKeys(pdump) + var lastStage string + //Loop over stages to find last successful one with at least one parser for i := len(stages) - 2; i >= 0; i-- { if len(pdump[stages[i]]) != 0 { @@ -333,11 +361,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) { break } } + parsers := make([]string, 0, len(pdump[lastStage])) + for k := range pdump[lastStage] { parsers = append(parsers, k) } + sort.Strings(parsers) + + if len(parsers) == 0 { + return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry") + } + lastParser := parsers[len(parsers)-1] for idx, result := range pdump[lastStage][lastParser] { @@ -357,47 +393,51 @@ type DumpOpts struct { ShowNotOkParsers bool } -func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts DumpOpts) { +func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) { //note : we can use line -> time as the unique identifier (of acquisition) - state := make(map[time.Time]map[string]map[string]ParserResult) assoc := make(map[time.Time]string, 0) - for stage, parsers := range parser_results { + for stage, parsers := range parserResults { for parser, results := range parsers { - for _, parser_res := range results { - evt := parser_res.Evt + for _, parserRes := range results { + evt := parserRes.Evt if _, ok := state[evt.Line.Time]; !ok { state[evt.Line.Time] = make(map[string]map[string]ParserResult) assoc[evt.Line.Time] = evt.Line.Raw } + if _, ok := state[evt.Line.Time][stage]; !ok { state[evt.Line.Time][stage] = make(map[string]ParserResult) } - state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parser_res.Success} - } + state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success} + } } } - for bname, evtlist := range bucket_pour { + for bname, evtlist := range bucketPour { for _, evt := range evtlist { if evt.Line.Raw == "" { continue } + //it might be bucket overflow being reprocessed, skip this if _, ok := state[evt.Line.Time]; !ok { state[evt.Line.Time] = make(map[string]map[string]ParserResult) assoc[evt.Line.Time] = evt.Line.Raw } + //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if _, ok := state[evt.Line.Time]["buckets"]; !ok { state[evt.Line.Time]["buckets"] = make(map[string]ParserResult) } + state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true} } } + yellow := color.New(color.FgYellow).SprintFunc() red := color.New(color.FgRed).SprintFunc() green := color.New(color.FgGreen).SprintFunc() @@ -409,19 +449,25 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum continue } } + fmt.Printf("line: %s\n", rawstr) + skeys := make([]string, 0, len(state[tstamp])) + for k := range state[tstamp] { //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if k == "buckets" { continue } + skeys = append(skeys, k) } + sort.Strings(skeys) - //iterate stage - var prev_item types.Event + + // iterate stage + var prevItem types.Event for _, stage := range skeys { parsers := state[tstamp][stage] @@ -431,18 +477,16 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum fmt.Printf("\t%s %s\n", sep, stage) - pkeys := make([]string, 0, len(parsers)) - for k := range parsers { - pkeys = append(pkeys, k) - } - sort.Strings(pkeys) + pkeys := sortedMapKeys(parsers) for idx, parser := range pkeys { res := parsers[parser].Success sep := "├" + if idx == len(pkeys)-1 { sep = "└" } + created := 0 updated := 0 deleted := 0 @@ -451,16 +495,19 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum detailsDisplay := "" if res { - changelog, _ := diff.Diff(prev_item, parsers[parser].Evt) + changelog, _ := diff.Diff(prevItem, parsers[parser].Evt) for _, change := range changelog { switch change.Type { case "create": created++ + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To)) case "update": detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To)) + if change.Path[0] == "Whitelisted" && change.To == true { whitelisted = true + if whitelistReason == "" { whitelistReason = parsers[parser].Evt.WhitelistReason } @@ -468,51 +515,64 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum updated++ case "delete": deleted++ + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, "."))) } } - prev_item = parsers[parser].Evt + + prevItem = parsers[parser].Evt } if created > 0 { changeStr += green(fmt.Sprintf("+%d", created)) } + if updated > 0 { if len(changeStr) > 0 { changeStr += " " } + changeStr += yellow(fmt.Sprintf("~%d", updated)) } + if deleted > 0 { if len(changeStr) > 0 { changeStr += " " } + changeStr += red(fmt.Sprintf("-%d", deleted)) } + if whitelisted { if len(changeStr) > 0 { changeStr += " " } + changeStr += red("[whitelisted]") } + if changeStr == "" { changeStr = yellow("unchanged") } + if res { fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr) + if opts.Details { fmt.Print(detailsDisplay) } } else if opts.ShowNotOkParsers { fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser) - } } } + sep := "└" + if len(state[tstamp]["buckets"]) > 0 { sep = "├" } + //did the event enter the bucket pour phase ? if _, ok := state[tstamp]["buckets"]["OK"]; ok { fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle) @@ -521,27 +581,35 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum } else { fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle) } + //now print bucket info if len(state[tstamp]["buckets"]) > 0 { fmt.Printf("\t├ Scenarios\n") } + bnames := make([]string, 0, len(state[tstamp]["buckets"])) + for k := range state[tstamp]["buckets"] { //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if k == "OK" { continue } + bnames = append(bnames, k) } + sort.Strings(bnames) + for idx, bname := range bnames { sep := "├" if idx == len(bnames)-1 { sep = "└" } + fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname) } + fmt.Println() } } diff --git a/pkg/hubtest/regexp.go b/pkg/hubtest/regexp.go new file mode 100644 index 000000000..f9165eae3 --- /dev/null +++ b/pkg/hubtest/regexp.go @@ -0,0 +1,11 @@ +package hubtest + +import ( + "regexp" +) + +var ( + variableRE = regexp.MustCompile(`(?P[^ =]+) == .*`) + parserResultRE = regexp.MustCompile(`^results\["[^"]+"\]\["(?P[^"]+)"\]\[[0-9]+\]\.Evt\..*`) + scenarioResultRE = regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P[^"]+)"`) +) diff --git a/pkg/hubtest/scenario_assert.go b/pkg/hubtest/scenario_assert.go index f5517c350..011d3dcfb 100644 --- a/pkg/hubtest/scenario_assert.go +++ b/pkg/hubtest/scenario_assert.go @@ -5,12 +5,10 @@ import ( "fmt" "io" "os" - "regexp" "sort" "strings" "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -42,6 +40,7 @@ func NewScenarioAssert(file string) *ScenarioAssert { TestData: &BucketResults{}, PourData: &BucketPourInfo{}, } + return ScenarioAssert } @@ -50,7 +49,9 @@ func (s *ScenarioAssert) AutoGenFromFile(filename string) (string, error) { if err != nil { return "", err } + ret := s.AutoGenScenarioAssert() + return ret, nil } @@ -59,6 +60,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error { if err != nil { return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err) } + s.TestData = bucketDump if bucketpour != "" { @@ -66,8 +68,10 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error { if err != nil { return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err) } + s.PourData = pourDump } + return nil } @@ -81,19 +85,26 @@ func (s *ScenarioAssert) AssertFile(testFile string) error { if err := s.LoadTest(testFile, ""); err != nil { return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err) } + scanner := bufio.NewScanner(file) scanner.Split(bufio.ScanLines) + nbLine := 0 + for scanner.Scan() { - nbLine += 1 + nbLine++ + if scanner.Text() == "" { continue } + ok, err := s.Run(scanner.Text()) if err != nil { return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) } - s.NbAssert += 1 + + s.NbAssert++ + if !ok { log.Debugf("%s is FALSE", scanner.Text()) failedAssert := &AssertFail{ @@ -102,31 +113,38 @@ func (s *ScenarioAssert) AssertFile(testFile string) error { Expression: scanner.Text(), Debug: make(map[string]string), } - variableRE := regexp.MustCompile(`(?P[^ ]+) == .*`) + match := variableRE.FindStringSubmatch(scanner.Text()) + if len(match) == 0 { log.Infof("Couldn't get variable of line '%s'", scanner.Text()) continue } + variable := match[1] + result, err := s.EvalExpression(variable) if err != nil { log.Errorf("unable to evaluate variable '%s': %s", variable, err) continue } + failedAssert.Debug[variable] = result s.Fails = append(s.Fails, *failedAssert) + continue } //fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text()) - } + file.Close() + if s.NbAssert == 0 { assertData, err := s.AutoGenFromFile(testFile) if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + s.AutoGenAssertData = assertData s.AutoGenAssert = true } @@ -139,15 +157,14 @@ func (s *ScenarioAssert) AssertFile(testFile string) error { } func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { - var err error //debug doesn't make much sense with the ability to evaluate "on the fly" //var debugFilter *exprhelpers.ExprDebugger - var runtimeFilter *vm.Program var output interface{} env := map[string]interface{}{"results": *s.TestData} - if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { + runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...) + if err != nil { return nil, err } // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil { @@ -161,8 +178,10 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) + return nil, fmt.Errorf("while running expression %s: %w", expression, err) } + return output, nil } @@ -171,10 +190,12 @@ func (s *ScenarioAssert) EvalExpression(expression string) (string, error) { if err != nil { return "", err } + ret, err := yaml.Marshal(output) if err != nil { return "", err } + return string(ret), nil } @@ -183,6 +204,7 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) { if err != nil { return false, err } + switch out := output.(type) { case bool: return out, nil @@ -192,9 +214,9 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) { } func (s *ScenarioAssert) AutoGenScenarioAssert() string { - //attempt to autogen parser asserts - var ret string - ret += fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData)) + // attempt to autogen scenario asserts + ret := fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData)) + for eventIndex, event := range *s.TestData { for ipSrc, source := range event.Overflow.Sources { ret += fmt.Sprintf(`"%s" in results[%d].Overflow.GetSources()`+"\n", ipSrc, eventIndex) @@ -203,15 +225,18 @@ func (s *ScenarioAssert) AutoGenScenarioAssert() string { ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetScope() == "%s"`+"\n", eventIndex, ipSrc, *source.Scope) ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetValue() == "%s"`+"\n", eventIndex, ipSrc, *source.Value) } + for evtIndex, evt := range event.Overflow.Alert.Events { for _, meta := range evt.Meta { ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value)) } } + ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetScenario() == "%s"`+"\n", eventIndex, *event.Overflow.Alert.Scenario) ret += fmt.Sprintf(`results[%d].Overflow.Alert.Remediation == %t`+"\n", eventIndex, event.Overflow.Alert.Remediation) ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetEventsCount() == %d`+"\n", eventIndex, *event.Overflow.Alert.EventsCount) } + return ret } @@ -228,8 +253,6 @@ func (b BucketResults) Swap(i, j int) { } func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { - var bucketDump BucketPourInfo - dumpData, err := os.Open(filepath) if err != nil { return nil, err @@ -241,6 +264,8 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { return nil, err } + var bucketDump BucketPourInfo + if err := yaml.Unmarshal(results, &bucketDump); err != nil { return nil, err } @@ -249,8 +274,6 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { } func LoadScenarioDump(filepath string) (*BucketResults, error) { - var bucketDump BucketResults - dumpData, err := os.Open(filepath) if err != nil { return nil, err @@ -262,6 +285,8 @@ func LoadScenarioDump(filepath string) (*BucketResults, error) { return nil, err } + var bucketDump BucketResults + if err := yaml.Unmarshal(results, &bucketDump); err != nil { return nil, err } diff --git a/pkg/hubtest/utils.go b/pkg/hubtest/utils.go index 5ccbbad39..090f1f85e 100644 --- a/pkg/hubtest/utils.go +++ b/pkg/hubtest/utils.go @@ -12,7 +12,9 @@ func sortedMapKeys[V any](m map[string]V) []string { for k := range m { keys = append(keys, k) } + sort.Strings(keys) + return keys } @@ -22,7 +24,7 @@ func Copy(src string, dst string) error { return err } - err = os.WriteFile(dst, content, 0644) + err = os.WriteFile(dst, content, 0o644) if err != nil { return err } @@ -43,16 +45,20 @@ func checkPathNotContained(path string, subpath string) error { } current := absSubPath + for { if current == absPath { return fmt.Errorf("cannot copy a folder onto itself") } + up := filepath.Dir(current) if current == up { break } + current = up } + return nil } diff --git a/pkg/hubtest/utils_test.go b/pkg/hubtest/utils_test.go index de4f1aac3..ce86785af 100644 --- a/pkg/hubtest/utils_test.go +++ b/pkg/hubtest/utils_test.go @@ -3,16 +3,16 @@ package hubtest import ( "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestCheckPathNotContained(t *testing.T) { - assert.Nil(t, checkPathNotContained("/foo", "/bar")) - assert.Nil(t, checkPathNotContained("/foo/bar", "/foo")) - assert.Nil(t, checkPathNotContained("/foo/bar", "/")) - assert.Nil(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else")) - assert.Nil(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else")) - assert.NotNil(t, checkPathNotContained("/foo", "/foo/bar")) - assert.NotNil(t, checkPathNotContained("/", "/foo")) - assert.NotNil(t, checkPathNotContained("/", "/foo/bar/baz")) + require.NoError(t, checkPathNotContained("/foo", "/bar")) + require.NoError(t, checkPathNotContained("/foo/bar", "/foo")) + require.NoError(t, checkPathNotContained("/foo/bar", "/")) + require.NoError(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else")) + require.NoError(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else")) + require.Error(t, checkPathNotContained("/foo", "/foo/bar")) + require.Error(t, checkPathNotContained("/", "/foo")) + require.Error(t, checkPathNotContained("/", "/foo/bar/baz")) } diff --git a/pkg/leakybucket/buckets_test.go b/pkg/leakybucket/buckets_test.go index e08887be8..f74fb8355 100644 --- a/pkg/leakybucket/buckets_test.go +++ b/pkg/leakybucket/buckets_test.go @@ -8,12 +8,14 @@ import ( "html/template" "io" "os" + "path/filepath" "reflect" "sync" "testing" "time" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -33,28 +35,45 @@ func TestBucket(t *testing.T) { envSetting = os.Getenv("TEST_ONLY") tomb = &tomb.Tomb{} ) - err := exprhelpers.Init(nil) + + testdata := "./tests" + + hubCfg := &csconfig.LocalHubCfg{ + HubDir: filepath.Join(testdata, "hub"), + HubIndexFile: filepath.Join(testdata, "hub", "index.json"), + InstallDataDir: testdata, + } + + hub, err := cwhub.NewHub(hubCfg, nil, false) + if err != nil { + t.Fatalf("failed to init hub: %s", err) + } + + err = exprhelpers.Init(nil) if err != nil { log.Fatalf("exprhelpers init failed: %s", err) } if envSetting != "" { - if err := testOneBucket(t, envSetting, tomb); err != nil { + if err := testOneBucket(t, hub, envSetting, tomb); err != nil { t.Fatalf("Test '%s' failed : %s", envSetting, err) } } else { wg := new(sync.WaitGroup) - fds, err := os.ReadDir("./tests/") + fds, err := os.ReadDir(testdata) if err != nil { t.Fatalf("Unable to read test directory : %s", err) } for _, fd := range fds { - fname := "./tests/" + fd.Name() + if fd.Name() == "hub" { + continue + } + fname := filepath.Join(testdata, fd.Name()) log.Infof("Running test on %s", fname) tomb.Go(func() error { wg.Add(1) defer wg.Done() - if err := testOneBucket(t, fname, tomb); err != nil { + if err := testOneBucket(t, hub, fname, tomb); err != nil { t.Fatalf("Test '%s' failed : %s", fname, err) } return nil @@ -76,7 +95,7 @@ func watchTomb(tomb *tomb.Tomb) { } } -func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error { +func testOneBucket(t *testing.T, hub *cwhub.Hub, dir string, tomb *tomb.Tomb) error { var ( holders []BucketFactory @@ -112,10 +131,8 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error { files = append(files, x.Filename) } - cscfg := &csconfig.CrowdsecServiceCfg{ - DataDir: "tests", - } - holders, response, err := LoadBuckets(cscfg, files, tomb, buckets, false) + cscfg := &csconfig.CrowdsecServiceCfg{} + holders, response, err := LoadBuckets(cscfg, hub, files, tomb, buckets, false) if err != nil { t.Fatalf("failed loading bucket : %s", err) } @@ -123,7 +140,7 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error { watchTomb(tomb) return nil }) - if !testFile(t, dir+"/test.json", dir+"/in-buckets_state.json", holders, response, buckets) { + if !testFile(t, filepath.Join(dir, "test.json"), filepath.Join(dir, "in-buckets_state.json"), holders, response, buckets) { return fmt.Errorf("tests from %s failed", dir) } return nil diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index 74643d0fd..9ccc1d326 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -178,7 +178,7 @@ func ValidateFactory(bucketFactory *BucketFactory) error { return nil } -func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) { +func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, hub *cwhub.Hub, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) { var ( ret = []BucketFactory{} response chan types.Event @@ -211,7 +211,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb. log.Tracef("End of yaml file") break } - bucketFactory.DataDir = cscfg.DataDir + bucketFactory.DataDir = hub.GetDataDir() //check empty if bucketFactory.Name == "" { log.Errorf("Won't load nameless bucket") @@ -234,7 +234,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb. bucketFactory.Filename = filepath.Clean(f) bucketFactory.BucketName = seed.Generate() bucketFactory.ret = response - hubItem, err := cwhub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename) + hubItem, err := hub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename) if err != nil { log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename) } else { @@ -242,8 +242,8 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb. bucketFactory.Simulated = cscfg.SimulationConfig.IsSimulated(hubItem.Name) } if hubItem != nil { - bucketFactory.ScenarioVersion = hubItem.LocalVersion - bucketFactory.hash = hubItem.LocalHash + bucketFactory.ScenarioVersion = hubItem.State.LocalVersion + bucketFactory.hash = hubItem.State.LocalHash } else { log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename) } diff --git a/pkg/leakybucket/tests/hub/index.json b/pkg/leakybucket/tests/hub/index.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/pkg/leakybucket/tests/hub/index.json @@ -0,0 +1 @@ +{} diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 2e4a8035b..617e46189 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -57,24 +57,25 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) { // Return new parsers // nodes and povfwnodes are already initialized in parser.LoadStages -func NewParsers() *Parsers { +func NewParsers(hub *cwhub.Hub) *Parsers { parsers := &Parsers{ Ctx: &UnixParserCtx{}, Povfwctx: &UnixParserCtx{}, StageFiles: make([]Stagefile, 0), PovfwStageFiles: make([]Stagefile, 0), } - for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} { - for _, hubParserItem := range cwhub.GetItemMap(itemType) { - if hubParserItem.Installed { + + for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} { + for _, hubParserItem := range hub.GetItemMap(itemType) { + if hubParserItem.State.Installed { stagefile := Stagefile{ - Filename: hubParserItem.LocalPath, + Filename: hubParserItem.State.LocalPath, Stage: hubParserItem.Stage, } if itemType == cwhub.PARSERS { parsers.StageFiles = append(parsers.StageFiles, stagefile) } - if itemType == cwhub.PARSERS_OVFLW { + if itemType == cwhub.POSTOVERFLOWS { parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile) } } @@ -97,16 +98,16 @@ func NewParsers() *Parsers { func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { var err error - patternsDir := filepath.Join(cConfig.Crowdsec.ConfigDir, "patterns/") + patternsDir := filepath.Join(cConfig.ConfigPaths.ConfigDir, "patterns/") log.Infof("Loading grok library %s", patternsDir) /* load base regexps for two grok parsers */ parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir, - "data": cConfig.Crowdsec.DataDir}) + "data": cConfig.ConfigPaths.DataDir}) if err != nil { return parsers, fmt.Errorf("failed to load parser patterns : %v", err) } parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir, - "data": cConfig.Crowdsec.DataDir}) + "data": cConfig.ConfigPaths.DataDir}) if err != nil { return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err) } @@ -116,7 +117,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { */ log.Infof("Loading enrich plugins") - parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir) + parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir) if err != nil { return parsers, fmt.Errorf("failed to load enrich plugin : %v", err) } diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 92a1968c8..fc922c5d1 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -10,7 +10,6 @@ import ( goccyyaml "github.com/goccy/go-yaml" "gopkg.in/yaml.v3" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -46,22 +45,12 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) { } // InstallHubItems installs the objects recommended in a setup file. -func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error { +func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error { setupEnvelope, err := decodeSetup(input, false) if err != nil { return err } - if err := csConfig.LoadHub(); err != nil { - return fmt.Errorf("loading hub: %w", err) - } - - cwhub.SetHubBranch() - - if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { - return fmt.Errorf("getting hub index: %w", err) - } - for _, setupItem := range setupEnvelope.Setup { forceAction := false downloadOnly := false @@ -73,14 +62,19 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error if len(install.Collections) > 0 { for _, collection := range setupItem.Install.Collections { + item := hub.GetItem(cwhub.COLLECTIONS, collection) + if item == nil { + return fmt.Errorf("collection %s not found", collection) + } + if dryRun { fmt.Println("dry-run: would install collection", collection) continue } - if err := cwhub.InstallItem(csConfig, collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing collection %s: %w", collection, err) + if err := item.Install(forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing collection %s: %w", item.Name, err) } } } @@ -93,8 +87,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing parser %s: %w", parser, err) + item := hub.GetItem(cwhub.PARSERS, parser) + if item == nil { + return fmt.Errorf("parser %s not found", parser) + } + + if err := item.Install(forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing parser %s: %w", item.Name, err) } } } @@ -107,8 +106,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing scenario %s: %w", scenario, err) + item := hub.GetItem(cwhub.SCENARIOS, scenario) + if item == nil { + return fmt.Errorf("scenario %s not found", scenario) + } + + if err := item.Install(forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing scenario %s: %w", item.Name, err) } } } @@ -121,8 +125,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) + item := hub.GetItem(cwhub.POSTOVERFLOWS, postoverflow) + if item == nil { + return fmt.Errorf("postoverflow %s not found", postoverflow) + } + + if err := item.Install(forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing postoverflow %s: %w", item.Name, err) } } } diff --git a/test/bats/00_wait_for.bats b/test/bats/00_wait_for.bats new file mode 100644 index 000000000..ffc6802d9 --- /dev/null +++ b/test/bats/00_wait_for.bats @@ -0,0 +1,71 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" +} + +setup() { + load "../lib/setup.sh" +} + +@test "run a command and capture its stdout" { + run -0 wait-for seq 1 3 + assert_output - <<-EOT + 1 + 2 + 3 + EOT +} + +@test "run a command and capture its stderr" { + rune -0 wait-for sh -c 'seq 1 3 >&2' + assert_stderr - <<-EOT + 1 + 2 + 3 + EOT +} + +@test "run a command until a pattern is found in stdout" { + run -0 wait-for --out "1[12]0" seq 1 200 + assert_line --index 0 "1" + assert_line --index -1 "110" + refute_line "111" +} + +@test "run a command until a pattern is found in stderr" { + rune -0 wait-for --err "10" sh -c 'seq 1 20 >&2' + assert_stderr - <<-EOT + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + EOT +} + +@test "run a command with timeout (no match)" { + # when the process is terminated without a match, it returns + # 256 - 15 (SIGTERM) = 241 + rune -241 wait-for --timeout 0.1 --out "10" sh -c 'echo 1; sleep 3; echo 2' + assert_line 1 + # there may be more, but we don't care +} + +@test "run a command with timeout (match)" { + # when the process is terminated with a match, return code is 128 + rune -128 wait-for --timeout .4 --out "2" sh -c 'echo 1; sleep .1; echo 2; echo 3; echo 4; sleep 10' + assert_output - <<-EOT + 1 + 2 + EOT +} + diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index 2e38e0e6c..7bcc35b1d 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -24,28 +24,22 @@ teardown() { #---------- @test "crowdsec (usage)" { - rune -0 timeout 2s "${CROWDSEC}" -h - assert_stderr_line --regexp "Usage of .*:" - - rune -0 timeout 2s "${CROWDSEC}" --help - assert_stderr_line --regexp "Usage of .*:" + rune -0 wait-for --out "Usage of " "${CROWDSEC}" -h + rune -0 wait-for --out "Usage of " "${CROWDSEC}" --help } @test "crowdsec (unknown flag)" { - rune -2 timeout 2s "${CROWDSEC}" --foobar - assert_stderr_line "flag provided but not defined: -foobar" - assert_stderr_line --regexp "Usage of .*" + rune -0 wait-for --err "flag provided but not defined: -foobar" "$CROWDSEC" --foobar } @test "crowdsec (unknown argument)" { - rune -2 timeout 2s "${CROWDSEC}" trololo - assert_stderr_line "argument provided but not defined: trololo" - assert_stderr_line --regexp "Usage of .*" + rune -0 wait-for --err "argument provided but not defined: trololo" "${CROWDSEC}" trololo } @test "crowdsec (no api and no agent)" { - rune -1 timeout 2s "${CROWDSEC}" -no-api -no-cs - assert_stderr_line --partial "You must run at least the API Server or crowdsec" + rune -0 wait-for \ + --err "You must run at least the API Server or crowdsec" \ + "${CROWDSEC}" -no-api -no-cs } @test "crowdsec - print error on exit" { @@ -55,20 +49,22 @@ teardown() { assert_stderr --partial "unable to create database client: unknown database type 'meh'" } -@test "crowdsec - bad configuration (empty/missing common section)" { +@test "crowdsec - default logging configuration (empty/missing common section)" { config_set '.common={}' - rune -1 "${CROWDSEC}" + rune -0 wait-for \ + --err "Starting processing data" \ + "${CROWDSEC}" refute_output - assert_stderr --partial "unable to load configuration: common section is empty" config_set 'del(.common)' - rune -1 "${CROWDSEC}" + rune -0 wait-for \ + --err "Starting processing data" \ + "${CROWDSEC}" refute_output - assert_stderr --partial "unable to load configuration: common section is empty" } @test "CS_LAPI_SECRET not strong enough" { - CS_LAPI_SECRET=foo rune -1 timeout 2s "${CROWDSEC}" + CS_LAPI_SECRET=foo rune -1 wait-for "${CROWDSEC}" assert_stderr --partial "api server init: unable to run local API: controller init: CS_LAPI_SECRET not strong enough" } @@ -138,8 +134,8 @@ teardown() { ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') rm -f "$ACQUIS_YAML" - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr_line --partial "acquis.yaml: no such file or directory" + rune -1 wait-for "${CROWDSEC}" + assert_stderr --partial "acquis.yaml: no such file or directory" } @test "crowdsec (error if acquisition_path is not defined and acquisition_dir is empty)" { @@ -151,7 +147,7 @@ teardown() { rm -f "$ACQUIS_DIR" config_set '.common.log_media="stdout"' - rune -1 timeout 2s "${CROWDSEC}" + rune -1 wait-for "${CROWDSEC}" # check warning assert_stderr --partial "no acquisition file found" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" @@ -167,13 +163,15 @@ teardown() { config_set '.crowdsec_service.acquisition_dir=""' config_set '.common.log_media="stdout"' - rune -1 timeout 2s "${CROWDSEC}" + rune -1 wait-for "${CROWDSEC}" # check warning assert_stderr --partial "no acquisition_path or acquisition_dir specified" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" { + config_set '.common.log_media="stdout"' + ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') config_set '.crowdsec_service.acquisition_path=""' @@ -181,13 +179,15 @@ teardown() { mkdir -p "$ACQUIS_DIR" mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml - rune -124 timeout 2s "${CROWDSEC}" + rune -0 wait-for \ + --err "Starting processing data" \ + "${CROWDSEC}" # now, if foo.yaml is empty instead, there won't be valid datasources. cat /dev/null >"$ACQUIS_DIR"/foo.yaml - rune -1 timeout 2s "${CROWDSEC}" + rune -1 wait-for "${CROWDSEC}" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } @@ -212,9 +212,10 @@ teardown() { type: syslog EOT - rune -124 timeout 2s env PATH='' "${CROWDSEC}" #shellcheck disable=SC2016 - assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH' + rune -0 wait-for \ + --err 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in ' \ + env PATH='' "${CROWDSEC}" # if all datasources are disabled, crowdsec should exit @@ -222,7 +223,7 @@ teardown() { rm -f "$ACQUIS_YAML" config_set '.crowdsec_service.acquisition_path=""' - rune -1 timeout 2s env PATH='' "${CROWDSEC}" + rune -1 wait-for env PATH='' "${CROWDSEC}" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index 3e61bd807..a49df68bc 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -110,6 +110,37 @@ teardown() { assert_json '["http://127.0.0.1:8080/","githubciXXXXXXXXXXXXXXXXXXXXXXXX"]' } +@test "cscli - required configuration paths" { + config=$(cat "${CONFIG_YAML}") + configdir=$(config_get '.config_paths.config_dir') + + # required configuration paths with no defaults + + config_set 'del(.config_paths)' + rune -1 cscli hub list + assert_stderr --partial 'no configuration paths provided' + echo "$config" > "${CONFIG_YAML}" + + config_set 'del(.config_paths.data_dir)' + rune -1 cscli hub list + assert_stderr --partial "please provide a data directory with the 'data_dir' directive in the 'config_paths' section" + echo "$config" > "${CONFIG_YAML}" + + # defaults + + config_set 'del(.config_paths.hub_dir)' + rune -0 cscli hub list + rune -0 cscli config show --key Config.ConfigPaths.HubDir + assert_output "$configdir/hub" + echo "$config" > "${CONFIG_YAML}" + + config_set 'del(.config_paths.index_path)' + rune -0 cscli hub list + rune -0 cscli config show --key Config.ConfigPaths.HubIndexFile + assert_output "$configdir/hub/.index.json" + echo "$config" > "${CONFIG_YAML}" +} + @test "cscli config show-yaml" { rune -0 cscli config show-yaml rune -0 yq .common.log_level <(output) @@ -245,50 +276,23 @@ teardown() { assert_output --partial "# bash completion for cscli" } -@test "cscli hub list" { - # we check for the presence of some objects. There may be others when we - # use $PACKAGE_TESTING, so the order is not important. - - rune -0 cscli hub list -o human - assert_line --regexp '^ crowdsecurity/linux' - assert_line --regexp '^ crowdsecurity/sshd' - assert_line --regexp '^ crowdsecurity/dateparse-enrich' - assert_line --regexp '^ crowdsecurity/geoip-enrich' - assert_line --regexp '^ crowdsecurity/sshd-logs' - assert_line --regexp '^ crowdsecurity/syslog-logs' - assert_line --regexp '^ crowdsecurity/ssh-bf' - assert_line --regexp '^ crowdsecurity/ssh-slow-bf' - - rune -0 cscli hub list -o raw - assert_line --regexp '^crowdsecurity/linux,enabled,[0-9]+\.[0-9]+,core linux support : syslog\+geoip\+ssh,collections$' - assert_line --regexp '^crowdsecurity/sshd,enabled,[0-9]+\.[0-9]+,sshd support : parser and brute-force detection,collections$' - assert_line --regexp '^crowdsecurity/dateparse-enrich,enabled,[0-9]+\.[0-9]+,,parsers$' - assert_line --regexp '^crowdsecurity/geoip-enrich,enabled,[0-9]+\.[0-9]+,"Populate event with geoloc info : as, country, coords, source range.",parsers$' - assert_line --regexp '^crowdsecurity/sshd-logs,enabled,[0-9]+\.[0-9]+,Parse openSSH logs,parsers$' - assert_line --regexp '^crowdsecurity/syslog-logs,enabled,[0-9]+\.[0-9]+,,parsers$' - assert_line --regexp '^crowdsecurity/ssh-bf,enabled,[0-9]+\.[0-9]+,Detect ssh bruteforce,scenarios$' - assert_line --regexp '^crowdsecurity/ssh-slow-bf,enabled,[0-9]+\.[0-9]+,Detect slow ssh bruteforce,scenarios$' - - rune -0 cscli hub list -o json - rune -0 jq -r '.collections[].name, .parsers[].name, .scenarios[].name' <(output) - assert_line 'crowdsecurity/linux' - assert_line 'crowdsecurity/sshd' - assert_line 'crowdsecurity/dateparse-enrich' - assert_line 'crowdsecurity/geoip-enrich' - assert_line 'crowdsecurity/sshd-logs' - assert_line 'crowdsecurity/syslog-logs' - assert_line 'crowdsecurity/ssh-bf' - assert_line 'crowdsecurity/ssh-slow-bf' -} - @test "cscli support dump (smoke test)" { rune -0 cscli support dump -f "$BATS_TEST_TMPDIR"/dump.zip assert_file_exists "$BATS_TEST_TMPDIR"/dump.zip } @test "cscli explain" { - rune -0 cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog --crowdsec "$CROWDSEC" + line="Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" + + rune -0 cscli parsers install crowdsecurity/syslog-logs + rune -0 cscli collections install crowdsecurity/sshd + + rune -0 cscli explain --log "$line" --type syslog --only-successful-parsers --crowdsec "$CROWDSEC" assert_output - <"$BATS_TEST_DIRNAME"/testdata/explain/explain-log.txt + + rune -0 cscli parsers remove --all --purge + rune -1 cscli explain --log "$line" --type syslog --crowdsec "$CROWDSEC" + assert_stderr --partial "unable to load parser dump result: no parser found. Please install the appropriate parser and retry" } @test 'Allow variable expansion and literal $ characters in passwords' { diff --git a/test/bats/02_nolapi.bats b/test/bats/02_nolapi.bats index a434e8a6d..dec94a86b 100644 --- a/test/bats/02_nolapi.bats +++ b/test/bats/02_nolapi.bats @@ -24,21 +24,23 @@ teardown() { #---------- @test "test without -no-api flag" { - rune -124 timeout 2s "${CROWDSEC}" - # from `man timeout`: If the command times out, and --preserve-status is not set, then exit with status 124. + config_set '.common.log_media="stdout"' + rune -0 wait-for \ + --err "CrowdSec Local API listening" \ + "${CROWDSEC}" } @test "crowdsec should not run without LAPI (-no-api flag)" { - # really needs 4 secs on slow boxes - rune -1 timeout 4s "${CROWDSEC}" -no-api + config_set '.common.log_media="stdout"' + rune -1 wait-for "${CROWDSEC}" -no-api } @test "crowdsec should not run without LAPI (no api.server in configuration file)" { config_disable_lapi config_log_stderr - # really needs 4 secs on slow boxes - rune -1 timeout 4s "${CROWDSEC}" - assert_stderr --partial "crowdsec local API is disabled" + rune -0 wait-for \ + --err "crowdsec local API is disabled" \ + "${CROWDSEC}" } @test "capi status shouldn't be ok without api.server" { diff --git a/test/bats/03_noagent.bats b/test/bats/03_noagent.bats index a91737f99..e75e375ad 100644 --- a/test/bats/03_noagent.bats +++ b/test/bats/03_noagent.bats @@ -23,20 +23,25 @@ teardown() { #---------- @test "with agent: test without -no-cs flag" { - rune -124 timeout 2s "${CROWDSEC}" - # from `man timeout`: If the command times out, and --preserve-status is not set, then exit with status 124. + config_set '.common.log_media="stdout"' + rune -0 wait-for \ + --err "Starting processing data" \ + "${CROWDSEC}" } @test "no agent: crowdsec LAPI should run (-no-cs flag)" { - rune -124 timeout 2s "${CROWDSEC}" -no-cs + config_set '.common.log_media="stdout"' + rune -0 wait-for \ + --err "CrowdSec Local API listening" \ + "${CROWDSEC}" -no-cs } @test "no agent: crowdsec LAPI should run (no crowdsec_service in configuration file)" { config_disable_agent config_log_stderr - rune -124 timeout 2s "${CROWDSEC}" - - assert_stderr --partial "crowdsec agent is disabled" + rune -0 wait-for \ + --err "crowdsec agent is disabled" \ + "${CROWDSEC}" } @test "no agent: cscli config show" { diff --git a/test/bats/04_capi.bats b/test/bats/04_capi.bats index ef933e10c..04d3a1aa0 100644 --- a/test/bats/04_capi.bats +++ b/test/bats/04_capi.bats @@ -22,6 +22,10 @@ setup() { @test "cscli capi status" { config_enable_capi rune -0 cscli capi register --schmilblick githubciXXXXXXXXXXXXXXXXXXXXXXXX + rune -1 cscli capi status + assert_stderr --partial "no scenarios installed, abort" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf rune -0 cscli capi status assert_stderr --partial "Loaded credentials from" assert_stderr --partial "Trying to authenticate with username" diff --git a/test/bats/04_nocapi.bats b/test/bats/04_nocapi.bats index 23994c43e..234db182a 100644 --- a/test/bats/04_nocapi.bats +++ b/test/bats/04_nocapi.bats @@ -25,16 +25,17 @@ teardown() { @test "without capi: crowdsec LAPI should run without capi (-no-capi flag)" { config_set '.common.log_media="stdout"' - rune -124 timeout 1s "${CROWDSEC}" -no-capi - assert_stderr --partial "Communication with CrowdSec Central API disabled from args" + rune -0 wait-for \ + --err "Communication with CrowdSec Central API disabled from args" \ + "${CROWDSEC}" -no-capi } @test "without capi: crowdsec LAPI should still work" { config_disable_capi config_set '.common.log_media="stdout"' - rune -124 timeout 1s "${CROWDSEC}" - # from `man timeout`: If the command times out, and --preserve-status is not set, then exit with status 124. - assert_stderr --partial "push and pull to Central API disabled" + rune -0 wait-for \ + --err "push and pull to Central API disabled" \ + "${CROWDSEC}" } @test "without capi: cscli capi status -> fail" { @@ -47,10 +48,7 @@ teardown() { @test "no capi: cscli config show" { config_disable_capi rune -0 cscli config show -o human - assert_output --partial "Global:" - assert_output --partial "cscli:" - assert_output --partial "Crowdsec:" - assert_output --partial "Local API Server:" + assert_output --regexp "Global:.*Crowdsec.*cscli:.*Local API Server:" } @test "no agent: cscli config backup" { diff --git a/test/bats/05_config_yaml_local.bats b/test/bats/05_config_yaml_local.bats index 3cc20819b..6f8b5c28b 100644 --- a/test/bats/05_config_yaml_local.bats +++ b/test/bats/05_config_yaml_local.bats @@ -56,28 +56,28 @@ teardown() { # disable the agent or we'll need to patch api client credentials too rune -0 config_disable_agent ./instance-crowdsec start - rune -0 ./bin/wait-for-port -q 8080 + rune -0 wait-for-port -q 8080 ./instance-crowdsec stop - rune -1 ./bin/wait-for-port -q 8080 + rune -1 wait-for-port -q 8080 echo "{'api':{'server':{'listen_uri':127.0.0.1:8083}}}" >"${CONFIG_YAML}.local" ./instance-crowdsec start - rune -0 ./bin/wait-for-port -q 8083 - rune -1 ./bin/wait-for-port -q 8080 + rune -0 wait-for-port -q 8083 + rune -1 wait-for-port -q 8080 ./instance-crowdsec stop rm -f "${CONFIG_YAML}.local" ./instance-crowdsec start - rune -1 ./bin/wait-for-port -q 8083 - rune -0 ./bin/wait-for-port -q 8080 + rune -1 wait-for-port -q 8083 + rune -0 wait-for-port -q 8080 } @test "local_api_credentials.yaml.local" { rune -0 config_disable_agent echo "{'api':{'server':{'listen_uri':127.0.0.1:8083}}}" >"${CONFIG_YAML}.local" ./instance-crowdsec start - rune -0 ./bin/wait-for-port -q 8083 + rune -0 wait-for-port -q 8083 rune -1 cscli decisions list echo "{'url':'http://127.0.0.1:8083'}" >"${LOCAL_API_CREDENTIALS}.local" @@ -127,6 +127,9 @@ teardown() { ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') echo -e "---\nfilename: ${tmpfile}\nlabels:\n type: syslog\n" >>"${ACQUIS_YAML}" + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli parsers install crowdsecurity/syslog-logs + ./instance-crowdsec start sleep .5 fake_log >>"${tmpfile}" diff --git a/test/bats/07_setup.bats b/test/bats/07_setup.bats index 9d6b32d15..9e3f55337 100644 --- a/test/bats/07_setup.bats +++ b/test/bats/07_setup.bats @@ -507,46 +507,49 @@ update-notifier-motd.timer enabled enabled @test "cscli setup install-hub (dry run)" { # it's not installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/apache2" + rune -0 cscli collections inspect crowdsecurity/apache2 -o json + rune -0 jq -e '.installed == false' <(output) # we install it rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' assert_output 'dry-run: would install collection crowdsecurity/apache2' # still not installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/apache2" + rune -0 cscli collections inspect crowdsecurity/apache2 -o json + rune -0 jq -e '.installed == false' <(output) + + # same with dependencies + rune -0 cscli collections remove --all + rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/linux"]}}]}' + assert_output 'dry-run: would install collection crowdsecurity/linux' } @test "cscli setup install-hub (dry run: install multiple collections)" { # it's not installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/apache2" + rune -0 cscli collections inspect crowdsecurity/apache2 -o json + rune -0 jq -e '.installed == false' <(output) # we install it rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' assert_output 'dry-run: would install collection crowdsecurity/apache2' # still not installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/apache2" + rune -0 cscli collections inspect crowdsecurity/apache2 -o json + rune -0 jq -e '.installed == false' <(output) } @test "cscli setup install-hub (dry run: install multiple collections, parsers, scenarios, postoverflows)" { - rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo","johndoe/bar"],"parsers":["crowdsecurity/fooparser","johndoe/barparser"],"scenarios":["crowdsecurity/fooscenario","johndoe/barscenario"],"postoverflows":["crowdsecurity/foopo","johndoe/barpo"]}}]}' - assert_line 'dry-run: would install collection crowdsecurity/foo' - assert_line 'dry-run: would install collection johndoe/bar' - assert_line 'dry-run: would install parser crowdsecurity/fooparser' - assert_line 'dry-run: would install parser johndoe/barparser' - assert_line 'dry-run: would install scenario crowdsecurity/fooscenario' - assert_line 'dry-run: would install scenario johndoe/barscenario' - assert_line 'dry-run: would install postoverflow crowdsecurity/foopo' - assert_line 'dry-run: would install postoverflow johndoe/barpo' + rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/aws-console","crowdsecurity/caddy"],"parsers":["crowdsecurity/asterisk-logs"],"scenarios":["crowdsecurity/smb-fs"],"postoverflows":["crowdsecurity/cdn-whitelist","crowdsecurity/rdns"]}}]}' + assert_line 'dry-run: would install collection crowdsecurity/aws-console' + assert_line 'dry-run: would install collection crowdsecurity/caddy' + assert_line 'dry-run: would install parser crowdsecurity/asterisk-logs' + assert_line 'dry-run: would install scenario crowdsecurity/smb-fs' + assert_line 'dry-run: would install postoverflow crowdsecurity/cdn-whitelist' + assert_line 'dry-run: would install postoverflow crowdsecurity/rdns' + + rune -1 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo"]}}]}' + assert_stderr --partial 'collection crowdsecurity/foo not found' + } @test "cscli setup datasources" { diff --git a/test/bats/08_metrics.bats b/test/bats/08_metrics.bats index 836e22048..0275d7fd4 100644 --- a/test/bats/08_metrics.bats +++ b/test/bats/08_metrics.bats @@ -25,8 +25,7 @@ teardown() { @test "cscli metrics (crowdsec not running)" { rune -1 cscli metrics # crowdsec is down - assert_stderr --partial "failed to fetch prometheus metrics" - assert_stderr --partial "connect: connection refused" + assert_stderr --partial 'failed to fetch prometheus metrics: executing GET request for URL \"http://127.0.0.1:6060/metrics\" failed: Get \"http://127.0.0.1:6060/metrics\": dial tcp 127.0.0.1:6060: connect: connection refused' } @test "cscli metrics (bad configuration)" { @@ -43,18 +42,20 @@ teardown() { @test "cscli metrics (missing listen_addr)" { config_set 'del(.prometheus.listen_addr)' - rune -1 cscli metrics - assert_stderr --partial "no prometheus url, please specify" + rune -0 ./instance-crowdsec start + rune -0 cscli metrics --debug + assert_stderr --partial "prometheus.listen_addr is empty, defaulting to 127.0.0.1" } @test "cscli metrics (missing listen_port)" { - config_set 'del(.prometheus.listen_addr)' - rune -1 cscli metrics - assert_stderr --partial "no prometheus url, please specify" + config_set 'del(.prometheus.listen_port)' + rune -0 ./instance-crowdsec start + rune -0 cscli metrics --debug + assert_stderr --partial "prometheus.listen_port is empty or zero, defaulting to 6060" } @test "cscli metrics (missing prometheus section)" { config_set 'del(.prometheus)' rune -1 cscli metrics - assert_stderr --partial "prometheus section missing, can't show metrics" + assert_stderr --partial "prometheus is not enabled, can't show metrics" } diff --git a/test/bats/13_capi_whitelists.bats b/test/bats/13_capi_whitelists.bats index 491de6498..61d0e641c 100644 --- a/test/bats/13_capi_whitelists.bats +++ b/test/bats/13_capi_whitelists.bats @@ -18,6 +18,7 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load + config_set '.common.log_media="stdout"' config_set '.api.server.capi_whitelists_path=strenv(CAPI_WHITELISTS_YAML)' } @@ -28,38 +29,51 @@ teardown() { #---------- @test "capi_whitelists: file missing" { - rune -1 timeout 1s "${CROWDSEC}" - assert_stderr --partial "capi whitelist file '$CAPI_WHITELISTS_YAML' does not exist" + rune -0 wait-for \ + --err "capi whitelist file '$CAPI_WHITELISTS_YAML' does not exist" \ + "${CROWDSEC}" } @test "capi_whitelists: error on open" { echo > "$CAPI_WHITELISTS_YAML" chmod 000 "$CAPI_WHITELISTS_YAML" - rune -1 timeout 1s "${CROWDSEC}" - assert_stderr --partial "while opening capi whitelist file: open $CAPI_WHITELISTS_YAML: permission denied" + if is_package_testing; then + rune -0 wait-for \ + --err "while parsing capi whitelist file .*: empty file" \ + "${CROWDSEC}" + else + rune -0 wait-for \ + --err "while opening capi whitelist file: open $CAPI_WHITELISTS_YAML: permission denied" \ + "${CROWDSEC}" + fi } @test "capi_whitelists: empty file" { echo > "$CAPI_WHITELISTS_YAML" - rune -1 timeout 1s "${CROWDSEC}" - assert_stderr --partial "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': empty file" + rune -0 wait-for \ + --err "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': empty file" \ + "${CROWDSEC}" } @test "capi_whitelists: empty lists" { echo '{"ips": [], "cidrs": []}' > "$CAPI_WHITELISTS_YAML" - rune -124 timeout 1s "${CROWDSEC}" + rune -0 wait-for \ + --err "Starting processing data" \ + "${CROWDSEC}" } @test "capi_whitelists: bad ip" { echo '{"ips": ["blahblah"], "cidrs": []}' > "$CAPI_WHITELISTS_YAML" - rune -1 timeout 1s "${CROWDSEC}" - assert_stderr --partial "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid IP address: blahblah" + rune -0 wait-for \ + --err "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid IP address: blahblah" \ + "${CROWDSEC}" } @test "capi_whitelists: bad cidr" { echo '{"ips": [], "cidrs": ["blahblah"]}' > "$CAPI_WHITELISTS_YAML" - rune -1 timeout 1s "${CROWDSEC}" - assert_stderr --partial "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid CIDR address: blahblah" + rune -0 wait-for \ + --err "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid CIDR address: blahblah" \ + "${CROWDSEC}" } @test "capi_whitelists: file with ip and cidr values" { diff --git a/test/bats/20_collections.bats b/test/bats/20_collections.bats deleted file mode 100644 index aa1fa6b21..000000000 --- a/test/bats/20_collections.bats +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bats -# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: - -set -u - -setup_file() { - load "../lib/setup_file.sh" -} - -teardown_file() { - load "../lib/teardown_file.sh" -} - -setup() { - load "../lib/setup.sh" - ./instance-data load - ./instance-crowdsec start -} - -teardown() { - ./instance-crowdsec stop -} - -#---------- - -@test "we can list collections" { - rune -0 cscli collections list -} - -@test "there are 2 collections (linux and sshd)" { - rune -0 cscli collections list -o json - rune -0 jq '.collections | length' <(output) - assert_output 2 -} - -@test "can install a collection (as a regular user) and remove it" { - # collection is not installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/mysql" - - # we install it - rune -0 cscli collections install crowdsecurity/mysql -o human - assert_stderr --partial "Enabled crowdsecurity/mysql" - - # it has been installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - assert_line "crowdsecurity/mysql" - - # we install it - rune -0 cscli collections remove crowdsecurity/mysql -o human - assert_stderr --partial "Removed symlink [crowdsecurity/mysql]" - - # it has been removed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/mysql" -} - -@test "must use --force to remove a collection that belongs to another, which becomes tainted" { - # we expect no error since we may have multiple collections, some removed and some not - rune -0 cscli collections remove crowdsecurity/sshd - assert_stderr --partial "crowdsecurity/sshd belongs to other collections" - assert_stderr --partial "[crowdsecurity/linux]" - - rune -0 cscli collections remove crowdsecurity/sshd --force - assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" - rune -0 cscli collections inspect crowdsecurity/linux -o json - rune -0 jq -r '.tainted' <(output) - assert_output "true" -} - -@test "can remove a collection" { - rune -0 cscli collections remove crowdsecurity/linux - assert_stderr --partial "Removed" - assert_stderr --regexp ".*for the new configuration to be effective." - rune -0 cscli collections inspect crowdsecurity/linux -o human - assert_line 'installed: false' -} - -@test "collections delete is an alias for collections remove" { - rune -0 cscli collections delete crowdsecurity/linux - assert_stderr --partial "Removed" - assert_stderr --regexp ".*for the new configuration to be effective." -} - -@test "removing a collection that does not exist is noop" { - rune -0 cscli collections remove crowdsecurity/apache2 - refute_stderr --partial "Removed" - assert_stderr --regexp ".*for the new configuration to be effective." -} - -@test "can remove a removed collection" { - rune -0 cscli collections install crowdsecurity/mysql - rune -0 cscli collections remove crowdsecurity/mysql - assert_stderr --partial "Removed" - rune -0 cscli collections remove crowdsecurity/mysql - refute_stderr --partial "Removed" -} - -@test "can remove all collections" { - # we may have this too, from package installs - rune cscli parsers delete crowdsecurity/whitelists - rune -0 cscli collections remove --all - assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" - assert_stderr --partial "Removed symlink [crowdsecurity/linux]" - rune -0 cscli hub list -o json - assert_json '{collections:[],parsers:[],postoverflows:[],scenarios:[]}' - rune -0 cscli collections remove --all - assert_stderr --partial 'Disabled 0 items' -} - -@test "a taint bubbles up to the top collection" { - coll=crowdsecurity/nginx - subcoll=crowdsecurity/base-http-scenarios - scenario=crowdsecurity/http-crawl-non_statics - - # install a collection with dependencies - rune -0 cscli collections install "$coll" - - # the collection, subcollection and scenario are installed and not tainted - # we have to default to false because tainted is (as of 1.4.6) returned - # only when true - rune -0 cscli collections inspect "$coll" -o json - rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) - rune -0 cscli collections inspect "$subcoll" -o json - rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) - rune -0 cscli scenarios inspect "$scenario" -o json - rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) - - # we taint the scenario - HUB_DIR=$(config_get '.config_paths.hub_dir') - yq e '.description="I am tainted"' -i "$HUB_DIR/scenarios/$scenario.yaml" - - # the collection, subcollection and scenario are now tainted - rune -0 cscli scenarios inspect "$scenario" -o json - rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) - rune -0 cscli collections inspect "$subcoll" -o json - rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) - rune -0 cscli collections inspect "$coll" -o json - rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) -} - -# TODO test download-only diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats new file mode 100644 index 000000000..f0a8fa4dc --- /dev/null +++ b/test/bats/20_hub.bats @@ -0,0 +1,121 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_strip_index +} + +teardown() { + : +} + +#---------- + +@test "cscli hub list" { + hub_purge_all + + # no items + rune -0 cscli hub list + assert_output --regexp ".*PARSERS.*POSTOVERFLOWS.*SCENARIOS.*COLLECTIONS.*" + rune -0 cscli hub list -o json + assert_json '{parsers:[],scenarios:[],collections:[],postoverflows:[]}' + rune -0 cscli hub list -o raw + assert_output 'name,status,version,description,type' + + # some items + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli scenarios install crowdsecurity/telnet-bf + rune -0 cscli hub list + assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*" + rune -0 cscli hub list -o json + rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output) + rune -0 cscli hub list -o raw + assert_output --partial 'crowdsecurity/whitelists' + assert_output --partial 'crowdsecurity/telnet-bf' + refute_output --partial 'crowdsecurity/iptables' + + # all items + rune -0 cscli hub list -a + assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*crowdsecurity/iptables.*" + rune -0 cscli hub list -a -o json + rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output) + rune -0 cscli hub list -a -o raw + assert_output --partial 'crowdsecurity/whitelists' + assert_output --partial 'crowdsecurity/telnet-bf' + assert_output --partial 'crowdsecurity/iptables' +} + +@test "missing reference in hub index" { + new_hub=$(jq <"$INDEX_PATH" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")') + echo "$new_hub" >"$INDEX_PATH" + rune -0 cscli hub list --error + assert_stderr --partial "can't find crowdsecurity/smb-logs in parsers, required by crowdsecurity/smb" + assert_stderr --partial "can't find crowdsecurity/mysql-bf in scenarios, required by crowdsecurity/mysql" +} + +@test "loading hub reports tainted items (subitem is tainted)" { + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli hub list + refute_stderr --partial "tainted" + rune -0 truncate -s0 "$CONFIG_DIR/parsers/s01-parse/sshd-logs.yaml" + rune -0 cscli hub list + assert_stderr --partial "crowdsecurity/sshd is tainted because parsers:crowdsecurity/sshd-logs is tainted" +} + +@test "loading hub reports tainted items (subitem is not installed)" { + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli hub list + refute_stderr --partial "tainted" + rune -0 rm "$CONFIG_DIR/parsers/s01-parse/sshd-logs.yaml" + rune -0 cscli hub list + assert_stderr --partial "crowdsecurity/sshd is tainted because parsers:crowdsecurity/sshd-logs is missing" +} + +@test "cscli hub update" { + rm -f "$INDEX_PATH" + rune -0 cscli hub update + assert_stderr --partial "Wrote index to $INDEX_PATH" + rune -0 cscli hub update + assert_stderr --partial "hub index is up to date" +} + +@test "cscli hub upgrade" { + rune -0 cscli hub upgrade + assert_stderr --partial "Upgrading parsers" + assert_stderr --partial "Upgraded 0 parsers" + assert_stderr --partial "Upgrading postoverflows" + assert_stderr --partial "Upgraded 0 postoverflows" + assert_stderr --partial "Upgrading scenarios" + assert_stderr --partial "Upgraded 0 scenarios" + assert_stderr --partial "Upgrading collections" + assert_stderr --partial "Upgraded 0 collections" + + rune -0 cscli parsers install crowdsecurity/syslog-logs + rune -0 cscli hub upgrade + assert_stderr --partial "crowdsecurity/syslog-logs: up-to-date" + + rune -0 cscli hub upgrade --force + assert_stderr --partial "crowdsecurity/syslog-logs: overwrite" + assert_stderr --partial "crowdsecurity/syslog-logs: updated" + assert_stderr --partial "Upgraded 1 parsers" + # this is used by the cron script to know if the hub was updated + assert_output --partial "updated crowdsecurity/syslog-logs" +} diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats new file mode 100644 index 000000000..f49d0e24b --- /dev/null +++ b/test/bats/20_hub_collections.bats @@ -0,0 +1,381 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_strip_index +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli collections list" { + hub_purge_all + + # no items + rune -0 cscli collections list + assert_output --partial "COLLECTIONS" + rune -0 cscli collections list -o json + assert_json '{collections:[]}' + rune -0 cscli collections list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + + rune -0 cscli collections list + assert_output --partial crowdsecurity/sshd + assert_output --partial crowdsecurity/smb + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli collections list -o json + assert_output --partial crowdsecurity/sshd + assert_output --partial crowdsecurity/smb + rune -0 jq '.collections | length' <(output) + assert_output "2" + + rune -0 cscli collections list -o raw + assert_output --partial crowdsecurity/sshd + assert_output --partial crowdsecurity/smb + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli collections list -a" { + expected=$(jq <"$INDEX_PATH" -r '.collections | length') + + rune -0 cscli collections list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli collections list -o json -a + rune -0 jq '.collections | length' <(output) + assert_output "$expected" + + rune -0 cscli collections list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" + + # the list should be the same in all formats, and sorted (not case sensitive) + + list_raw=$(cscli collections list -o raw -a | tail -n +2 | cut -d, -f1) + list_human=$(cscli collections list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) + list_json=$(cscli collections list -o json -a | jq -r '.collections[].name') + + rune -0 sort -f <<<"$list_raw" + assert_output "$list_raw" + + assert_equal "$list_raw" "$list_json" + assert_equal "$list_raw" "$list_human" +} + +@test "cscli collections list [collection]..." { + # non-existent + rune -1 cscli collections install foo/bar + assert_stderr --partial "can't find 'foo/bar' in collections" + + # not installed + rune -0 cscli collections list crowdsecurity/smb + assert_output --regexp 'crowdsecurity/smb.*disabled' + + # install two items + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + + # list an installed item + rune -0 cscli collections list crowdsecurity/sshd + assert_output --regexp "crowdsecurity/sshd" + refute_output --partial "crowdsecurity/smb" + + # list multiple installed and non installed items + rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb crowdsecurity/nginx + assert_output --partial "crowdsecurity/sshd" + assert_output --partial "crowdsecurity/smb" + assert_output --partial "crowdsecurity/nginx" + + rune -0 cscli collections list crowdsecurity/sshd -o json + rune -0 jq '.collections | length' <(output) + assert_output "1" + rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb crowdsecurity/nginx -o json + rune -0 jq '.collections | length' <(output) + assert_output "3" + + rune -0 cscli collections list crowdsecurity/sshd -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli collections install" { + rune -1 cscli collections install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli collections install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in collections" + + # simple install + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli collections install crowdsecurity/ssshd + assert_stderr --partial "can't find 'crowdsecurity/ssshd' in collections, did you mean 'crowdsecurity/sshd'?" + + # install multiple + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'installed: true' + rune -0 cscli collections inspect crowdsecurity/smb --no-metrics + assert_output --partial 'crowdsecurity/smb' + assert_output --partial 'installed: true' +} + +@test "cscli collections install (file location and download-only)" { + rune -0 cscli collections install crowdsecurity/linux --download-only + rune -0 cscli collections inspect crowdsecurity/linux --no-metrics + assert_output --partial 'crowdsecurity/linux' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/collections/crowdsecurity/linux.yaml" + assert_file_not_exists "$CONFIG_DIR/collections/linux.yaml" + + rune -0 cscli collections install crowdsecurity/linux + rune -0 cscli collections inspect crowdsecurity/linux --no-metrics + assert_output --partial 'installed: true' + assert_file_exists "$CONFIG_DIR/collections/linux.yaml" +} + +@test "cscli collections install --force (tainted)" { + rune -0 cscli collections install crowdsecurity/sshd + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + + rune -1 cscli collections install crowdsecurity/sshd + assert_stderr --partial "error while installing 'crowdsecurity/sshd': while enabling crowdsecurity/sshd: crowdsecurity/sshd is tainted, won't enable unless --force" + + rune -0 cscli collections install crowdsecurity/sshd --force + assert_stderr --partial "crowdsecurity/sshd: overwrite" + assert_stderr --partial "Enabled crowdsecurity/sshd" +} + +@test "cscli collections install --ignore (skip on errors)" { + rune -1 cscli collections install foo/bar crowdsecurity/sshd + assert_stderr --partial "can't find 'foo/bar' in collections" + refute_stderr --partial "Enabled collections: crowdsecurity/sshd" + + rune -0 cscli collections install foo/bar crowdsecurity/sshd --ignore + assert_stderr --partial "can't find 'foo/bar' in collections" + assert_stderr --partial "Enabled collections: crowdsecurity/sshd" +} + +@test "cscli collections inspect" { + rune -1 cscli collections inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + # required for metrics + ./instance-crowdsec start + + rune -1 cscli collections inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + + # one item + rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics + assert_line 'type: collections' + assert_line 'name: crowdsecurity/sshd' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: collections/crowdsecurity/sshd.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli collections inspect crowdsecurity/sshd + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output) + assert_json '["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",false]' + + # one item, raw + rune -0 cscli collections inspect crowdsecurity/sshd -o raw + assert_line 'type: collections' + assert_line 'name: crowdsecurity/sshd' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: collections/crowdsecurity/sshd.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb --no-metrics + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'crowdsecurity/smb' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o json + rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output) + assert_json '[["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",false],["collections","crowdsecurity/smb","crowdsecurity","collections/crowdsecurity/smb.yaml",false]]' + + # multiple items, raw + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o raw + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'crowdsecurity/smb' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli collections remove" { + rune -1 cscli collections remove + assert_stderr --partial "specify at least one collection to remove or '--all'" + rune -1 cscli collections remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + + rune -0 cscli collections install crowdsecurity/sshd --download-only + rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial 'removing crowdsecurity/sshd: not installed -- no need to remove' + + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial 'Removed crowdsecurity/sshd' + + rune -0 cscli collections remove crowdsecurity/sshd --purge + assert_stderr --partial 'Removed source file [crowdsecurity/sshd]' + + rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial 'removing crowdsecurity/sshd: not installed -- no need to remove' + + rune -0 cscli collections remove crowdsecurity/sshd --purge + assert_stderr --partial 'removing crowdsecurity/sshd: not downloaded -- no need to remove' + + # install, then remove, check files + rune -0 cscli collections install crowdsecurity/sshd + assert_file_exists "$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections remove crowdsecurity/sshd + assert_file_not_exists "$CONFIG_DIR/collections/sshd.yaml" + + # delete is an alias for remove + rune -0 cscli collections install crowdsecurity/sshd + assert_file_exists "$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections delete crowdsecurity/sshd + assert_file_not_exists "$CONFIG_DIR/collections/sshd.yaml" + + # purge + assert_file_exists "$HUB_DIR/collections/crowdsecurity/sshd.yaml" + rune -0 cscli collections remove crowdsecurity/sshd --purge + assert_file_not_exists "$HUB_DIR/collections/crowdsecurity/sshd.yaml" + + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + + # --all + rune -0 cscli collections list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli collections remove --all + + rune -0 cscli collections list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli collections remove --force" { + # remove a collections that belongs to a collection + rune -0 cscli collections install crowdsecurity/linux + rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial "crowdsecurity/sshd belongs to collections: [crowdsecurity/linux]" + assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this collection" +} + +@test "cscli collections upgrade" { + rune -1 cscli collections upgrade + assert_stderr --partial "specify at least one collection to upgrade or '--all'" + rune -1 cscli collections upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + rune -0 cscli collections remove crowdsecurity/exim --purge + rune -1 cscli collections upgrade crowdsecurity/exim + assert_stderr --partial "can't upgrade crowdsecurity/exim: not installed" + rune -0 cscli collections install crowdsecurity/exim --download-only + rune -1 cscli collections upgrade crowdsecurity/exim + assert_stderr --partial "can't upgrade crowdsecurity/exim: downloaded but not installed" + + # hash of the string "v0.0" + sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" + + # add version 0.0 to all collections + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.collections |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') + echo "$new_hub" >"$INDEX_PATH" + + rune -0 cscli collections install crowdsecurity/sshd + + echo "v0.0" > "$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli collections upgrade crowdsecurity/sshd + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + # XXX: should return error + rune -0 cscli collections upgrade crowdsecurity/sshd + assert_stderr --partial "crowdsecurity/sshd is tainted, --force to overwrite" + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli collections upgrade crowdsecurity/sshd --force + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli collections install crowdsecurity/smb + echo "v0.0" >"$CONFIG_DIR/collections/sshd.yaml" + echo "v0.0" >"$CONFIG_DIR/collections/smb.yaml" + rune -0 cscli collections list -o json + rune -0 jq -e '[.collections[].local_version]==["0.0","0.0"]' <(output) + rune -0 cscli collections upgrade crowdsecurity/sshd crowdsecurity/smb + rune -0 cscli collections list -o json + rune -0 jq -e 'any(.collections[].local_version; .=="0.0") | not' <(output) + + # upgrade all + echo "v0.0" >"$CONFIG_DIR/collections/sshd.yaml" + echo "v0.0" >"$CONFIG_DIR/collections/smb.yaml" + rune -0 cscli collections list -o json + rune -0 jq -e '[.collections[].local_version]==["0.0","0.0"]' <(output) + rune -0 cscli collections upgrade --all + rune -0 cscli collections list -o json + rune -0 jq -e 'any(.collections[].local_version; .=="0.0") | not' <(output) +} diff --git a/test/bats/20_hub_collections_dep.bats b/test/bats/20_hub_collections_dep.bats new file mode 100644 index 000000000..c3df948a3 --- /dev/null +++ b/test/bats/20_hub_collections_dep.bats @@ -0,0 +1,126 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_strip_index +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli collections (dependencies)" { + # inject a dependency: smb requires sshd + hub_dep=$(jq <"$INDEX_PATH" '. * {collections:{"crowdsecurity/smb":{collections:["crowdsecurity/sshd"]}}}') + echo "$hub_dep" >"$INDEX_PATH" + + # verify that installing smb brings sshd + rune -0 cscli collections install crowdsecurity/smb + rune -0 cscli collections list -o json + rune -0 jq -e '[.collections[].name]==["crowdsecurity/smb","crowdsecurity/sshd"]' <(output) + + # verify that removing smb removes sshd too + rune -0 cscli collections remove crowdsecurity/smb + rune -0 cscli collections list -o json + rune -0 jq -e '.collections | length == 0' <(output) + + # we can't remove sshd without --force + rune -0 cscli collections install crowdsecurity/smb + # XXX: should this be an error? + rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial "crowdsecurity/sshd belongs to collections: [crowdsecurity/smb]" + assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this collection" + rune -0 cscli collections list -o json + rune -0 jq -c '[.collections[].name]' <(output) + assert_json '["crowdsecurity/smb","crowdsecurity/sshd"]' + + # use the --force + rune -0 cscli collections remove crowdsecurity/sshd --force + rune -0 cscli collections list -o json + rune -0 jq -c '[.collections[].name]' <(output) + assert_json '["crowdsecurity/smb"]' + + # and now smb is tainted! + rune -0 cscli collections inspect crowdsecurity/smb -o json + rune -0 jq -e '.tainted==true' <(output) + rune -0 cscli collections remove crowdsecurity/smb --force + + # empty + rune -0 cscli collections list -o json + rune -0 jq -e '.collections | length == 0' <(output) + + # reinstall + rune -0 cscli collections install crowdsecurity/smb --force + + # taint on sshd means smb is tainted as well + rune -0 cscli collections inspect crowdsecurity/smb -o json + rune -0 jq -e '.tainted==false' <(output) + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections inspect crowdsecurity/smb -o json + rune -0 jq -e '.tainted==true' <(output) + + # now we can't remove smb without --force + rune -1 cscli collections remove crowdsecurity/smb + assert_stderr --partial "crowdsecurity/smb is tainted, use '--force' to remove" +} + +@test "cscli collections (dependencies II: the revenge)" { + rune -0 cscli collections install crowdsecurity/wireguard baudneo/gotify + rune -0 cscli collections remove crowdsecurity/wireguard + assert_stderr --partial "crowdsecurity/syslog-logs was not removed because it also belongs to baudneo/gotify" + rune -0 cscli collections inspect crowdsecurity/wireguard -o json + rune -0 jq -e '.installed==false' <(output) +} + +@test "cscli collections (dependencies III: origins)" { + # it is perfectly fine to remove an item belonging to a collection that we are removing anyway + + # inject a dependency: sshd requires the syslog-logs parsers, but linux does too + hub_dep=$(jq <"$INDEX_PATH" '. * {collections:{"crowdsecurity/sshd":{parsers:["crowdsecurity/syslog-logs"]}}}') + echo "$hub_dep" >"$INDEX_PATH" + + # verify that installing sshd brings syslog-logs + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json + rune -0 jq -e '.installed==true' <(output) + + rune -0 cscli collections install crowdsecurity/linux + + # removing linux should remove syslog-logs even though sshd depends on it + rune -0 cscli collections remove crowdsecurity/linux + refute_stderr --partial "crowdsecurity/syslog-logs was not removed" + # we must also consider indirect dependencies + refute_stderr --partial "crowdsecurity/ssh-bf was not removed" + rune -0 cscli parsers list -o json + rune -0 jq -e '.parsers | length == 0' <(output) +} + +@test "cscli collections (dependencies IV: looper)" { + hub_dep=$(jq <"$INDEX_PATH" '. * {collections:{"crowdsecurity/sshd":{collections:["crowdsecurity/linux"]}}}') + echo "$hub_dep" >"$INDEX_PATH" + + rune -1 cscli hub list + assert_stderr --partial "circular dependency detected" + rune -1 wait-for "${CROWDSEC}" + assert_stderr --partial "circular dependency detected" +} diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats new file mode 100644 index 000000000..3fb38c663 --- /dev/null +++ b/test/bats/20_hub_items.bats @@ -0,0 +1,149 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_strip_index +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- +# +# Tests that don't need to be repeated for each hub type +# + +@test "hub versions are correctly sorted during sync" { + # hash of an empty file + sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + # add two versions with the same hash, that don't sort the same way + # in a lexical vs semver sort. CrowdSec should report the latest version + + new_hub=$( \ + jq --arg DIGEST "$sha256_empty" <"$INDEX_PATH" \ + '. * {collections:{"crowdsecurity/sshd":{"versions":{"1.2":{"digest":$DIGEST, "deprecated": false}, "1.10": {"digest":$DIGEST, "deprecated": false}}}}}' \ + ) + echo "$new_hub" >"$INDEX_PATH" + + rune -0 cscli collections install crowdsecurity/sshd + + truncate -s 0 "$CONFIG_DIR/collections/sshd.yaml" + + rune -0 cscli collections inspect crowdsecurity/sshd -o json + # XXX: is this supposed to be tainted or up to date? + rune -0 jq -c '[.local_version,.up_to_date,.tainted]' <(output) + assert_json '["1.10",false,false]' +} + +@test "do not unmarshal state attributes" { + new_hub=$( \ + jq <"$INDEX_PATH" \ + '. * {parsers:{"crowdsecurity/syslog-logs":{"tainted":true, "installed":true, "local":true}}}' + ) + echo "$new_hub" >"$INDEX_PATH" + + rune -0 cscli parsers inspect crowdsecurity/syslog-logs --no-metrics + assert_output --partial 'tainted: false' + assert_output --partial 'installed: false' + assert_output --partial 'local: false' +} + +@test "hub index with invalid (non semver) version numbers" { + rune -0 cscli collections remove crowdsecurity/sshd --purge + + new_hub=$( \ + jq <"$INDEX_PATH" \ + '. * {collections:{"crowdsecurity/sshd":{"versions":{"1.2.3.4":{"digest":"foo", "deprecated": false}}}}}' \ + ) + echo "$new_hub" >"$INDEX_PATH" + + rune -0 cscli collections install crowdsecurity/sshd + rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics -o json + # XXX: we are on the verbose side here... + rune -0 jq -r ".msg" <(stderr) + assert_output --regexp "failed to read Hub index: failed to sync items: failed to scan .*: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again" +} + +@test "removing or purging an item already removed by hand" { + rune -0 cscli parsers install crowdsecurity/syslog-logs + rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json + rune -0 jq -r '.local_path' <(output) + rune -0 rm "$(output)" + + rune -0 cscli parsers remove crowdsecurity/syslog-logs --debug + assert_stderr --partial "removing crowdsecurity/syslog-logs: not installed -- no need to remove" + + rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json + rune -0 jq -r '.path' <(output) + rune -0 rm "$HUB_DIR/$(output)" + + rune -0 cscli parsers remove crowdsecurity/syslog-logs --purge + assert_stderr --partial "removing crowdsecurity/syslog-logs: not downloaded -- no need to remove" + + rune -0 cscli parsers remove crowdsecurity/linux --all --error --purge --force + rune -0 cscli collections remove crowdsecurity/linux --all --error --purge --force + refute_output + refute_stderr +} + +@test "a local item is not tainted" { + # not from cscli... inspect + rune -0 mkdir -p "$CONFIG_DIR/collections" + rune -0 touch "$CONFIG_DIR/collections/foobar.yaml" + rune -0 cscli collections inspect foobar.yaml -o json + rune -0 jq -e '[.tainted,.local==false,true]' <(output) + + rune -0 cscli collections install crowdsecurity/sshd + rune -0 truncate -s0 "$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '[.tainted,.local==true,false]' <(output) + + # and not from hub update + rune -0 cscli hub update + assert_stderr --partial "collection crowdsecurity/sshd is tainted" + refute_stderr --partial "collection foobar.yaml is tainted" +} + +@test "a local item's name defaults to its filename" { + rune -0 mkdir -p "$CONFIG_DIR/collections" + rune -0 touch "$CONFIG_DIR/collections/foobar.yaml" + rune -0 cscli collections list -o json + rune -0 jq -r '.[][].name' <(output) + assert_output "foobar.yaml" + rune -0 cscli collections list foobar.yaml + rune -0 cscli collections inspect foobar.yaml -o json + rune -0 jq -e '[.installed,.local==true,true]' <(output) +} + +@test "a local item can provide its own name" { + rune -0 mkdir -p "$CONFIG_DIR/collections" + echo "name: hi-its-me" > "$CONFIG_DIR/collections/foobar.yaml" + rune -0 cscli collections list -o json + rune -0 jq -r '.[][].name' <(output) + assert_output "hi-its-me" + rune -0 cscli collections list hi-its-me + rune -0 cscli collections inspect hi-its-me -o json + rune -0 jq -e '[.installed,.local]==[true,true]' <(output) +} diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats new file mode 100644 index 000000000..c780457b3 --- /dev/null +++ b/test/bats/20_hub_parsers.bats @@ -0,0 +1,383 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_strip_index +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli parsers list" { + hub_purge_all + + # no items + rune -0 cscli parsers list + assert_output --partial "PARSERS" + rune -0 cscli parsers list -o json + assert_json '{parsers:[]}' + rune -0 cscli parsers list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + rune -0 cscli parsers list + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli parsers list -o json + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 jq '.parsers | length' <(output) + assert_output "2" + + rune -0 cscli parsers list -o raw + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli parsers list -a" { + expected=$(jq <"$INDEX_PATH" -r '.parsers | length') + + rune -0 cscli parsers list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli parsers list -o json -a + rune -0 jq '.parsers | length' <(output) + assert_output "$expected" + + rune -0 cscli parsers list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" + + # the list should be the same in all formats, and sorted (not case sensitive) + + list_raw=$(cscli parsers list -o raw -a | tail -n +2 | cut -d, -f1) + list_human=$(cscli parsers list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) + list_json=$(cscli parsers list -o json -a | jq -r '.parsers[].name') + + rune -0 sort -f <<<"$list_raw" + assert_output "$list_raw" + + assert_equal "$list_raw" "$list_json" + assert_equal "$list_raw" "$list_human" +} + +@test "cscli parsers list [parser]..." { + # non-existent + rune -1 cscli parsers install foo/bar + assert_stderr --partial "can't find 'foo/bar' in parsers" + + # not installed + rune -0 cscli parsers list crowdsecurity/whitelists + assert_output --regexp 'crowdsecurity/whitelists.*disabled' + + # install two items + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + # list an installed item + rune -0 cscli parsers list crowdsecurity/whitelists + assert_output --regexp "crowdsecurity/whitelists.*enabled" + refute_output --partial "crowdsecurity/windows-auth" + + # list multiple installed and non installed items + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs + assert_output --partial "crowdsecurity/whitelists" + assert_output --partial "crowdsecurity/windows-auth" + assert_output --partial "crowdsecurity/traefik-logs" + + rune -0 cscli parsers list crowdsecurity/whitelists -o json + rune -0 jq '.parsers | length' <(output) + assert_output "1" + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o json + rune -0 jq '.parsers | length' <(output) + assert_output "3" + + rune -0 cscli parsers list crowdsecurity/whitelists -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "3" +} + +@test "cscli parsers install" { + rune -1 cscli parsers install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli parsers install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in parsers" + + # simple install + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics + assert_output --partial 'crowdsecurity/whitelists' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli parsers install crowdsecurity/sshd-logz + assert_stderr --partial "can't find 'crowdsecurity/sshd-logz' in parsers, did you mean 'crowdsecurity/sshd-logs'?" + + # install multiple + rune -0 cscli parsers install crowdsecurity/pgsql-logs crowdsecurity/postfix-logs + rune -0 cscli parsers inspect crowdsecurity/pgsql-logs --no-metrics + assert_output --partial 'crowdsecurity/pgsql-logs' + assert_output --partial 'installed: true' + rune -0 cscli parsers inspect crowdsecurity/postfix-logs --no-metrics + assert_output --partial 'crowdsecurity/postfix-logs' + assert_output --partial 'installed: true' +} + +@test "cscli parsers install (file location and download-only)" { + rune -0 cscli parsers install crowdsecurity/whitelists --download-only + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics + assert_output --partial 'crowdsecurity/whitelists' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics + assert_output --partial 'installed: true' + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" +} + +@test "cscli parsers install --force (tainted)" { + rune -0 cscli parsers install crowdsecurity/whitelists + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + rune -1 cscli parsers install crowdsecurity/whitelists + assert_stderr --partial "error while installing 'crowdsecurity/whitelists': while enabling crowdsecurity/whitelists: crowdsecurity/whitelists is tainted, won't enable unless --force" + + rune -0 cscli parsers install crowdsecurity/whitelists --force + assert_stderr --partial "crowdsecurity/whitelists: overwrite" + assert_stderr --partial "Enabled crowdsecurity/whitelists" +} + +@test "cscli parsers install --ignore (skip on errors)" { + rune -1 cscli parsers install foo/bar crowdsecurity/whitelists + assert_stderr --partial "can't find 'foo/bar' in parsers" + refute_stderr --partial "Enabled parsers: crowdsecurity/whitelists" + + rune -0 cscli parsers install foo/bar crowdsecurity/whitelists --ignore + assert_stderr --partial "can't find 'foo/bar' in parsers" + assert_stderr --partial "Enabled parsers: crowdsecurity/whitelists" +} + +@test "cscli parsers inspect" { + rune -1 cscli parsers inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + # required for metrics + ./instance-crowdsec start + + rune -1 cscli parsers inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + + # one item + rune -0 cscli parsers inspect crowdsecurity/sshd-logs --no-metrics + assert_line 'type: parsers' + assert_line 'stage: s01-parse' + assert_line 'name: crowdsecurity/sshd-logs' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli parsers inspect crowdsecurity/sshd-logs + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o json + rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output) + assert_json '["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",false]' + + # one item, raw + rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o raw + assert_line 'type: parsers' + assert_line 'name: crowdsecurity/sshd-logs' + assert_line 'stage: s01-parse' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists --no-metrics + assert_output --partial 'crowdsecurity/sshd-logs' + assert_output --partial 'crowdsecurity/whitelists' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o json + rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output) + assert_json '[["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",false],["parsers","s02-enrich","crowdsecurity/whitelists","crowdsecurity","parsers/s02-enrich/crowdsecurity/whitelists.yaml",false]]' + + # multiple items, raw + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o raw + assert_output --partial 'crowdsecurity/sshd-logs' + assert_output --partial 'crowdsecurity/whitelists' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli parsers remove" { + rune -1 cscli parsers remove + assert_stderr --partial "specify at least one parser to remove or '--all'" + rune -1 cscli parsers remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + + rune -0 cscli parsers install crowdsecurity/whitelists --download-only + rune -0 cscli parsers remove crowdsecurity/whitelists + assert_stderr --partial "removing crowdsecurity/whitelists: not installed -- no need to remove" + + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli parsers remove crowdsecurity/whitelists + assert_stderr --partial "Removed crowdsecurity/whitelists" + + rune -0 cscli parsers remove crowdsecurity/whitelists --purge + assert_stderr --partial 'Removed source file [crowdsecurity/whitelists]' + + rune -0 cscli parsers remove crowdsecurity/whitelists + assert_stderr --partial "removing crowdsecurity/whitelists: not installed -- no need to remove" + + rune -0 cscli parsers remove crowdsecurity/whitelists --purge + assert_stderr --partial 'removing crowdsecurity/whitelists: not downloaded -- no need to remove' + + # install, then remove, check files + rune -0 cscli parsers install crowdsecurity/whitelists + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + rune -0 cscli parsers remove crowdsecurity/whitelists + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + # delete is an alias for remove + rune -0 cscli parsers install crowdsecurity/whitelists + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + rune -0 cscli parsers delete crowdsecurity/whitelists + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + # purge + assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" + rune -0 cscli parsers remove crowdsecurity/whitelists --purge + assert_file_not_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" + + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + # --all + rune -0 cscli parsers list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli parsers remove --all + + rune -0 cscli parsers list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli parsers remove --force" { + # remove a parser that belongs to a collection + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli parsers remove crowdsecurity/sshd-logs + assert_stderr --partial "crowdsecurity/sshd-logs belongs to collections: [crowdsecurity/sshd]" + assert_stderr --partial "Run 'sudo cscli parsers remove crowdsecurity/sshd-logs --force' if you want to force remove this parser" +} + +@test "cscli parsers upgrade" { + rune -1 cscli parsers upgrade + assert_stderr --partial "specify at least one parser to upgrade or '--all'" + rune -1 cscli parsers upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + rune -0 cscli parsers remove crowdsecurity/pam-logs --purge + rune -1 cscli parsers upgrade crowdsecurity/pam-logs + assert_stderr --partial "can't upgrade crowdsecurity/pam-logs: not installed" + rune -0 cscli parsers install crowdsecurity/pam-logs --download-only + rune -1 cscli parsers upgrade crowdsecurity/pam-logs + assert_stderr --partial "can't upgrade crowdsecurity/pam-logs: downloaded but not installed" + + # hash of the string "v0.0" + sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" + + # add version 0.0 to all parsers + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.parsers |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') + echo "$new_hub" >"$INDEX_PATH" + + rune -0 cscli parsers install crowdsecurity/whitelists + + echo "v0.0" > "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli parsers upgrade crowdsecurity/whitelists + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + # XXX: should return error + rune -0 cscli parsers upgrade crowdsecurity/whitelists + assert_stderr --partial "crowdsecurity/whitelists is tainted, --force to overwrite" + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli parsers upgrade crowdsecurity/whitelists --force + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli parsers install crowdsecurity/windows-auth + echo "v0.0" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + echo "v0.0" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" + rune -0 cscli parsers list -o json + rune -0 jq -e '[.parsers[].local_version]==["0.0","0.0"]' <(output) + rune -0 cscli parsers upgrade crowdsecurity/whitelists crowdsecurity/windows-auth + rune -0 cscli parsers list -o json + rune -0 jq -e 'any(.parsers[].local_version; .=="0.0") | not' <(output) + + # upgrade all + echo "v0.0" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + echo "v0.0" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" + rune -0 cscli parsers list -o json + rune -0 jq -e '[.parsers[].local_version]==["0.0","0.0"]' <(output) + rune -0 cscli parsers upgrade --all + rune -0 cscli parsers list -o json + rune -0 jq -e 'any(.parsers[].local_version; .=="0.0") | not' <(output) +} diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats new file mode 100644 index 000000000..55c384942 --- /dev/null +++ b/test/bats/20_hub_postoverflows.bats @@ -0,0 +1,383 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_strip_index +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli postoverflows list" { + hub_purge_all + + # no items + rune -0 cscli postoverflows list + assert_output --partial "POSTOVERFLOWS" + rune -0 cscli postoverflows list -o json + assert_json '{postoverflows:[]}' + rune -0 cscli postoverflows list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + + rune -0 cscli postoverflows list + assert_output --partial crowdsecurity/rdns + assert_output --partial crowdsecurity/cdn-whitelist + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli postoverflows list -o json + assert_output --partial crowdsecurity/rdns + assert_output --partial crowdsecurity/cdn-whitelist + rune -0 jq '.postoverflows | length' <(output) + assert_output "2" + + rune -0 cscli postoverflows list -o raw + assert_output --partial crowdsecurity/rdns + assert_output --partial crowdsecurity/cdn-whitelist + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli postoverflows list -a" { + expected=$(jq <"$INDEX_PATH" -r '.postoverflows | length') + + rune -0 cscli postoverflows list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli postoverflows list -o json -a + rune -0 jq '.postoverflows | length' <(output) + assert_output "$expected" + + rune -0 cscli postoverflows list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" + + # the list should be the same in all formats, and sorted (not case sensitive) + + list_raw=$(cscli postoverflows list -o raw -a | tail -n +2 | cut -d, -f1) + list_human=$(cscli postoverflows list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) + list_json=$(cscli postoverflows list -o json -a | jq -r '.postoverflows[].name') + + rune -0 sort -f <<<"$list_raw" + assert_output "$list_raw" + + assert_equal "$list_raw" "$list_json" + assert_equal "$list_raw" "$list_human" +} + +@test "cscli postoverflows list [postoverflow]..." { + # non-existent + rune -1 cscli postoverflows install foo/bar + assert_stderr --partial "can't find 'foo/bar' in postoverflows" + + # not installed + rune -0 cscli postoverflows list crowdsecurity/rdns + assert_output --regexp 'crowdsecurity/rdns.*disabled' + + # install two items + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + + # list an installed item + rune -0 cscli postoverflows list crowdsecurity/rdns + assert_output --regexp "crowdsecurity/rdns.*enabled" + refute_output --partial "crowdsecurity/cdn-whitelist" + + # list multiple installed and non installed items + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist crowdsecurity/ipv6_to_range + assert_output --partial "crowdsecurity/rdns" + assert_output --partial "crowdsecurity/cdn-whitelist" + assert_output --partial "crowdsecurity/ipv6_to_range" + + rune -0 cscli postoverflows list crowdsecurity/rdns -o json + rune -0 jq '.postoverflows | length' <(output) + assert_output "1" + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist crowdsecurity/ipv6_to_range -o json + rune -0 jq '.postoverflows | length' <(output) + assert_output "3" + + rune -0 cscli postoverflows list crowdsecurity/rdns -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist crowdsecurity/ipv6_to_range -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "3" +} + +@test "cscli postoverflows install" { + rune -1 cscli postoverflows install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli postoverflows install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in postoverflows" + + # simple install + rune -0 cscli postoverflows install crowdsecurity/rdns + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli postoverflows install crowdsecurity/rdnf + assert_stderr --partial "can't find 'crowdsecurity/rdnf' in postoverflows, did you mean 'crowdsecurity/rdns'?" + + # install multiple + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'installed: true' + rune -0 cscli postoverflows inspect crowdsecurity/cdn-whitelist --no-metrics + assert_output --partial 'crowdsecurity/cdn-whitelist' + assert_output --partial 'installed: true' +} + +@test "cscli postoverflows install (file location and download-only)" { + rune -0 cscli postoverflows install crowdsecurity/rdns --download-only + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" + assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + + rune -0 cscli postoverflows install crowdsecurity/rdns + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_output --partial 'installed: true' + assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" +} + +@test "cscli postoverflows install --force (tainted)" { + rune -0 cscli postoverflows install crowdsecurity/rdns + echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + + rune -1 cscli postoverflows install crowdsecurity/rdns + assert_stderr --partial "error while installing 'crowdsecurity/rdns': while enabling crowdsecurity/rdns: crowdsecurity/rdns is tainted, won't enable unless --force" + + rune -0 cscli postoverflows install crowdsecurity/rdns --force + assert_stderr --partial "crowdsecurity/rdns: overwrite" + assert_stderr --partial "Enabled crowdsecurity/rdns" +} + +@test "cscli postoverflow install --ignore (skip on errors)" { + rune -1 cscli postoverflows install foo/bar crowdsecurity/rdns + assert_stderr --partial "can't find 'foo/bar' in postoverflows" + refute_stderr --partial "Enabled postoverflows: crowdsecurity/rdns" + + rune -0 cscli postoverflows install foo/bar crowdsecurity/rdns --ignore + assert_stderr --partial "can't find 'foo/bar' in postoverflows" + assert_stderr --partial "Enabled postoverflows: crowdsecurity/rdns" +} + +@test "cscli postoverflows inspect" { + rune -1 cscli postoverflows inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + # required for metrics + ./instance-crowdsec start + + rune -1 cscli postoverflows inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + + # one item + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_line 'type: postoverflows' + assert_line 'stage: s00-enrich' + assert_line 'name: crowdsecurity/rdns' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli postoverflows inspect crowdsecurity/rdns + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output) + assert_json '["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",false]' + + # one item, raw + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o raw + assert_line 'type: postoverflows' + assert_line 'name: crowdsecurity/rdns' + assert_line 'stage: s00-enrich' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'crowdsecurity/cdn-whitelist' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o json + rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output) + assert_json '[["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",false],["postoverflows","s01-whitelist","crowdsecurity/cdn-whitelist","crowdsecurity","postoverflows/s01-whitelist/crowdsecurity/cdn-whitelist.yaml",false]]' + + # multiple items, raw + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o raw + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'crowdsecurity/cdn-whitelist' + run -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli postoverflows remove" { + rune -1 cscli postoverflows remove + assert_stderr --partial "specify at least one postoverflow to remove or '--all'" + rune -1 cscli postoverflows remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + + rune -0 cscli postoverflows install crowdsecurity/rdns --download-only + rune -0 cscli postoverflows remove crowdsecurity/rdns + assert_stderr --partial "removing crowdsecurity/rdns: not installed -- no need to remove" + + rune -0 cscli postoverflows install crowdsecurity/rdns + rune -0 cscli postoverflows remove crowdsecurity/rdns + assert_stderr --partial 'Removed crowdsecurity/rdns' + + rune -0 cscli postoverflows remove crowdsecurity/rdns --purge + assert_stderr --partial 'Removed source file [crowdsecurity/rdns]' + + rune -0 cscli postoverflows remove crowdsecurity/rdns + assert_stderr --partial 'removing crowdsecurity/rdns: not installed -- no need to remove' + + rune -0 cscli postoverflows remove crowdsecurity/rdns --purge + assert_stderr --partial 'removing crowdsecurity/rdns: not downloaded -- no need to remove' + + # install, then remove, check files + rune -0 cscli postoverflows install crowdsecurity/rdns + assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + rune -0 cscli postoverflows remove crowdsecurity/rdns + assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + + # delete is an alias for remove + rune -0 cscli postoverflows install crowdsecurity/rdns + assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + rune -0 cscli postoverflows delete crowdsecurity/rdns + assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + + # purge + assert_file_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" + rune -0 cscli postoverflows remove crowdsecurity/rdns --purge + assert_file_not_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" + + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + + # --all + rune -0 cscli postoverflows list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli postoverflows remove --all + + rune -0 cscli postoverflows list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli postoverflows remove --force" { + # remove a postoverflow that belongs to a collection + rune -0 cscli collections install crowdsecurity/auditd + rune -0 cscli postoverflows remove crowdsecurity/auditd-whitelisted-process + assert_stderr --partial "crowdsecurity/auditd-whitelisted-process belongs to collections: [crowdsecurity/auditd]" + assert_stderr --partial "Run 'sudo cscli postoverflows remove crowdsecurity/auditd-whitelisted-process --force' if you want to force remove this postoverflow" +} + +@test "cscli postoverflows upgrade" { + rune -1 cscli postoverflows upgrade + assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'" + rune -1 cscli postoverflows upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + rune -0 cscli postoverflows remove crowdsecurity/discord-crawler-whitelist --purge + rune -1 cscli postoverflows upgrade crowdsecurity/discord-crawler-whitelist + assert_stderr --partial "can't upgrade crowdsecurity/discord-crawler-whitelist: not installed" + rune -0 cscli postoverflows install crowdsecurity/discord-crawler-whitelist --download-only + rune -1 cscli postoverflows upgrade crowdsecurity/discord-crawler-whitelist + assert_stderr --partial "can't upgrade crowdsecurity/discord-crawler-whitelist: downloaded but not installed" + + # hash of the string "v0.0" + sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" + + # add version 0.0 to all postoverflows + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.postoverflows |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') + echo "$new_hub" >"$INDEX_PATH" + + rune -0 cscli postoverflows install crowdsecurity/rdns + + echo "v0.0" > "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli postoverflows upgrade crowdsecurity/rdns + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + # XXX: should return error + rune -0 cscli postoverflows upgrade crowdsecurity/rdns + assert_stderr --partial "crowdsecurity/rdns is tainted, --force to overwrite" + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli postoverflows upgrade crowdsecurity/rdns --force + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli postoverflows install crowdsecurity/cdn-whitelist + echo "v0.0" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + echo "v0.0" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" + rune -0 cscli postoverflows list -o json + rune -0 jq -e '[.postoverflows[].local_version]==["0.0","0.0"]' <(output) + rune -0 cscli postoverflows upgrade crowdsecurity/rdns crowdsecurity/cdn-whitelist + rune -0 cscli postoverflows list -o json + rune -0 jq -e 'any(.postoverflows[].local_version; .=="0.0") | not' <(output) + + # upgrade all + echo "v0.0" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + echo "v0.0" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" + rune -0 cscli postoverflows list -o json + rune -0 jq -e '[.postoverflows[].local_version]==["0.0","0.0"]' <(output) + rune -0 cscli postoverflows upgrade --all + rune -0 cscli postoverflows list -o json + rune -0 jq -e 'any(.postoverflows[].local_version; .=="0.0") | not' <(output) +} diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats new file mode 100644 index 000000000..bf033c2f9 --- /dev/null +++ b/test/bats/20_hub_scenarios.bats @@ -0,0 +1,382 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_strip_index +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli scenarios list" { + hub_purge_all + + # no items + rune -0 cscli scenarios list + assert_output --partial "SCENARIOS" + rune -0 cscli scenarios list -o json + assert_json '{scenarios:[]}' + rune -0 cscli scenarios list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + + rune -0 cscli scenarios list + assert_output --partial crowdsecurity/ssh-bf + assert_output --partial crowdsecurity/telnet-bf + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli scenarios list -o json + assert_output --partial crowdsecurity/ssh-bf + assert_output --partial crowdsecurity/telnet-bf + rune -0 jq '.scenarios | length' <(output) + assert_output "2" + + rune -0 cscli scenarios list -o raw + assert_output --partial crowdsecurity/ssh-bf + assert_output --partial crowdsecurity/telnet-bf + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli scenarios list -a" { + expected=$(jq <"$INDEX_PATH" -r '.scenarios | length') + + rune -0 cscli scenarios list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli scenarios list -o json -a + rune -0 jq '.scenarios | length' <(output) + assert_output "$expected" + + rune -0 cscli scenarios list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" + + # the list should be the same in all formats, and sorted (not case sensitive) + + list_raw=$(cscli scenarios list -o raw -a | tail -n +2 | cut -d, -f1) + list_human=$(cscli scenarios list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) + list_json=$(cscli scenarios list -o json -a | jq -r '.scenarios[].name') + + rune -0 sort -f <<<"$list_raw" + assert_output "$list_raw" + + assert_equal "$list_raw" "$list_json" + assert_equal "$list_raw" "$list_human" +} + +@test "cscli scenarios list [scenario]..." { + # non-existent + rune -1 cscli scenario install foo/bar + assert_stderr --partial "can't find 'foo/bar' in scenarios" + + # not installed + rune -0 cscli scenarios list crowdsecurity/ssh-bf + assert_output --regexp 'crowdsecurity/ssh-bf.*disabled' + + # install two items + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + + # list an installed item + rune -0 cscli scenarios list crowdsecurity/ssh-bf + assert_output --regexp "crowdsecurity/ssh-bf.*enabled" + refute_output --partial "crowdsecurity/telnet-bf" + + # list multiple installed and non installed items + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf crowdsecurity/aws-bf + assert_output --partial "crowdsecurity/ssh-bf" + assert_output --partial "crowdsecurity/telnet-bf" + assert_output --partial "crowdsecurity/aws-bf" + + rune -0 cscli scenarios list crowdsecurity/ssh-bf -o json + rune -0 jq '.scenarios | length' <(output) + assert_output "1" + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o json + rune -0 jq '.scenarios | length' <(output) + assert_output "3" + + rune -0 cscli scenarios list crowdsecurity/ssh-bf -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "3" +} + +@test "cscli scenarios install" { + rune -1 cscli scenarios install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli scenarios install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in scenarios" + + # simple install + rune -0 cscli scenarios install crowdsecurity/ssh-bf + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli scenarios install crowdsecurity/ssh-tf + assert_stderr --partial "can't find 'crowdsecurity/ssh-tf' in scenarios, did you mean 'crowdsecurity/ssh-bf'?" + + # install multiple + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'installed: true' + rune -0 cscli scenarios inspect crowdsecurity/telnet-bf --no-metrics + assert_output --partial 'crowdsecurity/telnet-bf' + assert_output --partial 'installed: true' +} + +@test "cscli scenarios install (file location and download-only)" { + # simple install + rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" + assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_output --partial 'installed: true' + assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" +} + +@test "cscli scenarios install --force (tainted)" { + rune -0 cscli scenarios install crowdsecurity/ssh-bf + echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + + rune -1 cscli scenarios install crowdsecurity/ssh-bf + assert_stderr --partial "error while installing 'crowdsecurity/ssh-bf': while enabling crowdsecurity/ssh-bf: crowdsecurity/ssh-bf is tainted, won't enable unless --force" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf --force + assert_stderr --partial "crowdsecurity/ssh-bf: overwrite" + assert_stderr --partial "Enabled crowdsecurity/ssh-bf" +} + +@test "cscli scenarios install --ignore (skip on errors)" { + rune -1 cscli scenarios install foo/bar crowdsecurity/ssh-bf + assert_stderr --partial "can't find 'foo/bar' in scenarios" + refute_stderr --partial "Enabled scenarios: crowdsecurity/ssh-bf" + + rune -0 cscli scenarios install foo/bar crowdsecurity/ssh-bf --ignore + assert_stderr --partial "can't find 'foo/bar' in scenarios" + assert_stderr --partial "Enabled scenarios: crowdsecurity/ssh-bf" +} + +@test "cscli scenarios inspect" { + rune -1 cscli scenarios inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + # required for metrics + ./instance-crowdsec start + + rune -1 cscli scenarios inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + + # one item + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_line 'type: scenarios' + assert_line 'name: crowdsecurity/ssh-bf' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output) + assert_json '["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",false]' + + # one item, raw + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o raw + assert_line 'type: scenarios' + assert_line 'name: crowdsecurity/ssh-bf' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'crowdsecurity/telnet-bf' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json + rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output) + assert_json '[["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",false],["scenarios","crowdsecurity/telnet-bf","crowdsecurity","scenarios/crowdsecurity/telnet-bf.yaml",false]]' + + # multiple items, raw + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'crowdsecurity/telnet-bf' + run -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli scenarios remove" { + rune -1 cscli scenarios remove + assert_stderr --partial "specify at least one scenario to remove or '--all'" + rune -1 cscli scenarios remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_stderr --partial "removing crowdsecurity/ssh-bf: not installed -- no need to remove" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_stderr --partial "Removed crowdsecurity/ssh-bf" + + rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge + assert_stderr --partial 'Removed source file [crowdsecurity/ssh-bf]' + + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_stderr --partial "removing crowdsecurity/ssh-bf: not installed -- no need to remove" + + rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge + assert_stderr --partial 'removing crowdsecurity/ssh-bf: not downloaded -- no need to remove' + + # install, then remove, check files + rune -0 cscli scenarios install crowdsecurity/ssh-bf + assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + + # delete is an alias for remove + rune -0 cscli scenarios install crowdsecurity/ssh-bf + assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + rune -0 cscli scenarios delete crowdsecurity/ssh-bf + assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + + # purge + assert_file_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" + rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge + assert_file_not_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + + # --all + rune -0 cscli scenarios list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli scenarios remove --all + + rune -0 cscli scenarios list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli scenarios remove --force" { + # remove a scenario that belongs to a collection + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_stderr --partial "crowdsecurity/ssh-bf belongs to collections: [crowdsecurity/sshd]" + assert_stderr --partial "Run 'sudo cscli scenarios remove crowdsecurity/ssh-bf --force' if you want to force remove this scenario" +} + +@test "cscli scenarios upgrade" { + rune -1 cscli scenarios upgrade + assert_stderr --partial "specify at least one scenario to upgrade or '--all'" + rune -1 cscli scenarios upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + rune -0 cscli scenarios remove crowdsecurity/vsftpd-bf --purge + rune -1 cscli scenarios upgrade crowdsecurity/vsftpd-bf + assert_stderr --partial "can't upgrade crowdsecurity/vsftpd-bf: not installed" + rune -0 cscli scenarios install crowdsecurity/vsftpd-bf --download-only + rune -1 cscli scenarios upgrade crowdsecurity/vsftpd-bf + assert_stderr --partial "can't upgrade crowdsecurity/vsftpd-bf: downloaded but not installed" + + # hash of the string "v0.0" + sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" + + # add version 0.0 to all scenarios + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.scenarios |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') + echo "$new_hub" >"$INDEX_PATH" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf + + echo "v0.0" > "$CONFIG_DIR/scenarios/ssh-bf.yaml" + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + # XXX: should return error + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf + assert_stderr --partial "crowdsecurity/ssh-bf is tainted, --force to overwrite" + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf --force + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli scenarios install crowdsecurity/telnet-bf + echo "v0.0" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + echo "v0.0" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" + rune -0 cscli scenarios list -o json + rune -0 jq -e '[.scenarios[].local_version]==["0.0","0.0"]' <(output) + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/telnet-bf + rune -0 cscli scenarios list -o json + rune -0 jq -e 'any(.scenarios[].local_version; .=="0.0") | not' <(output) + + # upgrade all + echo "v0.0" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + echo "v0.0" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" + rune -0 cscli scenarios list -o json + rune -0 jq -e '[.scenarios[].local_version]==["0.0","0.0"]' <(output) + rune -0 cscli scenarios upgrade --all + rune -0 cscli scenarios list -o json + rune -0 jq -e 'any(.scenarios[].local_version; .=="0.0") | not' <(output) +} diff --git a/test/bats/30_machines_tls.bats b/test/bats/30_machines_tls.bats index 121cdecdf..535435336 100644 --- a/test/bats/30_machines_tls.bats +++ b/test/bats/30_machines_tls.bats @@ -78,15 +78,17 @@ teardown() { @test "missing key_file" { config_set '.api.server.tls.key_file=""' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "missing TLS key file" + rune -0 wait-for \ + --err "missing TLS key file" \ + "${CROWDSEC}" } @test "missing cert_file" { config_set '.api.server.tls.cert_file=""' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "missing TLS cert file" + rune -0 wait-for \ + --err "missing TLS cert file" \ + "${CROWDSEC}" } @test "invalid OU for agent" { diff --git a/test/bats/40_cold-logs.bats b/test/bats/40_cold-logs.bats index 21c0615c7..36220375b 100644 --- a/test/bats/40_cold-logs.bats +++ b/test/bats/40_cold-logs.bats @@ -11,9 +11,13 @@ fake_log() { setup_file() { load "../lib/setup_file.sh" - # we reset config and data, and only run the daemon once for all the tests in this file ./instance-data load + + cscli collections install crowdsecurity/sshd --error + cscli parsers install crowdsecurity/syslog-logs --error + cscli parsers install crowdsecurity/dateparse-enrich --error + ./instance-crowdsec start } diff --git a/test/bats/40_live-ban.bats b/test/bats/40_live-ban.bats index c410cbce5..c6b8ddf15 100644 --- a/test/bats/40_live-ban.bats +++ b/test/bats/40_live-ban.bats @@ -13,6 +13,11 @@ setup_file() { load "../lib/setup_file.sh" # we reset config and data, but run the daemon only in the tests that need it ./instance-data load + + cscli collections install crowdsecurity/sshd --error + cscli parsers install crowdsecurity/syslog-logs --error + cscli parsers install crowdsecurity/dateparse-enrich --error + } teardown_file() { diff --git a/test/bats/50_simulation.bats b/test/bats/50_simulation.bats index 0add1e816..0d29d6bfd 100644 --- a/test/bats/50_simulation.bats +++ b/test/bats/50_simulation.bats @@ -12,6 +12,11 @@ fake_log() { setup_file() { load "../lib/setup_file.sh" ./instance-data load + + cscli collections install crowdsecurity/sshd --error + cscli parsers install crowdsecurity/syslog-logs --error + cscli parsers install crowdsecurity/dateparse-enrich --error + ./instance-crowdsec start } diff --git a/test/bats/72_plugin_badconfig.bats b/test/bats/72_plugin_badconfig.bats index 4f325b0f9..c9a69b9fc 100644 --- a/test/bats/72_plugin_badconfig.bats +++ b/test/bats/72_plugin_badconfig.bats @@ -27,7 +27,7 @@ setup() { teardown() { ./instance-crowdsec stop rm -f "${PLUGIN_DIR}"/badname - chmod go-w "${PLUGIN_DIR}"/notification-http + chmod go-w "${PLUGIN_DIR}"/notification-http || true } #---------- @@ -35,36 +35,41 @@ teardown() { @test "misconfigured plugin, only user is empty" { config_set '.plugin_config.user="" | .plugin_config.group="nogroup"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" \ + "${CROWDSEC}" } @test "misconfigured plugin, only group is empty" { config_set '(.plugin_config.user="nobody") | (.plugin_config.group="")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" \ + "${CROWDSEC}" } @test "misconfigured plugin, user does not exist" { config_set '(.plugin_config.user="userdoesnotexist") | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" \ + "${CROWDSEC}" } @test "misconfigured plugin, group does not exist" { config_set '(.plugin_config.user=strenv(USER)) | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" \ + "${CROWDSEC}" } @test "bad plugin name" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' cp "${PLUGIN_DIR}"/notification-http "${PLUGIN_DIR}"/badname - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" \ + "${CROWDSEC}" } @test "duplicate notification config" { @@ -75,48 +80,55 @@ teardown() { config_set "${PROFILES_PATH}" '.notifications=["slack_default"]' # the slack plugin may fail or not, but we just need the logs config_set '.common.log_media="stdout"' - rune timeout 2s "${CROWDSEC}" - assert_stderr --partial "notification 'email_default' is defined multiple times" + rune wait-for \ + --err "notification 'email_default' is defined multiple times" \ + "${CROWDSEC}" } @test "bad plugin permission (group writable)" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod g+w "${PLUGIN_DIR}"/notification-http - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" \ + "${CROWDSEC}" } @test "bad plugin permission (world writable)" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod o+w "${PLUGIN_DIR}"/notification-http - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" \ + "${CROWDSEC}" } @test "config.yaml: missing .plugin_config section" { config_set 'del(.plugin_config)' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: plugins are enabled, but the plugin_config section is missing in the configuration" + rune -0 wait-for \ + --err "api server init: plugins are enabled, but the plugin_config section is missing in the configuration" \ + "${CROWDSEC}" } @test "config.yaml: missing config_paths.notification_dir" { config_set 'del(.config_paths.notification_dir)' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: plugins are enabled, but config_paths.notification_dir is not defined" + rune -0 wait-for \ + --err "api server init: plugins are enabled, but config_paths.notification_dir is not defined" \ + "${CROWDSEC}" } @test "config.yaml: missing config_paths.plugin_dir" { config_set 'del(.config_paths.plugin_dir)' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: plugins are enabled, but config_paths.plugin_dir is not defined" + rune -0 wait-for \ + --err "api server init: plugins are enabled, but config_paths.plugin_dir is not defined" \ + "${CROWDSEC}" } @test "unable to run plugin broker: while reading plugin config" { config_set '.config_paths.notification_dir="/this/path/does/not/exist"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin config: open /this/path/does/not/exist: no such file or directory" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin config: open /this/path/does/not/exist: no such file or directory" \ + "${CROWDSEC}" } diff --git a/test/bats/81_alert_context.bats b/test/bats/81_alert_context.bats index 6dd6100b9..df741f5f9 100644 --- a/test/bats/81_alert_context.bats +++ b/test/bats/81_alert_context.bats @@ -20,6 +20,9 @@ teardown_file() { setup() { load "../lib/setup.sh" ./instance-data load + cscli collections install crowdsecurity/sshd --error + cscli parsers install crowdsecurity/syslog-logs --error + cscli parsers install crowdsecurity/dateparse-enrich --error } teardown() { diff --git a/test/bats/testdata/explain/explain-log.txt b/test/bats/testdata/explain/explain-log.txt index aae9e8098..76247412c 100644 --- a/test/bats/testdata/explain/explain-log.txt +++ b/test/bats/testdata/explain/explain-log.txt @@ -2,15 +2,10 @@ line: Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authenticatio ├ s00-raw | └ 🟢 crowdsecurity/syslog-logs (+12 ~9) ├ s01-parse - | └ 🟢 crowdsecurity/sshd-logs (+8 ~1) - ├ s02-enrich - | ├ 🟢 crowdsecurity/dateparse-enrich (+2 ~2) - | ├ 🟢 crowdsecurity/geoip-enrich (+10) - | └ 🟢 crowdsecurity/whitelists (unchanged) + | └ 🟢 crowdsecurity/sshd-logs (+8) ├-------- parser success 🟢 ├ Scenarios ├ 🟢 crowdsecurity/ssh-bf ├ 🟢 crowdsecurity/ssh-bf_user-enum ├ 🟢 crowdsecurity/ssh-slow-bf └ 🟢 crowdsecurity/ssh-slow-bf_user-enum - diff --git a/test/bin/wait-for b/test/bin/wait-for new file mode 100755 index 000000000..6c6fdd5ce --- /dev/null +++ b/test/bin/wait-for @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +import asyncio +import argparse +import os +import re +import signal +import sys + +DEFAULT_TIMEOUT = 30 + +# TODO: signal handler to terminate spawned process group when wait-for is killed +# TODO: better return codes esp. when matches are found +# TODO: multiple patterns (multiple out, err, both) +# TODO: print unmatched patterns + + +async def terminate(p): + # Terminate the process group (shell, crowdsec plugins) + try: + os.killpg(os.getpgid(p.pid), signal.SIGTERM) + except ProcessLookupError: + pass + + +async def monitor(cmd, args, want_out, want_err, timeout): + """Monitor a process and terminate it if a pattern is matched in stdout or stderr. + + Args: + cmd: The command to run. + args: A list of arguments to pass to the command. + stdout: A regular expression pattern to search for in stdout. + stderr: A regular expression pattern to search for in stderr. + timeout: The maximum number of seconds to wait for the process to terminate. + + Returns: + The exit code of the process. + """ + + status = None + + async def read_stream(p, stream, outstream, pattern): + nonlocal status + if stream is None: + return + while True: + line = await stream.readline() + if line: + line = line.decode('utf-8') + outstream.write(line) + if pattern and pattern.search(line): + await terminate(process) + # this is nasty. + # if we timeout, we want to return a different exit code + # in case of a match, so that the caller can tell + # if the application was still running. + # XXX: still not good for match found, but return code != 0 + if timeout != DEFAULT_TIMEOUT: + status = 128 + else: + status = 0 + break + else: + break + + process = await asyncio.create_subprocess_exec( + cmd, + *args, + # capture stdout + stdout=asyncio.subprocess.PIPE, + # capture stderr + stderr=asyncio.subprocess.PIPE, + # disable buffering + bufsize=0, + # create a new process group + # (required to kill child processes when cmd is a shell) + preexec_fn=os.setsid) + + out_regex = re.compile(want_out) if want_out else None + err_regex = re.compile(want_err) if want_err else None + + # Apply a timeout + try: + await asyncio.wait_for( + asyncio.wait([ + asyncio.create_task(process.wait()), + asyncio.create_task(read_stream(process, process.stdout, sys.stdout, out_regex)), + asyncio.create_task(read_stream(process, process.stderr, sys.stderr, err_regex)) + ]), timeout) + if status is None: + status = process.returncode + except asyncio.TimeoutError: + await terminate(process) + status = 241 + + # Return the same exit code, stdout and stderr as the spawned process + return status + + +async def main(): + parser = argparse.ArgumentParser( + description='Monitor a process and terminate it if a pattern is matched in stdout or stderr.') + parser.add_argument('cmd', help='The command to run.') + parser.add_argument('args', nargs=argparse.REMAINDER, help='A list of arguments to pass to the command.') + parser.add_argument('--out', default='', help='A regular expression pattern to search for in stdout.') + parser.add_argument('--err', default='', help='A regular expression pattern to search for in stderr.') + parser.add_argument('--timeout', type=float, default=DEFAULT_TIMEOUT) + args = parser.parse_args() + + exit_code = await monitor(args.cmd, args.args, args.out, args.err, args.timeout) + + sys.exit(exit_code) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/test/lib/config/config-global b/test/lib/config/config-global index 592a927c2..46960bcab 100755 --- a/test/lib/config/config-global +++ b/test/lib/config/config-global @@ -38,6 +38,8 @@ DATA_DIR="${LOCAL_DIR}/${REL_DATA_DIR}" export DATA_DIR CONFIG_DIR="${LOCAL_DIR}/${REL_CONFIG_DIR}" export CONFIG_DIR +HUB_DIR="${CONFIG_DIR}/hub" +export HUB_DIR if [[ $(uname) == "OpenBSD" ]]; then TAR=gtar @@ -52,6 +54,51 @@ remove_init_data() { # we need a separate function for initializing config when testing package # because we want to test the configuration as well +preload_hub_items() { + # pre-download everything but don't install anything + # each test can install what it needs + + echo "Purging existing hub..." + + "$CSCLI" parsers delete --all --error --purge --force + "$CSCLI" scenarios delete --all --error --purge --force + "$CSCLI" postoverflows delete --all --error --purge --force + "$CSCLI" collections delete --all --error --purge --force + + echo "Pre-downloading hub content..." + + #shellcheck disable=SC2046 + "$CSCLI" collections install \ + $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \ + --download-only \ + --error + + #shellcheck disable=SC2046 + "$CSCLI" parsers install \ + $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \ + --download-only \ + --error + + #shellcheck disable=SC2046 + "$CSCLI" scenarios install \ + $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \ + --download-only \ + --error + + #shellcheck disable=SC2046 + "$CSCLI" postoverflows install \ + $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \ + --download-only \ + --error + + # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows. + # so we have to delete the links manually, and leave the downloaded files in place + + "$CSCLI" parsers delete --all --error + "$CSCLI" scenarios delete --all --error + "$CSCLI" postoverflows delete --all --error +} + make_init_data() { ./bin/assert-crowdsec-not-running || die "Cannot create fixture data." @@ -61,9 +108,11 @@ make_init_data() { # when installed packages are always using sqlite, so no need to regenerate # local credz for sqlite + preload_hub_items + [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto - mkdir -p "${LOCAL_INIT_DIR}" + mkdir -p "$LOCAL_INIT_DIR" ./instance-db dump "${LOCAL_INIT_DIR}/database" diff --git a/test/lib/config/config-local b/test/lib/config/config-local index 0e2c86692..625e6e5ce 100755 --- a/test/lib/config/config-local +++ b/test/lib/config/config-local @@ -101,6 +101,50 @@ config_generate() { ' ../config/config.yaml >"${CONFIG_DIR}/config.yaml" } +preload_hub_items() { + # pre-download everything but don't install anything + # each test can install what it needs + + echo "Purging existing hub..." + + "$CSCLI" parsers delete --all --error --purge --force + "$CSCLI" scenarios delete --all --error --purge --force + "$CSCLI" postoverflows delete --all --error --purge --force + "$CSCLI" collections delete --all --error --purge --force + + echo "Pre-downloading hub content..." + + #shellcheck disable=SC2046 + "$CSCLI" collections install \ + $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \ + --download-only \ + --error + + #shellcheck disable=SC2046 + "$CSCLI" parsers install \ + $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \ + --download-only \ + --error + + #shellcheck disable=SC2046 + "$CSCLI" scenarios install \ + $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \ + --download-only \ + --error + + #shellcheck disable=SC2046 + "$CSCLI" postoverflows install \ + $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \ + --download-only \ + --error + + # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows. + # so we have to delete the links manually, and leave the downloaded files in place + + "$CSCLI" parsers delete --all --error + "$CSCLI" scenarios delete --all --error + "$CSCLI" postoverflows delete --all --error +} make_init_data() { ./bin/assert-crowdsec-not-running || die "Cannot create fixture data." @@ -118,9 +162,8 @@ make_init_data() { "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto "$CSCLI" --warning hub update - "$CSCLI" --warning collections install crowdsecurity/linux - # the whitelists are installed by the deb & rpm packages, so we test with the same config - "$CSCLI" --warning parsers install crowdsecurity/whitelists + + preload_hub_items mkdir -p "$LOCAL_INIT_DIR" diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index 5e16340ec..256abbbc9 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -20,6 +20,7 @@ eval "$(debug)" # Allow tests to use relative paths for helper scripts. # shellcheck disable=SC2164 cd "${TEST_DIR}" +export PATH="${TEST_DIR}/bin:${PATH}" # complain if there's a crowdsec running system-wide or leftover from a previous test ./bin/assert-crowdsec-not-running @@ -238,6 +239,30 @@ assert_stderr_line() { } export -f assert_stderr_line +# remove all installed items and data +hub_purge_all() { + local CONFIG_DIR + CONFIG_DIR=$(dirname "$CONFIG_YAML") + rm -rf "$CONFIG_DIR"/collections/* "$CONFIG_DIR"/parsers/*/* "$CONFIG_DIR"/scenarios/* "$CONFIG_DIR"/postoverflows/* + rm -rf "$CONFIG_DIR"/hub/collections/* "$CONFIG_DIR"/hub/parsers/*/* "$CONFIG_DIR"/hub/scenarios/* "$CONFIG_DIR"/hub/postoverflows/* + local DATA_DIR + DATA_DIR=$(config_get .config_paths.data_dir) + # should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete), + # but don't play with fire if there is a misconfiguration + rm -rfv "$DATA_DIR"/GeoLite* +} +export -f hub_purge_all + +# remove unused data from the index, to make sure we don't rely on it in any way +hub_strip_index() { + local INDEX + INDEX=$(config_get .config_paths.index_path) + local hub_min + hub_min=$(jq <"$INDEX" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$INDEX" +} +export -f hub_strip_index + # remove color and style sequences from stdin plaintext() { sed -E 's/\x1B\[[0-9;]*[JKmsu]//g'