diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index ad3110011..2d139cd90 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -1,11 +1,13 @@ package main import ( + "encoding/json" "fmt" "github.com/fatih/color" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "gopkg.in/yaml.v3" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/cwhub" @@ -29,6 +31,7 @@ cscli hub upgrade`, cmdHub.AddCommand(NewHubListCmd()) cmdHub.AddCommand(NewHubUpdateCmd()) cmdHub.AddCommand(NewHubUpgradeCmd()) + cmdHub.AddCommand(NewHubTypesCmd()) return cmdHub } @@ -172,3 +175,40 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if return cmdHubUpgrade } + +func runHubTypes(cmd *cobra.Command, args []string) error { + switch csConfig.Cscli.Output { + case "human": + s, err := yaml.Marshal(cwhub.ItemTypes) + if err != nil { + return err + } + fmt.Print(string(s)) + case "json": + jsonStr, err := json.Marshal(cwhub.ItemTypes) + if err != nil { + return err + } + fmt.Println(string(jsonStr)) + case "raw": + for _, itemType := range cwhub.ItemTypes { + fmt.Println(itemType) + } + } + return nil +} + +func NewHubTypesCmd() *cobra.Command { + cmdHubTypes := &cobra.Command{ + Use: "types", + Short: "List supported item types", + Long: ` +List the types of supported hub items. +`, + Args: cobra.ExactArgs(0), + DisableAutoGenTag: true, + RunE: runHubTypes, + } + + return cmdHubTypes +} diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index 6ec53681d..a95803def 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -36,7 +36,7 @@ type hubItemType struct { var hubItemTypes = map[string]hubItemType{ "parsers": { - name: "parsers", + name: cwhub.PARSERS, singular: "parser", oneOrMore: "parser(s)", help: cmdHelp{ @@ -68,7 +68,7 @@ List only enabled parsers unless "-a" or names are specified.`, }, }, "postoverflows": { - name: "postoverflows", + name: cwhub.POSTOVERFLOWS, singular: "postoverflow", oneOrMore: "postoverflow(s)", help: cmdHelp{ @@ -100,7 +100,7 @@ List only enabled postoverflows unless "-a" or names are specified.`, }, }, "scenarios": { - name: "scenarios", + name: cwhub.SCENARIOS, singular: "scenario", oneOrMore: "scenario(s)", help: cmdHelp{ @@ -132,7 +132,7 @@ List only enabled scenarios unless "-a" or names are specified.`, }, }, "collections": { - name: "collections", + name: cwhub.COLLECTIONS, singular: "collection", oneOrMore: "collection(s)", help: cmdHelp{ diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 1cff0a21b..18218bd87 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -9,13 +9,14 @@ import ( "strings" log "github.com/sirupsen/logrus" + "slices" "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) // Hub is the main structure for the package. type Hub struct { - Items HubItems // Items read from HubDir and InstallDir + items HubItems // Items read from HubDir and InstallDir local *csconfig.LocalHubCfg remote *RemoteHubCfg Warnings []string // Warnings encountered during sync @@ -65,7 +66,7 @@ func (h *Hub) parseIndex() error { return fmt.Errorf("unable to read index file: %w", err) } - if err := json.Unmarshal(bidx, &h.Items); err != nil { + if err := json.Unmarshal(bidx, &h.items); err != nil { return fmt.Errorf("failed to unmarshal index: %w", err) } @@ -73,9 +74,9 @@ func (h *Hub) parseIndex() error { // Iterate over the different types to complete the struct for _, itemType := range ItemTypes { - log.Tracef("%s: %d items", itemType, len(h.Items[itemType])) + log.Tracef("%s: %d items", itemType, len(h.GetItemMap(itemType))) - for name, item := range h.Items[itemType] { + for name, item := range h.GetItemMap(itemType) { item.hub = h item.Name = name @@ -101,13 +102,13 @@ func (h *Hub) ItemStats() []string { tainted := 0 for _, itemType := range ItemTypes { - if len(h.Items[itemType]) == 0 { + if len(h.GetItemMap(itemType)) == 0 { continue } - loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) + loaded += fmt.Sprintf("%d %s, ", len(h.GetItemMap(itemType)), itemType) - for _, item := range h.Items[itemType] { + for _, item := range h.GetItemMap(itemType) { if item.IsLocal() { local++ } @@ -160,9 +161,17 @@ func (h *Hub) updateIndex() error { return nil } +func (h *Hub) addItem(item *Item) { + if h.items[item.Type] == nil { + h.items[item.Type] = make(map[string]*Item) + } + + h.items[item.Type][item.Name] = item +} + // GetItemMap returns the map of items for a given type. func (h *Hub) GetItemMap(itemType string) map[string]*Item { - return h.Items[itemType] + return h.items[itemType] } // GetItem returns an item from hub based on its type and full name (author/name). @@ -188,11 +197,12 @@ func (h *Hub) GetItemNames(itemType string) []string { // GetAllItems returns a slice of all the items of a given type, installed or not. func (h *Hub) GetAllItems(itemType string) ([]*Item, error) { - items, ok := h.Items[itemType] - if !ok { - return nil, fmt.Errorf("no %s in the hub index", itemType) + if !slices.Contains(ItemTypes, itemType) { + return nil, fmt.Errorf("invalid item type %s", itemType) } + items := h.items[itemType] + ret := make([]*Item, len(items)) idx := 0 @@ -207,11 +217,12 @@ func (h *Hub) GetAllItems(itemType string) ([]*Item, error) { // GetInstalledItems returns a slice of the installed items of a given type. func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { - items, ok := h.Items[itemType] - if !ok { - return nil, fmt.Errorf("no %s in the hub index", itemType) + if !slices.Contains(ItemTypes, itemType) { + return nil, fmt.Errorf("invalid item type %s", itemType) } + items := h.items[itemType] + retItems := make([]*Item, 0) for _, item := range items { diff --git a/pkg/cwhub/item_test.go b/pkg/cwhub/item_test.go index a4c2bc0a0..33a52e8e0 100644 --- a/pkg/cwhub/item_test.go +++ b/pkg/cwhub/item_test.go @@ -63,7 +63,7 @@ func TestGetters(t *testing.T) { // Add item and get it item.Name += "nope" - hub.Items[item.Type][item.Name] = item + hub.addItem(item) newitem := hub.GetItem(COLLECTIONS, item.Name) require.NotNil(t, newitem) diff --git a/pkg/cwhub/iteminstall_test.go b/pkg/cwhub/iteminstall_test.go index 36ba7c11d..cf17eede7 100644 --- a/pkg/cwhub/iteminstall_test.go +++ b/pkg/cwhub/iteminstall_test.go @@ -16,9 +16,9 @@ func testInstall(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name) + assert.True(t, item.State.UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, item.State.Installed, "%s should not be installed", item.Name) + assert.False(t, item.State.Tainted, "%s should not be tainted", item.Name) err = item.enable() require.NoError(t, err, "failed to enable %s", item.Name) @@ -26,11 +26,11 @@ func testInstall(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name) + assert.True(t, item.State.Installed, "%s should be installed", item.Name) } func testTaint(hub *Hub, t *testing.T, item *Item) { - assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name) + assert.False(t, item.State.Tainted, "%s should not be tainted", item.Name) // truncate the file f, err := os.Create(item.State.LocalPath) @@ -41,11 +41,11 @@ func testTaint(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should be tainted", item.Name) + assert.True(t, item.State.Tainted, "%s should be tainted", item.Name) } func testUpdate(hub *Hub, t *testing.T, item *Item) { - assert.False(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should not be up-to-date", item.Name) + assert.False(t, item.State.UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status _, err := item.downloadLatest(true, true) @@ -55,12 +55,12 @@ func testUpdate(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name) + assert.True(t, item.State.UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, item.State.Tainted, "%s should not be tainted anymore", item.Name) } func testDisable(hub *Hub, t *testing.T, item *Item) { - assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name) + assert.True(t, item.State.Installed, "%s should be installed", item.Name) // Remove _, err := item.disable(false, false) @@ -71,9 +71,9 @@ func testDisable(hub *Hub, t *testing.T, item *Item) { require.NoError(t, err, "failed to run localSync") require.Empty(t, hub.Warnings) - assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name) - assert.True(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should still be downloaded", item.Name) + assert.False(t, item.State.Tainted, "%s should not be tainted anymore", item.Name) + assert.False(t, item.State.Installed, "%s should not be installed anymore", item.Name) + assert.True(t, item.State.Downloaded, "%s should still be downloaded", item.Name) // Purge _, err = item.disable(true, false) @@ -84,8 +84,8 @@ func testDisable(hub *Hub, t *testing.T, item *Item) { require.NoError(t, err, "failed to run localSync") require.Empty(t, hub.Warnings) - assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should not be downloaded", item.Name) + assert.False(t, item.State.Installed, "%s should not be installed anymore", item.Name) + assert.False(t, item.State.Downloaded, "%s should not be downloaded", item.Name) } func TestInstallParser(t *testing.T) { @@ -101,15 +101,11 @@ func TestInstallParser(t *testing.T) { hub := envSetup(t) // map iteration is random by itself - for _, it := range hub.Items[PARSERS] { + for _, it := range hub.GetItemMap(PARSERS) { testInstall(hub, t, it) - it = hub.Items[PARSERS][it.Name] testTaint(hub, t, it) - it = hub.Items[PARSERS][it.Name] testUpdate(hub, t, it) - it = hub.Items[PARSERS][it.Name] testDisable(hub, t, it) - break } } @@ -127,15 +123,11 @@ func TestInstallCollection(t *testing.T) { hub := envSetup(t) // map iteration is random by itself - for _, it := range hub.Items[COLLECTIONS] { + for _, it := range hub.GetItemMap(COLLECTIONS) { testInstall(hub, t, it) - it = hub.Items[COLLECTIONS][it.Name] testTaint(hub, t, it) - it = hub.Items[COLLECTIONS][it.Name] testUpdate(hub, t, it) - it = hub.Items[COLLECTIONS][it.Name] testDisable(hub, t, it) - break } } diff --git a/pkg/cwhub/itemupgrade_test.go b/pkg/cwhub/itemupgrade_test.go index 60b70f491..4275e8f36 100644 --- a/pkg/cwhub/itemupgrade_test.go +++ b/pkg/cwhub/itemupgrade_test.go @@ -13,20 +13,21 @@ import ( func TestUpgradeItemNewScenarioInCollection(t *testing.T) { hub := envSetup(t) - // fresh install of collection - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) - item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + + // fresh install of collection + require.False(t, item.State.Downloaded) + require.False(t, item.State.Installed) + require.NoError(t, item.Install(false, false)) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, item.State.Downloaded) + require.True(t, item.State.Installed) + require.True(t, item.State.UpToDate) + require.False(t, item.State.Tainted) // This is the scenario that gets added in next version of collection - require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"]) + require.Nil(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario")) assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") @@ -44,19 +45,20 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { hub = getHubOrFail(t, hub.local, remote) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) - item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + + require.True(t, item.State.Downloaded) + require.True(t, item.State.Installed) + require.False(t, item.State.UpToDate) + require.False(t, item.State.Tainted) + didUpdate, err := item.Upgrade(false) require.NoError(t, err) require.True(t, didUpdate) assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Downloaded) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed) + require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Downloaded) + require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Installed) } // Install a collection, disable a scenario. @@ -64,19 +66,20 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { hub := envSetup(t) - // fresh install of collection - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) - item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + + // fresh install of collection + require.False(t, item.State.Downloaded) + require.False(t, item.State.Installed) + require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed) + require.NoError(t, item.Install(false, false)) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.True(t, item.State.Downloaded) + require.True(t, item.State.Installed) + require.True(t, item.State.UpToDate) + require.False(t, item.State.Tainted) + require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed) assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") @@ -92,11 +95,12 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { hub = getHubOrFail(t, hub.local, remote) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed) + + require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Tainted) + require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Downloaded) + require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Installed) + require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.UpToDate) hub, err = NewHub(hub.local, remote, true) require.NoError(t, err, "failed to download index: %s", err) @@ -107,7 +111,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.False(t, didUpdate) hub = getHubOrFail(t, hub.local, remote) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed) } // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test. @@ -124,19 +128,20 @@ func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCf func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { hub := envSetup(t) - // fresh install of collection - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) - item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + + // fresh install of collection + require.False(t, item.State.Downloaded) + require.False(t, item.State.Installed) + require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed) + require.NoError(t, item.Install(false, false)) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.True(t, item.State.Downloaded) + require.True(t, item.State.Installed) + require.True(t, item.State.UpToDate) + require.False(t, item.State.Tainted) + require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed) assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") @@ -152,12 +157,12 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te hub = getHubOrFail(t, hub.local, remote) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Downloaded) // this fails - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed) + require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Downloaded) // this fails + require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Tainted) + require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Downloaded) + require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Installed) + require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.UpToDate) // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" // we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario @@ -167,7 +172,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te hub, err = NewHub(hub.local, remote, true) require.NoError(t, err, "failed to download index: %s", err) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed) hub = getHubOrFail(t, hub.local, remote) item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") @@ -176,14 +181,14 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te require.True(t, didUpdate) hub = getHubOrFail(t, hub.local, remote) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed) + require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed) + require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Installed) } func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) { t.Helper() - c := hub.Items[COLLECTIONS][collection] + c := hub.GetItem(COLLECTIONS, collection) require.NoError(t, c.checkSubItemVersions()) } diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 7dc03e093..bbd8b2724 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -226,7 +226,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { if err != nil { return err } - h.Items[info.ftype][item.Name] = item + h.addItem(item) return nil } @@ -246,7 +246,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { // try to find which configuration item it is log.Tracef("check [%s] of %s", info.fname, info.ftype) - for name, item := range h.Items[info.ftype] { + for _, item := range h.GetItemMap(info.ftype) { if info.fname != item.FileName { continue } @@ -287,8 +287,6 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { return err } - h.Items[info.ftype][name] = item - return nil } @@ -395,7 +393,7 @@ func (h *Hub) localSync() error { warnings := make([]string, 0) - for _, item := range h.Items[COLLECTIONS] { + for _, item := range h.GetItemMap(COLLECTIONS) { // check for cyclic dependencies subs, err := item.descendants() if err != nil { diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index 5d3b79fe1..8148603d7 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -19,12 +19,12 @@ type Coverage struct { } func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { - if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok { + if len(h.HubIndex.GetItemMap(cwhub.PARSERS)) == 0 { return nil, fmt.Errorf("no parsers in hub index") } // populate from hub, iterate in alphabetical order - pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.PARSERS]) + pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.PARSERS)) coverage := make([]Coverage, len(pkeys)) for i, name := range pkeys { @@ -105,12 +105,12 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { } func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { - if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok { + if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0 { return nil, fmt.Errorf("no scenarios in hub index") } // populate from hub, iterate in alphabetical order - pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.SCENARIOS]) + pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) coverage := make([]Coverage, len(pkeys)) for i, name := range pkeys { diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 717c876ed..dd84fc022 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -146,7 +146,7 @@ func (t *HubTestItem) InstallHub() error { continue } - if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok { + if hubParser := t.HubIndex.GetItem(cwhub.PARSERS, parser); hubParser != nil { parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err) @@ -232,7 +232,7 @@ func (t *HubTestItem) InstallHub() error { continue } - if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok { + if hubScenario := t.HubIndex.GetItem(cwhub.SCENARIOS, scenario); hubScenario != nil { scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path to: %s", scenarioSource) @@ -303,7 +303,7 @@ func (t *HubTestItem) InstallHub() error { continue } - if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok { + if hubPostOverflow := t.HubIndex.GetItem(cwhub.POSTOVERFLOWS, postoverflow); hubPostOverflow != nil { postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err) diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index 3f2ce8c4b..5189044b9 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -126,3 +126,16 @@ teardown() { rune -0 cscli hub upgrade assert_stderr --partial "not upgrading foo.yaml: local item" } + +@test "cscli hub types" { + rune -0 cscli hub types -o raw + assert_line "parsers" + assert_line "postoverflows" + assert_line "scenarios" + assert_line "collections" + rune -0 cscli hub types -o human + rune -0 yq -o json <(output) + assert_json '["parsers","postoverflows","scenarios","collections"]' + rune -0 cscli hub types -o json + assert_json '["parsers","postoverflows","scenarios","collections"]' +} diff --git a/test/bin/preload-hub-items b/test/bin/preload-hub-items new file mode 100755 index 000000000..14e9cff99 --- /dev/null +++ b/test/bin/preload-hub-items @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -eu + +# shellcheck disable=SC1007 +THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +# shellcheck disable=SC1091 +. "${THIS_DIR}/../.environment.sh" + +# pre-download everything but don't install anything + +echo -n "Purging existing hub..." + +types=$("$CSCLI" hub types -o raw) + +for itemtype in $types; do + "$CSCLI" "${itemtype}" delete --all --error --purge --force +done + +echo " done." + +echo -n "Pre-downloading Hub content..." + +for itemtype in $types; do + ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | jq --arg itemtype "$itemtype" -r '.[$itemtype][].name') + if [[ -n "${ALL_ITEMS}" ]]; then + #shellcheck disable=SC2086 + "$CSCLI" "$itemtype" install \ + $ALL_ITEMS \ + --download-only \ + --error + fi +done + +# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows. +# so we have to delete the links manually, and leave the downloaded files in place + +for itemtype in $types; do + "$CSCLI" "$itemtype" delete --all --error +done + +echo " done." diff --git a/test/lib/config/config-global b/test/lib/config/config-global index 46960bcab..7814cefbb 100755 --- a/test/lib/config/config-global +++ b/test/lib/config/config-global @@ -54,50 +54,6 @@ remove_init_data() { # we need a separate function for initializing config when testing package # because we want to test the configuration as well -preload_hub_items() { - # pre-download everything but don't install anything - # each test can install what it needs - - echo "Purging existing hub..." - - "$CSCLI" parsers delete --all --error --purge --force - "$CSCLI" scenarios delete --all --error --purge --force - "$CSCLI" postoverflows delete --all --error --purge --force - "$CSCLI" collections delete --all --error --purge --force - - echo "Pre-downloading hub content..." - - #shellcheck disable=SC2046 - "$CSCLI" collections install \ - $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \ - --download-only \ - --error - - #shellcheck disable=SC2046 - "$CSCLI" parsers install \ - $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \ - --download-only \ - --error - - #shellcheck disable=SC2046 - "$CSCLI" scenarios install \ - $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \ - --download-only \ - --error - - #shellcheck disable=SC2046 - "$CSCLI" postoverflows install \ - $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \ - --download-only \ - --error - - # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows. - # so we have to delete the links manually, and leave the downloaded files in place - - "$CSCLI" parsers delete --all --error - "$CSCLI" scenarios delete --all --error - "$CSCLI" postoverflows delete --all --error -} make_init_data() { ./bin/assert-crowdsec-not-running || die "Cannot create fixture data." @@ -108,7 +64,7 @@ make_init_data() { # when installed packages are always using sqlite, so no need to regenerate # local credz for sqlite - preload_hub_items + ./bin/preload-hub-items [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto @@ -145,7 +101,6 @@ load_init_data() { ./instance-db restore "${LOCAL_INIT_DIR}/database" } - # --------------------------- [[ $# -lt 1 ]] && about diff --git a/test/lib/config/config-local b/test/lib/config/config-local index 625e6e5ce..0941484ff 100755 --- a/test/lib/config/config-local +++ b/test/lib/config/config-local @@ -101,51 +101,6 @@ config_generate() { ' ../config/config.yaml >"${CONFIG_DIR}/config.yaml" } -preload_hub_items() { - # pre-download everything but don't install anything - # each test can install what it needs - - echo "Purging existing hub..." - - "$CSCLI" parsers delete --all --error --purge --force - "$CSCLI" scenarios delete --all --error --purge --force - "$CSCLI" postoverflows delete --all --error --purge --force - "$CSCLI" collections delete --all --error --purge --force - - echo "Pre-downloading hub content..." - - #shellcheck disable=SC2046 - "$CSCLI" collections install \ - $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \ - --download-only \ - --error - - #shellcheck disable=SC2046 - "$CSCLI" parsers install \ - $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \ - --download-only \ - --error - - #shellcheck disable=SC2046 - "$CSCLI" scenarios install \ - $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \ - --download-only \ - --error - - #shellcheck disable=SC2046 - "$CSCLI" postoverflows install \ - $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \ - --download-only \ - --error - - # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows. - # so we have to delete the links manually, and leave the downloaded files in place - - "$CSCLI" parsers delete --all --error - "$CSCLI" scenarios delete --all --error - "$CSCLI" postoverflows delete --all --error -} - make_init_data() { ./bin/assert-crowdsec-not-running || die "Cannot create fixture data." @@ -163,7 +118,7 @@ make_init_data() { "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto "$CSCLI" --warning hub update - preload_hub_items + ./bin/preload-hub-items mkdir -p "$LOCAL_INIT_DIR" @@ -197,7 +152,6 @@ load_init_data() { ./instance-db restore "${LOCAL_INIT_DIR}/database" } - # --------------------------- [[ $# -lt 1 ]] && about diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index 256abbbc9..a70d14be3 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -243,8 +243,16 @@ export -f assert_stderr_line hub_purge_all() { local CONFIG_DIR CONFIG_DIR=$(dirname "$CONFIG_YAML") - rm -rf "$CONFIG_DIR"/collections/* "$CONFIG_DIR"/parsers/*/* "$CONFIG_DIR"/scenarios/* "$CONFIG_DIR"/postoverflows/* - rm -rf "$CONFIG_DIR"/hub/collections/* "$CONFIG_DIR"/hub/parsers/*/* "$CONFIG_DIR"/hub/scenarios/* "$CONFIG_DIR"/hub/postoverflows/* + rm -rf \ + "$CONFIG_DIR"/collections/* \ + "$CONFIG_DIR"/parsers/*/* \ + "$CONFIG_DIR"/scenarios/* \ + "$CONFIG_DIR"/postoverflows/* + rm -rf \ + "$CONFIG_DIR"/hub/collections/* \ + "$CONFIG_DIR"/hub/parsers/*/* \ + "$CONFIG_DIR"/hub/scenarios/* \ + "$CONFIG_DIR"/hub/postoverflows/* local DATA_DIR DATA_DIR=$(config_get .config_paths.data_dir) # should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete),