beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Refact cwhub: move methods from hub to item (#2585)

Browse source 
* Add back pointer Item.hub
* Hub.enableItem() -> Item.enable()
* Rename variable i -> idx (i is used for item instances)
* Move Hub.purgeItem() -> Item.purge()
* Move Hub.disableItem() -> Item.disable()
* Move Hub.downloadItem() -> Item.download()
* Move Hub.downloadLatest() -> Item.downloadLatest()
* Move Hub.DownloadDataIfNeeded() -> Item.DownloadDataIfNeeded()
* Move Hub.InstallItem() -> Item.Install()
* Move Hub.RemoveItem() -> Item.Remove()
* Move Hub.UpgradeItem() -> Item.Upgrade()
* store hub items as pointers
* No need to re-add items to the hub if we use pointers
* Fix parameter calling order + regression test
This commit is contained in:
mmetc 2023-11-09 15:19:38 +01:00 committed by GitHub
parent f80d841188
commit ab8de19506
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 269 additions and 286 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cmd/crowdsec-cli/config_restore.go
View file

@ -45,7 +45,12 @@ func restoreHub(dirPath string) error {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err) return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
} }
for _, toinstall := range upstreamList { for _, toinstall := range upstreamList {
err := hub.InstallItem(toinstall, itype, false, false) item := hub.GetItem(itype, toinstall)
if item == nil {
log.Errorf("Item %s/%s not found in hub", itype, toinstall)
continue
}
err := item.Install(false, false)
if err != nil { if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err) log.Errorf("Error while installing %s : %s", toinstall, err)
} }

2
cmd/crowdsec-cli/hub.go
View file

@ -145,7 +145,7 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error {
updated := 0 updated := 0
log.Infof("Upgrading %s", itemType) log.Infof("Upgrading %s", itemType)
for _, item := range items { for _, item := range items {
didUpdate, err := hub.UpgradeItem(itemType, item.Name, force) didUpdate, err := item.Upgrade(force)
if err != nil { if err != nil {
return err return err
} }

53
cmd/crowdsec-cli/itemcommands.go
View file

@ -203,7 +203,8 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string)
} }
for _, name := range args { for _, name := range args {
if hub.GetItem(it.name, name) == nil { item := hub.GetItem(it.name, name)
if item == nil {
msg := SuggestNearestMessage(hub, it.name, name) msg := SuggestNearestMessage(hub, it.name, name)
if !ignoreError { if !ignoreError {
return fmt.Errorf(msg) return fmt.Errorf(msg)
@ -213,11 +214,11 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string)
continue continue
} }
if err := hub.InstallItem(name, it.name, force, downloadOnly); err != nil { if err := item.Install(force, downloadOnly); err != nil {
if !ignoreError { if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err) return fmt.Errorf("error while installing '%s': %w", item.Name, err)
} }
log.Errorf("Error while installing '%s': %s", name, err) log.Errorf("Error while installing '%s': %s", item.Name, err)
} }
} }
@ -286,7 +287,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
removed := 0 removed := 0
for _, item := range items { for _, item := range items {
didRemove, err := hub.RemoveItem(it.name, item.Name, purge, force) didRemove, err := item.Remove(purge, force)
if err != nil { if err != nil {
return err return err
} }
@ -308,27 +309,25 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
} }
removed := 0 removed := 0
for _, name := range args { for _, itemName := range args {
if !force { item := hub.GetItem(it.name, itemName)
item := hub.GetItem(it.name, name) if item == nil {
if item == nil { return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
// XXX: this should be in GetItem?
return fmt.Errorf("can't find '%s' in %s", name, it.name)
}
if len(item.BelongsToCollections) > 0 {
log.Warningf("%s belongs to collections: %s", name, item.BelongsToCollections)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", it.name, name, it.singular)
continue
}
} }
didRemove, err := hub.RemoveItem(it.name, name, purge, force) if !force && len(item.BelongsToCollections) > 0 {
log.Warningf("%s belongs to collections: %s", item.Name, item.BelongsToCollections)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
continue
}
didRemove, err := item.Remove(purge, force)
if err != nil { if err != nil {
return err return err
} }
if didRemove { if didRemove {
log.Infof("Removed %s", name) log.Infof("Removed %s", item.Name)
removed++ removed++
} }
} }
@ -392,7 +391,7 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string)
updated := 0 updated := 0
for _, item := range items { for _, item := range items {
didUpdate, err := hub.UpgradeItem(it.name, item.Name, force) didUpdate, err := item.Upgrade(force)
if err != nil { if err != nil {
return err return err
} }
@ -400,7 +399,9 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string)
updated++ updated++
} }
} }
log.Infof("Updated %d %s", updated, it.name) log.Infof("Updated %d %s", updated, it.name)
if updated > 0 { if updated > 0 {
log.Infof(ReloadMessage()) log.Infof(ReloadMessage())
} }
@ -413,13 +414,19 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string)
} }
updated := 0 updated := 0
for _, name := range args { for _, itemName := range args {
didUpdate, err := hub.UpgradeItem(it.name, name, force) item := hub.GetItem(it.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
}
didUpdate, err := item.Upgrade(force)
if err != nil { if err != nil {
return err return err
} }
if didUpdate { if didUpdate {
log.Infof("Updated %s", name) log.Infof("Updated %s", item.Name)
updated++ updated++
} }
} }

103
pkg/cwhub/enable.go
View file

@ -10,24 +10,24 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
// enableItem creates a symlink between actual config file at hub.HubDir and hub.ConfigDir // enable creates a symlink between actual config file at hub.HubDir and hub.ConfigDir
// Handles collections recursively // Handles collections recursively
func (h *Hub) enableItem(target *Item) error { func (i *Item) enable() error {
parentDir := filepath.Clean(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/") parentDir := filepath.Clean(i.hub.local.InstallDir + "/" + i.Type + "/" + i.Stage + "/")
// create directories if needed // create directories if needed
if target.Installed { if i.Installed {
if target.Tainted { if i.Tainted {
return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
} }
if target.IsLocal() { if i.IsLocal() {
return fmt.Errorf("%s is local, won't enable", target.Name) return fmt.Errorf("%s is local, won't enable", i.Name)
} }
// if it's a collection, check sub-items even if the collection file itself is up-to-date // if it's a collection, check sub-items even if the collection file itself is up-to-date
if target.UpToDate && !target.HasSubItems() { if i.UpToDate && !i.HasSubItems() {
log.Tracef("%s is installed and up-to-date, skip.", target.Name) log.Tracef("%s is installed and up-to-date, skip.", i.Name)
return nil return nil
} }
} }
@ -41,30 +41,30 @@ func (h *Hub) enableItem(target *Item) error {
} }
// install sub-items if any // install sub-items if any
for _, sub := range target.SubItems() { for _, sub := range i.SubItems() {
val, ok := h.Items[sub.Type][sub.Name] val, ok := i.hub.Items[sub.Type][sub.Name]
if !ok { if !ok {
return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, i.Name)
} }
if err := h.enableItem(&val); err != nil { if err := val.enable(); err != nil {
return fmt.Errorf("while installing %s: %w", sub.Name, err) return fmt.Errorf("while installing %s: %w", sub.Name, err)
} }
} }
// check if file already exists where it should in configdir (eg /etc/crowdsec/collections/) // check if file already exists where it should in configdir (eg /etc/crowdsec/collections/)
if _, err := os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) { if _, err := os.Lstat(parentDir + "/" + i.FileName); !os.IsNotExist(err) {
log.Infof("%s already exists.", parentDir+"/"+target.FileName) log.Infof("%s already exists.", parentDir+"/"+i.FileName)
return nil return nil
} }
// hub.ConfigDir + target.RemotePath // hub.ConfigDir + target.RemotePath
srcPath, err := filepath.Abs(h.local.HubDir + "/" + target.RemotePath) srcPath, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath)
if err != nil { if err != nil {
return fmt.Errorf("while getting source path: %w", err) return fmt.Errorf("while getting source path: %w", err)
} }
dstPath, err := filepath.Abs(parentDir + "/" + target.FileName) dstPath, err := filepath.Abs(parentDir + "/" + i.FileName)
if err != nil { if err != nil {
return fmt.Errorf("while getting destination path: %w", err) return fmt.Errorf("while getting destination path: %w", err)
} }
@ -73,37 +73,36 @@ func (h *Hub) enableItem(target *Item) error {
return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err) return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err)
} }
log.Infof("Enabled %s: %s", target.Type, target.Name) log.Infof("Enabled %s: %s", i.Type, i.Name)
target.Installed = true i.Installed = true
h.Items[target.Type][target.Name] = *target
return nil return nil
} }
func (h *Hub) purgeItem(target Item) (Item, error) { // purge removes the actual config file that was downloaded
itempath := h.local.HubDir + "/" + target.RemotePath func (i *Item) purge() error {
itempath := i.hub.local.HubDir + "/" + i.RemotePath
// disable hub file // disable hub file
if err := os.Remove(itempath); err != nil { if err := os.Remove(itempath); err != nil {
return target, fmt.Errorf("while removing file: %w", err) return fmt.Errorf("while removing file: %w", err)
} }
target.Downloaded = false i.Downloaded = false
log.Infof("Removed source file [%s]: %s", target.Name, itempath) log.Infof("Removed source file [%s]: %s", i.Name, itempath)
h.Items[target.Type][target.Name] = target
return target, nil return nil
} }
// disableItem to disable an item managed by the hub, removes the symlink if purge is true // disable removes the symlink to the downloaded content, also removes the content if purge is true
func (h *Hub) disableItem(target *Item, purge bool, force bool) error { func (i *Item) disable(purge bool, force bool) error {
// XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not // XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not
var err error var err error
// already disabled, noop unless purge // already disabled, noop unless purge
if !target.Installed { if !i.Installed {
if purge { if purge {
*target, err = h.purgeItem(*target) err = i.purge()
if err != nil { if err != nil {
return err return err
} }
@ -112,20 +111,20 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error {
return nil return nil
} }
if target.IsLocal() { if i.IsLocal() {
return fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name) return fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
} }
if target.Tainted && !force { if i.Tainted && !force {
return fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name) return fmt.Errorf("%s is tainted, use '--force' to overwrite", i.Name)
} }
// disable sub-items if any - it's a collection // disable sub-items if any - it's a collection
for _, sub := range target.SubItems() { for _, sub := range i.SubItems() {
// XXX: we do this already when syncing, do we really need to do consistency checks here and there? // XXX: we do this already when syncing, do we really need to do consistency checks here and there?
val, ok := h.Items[sub.Type][sub.Name] val, ok := i.hub.Items[sub.Type][sub.Name]
if !ok { if !ok {
log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name) log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, i.Name)
continue continue
} }
@ -133,14 +132,14 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error {
toRemove := true toRemove := true
for _, collection := range val.BelongsToCollections { for _, collection := range val.BelongsToCollections {
if collection != target.Name { if collection != i.Name {
toRemove = false toRemove = false
break break
} }
} }
if toRemove { if toRemove {
if err = h.disableItem(&val, purge, force); err != nil { if err = val.disable(purge, force); err != nil {
return fmt.Errorf("while disabling %s: %w", sub.Name, err) return fmt.Errorf("while disabling %s: %w", sub.Name, err)
} }
} else { } else {
@ -148,7 +147,7 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error {
} }
} }
syml, err := filepath.Abs(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName) syml, err := filepath.Abs(i.hub.local.InstallDir + "/" + i.Type + "/" + i.Stage + "/" + i.FileName)
if err != nil { if err != nil {
return err return err
} }
@ -157,13 +156,13 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error {
if os.IsNotExist(err) { if os.IsNotExist(err) {
// we only accept to "delete" non existing items if it's a forced purge // we only accept to "delete" non existing items if it's a forced purge
if !purge && !force { if !purge && !force {
return fmt.Errorf("can't delete %s: %s doesn't exist", target.Name, syml) return fmt.Errorf("can't delete %s: %s doesn't exist", i.Name, syml)
} }
} else { } else {
// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ... // if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
if stat.Mode()&os.ModeSymlink == 0 { if stat.Mode()&os.ModeSymlink == 0 {
log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml) log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
return fmt.Errorf("%s isn't managed by hub", target.Name) return fmt.Errorf("%s isn't managed by hub", i.Name)
} }
hubpath, err := os.Readlink(syml) hubpath, err := os.Readlink(syml)
@ -171,14 +170,14 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error {
return fmt.Errorf("while reading symlink: %w", err) return fmt.Errorf("while reading symlink: %w", err)
} }
absPath, err := filepath.Abs(h.local.HubDir + "/" + target.RemotePath) absPath, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath)
if err != nil { if err != nil {
return fmt.Errorf("while abs path: %w", err) return fmt.Errorf("while abs path: %w", err)
} }
if hubpath != absPath { if hubpath != absPath {
log.Warningf("%s (%s) isn't a symlink to %s", target.Name, syml, absPath) log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, absPath)
return fmt.Errorf("%s isn't managed by hub", target.Name) return fmt.Errorf("%s isn't managed by hub", i.Name)
} }
// remove the symlink // remove the symlink
@ -186,19 +185,17 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error {
return fmt.Errorf("while removing symlink: %w", err) return fmt.Errorf("while removing symlink: %w", err)
} }
log.Infof("Removed symlink [%s]: %s", target.Name, syml) log.Infof("Removed symlink [%s]: %s", i.Name, syml)
} }
target.Installed = false i.Installed = false
if purge { if purge {
*target, err = h.purgeItem(*target) err = i.purge()
if err != nil { if err != nil {
return err return err
} }
} }
h.Items[target.Type][target.Name] = *target
return nil return nil
} }

18
pkg/cwhub/enable_test.go
View file

@ -8,9 +8,9 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func testInstall(hub *Hub, t *testing.T, item Item) { func testInstall(hub *Hub, t *testing.T, item *Item) {
// Install the parser // Install the parser
err := hub.downloadLatest(&item, false, false) err := item.downloadLatest(false, false)
require.NoError(t, err, "failed to download %s", item.Name) require.NoError(t, err, "failed to download %s", item.Name)
err = hub.localSync() err = hub.localSync()
@ -20,7 +20,7 @@ func testInstall(hub *Hub, t *testing.T, item Item) {
assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
err = hub.enableItem(&item) err = item.enable()
require.NoError(t, err, "failed to enable %s", item.Name) require.NoError(t, err, "failed to enable %s", item.Name)
err = hub.localSync() err = hub.localSync()
@ -29,7 +29,7 @@ func testInstall(hub *Hub, t *testing.T, item Item) {
assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
} }
func testTaint(hub *Hub, t *testing.T, item Item) { func testTaint(hub *Hub, t *testing.T, item *Item) {
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
@ -47,11 +47,11 @@ func testTaint(hub *Hub, t *testing.T, item Item) {
assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
} }
func testUpdate(hub *Hub, t *testing.T, item Item) { func testUpdate(hub *Hub, t *testing.T, item *Item) {
assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
// Update it + check status // Update it + check status
err := hub.downloadLatest(&item, true, true) err := item.downloadLatest(true, true)
require.NoError(t, err, "failed to update %s", item.Name) require.NoError(t, err, "failed to update %s", item.Name)
// Local sync and check status // Local sync and check status
@ -62,11 +62,11 @@ func testUpdate(hub *Hub, t *testing.T, item Item) {
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
} }
func testDisable(hub *Hub, t *testing.T, item Item) { func testDisable(hub *Hub, t *testing.T, item *Item) {
assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
// Remove // Remove
err := hub.disableItem(&item, false, false) err := item.disable(false, false)
require.NoError(t, err, "failed to disable %s", item.Name) require.NoError(t, err, "failed to disable %s", item.Name)
// Local sync and check status // Local sync and check status
@ -79,7 +79,7 @@ func testDisable(hub *Hub, t *testing.T, item Item) {
assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
// Purge // Purge
err = hub.disableItem(&item, true, false) err = item.disable(true, false)
require.NoError(t, err, "failed to purge %s", item.Name) require.NoError(t, err, "failed to purge %s", item.Name)
// Local sync and check status // Local sync and check status

199
pkg/cwhub/helpers.go
View file

@ -19,15 +19,10 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
// InstallItem installs an item from the hub // Install installs the item from the hub, downloading it if needed
func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error { func (i *Item) Install(force bool, downloadOnly bool) error {
item := h.GetItem(itemType, name) if downloadOnly && i.Downloaded && i.UpToDate {
if item == nil { log.Warningf("%s is already downloaded and up-to-date", i.Name)
return fmt.Errorf("unable to retrieve item: %s", name)
}
if downloadOnly && item.Downloaded && item.UpToDate {
log.Warningf("%s is already downloaded and up-to-date", item.Name)
if !force { if !force {
return nil return nil
@ -35,90 +30,68 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly
} }
// XXX: confusing semantic between force and updateOnly? // XXX: confusing semantic between force and updateOnly?
if err := h.downloadLatest(item, force, true); err != nil { if err := i.downloadLatest(force, true); err != nil {
return fmt.Errorf("while downloading %s: %w", item.Name, err) return fmt.Errorf("while downloading %s: %w", i.Name, err)
}
if err := h.AddItem(*item); err != nil {
return fmt.Errorf("while adding %s: %w", item.Name, err)
} }
if downloadOnly { if downloadOnly {
// XXX: should get the path from downloadLatest // XXX: should get the path from downloadLatest
log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.local.HubDir, item.RemotePath)) log.Infof("Downloaded %s to %s", i.Name, filepath.Join(i.hub.local.HubDir, i.RemotePath))
return nil return nil
} }
// XXX: should we stop here if the item is already installed? // XXX: should we stop here if the item is already installed?
if err := h.enableItem(item); err != nil { if err := i.enable(); err != nil {
return fmt.Errorf("while enabling %s: %w", item.Name, err) return fmt.Errorf("while enabling %s: %w", i.Name, err)
} }
if err := h.AddItem(*item); err != nil { log.Infof("Enabled %s", i.Name)
return fmt.Errorf("while adding %s: %w", item.Name, err)
}
log.Infof("Enabled %s", item.Name)
return nil return nil
} }
// RemoveItem disables one item, optionally removing the downloaded content // Remove disables the item, optionally removing the downloaded content
func (h *Hub) RemoveItem(itemType string, name string, purge bool, forceAction bool) (bool, error) { func (i *Item) Remove(purge bool, forceAction bool) (bool, error) {
removed := false removed := false
item := h.GetItem(itemType, name) if !i.Downloaded {
if item == nil { log.Infof("removing %s: not downloaded -- no removal required", i.Name)
return false, fmt.Errorf("can't find '%s' in %s", name, itemType)
}
if !item.Downloaded {
log.Infof("removing %s: not downloaded -- no removal required", item.Name)
return false, nil return false, nil
} }
if !item.Installed && !purge { if !i.Installed && !purge {
log.Infof("removing %s: already uninstalled", item.Name) log.Infof("removing %s: already uninstalled", i.Name)
return false, nil return false, nil
} }
if err := h.disableItem(item, purge, forceAction); err != nil { if err := i.disable(purge, forceAction); err != nil {
return false, fmt.Errorf("unable to disable %s: %w", item.Name, err) return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
} }
// XXX: should take the value from disableItem // XXX: should take the value from disable()
removed = true removed = true
if err := h.AddItem(*item); err != nil {
return false, fmt.Errorf("unable to refresh item state %s: %w", item.Name, err)
}
return removed, nil return removed, nil
} }
// UpgradeItem upgrades an item from the hub // Upgrade downloads and applies the last version from the hub
func (h *Hub) UpgradeItem(itemType string, name string, force bool) (bool, error) { func (i *Item) Upgrade(force bool) (bool, error) {
updated := false updated := false
item := h.GetItem(itemType, name) if !i.Downloaded {
if item == nil { return false, fmt.Errorf("can't upgrade %s: not installed", i.Name)
return false, fmt.Errorf("can't find '%s' in %s", name, itemType)
} }
if !item.Downloaded { if !i.Installed {
return false, fmt.Errorf("can't upgrade %s: not installed", item.Name) return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", i.Name)
} }
if !item.Installed { if i.UpToDate {
return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", item.Name) log.Infof("%s: up-to-date", i.Name)
}
if item.UpToDate { if err := i.DownloadDataIfNeeded(force); err != nil {
log.Infof("%s: up-to-date", item.Name) return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
if err := h.DownloadDataIfNeeded(*item, force); err != nil {
return false, fmt.Errorf("%s: download failed: %w", item.Name, err)
} }
if !force { if !force {
@ -127,112 +100,106 @@ func (h *Hub) UpgradeItem(itemType string, name string, force bool) (bool, error
} }
} }
if err := h.downloadLatest(item, force, true); err != nil { if err := i.downloadLatest(force, true); err != nil {
return false, fmt.Errorf("%s: download failed: %w", item.Name, err) return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
} }
if !item.UpToDate { if !i.UpToDate {
if item.Tainted { if i.Tainted {
log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, item.Name) log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name)
} else if item.IsLocal() { } else if i.IsLocal() {
log.Infof("%v %s is local", emoji.Prohibited, item.Name) log.Infof("%v %s is local", emoji.Prohibited, i.Name)
} }
} else { } else {
// a check on stdout is used while scripting to know if the hub has been upgraded // a check on stdout is used while scripting to know if the hub has been upgraded
// and a configuration reload is required // and a configuration reload is required
// TODO: use a better way to communicate this // TODO: use a better way to communicate this
fmt.Printf("updated %s\n", item.Name) fmt.Printf("updated %s\n", i.Name)
log.Infof("%v %s: updated", emoji.Package, item.Name) log.Infof("%v %s: updated", emoji.Package, i.Name)
updated = true updated = true
} }
if err := h.AddItem(*item); err != nil {
return false, fmt.Errorf("unable to refresh item state %s: %w", item.Name, err)
}
return updated, nil return updated, nil
} }
// downloadLatest will download the latest version of Item to the tdir directory // downloadLatest will download the latest version of Item to the tdir directory
func (h *Hub) downloadLatest(target *Item, overwrite bool, updateOnly bool) error { func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error {
// XXX: should return the path of the downloaded file (taken from downloadItem) // XXX: should return the path of the downloaded file (taken from download())
log.Debugf("Downloading %s %s", target.Type, target.Name) log.Debugf("Downloading %s %s", i.Type, i.Name)
if !target.HasSubItems() { if !i.HasSubItems() {
if !target.Installed && updateOnly && target.Downloaded { if !i.Installed && updateOnly && i.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", target.Name) log.Debugf("skipping upgrade of %s: not installed", i.Name)
return nil return nil
} }
// XXX: // XXX:
return h.downloadItem(target, overwrite) return i.download(overwrite)
} }
// collection // collection
for _, sub := range target.SubItems() { for _, sub := range i.SubItems() {
val, ok := h.Items[sub.Type][sub.Name] val, ok := i.hub.Items[sub.Type][sub.Name]
if !ok { if !ok {
return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, i.Name)
} }
if !val.Installed && updateOnly && val.Downloaded { if !val.Installed && updateOnly && val.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", target.Name) log.Debugf("skipping upgrade of %s: not installed", i.Name)
continue continue
} }
log.Debugf("Download %s sub-item: %s %s (%t -> %t)", target.Name, sub.Type, sub.Name, target.Installed, updateOnly) log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.Installed, updateOnly)
// recurse as it's a collection // recurse as it's a collection
if sub.HasSubItems() { if sub.HasSubItems() {
log.Tracef("collection, recurse") log.Tracef("collection, recurse")
if err := h.downloadLatest(&val, overwrite, updateOnly); err != nil { if err := val.downloadLatest(overwrite, updateOnly); err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err) return fmt.Errorf("while downloading %s: %w", val.Name, err)
} }
} }
downloaded := val.Downloaded downloaded := val.Downloaded
if err := h.downloadItem(&val, overwrite); err != nil { if err := val.download(overwrite); err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err) return fmt.Errorf("while downloading %s: %w", val.Name, err)
} }
// We need to enable an item when it has been added to a collection since latest release of the collection. // We need to enable an item when it has been added to a collection since latest release of the collection.
// We check if val.Downloaded is false because maybe the item has been disabled by the user. // We check if val.Downloaded is false because maybe the item has been disabled by the user.
if !val.Installed && !downloaded { if !val.Installed && !downloaded {
if err := h.enableItem(&val); err != nil { if err := val.enable(); err != nil {
return fmt.Errorf("enabling '%s': %w", val.Name, err) return fmt.Errorf("enabling '%s': %w", val.Name, err)
} }
} }
h.Items[sub.Type][sub.Name] = val
} }
if err := h.downloadItem(target, overwrite); err != nil { if err := i.download(overwrite); err != nil {
return fmt.Errorf("failed to download item: %w", err) return fmt.Errorf("failed to download item: %w", err)
} }
return nil return nil
} }
func (h *Hub) downloadItem(target *Item, overwrite bool) error { func (i *Item) download(overwrite bool) error {
url, err := h.remote.urlTo(target.RemotePath) url, err := i.hub.remote.urlTo(i.RemotePath)
if err != nil { if err != nil {
return fmt.Errorf("failed to build hub item request: %w", err) return fmt.Errorf("failed to build hub item request: %w", err)
} }
tdir := h.local.HubDir tdir := i.hub.local.HubDir
// if user didn't --force, don't overwrite local, tainted, up-to-date files // if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite { if !overwrite {
if target.Tainted { if i.Tainted {
log.Debugf("%s: tainted, not updated", target.Name) log.Debugf("%s: tainted, not updated", i.Name)
return nil return nil
} }
if target.UpToDate { if i.UpToDate {
// We still have to check if data files are present // We still have to check if data files are present
log.Debugf("%s: up-to-date, not updated", target.Name) log.Debugf("%s: up-to-date, not updated", i.Name)
} }
} }
@ -253,30 +220,30 @@ func (h *Hub) downloadItem(target *Item, overwrite bool) error {
hash := sha256.New() hash := sha256.New()
if _, err = hash.Write(body); err != nil { if _, err = hash.Write(body); err != nil {
return fmt.Errorf("while hashing %s: %w", target.Name, err) return fmt.Errorf("while hashing %s: %w", i.Name, err)
} }
meow := hex.EncodeToString(hash.Sum(nil)) meow := hex.EncodeToString(hash.Sum(nil))
if meow != target.Versions[target.Version].Digest { if meow != i.Versions[i.Version].Digest {
log.Errorf("Downloaded version doesn't match index, please 'hub update'") log.Errorf("Downloaded version doesn't match index, please 'hub update'")
log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
return fmt.Errorf("invalid download hash for %s", target.Name) return fmt.Errorf("invalid download hash for %s", i.Name)
} }
//all good, install //all good, install
//check if parent dir exists //check if parent dir exists
tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") tmpdirs := strings.Split(tdir+"/"+i.RemotePath, "/")
parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
// ensure that target file is within target dir // ensure that target file is within target dir
finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath) finalPath, err := filepath.Abs(tdir + "/" + i.RemotePath)
if err != nil { if err != nil {
return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err) return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+i.RemotePath, err)
} }
if !strings.HasPrefix(finalPath, tdir) { if !strings.HasPrefix(finalPath, tdir) {
return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir) return fmt.Errorf("path %s escapes %s, abort", i.RemotePath, tdir)
} }
// check dir // check dir
@ -290,13 +257,13 @@ func (h *Hub) downloadItem(target *Item, overwrite bool) error {
// check actual file // check actual file
if _, err = os.Stat(finalPath); !os.IsNotExist(err) { if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
log.Warningf("%s: overwrite", target.Name) log.Warningf("%s: overwrite", i.Name)
log.Debugf("target: %s/%s", tdir, target.RemotePath) log.Debugf("target: %s/%s", tdir, i.RemotePath)
} else { } else {
log.Infof("%s: OK", target.Name) log.Infof("%s: OK", i.Name)
} }
f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) f, err := os.OpenFile(tdir+"/"+i.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
if err != nil { if err != nil {
return fmt.Errorf("while opening file: %w", err) return fmt.Errorf("while opening file: %w", err)
} }
@ -308,22 +275,20 @@ func (h *Hub) downloadItem(target *Item, overwrite bool) error {
return fmt.Errorf("while writing file: %w", err) return fmt.Errorf("while writing file: %w", err)
} }
target.Downloaded = true i.Downloaded = true
target.Tainted = false i.Tainted = false
target.UpToDate = true i.UpToDate = true
if err = downloadData(h.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { if err = downloadData(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) return fmt.Errorf("while downloading data for %s: %w", i.FileName, err)
} }
h.Items[target.Type][target.Name] = *target
return nil return nil
} }
// DownloadDataIfNeeded downloads the data files for an item // DownloadDataIfNeeded downloads the data files for the item
func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { func (i *Item) DownloadDataIfNeeded(force bool) error {
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.local.InstallDir, target.Type, target.Stage, target.FileName) itemFilePath := fmt.Sprintf("%s/%s/%s/%s", i.hub.local.InstallDir, i.Type, i.Stage, i.FileName)
itemFile, err := os.Open(itemFilePath) itemFile, err := os.Open(itemFilePath)
if err != nil { if err != nil {
@ -332,7 +297,7 @@ func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error {
defer itemFile.Close() defer itemFile.Close()
if err = downloadData(h.local.InstallDataDir, force, itemFile); err != nil { if err = downloadData(i.hub.local.InstallDataDir, force, itemFile); err != nil {
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
} }

29
pkg/cwhub/helpers_test.go
View file

@ -17,7 +17,8 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.NoError(t, item.Install(false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
@ -25,8 +26,7 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
// This is the scenario that gets added in next version of collection // This is the scenario that gets added in next version of collection
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"])
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
@ -49,7 +49,8 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false) item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
didUpdate, err := item.Upgrade(false)
require.NoError(t, err) require.NoError(t, err)
require.True(t, didUpdate) require.True(t, didUpdate)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
@ -68,7 +69,8 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.NoError(t, item.Install(false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
@ -77,7 +79,8 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
didRemove, err := hub.RemoveItem(SCENARIOS, "crowdsecurity/foobar_scenario", false, false) item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
didRemove, err := item.Remove(false, false)
require.NoError(t, err) require.NoError(t, err)
require.True(t, didRemove) require.True(t, didRemove)
@ -98,7 +101,8 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
hub, err = NewHub(hub.local, remote, true) hub, err = NewHub(hub.local, remote, true)
require.NoError(t, err, "failed to download index: %s", err) require.NoError(t, err, "failed to download index: %s", err)
didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false) item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
didUpdate, err := item.Upgrade(false)
require.NoError(t, err) require.NoError(t, err)
require.False(t, didUpdate) require.False(t, didUpdate)
@ -125,7 +129,8 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.NoError(t, item.Install(false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
@ -134,7 +139,8 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
didRemove, err := hub.RemoveItem(SCENARIOS, "crowdsecurity/foobar_scenario", false, false) item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
didRemove, err := item.Remove(false, false)
require.NoError(t, err) require.NoError(t, err)
require.True(t, didRemove) require.True(t, didRemove)
@ -164,7 +170,8 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
hub = getHubOrFail(t, hub.local, remote) hub = getHubOrFail(t, hub.local, remote)
didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false) item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
didUpdate, err := item.Upgrade(false)
require.NoError(t, err) require.NoError(t, err)
require.True(t, didUpdate) require.True(t, didUpdate)
@ -180,7 +187,7 @@ func assertCollectionDepsInstalled(t *testing.T, collection string) {
require.NoError(t, err) require.NoError(t, err)
c := hub.Items[COLLECTIONS][collection] c := hub.Items[COLLECTIONS][collection]
require.NoError(t, hub.checkSubItems(&c)) require.NoError(t, hub.checkSubItems(c))
} }
func pushUpdateToCollectionInHub() { func pushUpdateToCollectionInHub() {

2
pkg/cwhub/hub.go
View file

@ -82,6 +82,7 @@ func (h *Hub) parseIndex() error {
log.Tracef("%s: %d items", itemType, len(h.Items[itemType])) log.Tracef("%s: %d items", itemType, len(h.Items[itemType]))
for name, item := range h.Items[itemType] { for name, item := range h.Items[itemType] {
item.hub = h
item.Name = name item.Name = name
// if the item has no (redundant) author, take it from the json key // if the item has no (redundant) author, take it from the json key
@ -92,7 +93,6 @@ func (h *Hub) parseIndex() error {
item.Type = itemType item.Type = itemType
x := strings.Split(item.RemotePath, "/") x := strings.Split(item.RemotePath, "/")
item.FileName = x[len(x)-1] item.FileName = x[len(x)-1]
h.Items[itemType][name] = item
// if it's a collection, check its sub-items are present // if it's a collection, check its sub-items are present
// XXX should be done later, maybe report all missing at once? // XXX should be done later, maybe report all missing at once?

42
pkg/cwhub/items.go
View file

@ -26,7 +26,7 @@ const (
// The order is important, as it is used to range over sub-items in collections // The order is important, as it is used to range over sub-items in collections
var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
type HubItems map[string]map[string]Item type HubItems map[string]map[string]*Item
// ItemVersion is used to detect the version of a given item // ItemVersion is used to detect the version of a given item
// by comparing the hash of each version to the local file. // by comparing the hash of each version to the local file.
@ -38,6 +38,9 @@ type ItemVersion struct {
// Item represents an object managed in the hub. It can be a parser, scenario, collection.. // Item represents an object managed in the hub. It can be a parser, scenario, collection..
type Item struct { type Item struct {
// back pointer to the hub, to retrieve subitems and call install/remove methods
hub *Hub
// descriptive info // descriptive info
Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
@ -220,23 +223,13 @@ func (i *Item) validPath(dirName, fileName string) bool {
} }
// GetItemMap returns the map of items for a given type // GetItemMap returns the map of items for a given type
func (h *Hub) GetItemMap(itemType string) map[string]Item { func (h *Hub) GetItemMap(itemType string) map[string]*Item {
m, ok := h.Items[itemType] return h.Items[itemType]
if !ok {
return nil
}
return m
} }
// GetItem returns the item from hub based on its type and full name (author/name) // GetItem returns the item from hub based on its type and full name (author/name)
func (h *Hub) GetItem(itemType string, itemName string) *Item { func (h *Hub) GetItem(itemType string, itemName string) *Item {
m, ok := h.GetItemMap(itemType)[itemName] return h.GetItemMap(itemType)[itemName]
if !ok {
return nil
}
return &m
} }
// GetItemNames returns the list of item (full) names for a given type // GetItemNames returns the list of item (full) names for a given type
@ -256,27 +249,14 @@ func (h *Hub) GetItemNames(itemType string) []string {
return names return names
} }
// AddItem adds an item to the hub index
func (h *Hub) AddItem(item Item) error {
for _, t := range ItemTypes {
if t == item.Type {
h.Items[t][item.Name] = item
return nil
}
}
// XXX: can this happen?
return fmt.Errorf("ItemType %s is unknown", item.Type)
}
// GetInstalledItems returns the list of installed items // GetInstalledItems returns the list of installed items
func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) { func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType] items, ok := h.Items[itemType]
if !ok { if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType) return nil, fmt.Errorf("no %s in the hub index", itemType)
} }
retItems := make([]Item, 0) retItems := make([]*Item, 0)
for _, item := range items { for _, item := range items {
if item.Installed { if item.Installed {
@ -296,8 +276,8 @@ func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) {
retStr := make([]string, len(items)) retStr := make([]string, len(items))
for i, it := range items { for idx, it := range items {
retStr[i] = it.Name retStr[idx] = it.Name
} }
return retStr, nil return retStr, nil

9
pkg/cwhub/items_test.go
View file

@ -4,8 +4,6 @@ import (
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
) )
func TestItemStatus(t *testing.T) { func TestItemStatus(t *testing.T) {
@ -62,14 +60,9 @@ func TestGetters(t *testing.T) {
// Add item and get it // Add item and get it
item.Name += "nope" item.Name += "nope"
err := hub.AddItem(*item) hub.Items[item.Type][item.Name] = item
require.NoError(t, err)
newitem := hub.GetItem(COLLECTIONS, item.Name) newitem := hub.GetItem(COLLECTIONS, item.Name)
require.NotNil(t, newitem) require.NotNil(t, newitem)
item.Type = "ratata"
err = hub.AddItem(*item)
cstest.RequireErrorContains(t, err, "ItemType ratata is unknown")
} }
} }

24
pkg/cwhub/sync.go
View file

@ -129,20 +129,20 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
// sortedVersions returns the input data, sorted in reverse order by semver // sortedVersions returns the input data, sorted in reverse order by semver
func sortedVersions(raw []string) ([]string, error) { func sortedVersions(raw []string) ([]string, error) {
vs := make([]*semver.Version, len(raw)) vs := make([]*semver.Version, len(raw))
for i, r := range raw { for idx, r := range raw {
v, err := semver.NewVersion(r) v, err := semver.NewVersion(r)
if err != nil { if err != nil {
return nil, fmt.Errorf("%s: %w", r, err) return nil, fmt.Errorf("%s: %w", r, err)
} }
vs[i] = v vs[idx] = v
} }
sort.Sort(sort.Reverse(semver.Collection(vs))) sort.Sort(sort.Reverse(semver.Collection(vs)))
ret := make([]string, len(vs)) ret := make([]string, len(vs))
for i, v := range vs { for idx, v := range vs {
ret[i] = v.Original() ret[idx] = v.Original()
} }
return ret, nil return ret, nil
@ -209,7 +209,8 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
_, fileName := filepath.Split(path) _, fileName := filepath.Split(path)
h.Items[info.ftype][info.fname] = Item{ h.Items[info.ftype][info.fname] = &Item{
hub: h,
Name: info.fname, Name: info.fname,
Stage: info.stage, Stage: info.stage,
Installed: true, Installed: true,
@ -360,7 +361,11 @@ func (h *Hub) checkSubItems(v *Item) error {
continue continue
} }
if err := h.checkSubItems(&subItem); err != nil { if err := h.checkSubItems(subItem); err != nil {
if subItem.Tainted {
v.Tainted = true
}
return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
} }
@ -383,8 +388,6 @@ func (h *Hub) checkSubItems(v *Item) error {
subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
} }
h.Items[sub.Type][sub.Name] = subItem
log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate) log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate)
} }
@ -412,7 +415,7 @@ func (h *Hub) syncDir(dir string) ([]string, error) {
} }
} }
for name, item := range h.Items[COLLECTIONS] { for _, item := range h.Items[COLLECTIONS] {
if !item.Installed { if !item.Installed {
continue continue
} }
@ -420,9 +423,8 @@ func (h *Hub) syncDir(dir string) ([]string, error) {
vs := item.versionStatus() vs := item.versionStatus()
switch vs { switch vs {
case VersionUpToDate: // latest case VersionUpToDate: // latest
if err := h.checkSubItems(&item); err != nil { if err := h.checkSubItems(item); err != nil {
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
h.Items[COLLECTIONS][name] = item
} }
case VersionUpdateAvailable: // not up-to-date case VersionUpdateAvailable: // not up-to-date
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))

6
pkg/hubtest/hubtest_item.go
View file

@ -410,7 +410,7 @@ func (t *HubTestItem) InstallHub() error {
ret := hub.GetItemMap(cwhub.PARSERS) ret := hub.GetItemMap(cwhub.PARSERS)
for parserName, item := range ret { for parserName, item := range ret {
if item.Installed { if item.Installed {
if err := hub.DownloadDataIfNeeded(item, true); err != nil { if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err) return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err)
} }
@ -422,7 +422,7 @@ func (t *HubTestItem) InstallHub() error {
ret = hub.GetItemMap(cwhub.SCENARIOS) ret = hub.GetItemMap(cwhub.SCENARIOS)
for scenarioName, item := range ret { for scenarioName, item := range ret {
if item.Installed { if item.Installed {
if err := hub.DownloadDataIfNeeded(item, true); err != nil { if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err) return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err)
} }
@ -434,7 +434,7 @@ func (t *HubTestItem) InstallHub() error {
ret = hub.GetItemMap(cwhub.POSTOVERFLOWS) ret = hub.GetItemMap(cwhub.POSTOVERFLOWS)
for postoverflowName, item := range ret { for postoverflowName, item := range ret {
if item.Installed { if item.Installed {
if err := hub.DownloadDataIfNeeded(item, true); err != nil { if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err) return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err)
} }

36
pkg/setup/install.go
View file

@ -62,14 +62,19 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
if len(install.Collections) > 0 { if len(install.Collections) > 0 {
for _, collection := range setupItem.Install.Collections { for _, collection := range setupItem.Install.Collections {
item := hub.GetItem(cwhub.COLLECTIONS, collection)
if item == nil {
return fmt.Errorf("collection %s not found", collection)
}
if dryRun { if dryRun {
fmt.Println("dry-run: would install collection", collection) fmt.Println("dry-run: would install collection", collection)
continue continue
} }
if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing collection %s: %w", collection, err) return fmt.Errorf("while installing collection %s: %w", item.Name, err)
} }
} }
} }
@ -82,8 +87,13 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
continue continue
} }
if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { item := hub.GetItem(cwhub.PARSERS, parser)
return fmt.Errorf("while installing parser %s: %w", parser, err) if item == nil {
return fmt.Errorf("parser %s not found", parser)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing parser %s: %w", item.Name, err)
} }
} }
} }
@ -96,8 +106,13 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
continue continue
} }
if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { item := hub.GetItem(cwhub.SCENARIOS, scenario)
return fmt.Errorf("while installing scenario %s: %w", scenario, err) if item == nil {
return fmt.Errorf("scenario %s not found", scenario)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing scenario %s: %w", item.Name, err)
} }
} }
} }
@ -110,8 +125,13 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
continue continue
} }
if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil { item := hub.GetItem(cwhub.POSTOVERFLOWS, postoverflow)
return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) if item == nil {
return fmt.Errorf("postoverflow %s not found", postoverflow)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing postoverflow %s: %w", item.Name, err)
} }
} }
} }

25
test/bats/07_setup.bats
View file

@ -519,6 +519,11 @@ update-notifier-motd.timer enabled enabled
rune -0 cscli collections list -o json rune -0 cscli collections list -o json
rune -0 jq -r '.collections[].name' <(output) rune -0 jq -r '.collections[].name' <(output)
refute_line "crowdsecurity/apache2" refute_line "crowdsecurity/apache2"
# same with dependencies
rune -0 cscli collections remove --all
rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/linux"]}}]}'
assert_output 'dry-run: would install collection crowdsecurity/linux'
} }
@test "cscli setup install-hub (dry run: install multiple collections)" { @test "cscli setup install-hub (dry run: install multiple collections)" {
@ -538,15 +543,17 @@ update-notifier-motd.timer enabled enabled
} }
@test "cscli setup install-hub (dry run: install multiple collections, parsers, scenarios, postoverflows)" { @test "cscli setup install-hub (dry run: install multiple collections, parsers, scenarios, postoverflows)" {
rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo","johndoe/bar"],"parsers":["crowdsecurity/fooparser","johndoe/barparser"],"scenarios":["crowdsecurity/fooscenario","johndoe/barscenario"],"postoverflows":["crowdsecurity/foopo","johndoe/barpo"]}}]}' rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/aws-console","crowdsecurity/caddy"],"parsers":["crowdsecurity/asterisk-logs"],"scenarios":["crowdsecurity/smb-fs"],"postoverflows":["crowdsecurity/cdn-whitelist","crowdsecurity/rdns"]}}]}'
assert_line 'dry-run: would install collection crowdsecurity/foo' assert_line 'dry-run: would install collection crowdsecurity/aws-console'
assert_line 'dry-run: would install collection johndoe/bar' assert_line 'dry-run: would install collection crowdsecurity/caddy'
assert_line 'dry-run: would install parser crowdsecurity/fooparser' assert_line 'dry-run: would install parser crowdsecurity/asterisk-logs'
assert_line 'dry-run: would install parser johndoe/barparser' assert_line 'dry-run: would install scenario crowdsecurity/smb-fs'
assert_line 'dry-run: would install scenario crowdsecurity/fooscenario' assert_line 'dry-run: would install postoverflow crowdsecurity/cdn-whitelist'
assert_line 'dry-run: would install scenario johndoe/barscenario' assert_line 'dry-run: would install postoverflow crowdsecurity/rdns'
assert_line 'dry-run: would install postoverflow crowdsecurity/foopo'
assert_line 'dry-run: would install postoverflow johndoe/barpo' rune -1 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo"]}}]}'
assert_stderr --partial 'collection crowdsecurity/foo not found'
} }
@test "cscli setup datasources" { @test "cscli setup datasources" {