2023-10-17 14:17:37 +00:00
|
|
|
package cwhub
|
|
|
|
|
|
|
|
import (
|
|
|
|
"encoding/json"
|
2023-10-20 12:32:35 +00:00
|
|
|
"errors"
|
2023-10-17 14:17:37 +00:00
|
|
|
"fmt"
|
2023-10-20 12:32:35 +00:00
|
|
|
"os"
|
2023-10-17 14:17:37 +00:00
|
|
|
"strings"
|
|
|
|
|
|
|
|
log "github.com/sirupsen/logrus"
|
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
|
|
|
)
|
2023-10-17 14:17:37 +00:00
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
type Hub struct {
|
|
|
|
Items HubItems
|
2023-10-30 16:23:50 +00:00
|
|
|
local *csconfig.LocalHubCfg
|
|
|
|
remote *RemoteHubCfg
|
2023-10-17 14:17:37 +00:00
|
|
|
skippedLocal int
|
|
|
|
skippedTainted int
|
|
|
|
}
|
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
var (
|
|
|
|
theHub *Hub
|
|
|
|
ErrIndexNotFound = fmt.Errorf("index not found")
|
|
|
|
)
|
2023-10-19 10:04:29 +00:00
|
|
|
|
|
|
|
// GetHub returns the hub singleton
|
|
|
|
// it returns an error if it's not initialized to avoid nil dereference
|
2023-10-30 16:23:50 +00:00
|
|
|
// XXX: convenience function that we should get rid of at some point
|
2023-10-19 10:04:29 +00:00
|
|
|
func GetHub() (*Hub, error) {
|
|
|
|
if theHub == nil {
|
|
|
|
return nil, fmt.Errorf("hub not initialized")
|
|
|
|
}
|
|
|
|
|
|
|
|
return theHub, nil
|
|
|
|
}
|
|
|
|
|
2023-10-31 11:47:39 +00:00
|
|
|
// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state
|
|
|
|
// It also downloads the index if downloadIndex is true
|
|
|
|
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, downloadIndex bool) (*Hub, error) {
|
2023-10-30 16:23:50 +00:00
|
|
|
if local == nil {
|
|
|
|
return nil, fmt.Errorf("no hub configuration found")
|
2023-10-17 14:17:37 +00:00
|
|
|
}
|
|
|
|
|
2023-10-31 11:47:39 +00:00
|
|
|
if downloadIndex {
|
2023-10-31 15:32:29 +00:00
|
|
|
if err := remote.DownloadIndex(local.HubIndexFile); err != nil {
|
2023-10-31 11:47:39 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-10-30 16:23:50 +00:00
|
|
|
log.Debugf("loading hub idx %s", local.HubIndexFile)
|
2023-10-20 12:32:35 +00:00
|
|
|
|
2023-10-30 16:23:50 +00:00
|
|
|
bidx, err := os.ReadFile(local.HubIndexFile)
|
2023-10-20 12:32:35 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("unable to read index file: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
ret, err := ParseIndex(bidx)
|
|
|
|
if err != nil {
|
|
|
|
if !errors.Is(err, ErrMissingReference) {
|
2023-10-31 11:47:39 +00:00
|
|
|
return nil, fmt.Errorf("failed to load index: %w", err)
|
2023-10-20 12:32:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// XXX: why the error check if we bail out anyway?
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
theHub = &Hub{
|
2023-10-30 16:23:50 +00:00
|
|
|
Items: ret,
|
|
|
|
local: local,
|
|
|
|
remote: remote,
|
2023-10-17 14:17:37 +00:00
|
|
|
}
|
2023-10-20 12:32:35 +00:00
|
|
|
|
2023-10-31 15:32:29 +00:00
|
|
|
if _, err = theHub.LocalSync(); err != nil {
|
2023-10-31 11:47:39 +00:00
|
|
|
return nil, fmt.Errorf("failed to sync hub index: %w", err)
|
2023-10-20 12:32:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return theHub, nil
|
2023-10-17 14:17:37 +00:00
|
|
|
}
|
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
// ParseIndex takes the content of an index file and returns the map of associated parsers/scenarios/collections
|
2023-10-17 14:17:37 +00:00
|
|
|
func ParseIndex(buff []byte) (HubItems, error) {
|
|
|
|
var (
|
|
|
|
RawIndex HubItems
|
|
|
|
missingItems []string
|
|
|
|
)
|
|
|
|
|
|
|
|
if err := json.Unmarshal(buff, &RawIndex); err != nil {
|
|
|
|
return nil, fmt.Errorf("failed to unmarshal index: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Debugf("%d item types in hub index", len(ItemTypes))
|
|
|
|
|
|
|
|
// Iterate over the different types to complete the struct
|
|
|
|
for _, itemType := range ItemTypes {
|
|
|
|
log.Tracef("%s: %d items", itemType, len(RawIndex[itemType]))
|
|
|
|
|
|
|
|
for name, item := range RawIndex[itemType] {
|
|
|
|
item.Name = name
|
|
|
|
item.Type = itemType
|
|
|
|
x := strings.Split(item.RemotePath, "/")
|
|
|
|
item.FileName = x[len(x)-1]
|
|
|
|
RawIndex[itemType][name] = item
|
|
|
|
|
|
|
|
if itemType != COLLECTIONS {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// if it's a collection, check its sub-items are present
|
|
|
|
// XXX should be done later
|
2023-10-20 12:32:35 +00:00
|
|
|
for _, sub := range item.SubItems() {
|
|
|
|
if _, ok := RawIndex[sub.Type][sub.Name]; !ok {
|
|
|
|
log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, item.Name)
|
|
|
|
missingItems = append(missingItems, sub.Name)
|
2023-10-17 14:17:37 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(missingItems) > 0 {
|
|
|
|
return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference)
|
|
|
|
}
|
|
|
|
|
|
|
|
return RawIndex, nil
|
|
|
|
}
|
2023-10-20 12:32:35 +00:00
|
|
|
|
|
|
|
// ItemStats returns total counts of the hub items
|
2023-10-27 08:25:29 +00:00
|
|
|
func (h *Hub) ItemStats() []string {
|
2023-10-20 12:32:35 +00:00
|
|
|
loaded := ""
|
2023-10-30 16:23:50 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
for _, itemType := range ItemTypes {
|
|
|
|
// ensure the order is always the same
|
|
|
|
if h.Items[itemType] == nil {
|
|
|
|
continue
|
|
|
|
}
|
2023-10-30 16:23:50 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
if len(h.Items[itemType]) == 0 {
|
|
|
|
continue
|
|
|
|
}
|
2023-10-30 16:23:50 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
|
|
|
|
}
|
|
|
|
|
|
|
|
loaded = strings.Trim(loaded, ", ")
|
|
|
|
if loaded == "" {
|
|
|
|
// empty hub
|
|
|
|
loaded = "0 items"
|
|
|
|
}
|
|
|
|
|
|
|
|
ret := []string{
|
|
|
|
fmt.Sprintf("Loaded: %s", loaded),
|
|
|
|
}
|
|
|
|
|
|
|
|
if h.skippedLocal > 0 || h.skippedTainted > 0 {
|
|
|
|
ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted))
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret
|
|
|
|
}
|