lint: pkg/cwhub (#2510)
no functional changes - reformat - comments - whitespace - removed a dot or two in log messages - some "var x=y" -> x:=y
This commit is contained in:
mmetc
GitHub
parent
6dadfcb2ef
commit
8b5ad6990d
|
|
@ -15,7 +15,7 @@ import (
|
|||
"golang.org/x/mod/semver"
|
||||
)
|
||||
|
||||
/*managed configuration types*/
|
||||
// managed configuration types
|
||||
var PARSERS = "parsers"
|
||||
var PARSERS_OVFLW = "postoverflows"
|
||||
var SCENARIOS = "scenarios"
|
||||
|
|
@ -42,37 +42,37 @@ type ItemHubStatus struct {
|
|||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
//Item can be : parsed, scenario, collection
|
||||
// Item can be: parsed, scenario, collection
|
||||
type Item struct {
|
||||
/*descriptive info*/
|
||||
Type string `yaml:"type,omitempty" json:"type,omitempty"` //parser|postoverflows|scenario|collection(|enrich)
|
||||
Stage string `json:"stage,omitempty" yaml:"stage,omitempty,omitempty"` //Stage for parser|postoverflow : s00-raw/s01-...
|
||||
Name string `json:"name,omitempty"` //as seen in .config.json, usually "author/name"
|
||||
FileName string `json:"file_name,omitempty"` //the filename, ie. apache2-logs.yaml
|
||||
Description string `yaml:"description,omitempty" json:"description,omitempty"` //as seen in .config.json
|
||||
Author string `json:"author,omitempty"` //as seen in .config.json
|
||||
References []string `yaml:"references,omitempty" json:"references,omitempty"` //as seen in .config.json
|
||||
BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` /*if it's part of collections, track name here*/
|
||||
// descriptive info
|
||||
Type string `yaml:"type,omitempty" json:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich)
|
||||
Stage string `json:"stage,omitempty" yaml:"stage,omitempty,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
|
||||
Name string `json:"name,omitempty"` // as seen in .config.json, usually "author/name"
|
||||
FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml
|
||||
Description string `yaml:"description,omitempty" json:"description,omitempty"` // as seen in .config.json
|
||||
Author string `json:"author,omitempty"` // as seen in .config.json
|
||||
References []string `yaml:"references,omitempty" json:"references,omitempty"` // as seen in .config.json
|
||||
BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` // if it's part of collections, track name here
|
||||
|
||||
/*remote (hub) infos*/
|
||||
RemoteURL string `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` //the full remote uri of file in http
|
||||
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` //the path relative to git ie. /parsers/stage/author/file.yaml
|
||||
RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` //the meow
|
||||
Version string `json:"version,omitempty"` //the last version
|
||||
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` //the list of existing versions
|
||||
// remote (hub) infos
|
||||
RemoteURL string `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` // the full remote uri of file in http
|
||||
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to git ie. /parsers/stage/author/file.yaml
|
||||
RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` // the meow
|
||||
Version string `json:"version,omitempty"` // the last version
|
||||
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions
|
||||
|
||||
/*local (deployed) infos*/
|
||||
LocalPath string `yaml:"local_path,omitempty" json:"local_path,omitempty"` //the local path relative to ${CFG_DIR}
|
||||
//LocalHubPath string
|
||||
// local (deployed) infos
|
||||
LocalPath string `yaml:"local_path,omitempty" json:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
|
||||
// LocalHubPath string
|
||||
LocalVersion string `json:"local_version,omitempty"`
|
||||
LocalHash string `json:"local_hash,omitempty"` //the local meow
|
||||
LocalHash string `json:"local_hash,omitempty"` // the local meow
|
||||
Installed bool `json:"installed,omitempty"`
|
||||
Downloaded bool `json:"downloaded,omitempty"`
|
||||
UpToDate bool `json:"up_to_date,omitempty"`
|
||||
Tainted bool `json:"tainted,omitempty"` //has it been locally modified
|
||||
Local bool `json:"local,omitempty"` //if it's a non versioned control one
|
||||
Tainted bool `json:"tainted,omitempty"` // has it been locally modified
|
||||
Local bool `json:"local,omitempty"` // if it's a non versioned control one
|
||||
|
||||
/*if it's a collection, it not a single file*/
|
||||
// if it's a collection, it not a single file
|
||||
Parsers []string `yaml:"parsers,omitempty" json:"parsers,omitempty"`
|
||||
PostOverflows []string `yaml:"postoverflows,omitempty" json:"postoverflows,omitempty"`
|
||||
Scenarios []string `yaml:"scenarios,omitempty" json:"scenarios,omitempty"`
|
||||
|
|
@ -88,6 +88,7 @@ func (i *Item) toHubStatus() ItemHubStatus {
|
|||
|
||||
status, ok, warning, managed := ItemStatus(*i)
|
||||
hubStatus.Status = status
|
||||
|
||||
if !managed {
|
||||
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.House, status)
|
||||
} else if !i.Installed {
|
||||
|
|
@ -97,27 +98,28 @@ func (i *Item) toHubStatus() ItemHubStatus {
|
|||
} else if ok {
|
||||
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.CheckMark, status)
|
||||
}
|
||||
|
||||
return hubStatus
|
||||
}
|
||||
|
||||
var skippedLocal = 0
|
||||
var skippedTainted = 0
|
||||
|
||||
/*To be used when reference(s) (is/are) missing in a collection*/
|
||||
// To be used when reference(s) (is/are) missing in a collection
|
||||
var ReferenceMissingError = errors.New("Reference(s) missing in collection")
|
||||
var MissingHubIndex = errors.New("hub index can't be found")
|
||||
|
||||
//GetVersionStatus : semver requires 'v' prefix
|
||||
// GetVersionStatus: semver requires 'v' prefix
|
||||
func GetVersionStatus(v *Item) int {
|
||||
return semver.Compare("v"+v.Version, "v"+v.LocalVersion)
|
||||
}
|
||||
|
||||
// calculate sha256 of a file
|
||||
func getSHA256(filepath string) (string, error) {
|
||||
/* Digest of file */
|
||||
// Digest of file
|
||||
f, err := os.Open(filepath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to open '%s' : %s", filepath, err)
|
||||
return "", fmt.Errorf("unable to open '%s': %s", filepath, err)
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
|
@ -131,49 +133,55 @@ func getSHA256(filepath string) (string, error) {
|
|||
}
|
||||
|
||||
func GetItemMap(itemType string) map[string]Item {
|
||||
var m map[string]Item
|
||||
var ok bool
|
||||
var (
|
||||
m map[string]Item
|
||||
ok bool
|
||||
)
|
||||
|
||||
if m, ok = hubIdx[itemType]; !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
//GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
|
||||
// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
|
||||
func GetItemByPath(itemType string, itemPath string) (*Item, error) {
|
||||
/*try to resolve symlink*/
|
||||
// try to resolve symlink
|
||||
finalName := ""
|
||||
|
||||
f, err := os.Lstat(itemPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
|
||||
}
|
||||
|
||||
if f.Mode()&os.ModeSymlink == 0 {
|
||||
/*it's not a symlink, it should be the filename itsef the key*/
|
||||
// it's not a symlink, it should be the filename itsef the key
|
||||
finalName = filepath.Base(itemPath)
|
||||
} else {
|
||||
/*resolve the symlink to hub file*/
|
||||
// resolve the symlink to hub file
|
||||
pathInHub, err := os.Readlink(itemPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
|
||||
}
|
||||
//extract author from path
|
||||
// extract author from path
|
||||
fname := filepath.Base(pathInHub)
|
||||
author := filepath.Base(filepath.Dir(pathInHub))
|
||||
//trim yaml suffix
|
||||
// trim yaml suffix
|
||||
fname = strings.TrimSuffix(fname, ".yaml")
|
||||
fname = strings.TrimSuffix(fname, ".yml")
|
||||
finalName = fmt.Sprintf("%s/%s", author, fname)
|
||||
}
|
||||
|
||||
/*it's not a symlink, it should be the filename itsef the key*/
|
||||
// it's not a symlink, it should be the filename itsef the key
|
||||
if m := GetItemMap(itemType); m != nil {
|
||||
if v, ok := m[finalName]; ok {
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("%s not found in %s", finalName, itemType)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("item type %s doesn't exist", itemType)
|
||||
}
|
||||
|
||||
|
|
@ -181,35 +189,42 @@ func GetItem(itemType string, itemName string) *Item {
|
|||
if m, ok := GetItemMap(itemType)[itemName]; ok {
|
||||
return &m
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func AddItem(itemType string, item Item) error {
|
||||
in := false
|
||||
|
||||
for _, itype := range ItemTypes {
|
||||
if itype == itemType {
|
||||
in = true
|
||||
}
|
||||
}
|
||||
|
||||
if !in {
|
||||
return fmt.Errorf("ItemType %s is unknown", itemType)
|
||||
}
|
||||
|
||||
hubIdx[itemType][item.Name] = item
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func DisplaySummary() {
|
||||
log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
|
||||
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
|
||||
|
||||
if skippedLocal > 0 || skippedTainted > 0 {
|
||||
log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted)
|
||||
log.Printf("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted)
|
||||
}
|
||||
}
|
||||
|
||||
//returns: human-text, Enabled, Warning, Unmanaged
|
||||
// returns: human-text, Enabled, Warning, Unmanaged
|
||||
func ItemStatus(v Item) (string, bool, bool, bool) {
|
||||
strret := "disabled"
|
||||
Ok := false
|
||||
|
||||
if v.Installed {
|
||||
Ok = true
|
||||
strret = "enabled"
|
||||
|
|
@ -221,7 +236,7 @@ func ItemStatus(v Item) (string, bool, bool, bool) {
|
|||
strret += ",local"
|
||||
}
|
||||
|
||||
//tainted or out of date
|
||||
// tainted or out of date
|
||||
Warning := false
|
||||
if v.Tainted {
|
||||
Warning = true
|
||||
|
|
@ -230,6 +245,7 @@ func ItemStatus(v Item) (string, bool, bool, bool) {
|
|||
strret += ",update-available"
|
||||
Warning = true
|
||||
}
|
||||
|
||||
return strret, Ok, Warning, Managed
|
||||
}
|
||||
|
||||
|
|
@ -240,9 +256,11 @@ func GetInstalledScenariosAsString() ([]string, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("while fetching scenarios: %w", err)
|
||||
}
|
||||
|
||||
for _, it := range items {
|
||||
retStr = append(retStr, it.Name)
|
||||
}
|
||||
|
||||
return retStr, nil
|
||||
}
|
||||
|
||||
|
|
@ -252,11 +270,13 @@ func GetInstalledScenarios() ([]Item, error) {
|
|||
if _, ok := hubIdx[SCENARIOS]; !ok {
|
||||
return nil, fmt.Errorf("no scenarios in hubIdx")
|
||||
}
|
||||
|
||||
for _, item := range hubIdx[SCENARIOS] {
|
||||
if item.Installed {
|
||||
retItems = append(retItems, item)
|
||||
}
|
||||
}
|
||||
|
||||
return retItems, nil
|
||||
}
|
||||
|
||||
|
|
@ -266,11 +286,13 @@ func GetInstalledParsers() ([]Item, error) {
|
|||
if _, ok := hubIdx[PARSERS]; !ok {
|
||||
return nil, fmt.Errorf("no parsers in hubIdx")
|
||||
}
|
||||
|
||||
for _, item := range hubIdx[PARSERS] {
|
||||
if item.Installed {
|
||||
retItems = append(retItems, item)
|
||||
}
|
||||
}
|
||||
|
||||
return retItems, nil
|
||||
}
|
||||
|
||||
|
|
@ -281,9 +303,11 @@ func GetInstalledParsersAsString() ([]string, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("while fetching parsers: %w", err)
|
||||
}
|
||||
|
||||
for _, it := range items {
|
||||
retStr = append(retStr, it.Name)
|
||||
}
|
||||
|
||||
return retStr, nil
|
||||
}
|
||||
|
||||
|
|
@ -293,11 +317,13 @@ func GetInstalledPostOverflows() ([]Item, error) {
|
|||
if _, ok := hubIdx[PARSERS_OVFLW]; !ok {
|
||||
return nil, fmt.Errorf("no post overflows in hubIdx")
|
||||
}
|
||||
|
||||
for _, item := range hubIdx[PARSERS_OVFLW] {
|
||||
if item.Installed {
|
||||
retItems = append(retItems, item)
|
||||
}
|
||||
}
|
||||
|
||||
return retItems, nil
|
||||
}
|
||||
|
||||
|
|
@ -308,9 +334,11 @@ func GetInstalledPostOverflowsAsString() ([]string, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("while fetching post overflows: %w", err)
|
||||
}
|
||||
|
||||
for _, it := range items {
|
||||
retStr = append(retStr, it.Name)
|
||||
}
|
||||
|
||||
return retStr, nil
|
||||
}
|
||||
|
||||
|
|
@ -325,6 +353,7 @@ func GetInstalledCollectionsAsString() ([]string, error) {
|
|||
for _, it := range items {
|
||||
retStr = append(retStr, it.Name)
|
||||
}
|
||||
|
||||
return retStr, nil
|
||||
}
|
||||
|
||||
|
|
@ -334,15 +363,17 @@ func GetInstalledCollections() ([]Item, error) {
|
|||
if _, ok := hubIdx[COLLECTIONS]; !ok {
|
||||
return nil, fmt.Errorf("no collection in hubIdx")
|
||||
}
|
||||
|
||||
for _, item := range hubIdx[COLLECTIONS] {
|
||||
if item.Installed {
|
||||
retItems = append(retItems, item)
|
||||
}
|
||||
}
|
||||
|
||||
return retItems, nil
|
||||
}
|
||||
|
||||
//Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy)
|
||||
// Returns a list of entries for packages: name, status, local_path, local_version, utf8_status (fancy)
|
||||
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
|
||||
if _, ok := hubIdx[itemType]; !ok {
|
||||
log.Errorf("type %s doesn't exist", itemType)
|
||||
|
|
@ -351,19 +382,22 @@ func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubSt
|
|||
}
|
||||
|
||||
var ret = make([]ItemHubStatus, 0)
|
||||
/*remember, you do it for the user :)*/
|
||||
|
||||
// remember, you do it for the user :)
|
||||
for _, item := range hubIdx[itemType] {
|
||||
if name != "" && name != item.Name {
|
||||
//user has requested a specific name
|
||||
// user has requested a specific name
|
||||
continue
|
||||
}
|
||||
//Only enabled items ?
|
||||
// Only enabled items ?
|
||||
if !all && !item.Installed {
|
||||
continue
|
||||
}
|
||||
//Check the item status
|
||||
// Check the item status
|
||||
ret = append(ret, item.toHubStatus())
|
||||
}
|
||||
|
||||
sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
|
||||
|
||||
return ret
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,8 +9,9 @@ import (
|
|||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
)
|
||||
|
||||
/*
|
||||
|
|
@ -29,30 +30,33 @@ func TestItemStatus(t *testing.T) {
|
|||
defer envTearDown(cfg)
|
||||
|
||||
err := UpdateHubIdx(cfg.Hub)
|
||||
//DownloadHubIdx()
|
||||
// DownloadHubIdx()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to download index : %s", err)
|
||||
}
|
||||
|
||||
if err := GetHubIdx(cfg.Hub); err != nil {
|
||||
t.Fatalf("failed to load hub index : %s", err)
|
||||
}
|
||||
|
||||
//get existing map
|
||||
// get existing map
|
||||
x := GetItemMap(COLLECTIONS)
|
||||
if len(x) == 0 {
|
||||
t.Fatalf("expected non empty result")
|
||||
}
|
||||
|
||||
//Get item : good and bad
|
||||
// Get item : good and bad
|
||||
for k := range x {
|
||||
item := GetItem(COLLECTIONS, k)
|
||||
if item == nil {
|
||||
t.Fatalf("expected item")
|
||||
}
|
||||
|
||||
item.Installed = true
|
||||
item.UpToDate = false
|
||||
item.Local = false
|
||||
item.Tainted = false
|
||||
|
||||
txt, _, _, _ := ItemStatus(*item)
|
||||
if txt != "enabled,update-available" {
|
||||
t.Fatalf("got '%s'", txt)
|
||||
|
|
@ -62,6 +66,7 @@ func TestItemStatus(t *testing.T) {
|
|||
item.UpToDate = false
|
||||
item.Local = true
|
||||
item.Tainted = false
|
||||
|
||||
txt, _, _, _ = ItemStatus(*item)
|
||||
if txt != "disabled,local" {
|
||||
t.Fatalf("got '%s'", txt)
|
||||
|
|
@ -69,6 +74,7 @@ func TestItemStatus(t *testing.T) {
|
|||
|
||||
break
|
||||
}
|
||||
|
||||
DisplaySummary()
|
||||
}
|
||||
|
||||
|
|
@ -77,26 +83,28 @@ func TestGetters(t *testing.T) {
|
|||
defer envTearDown(cfg)
|
||||
|
||||
err := UpdateHubIdx(cfg.Hub)
|
||||
//DownloadHubIdx()
|
||||
// DownloadHubIdx()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to download index : %s", err)
|
||||
}
|
||||
|
||||
if err := GetHubIdx(cfg.Hub); err != nil {
|
||||
t.Fatalf("failed to load hub index : %s", err)
|
||||
}
|
||||
|
||||
//get non existing map
|
||||
// get non existing map
|
||||
empty := GetItemMap("ratata")
|
||||
if empty != nil {
|
||||
t.Fatalf("expected nil result")
|
||||
}
|
||||
//get existing map
|
||||
|
||||
// get existing map
|
||||
x := GetItemMap(COLLECTIONS)
|
||||
if len(x) == 0 {
|
||||
t.Fatalf("expected non empty result")
|
||||
}
|
||||
|
||||
//Get item : good and bad
|
||||
// Get item : good and bad
|
||||
for k := range x {
|
||||
empty := GetItem(COLLECTIONS, k+"nope")
|
||||
if empty != nil {
|
||||
|
|
@ -108,7 +116,7 @@ func TestGetters(t *testing.T) {
|
|||
t.Fatalf("expected non empty item")
|
||||
}
|
||||
|
||||
//Add item and get it
|
||||
// Add item and get it
|
||||
item.Name += "nope"
|
||||
if err := AddItem(COLLECTIONS, *item); err != nil {
|
||||
t.Fatalf("didn't expect error : %s", err)
|
||||
|
|
@ -119,7 +127,7 @@ func TestGetters(t *testing.T) {
|
|||
t.Fatalf("expected non empty item")
|
||||
}
|
||||
|
||||
//Add bad item
|
||||
// Add bad item
|
||||
if err := AddItem("ratata", *item); err != nil {
|
||||
if fmt.Sprintf("%s", err) != "ItemType ratata is unknown" {
|
||||
t.Fatalf("unexpected error")
|
||||
|
|
@ -130,7 +138,6 @@ func TestGetters(t *testing.T) {
|
|||
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestIndexDownload(t *testing.T) {
|
||||
|
|
@ -138,10 +145,11 @@ func TestIndexDownload(t *testing.T) {
|
|||
defer envTearDown(cfg)
|
||||
|
||||
err := UpdateHubIdx(cfg.Hub)
|
||||
//DownloadHubIdx()
|
||||
// DownloadHubIdx()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to download index : %s", err)
|
||||
}
|
||||
|
||||
if err := GetHubIdx(cfg.Hub); err != nil {
|
||||
t.Fatalf("failed to load hub index : %s", err)
|
||||
}
|
||||
|
|
@ -152,20 +160,23 @@ func getTestCfg() (cfg *csconfig.Config) {
|
|||
cfg.Hub.ConfigDir, _ = filepath.Abs("./install")
|
||||
cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
|
||||
cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func envSetup(t *testing.T) *csconfig.Config {
|
||||
resetResponseByPath()
|
||||
log.SetLevel(log.DebugLevel)
|
||||
|
||||
cfg := getTestCfg()
|
||||
|
||||
defaultTransport := http.DefaultClient.Transport
|
||||
|
||||
t.Cleanup(func() {
|
||||
http.DefaultClient.Transport = defaultTransport
|
||||
})
|
||||
|
||||
//Mock the http client
|
||||
// Mock the http client
|
||||
http.DefaultClient.Transport = newMockTransport()
|
||||
|
||||
if err := os.MkdirAll(cfg.Hub.ConfigDir, 0700); err != nil {
|
||||
|
|
@ -189,7 +200,6 @@ func envSetup(t *testing.T) *csconfig.Config {
|
|||
return cfg
|
||||
}
|
||||
|
||||
|
||||
func envTearDown(cfg *csconfig.Config) {
|
||||
if err := os.RemoveAll(cfg.Hub.ConfigDir); err != nil {
|
||||
log.Fatalf("failed to remove %s : %s", cfg.Hub.ConfigDir, err)
|
||||
|
|
@ -200,23 +210,25 @@ func envTearDown(cfg *csconfig.Config) {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
||||
|
||||
//Install the parser
|
||||
// Install the parser
|
||||
item, err := DownloadLatest(cfg, item, false, false)
|
||||
if err != nil {
|
||||
t.Fatalf("error while downloading %s : %v", item.Name, err)
|
||||
}
|
||||
|
||||
if err, _ := LocalSync(cfg); err != nil {
|
||||
t.Fatalf("taint: failed to run localSync : %s", err)
|
||||
}
|
||||
|
||||
if !hubIdx[item.Type][item.Name].UpToDate {
|
||||
t.Fatalf("download: %s should be up-to-date", item.Name)
|
||||
}
|
||||
|
||||
if hubIdx[item.Type][item.Name].Installed {
|
||||
t.Fatalf("download: %s should not be installed", item.Name)
|
||||
}
|
||||
|
||||
if hubIdx[item.Type][item.Name].Tainted {
|
||||
t.Fatalf("download: %s should not be tainted", item.Name)
|
||||
}
|
||||
|
|
@ -225,9 +237,11 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
|||
if err != nil {
|
||||
t.Fatalf("error while enabling %s : %v.", item.Name, err)
|
||||
}
|
||||
|
||||
if err, _ := LocalSync(cfg); err != nil {
|
||||
t.Fatalf("taint: failed to run localSync : %s", err)
|
||||
}
|
||||
|
||||
if !hubIdx[item.Type][item.Name].Installed {
|
||||
t.Fatalf("install: %s should be installed", item.Name)
|
||||
}
|
||||
|
|
@ -237,6 +251,7 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
|||
if hubIdx[item.Type][item.Name].Tainted {
|
||||
t.Fatalf("pre-taint: %s should not be tainted", item.Name)
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
t.Fatalf("(taint) opening %s (%s) : %s", item.LocalPath, item.Name, err)
|
||||
|
|
@ -246,32 +261,37 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
|||
if _, err = f.WriteString("tainted"); err != nil {
|
||||
t.Fatalf("tainting %s : %s", item.Name, err)
|
||||
}
|
||||
//Local sync and check status
|
||||
|
||||
// Local sync and check status
|
||||
if err, _ := LocalSync(cfg); err != nil {
|
||||
t.Fatalf("taint: failed to run localSync : %s", err)
|
||||
}
|
||||
|
||||
if !hubIdx[item.Type][item.Name].Tainted {
|
||||
t.Fatalf("taint: %s should be tainted", item.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
||||
|
||||
if hubIdx[item.Type][item.Name].UpToDate {
|
||||
t.Fatalf("update: %s should NOT be up-to-date", item.Name)
|
||||
}
|
||||
//Update it + check status
|
||||
|
||||
// Update it + check status
|
||||
item, err := DownloadLatest(cfg, item, true, true)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to update %s : %s", item.Name, err)
|
||||
}
|
||||
//Local sync and check status
|
||||
|
||||
// Local sync and check status
|
||||
if err, _ := LocalSync(cfg); err != nil {
|
||||
t.Fatalf("failed to run localSync : %s", err)
|
||||
}
|
||||
|
||||
if !hubIdx[item.Type][item.Name].UpToDate {
|
||||
t.Fatalf("update: %s should be up-to-date", item.Name)
|
||||
}
|
||||
|
||||
if hubIdx[item.Type][item.Name].Tainted {
|
||||
t.Fatalf("update: %s should not be tainted anymore", item.Name)
|
||||
}
|
||||
|
|
@ -281,43 +301,51 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
|||
if !item.Installed {
|
||||
t.Fatalf("disable: %s should be installed", item.Name)
|
||||
}
|
||||
//Remove
|
||||
|
||||
// Remove
|
||||
item, err := DisableItem(cfg, item, false, false)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to disable item : %v", err)
|
||||
}
|
||||
//Local sync and check status
|
||||
|
||||
// Local sync and check status
|
||||
if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 {
|
||||
t.Fatalf("failed to run localSync : %s (%+v)", err, warns)
|
||||
}
|
||||
|
||||
if hubIdx[item.Type][item.Name].Tainted {
|
||||
t.Fatalf("disable: %s should not be tainted anymore", item.Name)
|
||||
}
|
||||
|
||||
if hubIdx[item.Type][item.Name].Installed {
|
||||
t.Fatalf("disable: %s should not be installed anymore", item.Name)
|
||||
}
|
||||
|
||||
if !hubIdx[item.Type][item.Name].Downloaded {
|
||||
t.Fatalf("disable: %s should still be downloaded", item.Name)
|
||||
}
|
||||
//Purge
|
||||
|
||||
// Purge
|
||||
item, err = DisableItem(cfg, item, true, false)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to purge item : %v", err)
|
||||
}
|
||||
//Local sync and check status
|
||||
|
||||
// Local sync and check status
|
||||
if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 {
|
||||
t.Fatalf("failed to run localSync : %s (%+v)", err, warns)
|
||||
}
|
||||
|
||||
if hubIdx[item.Type][item.Name].Installed {
|
||||
t.Fatalf("disable: %s should not be installed anymore", item.Name)
|
||||
}
|
||||
|
||||
if hubIdx[item.Type][item.Name].Downloaded {
|
||||
t.Fatalf("disable: %s should not be downloaded", item.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallParser(t *testing.T) {
|
||||
|
||||
/*
|
||||
- install a random parser
|
||||
- check its status
|
||||
|
|
@ -331,7 +359,7 @@ func TestInstallParser(t *testing.T) {
|
|||
defer envTearDown(cfg)
|
||||
|
||||
getHubIdxOrFail(t)
|
||||
//map iteration is random by itself
|
||||
// map iteration is random by itself
|
||||
for _, it := range hubIdx[PARSERS] {
|
||||
testInstallItem(cfg.Hub, t, it)
|
||||
it = hubIdx[PARSERS][it.Name]
|
||||
|
|
@ -349,7 +377,6 @@ func TestInstallParser(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestInstallCollection(t *testing.T) {
|
||||
|
||||
/*
|
||||
- install a random parser
|
||||
- check its status
|
||||
|
|
@ -363,7 +390,7 @@ func TestInstallCollection(t *testing.T) {
|
|||
defer envTearDown(cfg)
|
||||
|
||||
getHubIdxOrFail(t)
|
||||
//map iteration is random by itself
|
||||
// map iteration is random by itself
|
||||
for _, it := range hubIdx[COLLECTIONS] {
|
||||
testInstallItem(cfg.Hub, t, it)
|
||||
it = hubIdx[COLLECTIONS][it.Name]
|
||||
|
|
@ -376,6 +403,7 @@ func TestInstallCollection(t *testing.T) {
|
|||
it = hubIdx[COLLECTIONS][it.Name]
|
||||
x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
|
||||
log.Printf("%+v", x)
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
|
@ -395,10 +423,12 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
|||
StatusCode: http.StatusOK,
|
||||
}
|
||||
response.Header.Set("Content-Type", "application/json")
|
||||
|
||||
responseBody := ""
|
||||
|
||||
log.Printf("---> %s", req.URL.Path)
|
||||
|
||||
/*FAKE PARSER*/
|
||||
// FAKE PARSER
|
||||
if resp, ok := responseByPath[req.URL.Path]; ok {
|
||||
responseBody = resp
|
||||
} else {
|
||||
|
|
@ -406,12 +436,14 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
|||
}
|
||||
|
||||
response.Body = io.NopCloser(strings.NewReader(responseBody))
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func fileToStringX(path string) string {
|
||||
if f, err := os.Open(path); err == nil {
|
||||
defer f.Close()
|
||||
|
||||
if data, err := io.ReadAll(f); err == nil {
|
||||
return strings.ReplaceAll(string(data), "\r\n", "\n")
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ type DataSet struct {
|
|||
|
||||
func downloadFile(url string, destPath string) error {
|
||||
log.Debugf("downloading %s in %s", url, destPath)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -60,6 +61,7 @@ func GetData(data []*types.DataSource, dataDir string) error {
|
|||
for _, dataS := range data {
|
||||
destPath := filepath.Join(dataDir, dataS.DestPath)
|
||||
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
|
||||
|
||||
err := downloadFile(dataS.SourceURL, destPath)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -4,9 +4,8 @@ import (
|
|||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/jarcoal/httpmock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestDownloadFile(t *testing.T) {
|
||||
|
|
@ -26,6 +25,7 @@ func TestDownloadFile(t *testing.T) {
|
|||
"https://example.com/x",
|
||||
httpmock.NewStringResponder(404, "not found"),
|
||||
)
|
||||
|
||||
err := downloadFile("https://example.com/xx", examplePath)
|
||||
assert.NoError(t, err)
|
||||
content, err := os.ReadFile(examplePath)
|
||||
|
|
|
|||
|
|
@ -24,36 +24,45 @@ func UpdateHubIdx(hub *csconfig.Hub) error {
|
|||
if err != nil {
|
||||
return fmt.Errorf("failed to download index: %w", err)
|
||||
}
|
||||
|
||||
ret, err := LoadPkgIndex(bidx)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ReferenceMissingError) {
|
||||
return fmt.Errorf("failed to read index: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
hubIdx = ret
|
||||
|
||||
if err, _ := LocalSync(hub); err != nil {
|
||||
return fmt.Errorf("failed to sync: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
|
||||
log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build request for hub index: %w", err)
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed http request for hub index: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, ErrIndexNotFound
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String())
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
|
||||
|
|
@ -80,7 +89,9 @@ func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("while writing hub index file: %w", err)
|
||||
}
|
||||
|
||||
log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
|
|
@ -89,11 +100,13 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b
|
|||
var err error
|
||||
|
||||
log.Debugf("Downloading %s %s", target.Type, target.Name)
|
||||
|
||||
if target.Type != COLLECTIONS {
|
||||
if !target.Installed && updateOnly && target.Downloaded {
|
||||
log.Debugf("skipping upgrade of %s : not installed", target.Name)
|
||||
return target, nil
|
||||
}
|
||||
|
||||
return DownloadItem(hub, target, overwrite)
|
||||
}
|
||||
|
||||
|
|
@ -116,11 +129,13 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b
|
|||
//recurse as it's a collection
|
||||
if ptrtype == COLLECTIONS {
|
||||
log.Tracef("collection, recurse")
|
||||
|
||||
hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly)
|
||||
if err != nil {
|
||||
return target, fmt.Errorf("while downloading %s: %w", val.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
item, err := DownloadItem(hub, val, overwrite)
|
||||
if err != nil {
|
||||
return target, fmt.Errorf("while downloading %s: %w", val.Name, err)
|
||||
|
|
@ -133,77 +148,95 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b
|
|||
return target, fmt.Errorf("enabling '%s': %w", item.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
hubIdx[ptrtype][p] = item
|
||||
}
|
||||
}
|
||||
|
||||
target, err = DownloadItem(hub, target, overwrite)
|
||||
if err != nil {
|
||||
return target, fmt.Errorf("failed to download item : %s", err)
|
||||
}
|
||||
|
||||
return target, nil
|
||||
}
|
||||
|
||||
func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) {
|
||||
var tdir = hub.HubDir
|
||||
var dataFolder = hub.DataDir
|
||||
/*if user didn't --force, don't overwrite local, tainted, up-to-date files*/
|
||||
tdir := hub.HubDir
|
||||
dataFolder := hub.DataDir
|
||||
|
||||
// if user didn't --force, don't overwrite local, tainted, up-to-date files
|
||||
if !overwrite {
|
||||
if target.Tainted {
|
||||
log.Debugf("%s : tainted, not updated", target.Name)
|
||||
return target, nil
|
||||
}
|
||||
|
||||
if target.UpToDate {
|
||||
log.Debugf("%s : up-to-date, not updated", target.Name)
|
||||
// We still have to check if data files are present
|
||||
log.Debugf("%s : up-to-date, not updated", target.Name)
|
||||
}
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
|
||||
if err != nil {
|
||||
return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return target, fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return target, fmt.Errorf("while reading %s: %w", req.URL.String(), err)
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := h.Write(body); err != nil {
|
||||
return target, fmt.Errorf("while hashing %s: %w", target.Name, err)
|
||||
}
|
||||
|
||||
meow := fmt.Sprintf("%x", h.Sum(nil))
|
||||
if meow != target.Versions[target.Version].Digest {
|
||||
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
|
||||
log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
|
||||
|
||||
return target, fmt.Errorf("invalid download hash for %s", target.Name)
|
||||
}
|
||||
|
||||
//all good, install
|
||||
//check if parent dir exists
|
||||
tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/")
|
||||
parent_dir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
|
||||
|
||||
/*ensure that target file is within target dir*/
|
||||
// ensure that target file is within target dir
|
||||
finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath)
|
||||
if err != nil {
|
||||
return target, fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err)
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(finalPath, tdir) {
|
||||
return target, fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir)
|
||||
}
|
||||
/*check dir*/
|
||||
|
||||
// check dir
|
||||
if _, err = os.Stat(parent_dir); os.IsNotExist(err) {
|
||||
log.Debugf("%s doesn't exist, create", parent_dir)
|
||||
|
||||
if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil {
|
||||
return target, fmt.Errorf("while creating parent directories: %w", err)
|
||||
}
|
||||
}
|
||||
/*check actual file*/
|
||||
|
||||
// check actual file
|
||||
if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
|
||||
log.Warningf("%s : overwrite", target.Name)
|
||||
log.Debugf("target: %s/%s", tdir, target.RemotePath)
|
||||
|
|
@ -215,11 +248,14 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error)
|
|||
if err != nil {
|
||||
return target, fmt.Errorf("while opening file: %w", err)
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
||||
_, err = f.Write(body)
|
||||
if err != nil {
|
||||
return target, fmt.Errorf("while writing file: %w", err)
|
||||
}
|
||||
|
||||
target.Downloaded = true
|
||||
target.Tainted = false
|
||||
target.UpToDate = true
|
||||
|
|
@ -229,6 +265,7 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error)
|
|||
}
|
||||
|
||||
hubIdx[target.Type][target.Name] = target
|
||||
|
||||
return target, nil
|
||||
}
|
||||
|
||||
|
|
@ -238,37 +275,47 @@ func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
|
|||
itemFile *os.File
|
||||
err error
|
||||
)
|
||||
|
||||
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.ConfigDir, target.Type, target.Stage, target.FileName)
|
||||
|
||||
if itemFile, err = os.Open(itemFilePath); err != nil {
|
||||
return fmt.Errorf("while opening %s: %w", itemFilePath, err)
|
||||
}
|
||||
|
||||
defer itemFile.Close()
|
||||
|
||||
if err = downloadData(dataFolder, force, itemFile); err != nil {
|
||||
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func downloadData(dataFolder string, force bool, reader io.Reader) error {
|
||||
var err error
|
||||
|
||||
dec := yaml.NewDecoder(reader)
|
||||
|
||||
for {
|
||||
data := &DataSet{}
|
||||
|
||||
err = dec.Decode(data)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
return fmt.Errorf("while reading file: %w", err)
|
||||
}
|
||||
|
||||
download := false
|
||||
|
||||
for _, dataS := range data.Data {
|
||||
if _, err := os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
|
||||
download = true
|
||||
}
|
||||
}
|
||||
|
||||
if download || force {
|
||||
err = GetData(data.Data, dataFolder)
|
||||
if err != nil {
|
||||
|
|
@ -276,5 +323,6 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,38 +5,48 @@ import (
|
|||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
)
|
||||
|
||||
func TestDownloadHubIdx(t *testing.T) {
|
||||
back := RawFileURLTemplate
|
||||
//bad url template
|
||||
// bad url template
|
||||
fmt.Println("Test 'bad URL'")
|
||||
|
||||
RawFileURLTemplate = "x"
|
||||
|
||||
ret, err := DownloadHubIdx(&csconfig.Hub{})
|
||||
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
|
||||
log.Errorf("unexpected error %s", err)
|
||||
}
|
||||
|
||||
fmt.Printf("->%+v", ret)
|
||||
|
||||
//bad domain
|
||||
// bad domain
|
||||
fmt.Println("Test 'bad domain'")
|
||||
|
||||
RawFileURLTemplate = "https://baddomain/%s/%s"
|
||||
|
||||
ret, err = DownloadHubIdx(&csconfig.Hub{})
|
||||
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
|
||||
log.Errorf("unexpected error %s", err)
|
||||
}
|
||||
|
||||
fmt.Printf("->%+v", ret)
|
||||
|
||||
//bad target path
|
||||
// bad target path
|
||||
fmt.Println("Test 'bad target path'")
|
||||
|
||||
RawFileURLTemplate = back
|
||||
|
||||
ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
|
||||
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
|
||||
log.Errorf("unexpected error %s", err)
|
||||
}
|
||||
|
||||
RawFileURLTemplate = back
|
||||
|
||||
fmt.Printf("->%+v", ret)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ func chooseHubBranch() (string, error) {
|
|||
latest, err := cwversion.Latest()
|
||||
if err != nil {
|
||||
log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
|
||||
//lint:ignore nilerr reason
|
||||
//lint:ignore nilerr
|
||||
return "master", nil // ignore
|
||||
}
|
||||
|
||||
|
|
@ -41,8 +41,10 @@ func chooseHubBranch() (string, error) {
|
|||
log.Warnf("Crowdsec is not the latest version. "+
|
||||
"Current version is '%s' and the latest stable version is '%s'. Please update it!",
|
||||
csVersion, latest)
|
||||
|
||||
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
|
||||
"added to Crowdsec Hub after CrowdSec %s", latest)
|
||||
|
||||
return csVersion, nil
|
||||
}
|
||||
|
||||
|
|
@ -58,8 +60,10 @@ func SetHubBranch() error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
HubBranch = branch
|
||||
log.Debugf("Using branch '%s' for the hub", HubBranch)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -72,6 +76,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo
|
|||
item := *it
|
||||
if downloadOnly && item.Downloaded && item.UpToDate {
|
||||
log.Warningf("%s is already downloaded and up-to-date", item.Name)
|
||||
|
||||
if !force {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -120,6 +125,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
|
|||
|
||||
item := *it
|
||||
item, err = DisableItem(csConfig.Hub, item, purge, forceAction)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("unable to disable %s : %v", item.Name, err)
|
||||
}
|
||||
|
|
@ -127,6 +133,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
|
|||
if err := AddItem(itemType, item); err != nil {
|
||||
log.Fatalf("unable to add %s: %v", item.Name, err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -139,6 +146,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
|
|||
if !v.Installed {
|
||||
continue
|
||||
}
|
||||
|
||||
v, err = DisableItem(csConfig.Hub, v, purge, forceAction)
|
||||
if err != nil {
|
||||
log.Fatalf("unable to disable %s : %v", v.Name, err)
|
||||
|
|
@ -149,6 +157,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
|
|||
}
|
||||
disabled++
|
||||
}
|
||||
|
||||
log.Infof("Disabled %d items", disabled)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
//Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
|
||||
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
|
||||
// We expect the new scenario to be installed
|
||||
func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
|
||||
cfg := envSetup(t)
|
||||
|
|
@ -37,6 +37,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
|
|||
if err := UpdateHubIdx(cfg.Hub); err != nil {
|
||||
t.Fatalf("failed to download index : %s", err)
|
||||
}
|
||||
|
||||
getHubIdxOrFail(t)
|
||||
|
||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
||||
|
|
@ -49,7 +50,6 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
|
|||
|
||||
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
|
||||
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
|
||||
|
||||
}
|
||||
|
||||
// Install a collection, disable a scenario.
|
||||
|
|
@ -140,6 +140,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
|
|||
if err := UpdateHubIdx(cfg.Hub); err != nil {
|
||||
t.Fatalf("failed to download index : %s", err)
|
||||
}
|
||||
|
||||
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
||||
getHubIdxOrFail(t)
|
||||
|
||||
|
|
@ -151,6 +152,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
|
|||
|
||||
func assertCollectionDepsInstalled(t *testing.T, collection string) {
|
||||
t.Helper()
|
||||
|
||||
c := hubIdx[COLLECTIONS][collection]
|
||||
require.NoError(t, CollecDepsCheck(&c))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,15 +22,17 @@ func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
|
|||
target.Downloaded = false
|
||||
log.Infof("Removed source file [%s] : %s", target.Name, hubpath)
|
||||
hubIdx[target.Type][target.Name] = target
|
||||
|
||||
return target, nil
|
||||
}
|
||||
|
||||
//DisableItem to disable an item managed by the hub, removes the symlink if purge is true
|
||||
// DisableItem to disable an item managed by the hub, removes the symlink if purge is true
|
||||
func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, error) {
|
||||
var tdir = hub.ConfigDir
|
||||
var hdir = hub.HubDir
|
||||
var err error
|
||||
|
||||
tdir := hub.ConfigDir
|
||||
hdir := hub.HubDir
|
||||
|
||||
if !target.Installed {
|
||||
if purge {
|
||||
target, err = purgeItem(hub, target)
|
||||
|
|
@ -38,6 +40,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
|||
return target, err
|
||||
}
|
||||
}
|
||||
|
||||
return target, nil
|
||||
}
|
||||
|
||||
|
|
@ -54,7 +57,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
|||
return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
|
||||
}
|
||||
|
||||
/*for a COLLECTIONS, disable sub-items*/
|
||||
// for a COLLECTIONS, disable sub-items
|
||||
if target.Type == COLLECTIONS {
|
||||
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
|
||||
for idx, ptr := range tmp {
|
||||
|
|
@ -63,12 +66,14 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
|||
if val, ok := hubIdx[ptrtype][p]; ok {
|
||||
// check if the item doesn't belong to another collection before removing it
|
||||
toRemove := true
|
||||
|
||||
for _, collection := range val.BelongsToCollections {
|
||||
if collection != target.Name {
|
||||
toRemove = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if toRemove {
|
||||
hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
|
||||
if err != nil {
|
||||
|
|
@ -114,6 +119,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
|||
}
|
||||
log.Infof("Removed symlink [%s] : %s", target.Name, syml)
|
||||
}
|
||||
|
||||
target.Installed = false
|
||||
|
||||
if purge {
|
||||
|
|
@ -122,39 +128,46 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
|||
return target, err
|
||||
}
|
||||
}
|
||||
|
||||
hubIdx[target.Type][target.Name] = target
|
||||
|
||||
return target, nil
|
||||
}
|
||||
|
||||
// creates symlink between actual config file at hub.HubDir and hub.ConfigDir
|
||||
// Handles collections recursively
|
||||
func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
|
||||
var tdir = hub.ConfigDir
|
||||
var hdir = hub.HubDir
|
||||
var err error
|
||||
|
||||
tdir := hub.ConfigDir
|
||||
hdir := hub.HubDir
|
||||
|
||||
parent_dir := filepath.Clean(tdir + "/" + target.Type + "/" + target.Stage + "/")
|
||||
/*create directories if needed*/
|
||||
// create directories if needed
|
||||
if target.Installed {
|
||||
if target.Tainted {
|
||||
return target, fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
|
||||
}
|
||||
|
||||
if target.Local {
|
||||
return target, fmt.Errorf("%s is local, won't enable", target.Name)
|
||||
}
|
||||
/* if it's a collection, check sub-items even if the collection file itself is up-to-date */
|
||||
// if it's a collection, check sub-items even if the collection file itself is up-to-date
|
||||
if target.UpToDate && target.Type != COLLECTIONS {
|
||||
log.Tracef("%s is installed and up-to-date, skip.", target.Name)
|
||||
return target, nil
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := os.Stat(parent_dir); os.IsNotExist(err) {
|
||||
log.Printf("%s doesn't exist, create", parent_dir)
|
||||
|
||||
if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil {
|
||||
return target, fmt.Errorf("while creating directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
/*install sub-items if it's a collection*/
|
||||
// install sub-items if it's a collection
|
||||
if target.Type == COLLECTIONS {
|
||||
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
|
||||
for idx, ptr := range tmp {
|
||||
|
|
@ -179,7 +192,7 @@ func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
|
|||
return target, nil
|
||||
}
|
||||
|
||||
//tdir+target.RemotePath
|
||||
// tdir+target.RemotePath
|
||||
srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath)
|
||||
if err != nil {
|
||||
return target, fmt.Errorf("while getting source path: %w", err)
|
||||
|
|
@ -197,5 +210,6 @@ func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
|
|||
log.Printf("Enabled %s : %s", target.Type, target.Name)
|
||||
target.Installed = true
|
||||
hubIdx[target.Type][target.Name] = target
|
||||
|
||||
return target, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,19 +15,20 @@ import (
|
|||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
)
|
||||
|
||||
/*the walk/parser_visit function can't receive extra args*/
|
||||
// the walk/parser_visit function can't receive extra args
|
||||
var hubdir, installdir string
|
||||
|
||||
func parser_visit(path string, f os.DirEntry, err error) error {
|
||||
|
||||
var target Item
|
||||
var local bool
|
||||
var hubpath string
|
||||
var inhub bool
|
||||
var fname string
|
||||
var ftype string
|
||||
var fauthor string
|
||||
var stage string
|
||||
var (
|
||||
target Item
|
||||
local bool
|
||||
hubpath string
|
||||
inhub bool
|
||||
fname string
|
||||
ftype string
|
||||
fauthor string
|
||||
stage string
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Debugf("while syncing hub dir: %s", err)
|
||||
|
|
@ -39,11 +40,11 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
//we only care about files
|
||||
// we only care about files
|
||||
if f == nil || f.IsDir() {
|
||||
return nil
|
||||
}
|
||||
//we only care about yaml files
|
||||
// we only care about yaml files
|
||||
if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -52,9 +53,10 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
|||
|
||||
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubdir, installdir)
|
||||
log.Tracef("subs:%v", subs)
|
||||
/*we're in hub (~/.hub/hub/)*/
|
||||
// we're in hub (~/.hub/hub/)
|
||||
if strings.HasPrefix(path, hubdir) {
|
||||
log.Tracef("in hub dir")
|
||||
|
||||
inhub = true
|
||||
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
|
||||
//.../hub/scenarios/crowdsec/ssh_bf.yaml
|
||||
|
|
@ -62,11 +64,12 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
|||
if len(subs) < 4 {
|
||||
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
||||
}
|
||||
|
||||
fname = subs[len(subs)-1]
|
||||
fauthor = subs[len(subs)-2]
|
||||
stage = subs[len(subs)-3]
|
||||
ftype = subs[len(subs)-4]
|
||||
} else if strings.HasPrefix(path, installdir) { /*we're in install /etc/crowdsec/<type>/... */
|
||||
} else if strings.HasPrefix(path, installdir) { // we're in install /etc/crowdsec/<type>/...
|
||||
log.Tracef("in install dir")
|
||||
if len(subs) < 3 {
|
||||
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
||||
|
|
@ -84,14 +87,15 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
|||
}
|
||||
|
||||
log.Tracef("stage:%s ftype:%s", stage, ftype)
|
||||
//log.Printf("%s -> name:%s stage:%s", path, fname, stage)
|
||||
// log.Printf("%s -> name:%s stage:%s", path, fname, stage)
|
||||
if stage == SCENARIOS {
|
||||
ftype = SCENARIOS
|
||||
stage = ""
|
||||
} else if stage == COLLECTIONS {
|
||||
ftype = COLLECTIONS
|
||||
stage = ""
|
||||
} else if ftype != PARSERS && ftype != PARSERS_OVFLW /*its a PARSER / PARSER_OVFLW with a stage */ {
|
||||
} else if ftype != PARSERS && ftype != PARSERS_OVFLW {
|
||||
// its a PARSER / PARSER_OVFLW with a stage
|
||||
return fmt.Errorf("unknown configuration type for file '%s'", path)
|
||||
}
|
||||
|
||||
|
|
@ -102,20 +106,21 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
|||
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
|
||||
when the collection is installed, both files are created
|
||||
*/
|
||||
//non symlinks are local user files or hub files
|
||||
// non symlinks are local user files or hub files
|
||||
if f.Type()&os.ModeSymlink == 0 {
|
||||
local = true
|
||||
|
||||
log.Tracef("%s isn't a symlink", path)
|
||||
} else {
|
||||
hubpath, err = os.Readlink(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read symlink of %s", path)
|
||||
}
|
||||
//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
|
||||
// the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
|
||||
_, err := os.Lstat(hubpath)
|
||||
if os.IsNotExist(err) {
|
||||
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
|
||||
//remove the symlink
|
||||
// remove the symlink
|
||||
if err = os.Remove(path); err != nil {
|
||||
return fmt.Errorf("failed to unlink %s: %+v", path, err)
|
||||
}
|
||||
|
|
@ -124,7 +129,7 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
|||
log.Tracef("%s points to %s", path, hubpath)
|
||||
}
|
||||
|
||||
//if it's not a symlink and not in hub, it's a local file, don't bother
|
||||
// if it's not a symlink and not in hub, it's a local file, don't bother
|
||||
if local && !inhub {
|
||||
log.Tracef("%s is a local file, skip", path)
|
||||
skippedLocal++
|
||||
|
|
@ -139,29 +144,34 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
|||
_, target.FileName = filepath.Split(path)
|
||||
|
||||
hubIdx[ftype][fname] = target
|
||||
|
||||
return nil
|
||||
}
|
||||
//try to find which configuration item it is
|
||||
// try to find which configuration item it is
|
||||
log.Tracef("check [%s] of %s", fname, ftype)
|
||||
|
||||
match := false
|
||||
|
||||
for k, v := range hubIdx[ftype] {
|
||||
log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml")
|
||||
|
||||
if fname != v.FileName {
|
||||
log.Tracef("%s != %s (filename)", fname, v.FileName)
|
||||
continue
|
||||
}
|
||||
//wrong stage
|
||||
|
||||
// wrong stage
|
||||
if v.Stage != stage {
|
||||
continue
|
||||
}
|
||||
/*if we are walking hub dir, just mark present files as downloaded*/
|
||||
|
||||
// if we are walking hub dir, just mark present files as downloaded
|
||||
if inhub {
|
||||
//wrong author
|
||||
// wrong author
|
||||
if fauthor != v.Author {
|
||||
continue
|
||||
}
|
||||
//wrong file
|
||||
// wrong file
|
||||
if CheckName(v.Name, fauthor, fname) {
|
||||
continue
|
||||
}
|
||||
|
|
@ -171,35 +181,38 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
|||
v.Downloaded = true
|
||||
}
|
||||
} else if CheckSuffix(hubpath, v.RemotePath) {
|
||||
//wrong file
|
||||
//<type>/<stage>/<author>/<name>.yaml
|
||||
// wrong file
|
||||
// <type>/<stage>/<author>/<name>.yaml
|
||||
continue
|
||||
}
|
||||
|
||||
sha, err := getSHA256(path)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get sha of %s : %v", path, err)
|
||||
}
|
||||
//let's reverse sort the versions to deal with hash collisions (#154)
|
||||
|
||||
// let's reverse sort the versions to deal with hash collisions (#154)
|
||||
versions := make([]string, 0, len(v.Versions))
|
||||
for k := range v.Versions {
|
||||
versions = append(versions, k)
|
||||
}
|
||||
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
|
||||
|
||||
for _, version := range versions {
|
||||
val := v.Versions[version]
|
||||
if sha != val.Digest {
|
||||
//log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
|
||||
// log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
|
||||
continue
|
||||
}
|
||||
/*we got an exact match, update struct*/
|
||||
// we got an exact match, update struct
|
||||
if !inhub {
|
||||
log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version)
|
||||
v.LocalPath = path
|
||||
v.LocalVersion = version
|
||||
v.Tainted = false
|
||||
v.Downloaded = true
|
||||
/*if we're walking the hub, present file doesn't means installed file*/
|
||||
// if we're walking the hub, present file doesn't means installed file
|
||||
v.Installed = true
|
||||
v.LocalHash = sha
|
||||
_, target.FileName = filepath.Split(path)
|
||||
|
|
@ -207,29 +220,34 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
|||
v.Downloaded = true
|
||||
v.LocalHash = sha
|
||||
}
|
||||
|
||||
if version == v.Version {
|
||||
log.Tracef("%s is up-to-date", v.Name)
|
||||
v.UpToDate = true
|
||||
}
|
||||
|
||||
match = true
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
if !match {
|
||||
log.Tracef("got tainted match for %s : %s", v.Name, path)
|
||||
|
||||
skippedTainted += 1
|
||||
//the file and the stage is right, but the hash is wrong, it has been tainted by user
|
||||
// the file and the stage is right, but the hash is wrong, it has been tainted by user
|
||||
if !inhub {
|
||||
v.LocalPath = path
|
||||
v.Installed = true
|
||||
}
|
||||
|
||||
v.UpToDate = false
|
||||
v.LocalVersion = "?"
|
||||
v.Tainted = true
|
||||
v.LocalHash = sha
|
||||
_, target.FileName = filepath.Split(path)
|
||||
|
||||
}
|
||||
//update the entry if appropriate
|
||||
// update the entry if appropriate
|
||||
// if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D {
|
||||
// fmt.Printf("Updating %s", k)
|
||||
// hubIdx[ftype][k] = v
|
||||
|
|
@ -237,22 +255,25 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
|||
|
||||
// } else if
|
||||
hubIdx[ftype][k] = v
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof("Ignoring file %s of type %s", path, ftype)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func CollecDepsCheck(v *Item) error {
|
||||
|
||||
if GetVersionStatus(v) != 0 { //not up-to-date
|
||||
if GetVersionStatus(v) != 0 { // not up-to-date
|
||||
log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
/*if it's a collection, ensure all the items are installed, or tag it as tainted*/
|
||||
// if it's a collection, ensure all the items are installed, or tag it as tainted
|
||||
if v.Type == COLLECTIONS {
|
||||
log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
|
||||
|
||||
var tmp = [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections}
|
||||
for idx, ptr := range tmp {
|
||||
ptrtype := ItemTypes[idx]
|
||||
|
|
@ -261,48 +282,62 @@ func CollecDepsCheck(v *Item) error {
|
|||
if !ok {
|
||||
log.Fatalf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, v.Name)
|
||||
}
|
||||
|
||||
log.Tracef("check %s installed:%t", val.Name, val.Installed)
|
||||
|
||||
if !v.Installed {
|
||||
continue
|
||||
}
|
||||
|
||||
if val.Type == COLLECTIONS {
|
||||
log.Tracef("collec, recurse.")
|
||||
|
||||
if err := CollecDepsCheck(&val); err != nil {
|
||||
if val.Tainted {
|
||||
v.Tainted = true
|
||||
}
|
||||
return fmt.Errorf("sub collection %s is broken : %s", val.Name, err)
|
||||
|
||||
return fmt.Errorf("sub collection %s is broken: %s", val.Name, err)
|
||||
}
|
||||
|
||||
hubIdx[ptrtype][p] = val
|
||||
}
|
||||
|
||||
//propagate the state of sub-items to set
|
||||
// propagate the state of sub-items to set
|
||||
if val.Tainted {
|
||||
v.Tainted = true
|
||||
return fmt.Errorf("tainted %s %s, tainted.", ptrtype, p)
|
||||
return fmt.Errorf("tainted %s %s, tainted", ptrtype, p)
|
||||
}
|
||||
|
||||
if !val.Installed && v.Installed {
|
||||
v.Tainted = true
|
||||
return fmt.Errorf("missing %s %s, tainted.", ptrtype, p)
|
||||
return fmt.Errorf("missing %s %s, tainted", ptrtype, p)
|
||||
}
|
||||
|
||||
if !val.UpToDate {
|
||||
v.UpToDate = false
|
||||
return fmt.Errorf("outdated %s %s", ptrtype, p)
|
||||
}
|
||||
|
||||
skip := false
|
||||
|
||||
for idx := range val.BelongsToCollections {
|
||||
if val.BelongsToCollections[idx] == v.Name {
|
||||
skip = true
|
||||
}
|
||||
}
|
||||
|
||||
if !skip {
|
||||
val.BelongsToCollections = append(val.BelongsToCollections, v.Name)
|
||||
}
|
||||
|
||||
hubIdx[ptrtype][p] = val
|
||||
|
||||
log.Tracef("checking for %s - tainted:%t uptodate:%t", p, v.Tainted, v.UpToDate)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -311,39 +346,41 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
|
|||
installdir = hub.ConfigDir
|
||||
warnings := []string{}
|
||||
|
||||
/*For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last*/
|
||||
// For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last
|
||||
for _, scan := range ItemTypes {
|
||||
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
|
||||
if err != nil {
|
||||
log.Errorf("failed %s : %s", cpath, err)
|
||||
}
|
||||
|
||||
err = filepath.WalkDir(cpath, parser_visit)
|
||||
if err != nil {
|
||||
return err, warnings
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
for k, v := range hubIdx[COLLECTIONS] {
|
||||
if v.Installed {
|
||||
versStat := GetVersionStatus(&v)
|
||||
if versStat == 0 { //latest
|
||||
if versStat == 0 { // latest
|
||||
if err := CollecDepsCheck(&v); err != nil {
|
||||
warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", v.Name, err))
|
||||
hubIdx[COLLECTIONS][k] = v
|
||||
}
|
||||
} else if versStat == 1 { //not up-to-date
|
||||
} else if versStat == 1 { // not up-to-date
|
||||
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version))
|
||||
} else { //version is higher than the highest available from hub?
|
||||
} else { // version is higher than the highest available from hub?
|
||||
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version))
|
||||
}
|
||||
|
||||
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", v.Name, semver.Compare("v"+v.Version, "v"+v.LocalVersion), v.LocalVersion, v.Version, v.Versions)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, warnings
|
||||
}
|
||||
|
||||
/* Updates the infos from HubInit() with the local state */
|
||||
// Updates the infos from HubInit() with the local state
|
||||
func LocalSync(hub *csconfig.Hub) (error, []string) {
|
||||
skippedLocal = 0
|
||||
skippedTainted = 0
|
||||
|
|
@ -352,10 +389,12 @@ func LocalSync(hub *csconfig.Hub) (error, []string) {
|
|||
if err != nil {
|
||||
return fmt.Errorf("failed to scan %s : %s", hub.ConfigDir, err), warnings
|
||||
}
|
||||
|
||||
err, _ = SyncDir(hub, hub.HubDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings
|
||||
}
|
||||
|
||||
return nil, warnings
|
||||
}
|
||||
|
||||
|
|
@ -363,49 +402,61 @@ func GetHubIdx(hub *csconfig.Hub) error {
|
|||
if hub == nil {
|
||||
return fmt.Errorf("no configuration found for hub")
|
||||
}
|
||||
|
||||
log.Debugf("loading hub idx %s", hub.HubIndexFile)
|
||||
|
||||
bidx, err := os.ReadFile(hub.HubIndexFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read index file: %w", err)
|
||||
}
|
||||
|
||||
ret, err := LoadPkgIndex(bidx)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ReferenceMissingError) {
|
||||
log.Fatalf("Unable to load existing index : %v.", err)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
hubIdx = ret
|
||||
|
||||
err, _ = LocalSync(hub)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to sync Hub index with local deployment : %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
/*LoadPkgIndex loads a local .index.json file and returns the map of parsers/scenarios/collections associated*/
|
||||
// LoadPkgIndex loads a local .index.json file and returns the map of parsers/scenarios/collections associated
|
||||
func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
|
||||
var err error
|
||||
var RawIndex map[string]map[string]Item
|
||||
var missingItems []string
|
||||
var (
|
||||
err error
|
||||
RawIndex map[string]map[string]Item
|
||||
missingItems []string
|
||||
)
|
||||
|
||||
if err = json.Unmarshal(buff, &RawIndex); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal index : %v", err)
|
||||
}
|
||||
|
||||
log.Debugf("%d item types in hub index", len(ItemTypes))
|
||||
/*Iterate over the different types to complete struct */
|
||||
|
||||
// Iterate over the different types to complete struct
|
||||
for _, itemType := range ItemTypes {
|
||||
/*complete struct*/
|
||||
// complete struct
|
||||
log.Tracef("%d item", len(RawIndex[itemType]))
|
||||
|
||||
for idx, item := range RawIndex[itemType] {
|
||||
item.Name = idx
|
||||
item.Type = itemType
|
||||
x := strings.Split(item.RemotePath, "/")
|
||||
item.FileName = x[len(x)-1]
|
||||
RawIndex[itemType][idx] = item
|
||||
/*if it's a collection, check its sub-items are present*/
|
||||
//XX should be done later
|
||||
|
||||
// if it's a collection, check its sub-items are present
|
||||
// XXX should be done later
|
||||
if itemType == COLLECTIONS {
|
||||
var tmp = [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections}
|
||||
for idx, ptr := range tmp {
|
||||
|
|
@ -420,8 +471,9 @@ func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(missingItems) > 0 {
|
||||
return RawIndex, fmt.Errorf("%q : %w", missingItems, ReferenceMissingError)
|
||||
return RawIndex, fmt.Errorf("%q: %w", missingItems, ReferenceMissingError)
|
||||
}
|
||||
|
||||
return RawIndex, nil
|
||||
|
|
|
|||
Loading…
Reference in a new issue