lint: pkg/cwhub (#2510)
no functional changes - reformat - comments - whitespace - removed a dot or two in log messages - some "var x=y" -> x:=y
This commit is contained in:
parent
6dadfcb2ef
commit
8b5ad6990d
|
@ -15,7 +15,7 @@ import (
|
||||||
"golang.org/x/mod/semver"
|
"golang.org/x/mod/semver"
|
||||||
)
|
)
|
||||||
|
|
||||||
/*managed configuration types*/
|
// managed configuration types
|
||||||
var PARSERS = "parsers"
|
var PARSERS = "parsers"
|
||||||
var PARSERS_OVFLW = "postoverflows"
|
var PARSERS_OVFLW = "postoverflows"
|
||||||
var SCENARIOS = "scenarios"
|
var SCENARIOS = "scenarios"
|
||||||
|
@ -42,37 +42,37 @@ type ItemHubStatus struct {
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
}
|
}
|
||||||
|
|
||||||
//Item can be : parsed, scenario, collection
|
// Item can be: parsed, scenario, collection
|
||||||
type Item struct {
|
type Item struct {
|
||||||
/*descriptive info*/
|
// descriptive info
|
||||||
Type string `yaml:"type,omitempty" json:"type,omitempty"` //parser|postoverflows|scenario|collection(|enrich)
|
Type string `yaml:"type,omitempty" json:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich)
|
||||||
Stage string `json:"stage,omitempty" yaml:"stage,omitempty,omitempty"` //Stage for parser|postoverflow : s00-raw/s01-...
|
Stage string `json:"stage,omitempty" yaml:"stage,omitempty,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
|
||||||
Name string `json:"name,omitempty"` //as seen in .config.json, usually "author/name"
|
Name string `json:"name,omitempty"` // as seen in .config.json, usually "author/name"
|
||||||
FileName string `json:"file_name,omitempty"` //the filename, ie. apache2-logs.yaml
|
FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml
|
||||||
Description string `yaml:"description,omitempty" json:"description,omitempty"` //as seen in .config.json
|
Description string `yaml:"description,omitempty" json:"description,omitempty"` // as seen in .config.json
|
||||||
Author string `json:"author,omitempty"` //as seen in .config.json
|
Author string `json:"author,omitempty"` // as seen in .config.json
|
||||||
References []string `yaml:"references,omitempty" json:"references,omitempty"` //as seen in .config.json
|
References []string `yaml:"references,omitempty" json:"references,omitempty"` // as seen in .config.json
|
||||||
BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` /*if it's part of collections, track name here*/
|
BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` // if it's part of collections, track name here
|
||||||
|
|
||||||
/*remote (hub) infos*/
|
// remote (hub) infos
|
||||||
RemoteURL string `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` //the full remote uri of file in http
|
RemoteURL string `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` // the full remote uri of file in http
|
||||||
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` //the path relative to git ie. /parsers/stage/author/file.yaml
|
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to git ie. /parsers/stage/author/file.yaml
|
||||||
RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` //the meow
|
RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` // the meow
|
||||||
Version string `json:"version,omitempty"` //the last version
|
Version string `json:"version,omitempty"` // the last version
|
||||||
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` //the list of existing versions
|
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions
|
||||||
|
|
||||||
/*local (deployed) infos*/
|
// local (deployed) infos
|
||||||
LocalPath string `yaml:"local_path,omitempty" json:"local_path,omitempty"` //the local path relative to ${CFG_DIR}
|
LocalPath string `yaml:"local_path,omitempty" json:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
|
||||||
//LocalHubPath string
|
// LocalHubPath string
|
||||||
LocalVersion string `json:"local_version,omitempty"`
|
LocalVersion string `json:"local_version,omitempty"`
|
||||||
LocalHash string `json:"local_hash,omitempty"` //the local meow
|
LocalHash string `json:"local_hash,omitempty"` // the local meow
|
||||||
Installed bool `json:"installed,omitempty"`
|
Installed bool `json:"installed,omitempty"`
|
||||||
Downloaded bool `json:"downloaded,omitempty"`
|
Downloaded bool `json:"downloaded,omitempty"`
|
||||||
UpToDate bool `json:"up_to_date,omitempty"`
|
UpToDate bool `json:"up_to_date,omitempty"`
|
||||||
Tainted bool `json:"tainted,omitempty"` //has it been locally modified
|
Tainted bool `json:"tainted,omitempty"` // has it been locally modified
|
||||||
Local bool `json:"local,omitempty"` //if it's a non versioned control one
|
Local bool `json:"local,omitempty"` // if it's a non versioned control one
|
||||||
|
|
||||||
/*if it's a collection, it not a single file*/
|
// if it's a collection, it not a single file
|
||||||
Parsers []string `yaml:"parsers,omitempty" json:"parsers,omitempty"`
|
Parsers []string `yaml:"parsers,omitempty" json:"parsers,omitempty"`
|
||||||
PostOverflows []string `yaml:"postoverflows,omitempty" json:"postoverflows,omitempty"`
|
PostOverflows []string `yaml:"postoverflows,omitempty" json:"postoverflows,omitempty"`
|
||||||
Scenarios []string `yaml:"scenarios,omitempty" json:"scenarios,omitempty"`
|
Scenarios []string `yaml:"scenarios,omitempty" json:"scenarios,omitempty"`
|
||||||
|
@ -88,6 +88,7 @@ func (i *Item) toHubStatus() ItemHubStatus {
|
||||||
|
|
||||||
status, ok, warning, managed := ItemStatus(*i)
|
status, ok, warning, managed := ItemStatus(*i)
|
||||||
hubStatus.Status = status
|
hubStatus.Status = status
|
||||||
|
|
||||||
if !managed {
|
if !managed {
|
||||||
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.House, status)
|
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.House, status)
|
||||||
} else if !i.Installed {
|
} else if !i.Installed {
|
||||||
|
@ -97,27 +98,28 @@ func (i *Item) toHubStatus() ItemHubStatus {
|
||||||
} else if ok {
|
} else if ok {
|
||||||
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.CheckMark, status)
|
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.CheckMark, status)
|
||||||
}
|
}
|
||||||
|
|
||||||
return hubStatus
|
return hubStatus
|
||||||
}
|
}
|
||||||
|
|
||||||
var skippedLocal = 0
|
var skippedLocal = 0
|
||||||
var skippedTainted = 0
|
var skippedTainted = 0
|
||||||
|
|
||||||
/*To be used when reference(s) (is/are) missing in a collection*/
|
// To be used when reference(s) (is/are) missing in a collection
|
||||||
var ReferenceMissingError = errors.New("Reference(s) missing in collection")
|
var ReferenceMissingError = errors.New("Reference(s) missing in collection")
|
||||||
var MissingHubIndex = errors.New("hub index can't be found")
|
var MissingHubIndex = errors.New("hub index can't be found")
|
||||||
|
|
||||||
//GetVersionStatus : semver requires 'v' prefix
|
// GetVersionStatus: semver requires 'v' prefix
|
||||||
func GetVersionStatus(v *Item) int {
|
func GetVersionStatus(v *Item) int {
|
||||||
return semver.Compare("v"+v.Version, "v"+v.LocalVersion)
|
return semver.Compare("v"+v.Version, "v"+v.LocalVersion)
|
||||||
}
|
}
|
||||||
|
|
||||||
// calculate sha256 of a file
|
// calculate sha256 of a file
|
||||||
func getSHA256(filepath string) (string, error) {
|
func getSHA256(filepath string) (string, error) {
|
||||||
/* Digest of file */
|
// Digest of file
|
||||||
f, err := os.Open(filepath)
|
f, err := os.Open(filepath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("unable to open '%s' : %s", filepath, err)
|
return "", fmt.Errorf("unable to open '%s': %s", filepath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
|
@ -131,49 +133,55 @@ func getSHA256(filepath string) (string, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetItemMap(itemType string) map[string]Item {
|
func GetItemMap(itemType string) map[string]Item {
|
||||||
var m map[string]Item
|
var (
|
||||||
var ok bool
|
m map[string]Item
|
||||||
|
ok bool
|
||||||
|
)
|
||||||
|
|
||||||
if m, ok = hubIdx[itemType]; !ok {
|
if m, ok = hubIdx[itemType]; !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
//GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
|
// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
|
||||||
func GetItemByPath(itemType string, itemPath string) (*Item, error) {
|
func GetItemByPath(itemType string, itemPath string) (*Item, error) {
|
||||||
/*try to resolve symlink*/
|
// try to resolve symlink
|
||||||
finalName := ""
|
finalName := ""
|
||||||
|
|
||||||
f, err := os.Lstat(itemPath)
|
f, err := os.Lstat(itemPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
|
return nil, fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if f.Mode()&os.ModeSymlink == 0 {
|
if f.Mode()&os.ModeSymlink == 0 {
|
||||||
/*it's not a symlink, it should be the filename itsef the key*/
|
// it's not a symlink, it should be the filename itsef the key
|
||||||
finalName = filepath.Base(itemPath)
|
finalName = filepath.Base(itemPath)
|
||||||
} else {
|
} else {
|
||||||
/*resolve the symlink to hub file*/
|
// resolve the symlink to hub file
|
||||||
pathInHub, err := os.Readlink(itemPath)
|
pathInHub, err := os.Readlink(itemPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
|
return nil, fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
|
||||||
}
|
}
|
||||||
//extract author from path
|
// extract author from path
|
||||||
fname := filepath.Base(pathInHub)
|
fname := filepath.Base(pathInHub)
|
||||||
author := filepath.Base(filepath.Dir(pathInHub))
|
author := filepath.Base(filepath.Dir(pathInHub))
|
||||||
//trim yaml suffix
|
// trim yaml suffix
|
||||||
fname = strings.TrimSuffix(fname, ".yaml")
|
fname = strings.TrimSuffix(fname, ".yaml")
|
||||||
fname = strings.TrimSuffix(fname, ".yml")
|
fname = strings.TrimSuffix(fname, ".yml")
|
||||||
finalName = fmt.Sprintf("%s/%s", author, fname)
|
finalName = fmt.Sprintf("%s/%s", author, fname)
|
||||||
}
|
}
|
||||||
|
|
||||||
/*it's not a symlink, it should be the filename itsef the key*/
|
// it's not a symlink, it should be the filename itsef the key
|
||||||
if m := GetItemMap(itemType); m != nil {
|
if m := GetItemMap(itemType); m != nil {
|
||||||
if v, ok := m[finalName]; ok {
|
if v, ok := m[finalName]; ok {
|
||||||
return &v, nil
|
return &v, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, fmt.Errorf("%s not found in %s", finalName, itemType)
|
return nil, fmt.Errorf("%s not found in %s", finalName, itemType)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, fmt.Errorf("item type %s doesn't exist", itemType)
|
return nil, fmt.Errorf("item type %s doesn't exist", itemType)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -181,35 +189,42 @@ func GetItem(itemType string, itemName string) *Item {
|
||||||
if m, ok := GetItemMap(itemType)[itemName]; ok {
|
if m, ok := GetItemMap(itemType)[itemName]; ok {
|
||||||
return &m
|
return &m
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func AddItem(itemType string, item Item) error {
|
func AddItem(itemType string, item Item) error {
|
||||||
in := false
|
in := false
|
||||||
|
|
||||||
for _, itype := range ItemTypes {
|
for _, itype := range ItemTypes {
|
||||||
if itype == itemType {
|
if itype == itemType {
|
||||||
in = true
|
in = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !in {
|
if !in {
|
||||||
return fmt.Errorf("ItemType %s is unknown", itemType)
|
return fmt.Errorf("ItemType %s is unknown", itemType)
|
||||||
}
|
}
|
||||||
|
|
||||||
hubIdx[itemType][item.Name] = item
|
hubIdx[itemType][item.Name] = item
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func DisplaySummary() {
|
func DisplaySummary() {
|
||||||
log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
|
log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
|
||||||
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
|
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
|
||||||
|
|
||||||
if skippedLocal > 0 || skippedTainted > 0 {
|
if skippedLocal > 0 || skippedTainted > 0 {
|
||||||
log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted)
|
log.Printf("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//returns: human-text, Enabled, Warning, Unmanaged
|
// returns: human-text, Enabled, Warning, Unmanaged
|
||||||
func ItemStatus(v Item) (string, bool, bool, bool) {
|
func ItemStatus(v Item) (string, bool, bool, bool) {
|
||||||
strret := "disabled"
|
strret := "disabled"
|
||||||
Ok := false
|
Ok := false
|
||||||
|
|
||||||
if v.Installed {
|
if v.Installed {
|
||||||
Ok = true
|
Ok = true
|
||||||
strret = "enabled"
|
strret = "enabled"
|
||||||
|
@ -221,7 +236,7 @@ func ItemStatus(v Item) (string, bool, bool, bool) {
|
||||||
strret += ",local"
|
strret += ",local"
|
||||||
}
|
}
|
||||||
|
|
||||||
//tainted or out of date
|
// tainted or out of date
|
||||||
Warning := false
|
Warning := false
|
||||||
if v.Tainted {
|
if v.Tainted {
|
||||||
Warning = true
|
Warning = true
|
||||||
|
@ -230,6 +245,7 @@ func ItemStatus(v Item) (string, bool, bool, bool) {
|
||||||
strret += ",update-available"
|
strret += ",update-available"
|
||||||
Warning = true
|
Warning = true
|
||||||
}
|
}
|
||||||
|
|
||||||
return strret, Ok, Warning, Managed
|
return strret, Ok, Warning, Managed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -240,9 +256,11 @@ func GetInstalledScenariosAsString() ([]string, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("while fetching scenarios: %w", err)
|
return nil, fmt.Errorf("while fetching scenarios: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, it := range items {
|
for _, it := range items {
|
||||||
retStr = append(retStr, it.Name)
|
retStr = append(retStr, it.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
return retStr, nil
|
return retStr, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -252,11 +270,13 @@ func GetInstalledScenarios() ([]Item, error) {
|
||||||
if _, ok := hubIdx[SCENARIOS]; !ok {
|
if _, ok := hubIdx[SCENARIOS]; !ok {
|
||||||
return nil, fmt.Errorf("no scenarios in hubIdx")
|
return nil, fmt.Errorf("no scenarios in hubIdx")
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, item := range hubIdx[SCENARIOS] {
|
for _, item := range hubIdx[SCENARIOS] {
|
||||||
if item.Installed {
|
if item.Installed {
|
||||||
retItems = append(retItems, item)
|
retItems = append(retItems, item)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return retItems, nil
|
return retItems, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -266,11 +286,13 @@ func GetInstalledParsers() ([]Item, error) {
|
||||||
if _, ok := hubIdx[PARSERS]; !ok {
|
if _, ok := hubIdx[PARSERS]; !ok {
|
||||||
return nil, fmt.Errorf("no parsers in hubIdx")
|
return nil, fmt.Errorf("no parsers in hubIdx")
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, item := range hubIdx[PARSERS] {
|
for _, item := range hubIdx[PARSERS] {
|
||||||
if item.Installed {
|
if item.Installed {
|
||||||
retItems = append(retItems, item)
|
retItems = append(retItems, item)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return retItems, nil
|
return retItems, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -281,9 +303,11 @@ func GetInstalledParsersAsString() ([]string, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("while fetching parsers: %w", err)
|
return nil, fmt.Errorf("while fetching parsers: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, it := range items {
|
for _, it := range items {
|
||||||
retStr = append(retStr, it.Name)
|
retStr = append(retStr, it.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
return retStr, nil
|
return retStr, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -293,11 +317,13 @@ func GetInstalledPostOverflows() ([]Item, error) {
|
||||||
if _, ok := hubIdx[PARSERS_OVFLW]; !ok {
|
if _, ok := hubIdx[PARSERS_OVFLW]; !ok {
|
||||||
return nil, fmt.Errorf("no post overflows in hubIdx")
|
return nil, fmt.Errorf("no post overflows in hubIdx")
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, item := range hubIdx[PARSERS_OVFLW] {
|
for _, item := range hubIdx[PARSERS_OVFLW] {
|
||||||
if item.Installed {
|
if item.Installed {
|
||||||
retItems = append(retItems, item)
|
retItems = append(retItems, item)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return retItems, nil
|
return retItems, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -308,9 +334,11 @@ func GetInstalledPostOverflowsAsString() ([]string, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("while fetching post overflows: %w", err)
|
return nil, fmt.Errorf("while fetching post overflows: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, it := range items {
|
for _, it := range items {
|
||||||
retStr = append(retStr, it.Name)
|
retStr = append(retStr, it.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
return retStr, nil
|
return retStr, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -325,6 +353,7 @@ func GetInstalledCollectionsAsString() ([]string, error) {
|
||||||
for _, it := range items {
|
for _, it := range items {
|
||||||
retStr = append(retStr, it.Name)
|
retStr = append(retStr, it.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
return retStr, nil
|
return retStr, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -334,15 +363,17 @@ func GetInstalledCollections() ([]Item, error) {
|
||||||
if _, ok := hubIdx[COLLECTIONS]; !ok {
|
if _, ok := hubIdx[COLLECTIONS]; !ok {
|
||||||
return nil, fmt.Errorf("no collection in hubIdx")
|
return nil, fmt.Errorf("no collection in hubIdx")
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, item := range hubIdx[COLLECTIONS] {
|
for _, item := range hubIdx[COLLECTIONS] {
|
||||||
if item.Installed {
|
if item.Installed {
|
||||||
retItems = append(retItems, item)
|
retItems = append(retItems, item)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return retItems, nil
|
return retItems, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
//Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy)
|
// Returns a list of entries for packages: name, status, local_path, local_version, utf8_status (fancy)
|
||||||
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
|
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
|
||||||
if _, ok := hubIdx[itemType]; !ok {
|
if _, ok := hubIdx[itemType]; !ok {
|
||||||
log.Errorf("type %s doesn't exist", itemType)
|
log.Errorf("type %s doesn't exist", itemType)
|
||||||
|
@ -351,19 +382,22 @@ func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubSt
|
||||||
}
|
}
|
||||||
|
|
||||||
var ret = make([]ItemHubStatus, 0)
|
var ret = make([]ItemHubStatus, 0)
|
||||||
/*remember, you do it for the user :)*/
|
|
||||||
|
// remember, you do it for the user :)
|
||||||
for _, item := range hubIdx[itemType] {
|
for _, item := range hubIdx[itemType] {
|
||||||
if name != "" && name != item.Name {
|
if name != "" && name != item.Name {
|
||||||
//user has requested a specific name
|
// user has requested a specific name
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
//Only enabled items ?
|
// Only enabled items ?
|
||||||
if !all && !item.Installed {
|
if !all && !item.Installed {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
//Check the item status
|
// Check the item status
|
||||||
ret = append(ret, item.toHubStatus())
|
ret = append(ret, item.toHubStatus())
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
|
sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
|
||||||
|
|
||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,8 +9,9 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
)
|
)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -29,30 +30,33 @@ func TestItemStatus(t *testing.T) {
|
||||||
defer envTearDown(cfg)
|
defer envTearDown(cfg)
|
||||||
|
|
||||||
err := UpdateHubIdx(cfg.Hub)
|
err := UpdateHubIdx(cfg.Hub)
|
||||||
//DownloadHubIdx()
|
// DownloadHubIdx()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to download index : %s", err)
|
t.Fatalf("failed to download index : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := GetHubIdx(cfg.Hub); err != nil {
|
if err := GetHubIdx(cfg.Hub); err != nil {
|
||||||
t.Fatalf("failed to load hub index : %s", err)
|
t.Fatalf("failed to load hub index : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
//get existing map
|
// get existing map
|
||||||
x := GetItemMap(COLLECTIONS)
|
x := GetItemMap(COLLECTIONS)
|
||||||
if len(x) == 0 {
|
if len(x) == 0 {
|
||||||
t.Fatalf("expected non empty result")
|
t.Fatalf("expected non empty result")
|
||||||
}
|
}
|
||||||
|
|
||||||
//Get item : good and bad
|
// Get item : good and bad
|
||||||
for k := range x {
|
for k := range x {
|
||||||
item := GetItem(COLLECTIONS, k)
|
item := GetItem(COLLECTIONS, k)
|
||||||
if item == nil {
|
if item == nil {
|
||||||
t.Fatalf("expected item")
|
t.Fatalf("expected item")
|
||||||
}
|
}
|
||||||
|
|
||||||
item.Installed = true
|
item.Installed = true
|
||||||
item.UpToDate = false
|
item.UpToDate = false
|
||||||
item.Local = false
|
item.Local = false
|
||||||
item.Tainted = false
|
item.Tainted = false
|
||||||
|
|
||||||
txt, _, _, _ := ItemStatus(*item)
|
txt, _, _, _ := ItemStatus(*item)
|
||||||
if txt != "enabled,update-available" {
|
if txt != "enabled,update-available" {
|
||||||
t.Fatalf("got '%s'", txt)
|
t.Fatalf("got '%s'", txt)
|
||||||
|
@ -62,6 +66,7 @@ func TestItemStatus(t *testing.T) {
|
||||||
item.UpToDate = false
|
item.UpToDate = false
|
||||||
item.Local = true
|
item.Local = true
|
||||||
item.Tainted = false
|
item.Tainted = false
|
||||||
|
|
||||||
txt, _, _, _ = ItemStatus(*item)
|
txt, _, _, _ = ItemStatus(*item)
|
||||||
if txt != "disabled,local" {
|
if txt != "disabled,local" {
|
||||||
t.Fatalf("got '%s'", txt)
|
t.Fatalf("got '%s'", txt)
|
||||||
|
@ -69,6 +74,7 @@ func TestItemStatus(t *testing.T) {
|
||||||
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
DisplaySummary()
|
DisplaySummary()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -77,26 +83,28 @@ func TestGetters(t *testing.T) {
|
||||||
defer envTearDown(cfg)
|
defer envTearDown(cfg)
|
||||||
|
|
||||||
err := UpdateHubIdx(cfg.Hub)
|
err := UpdateHubIdx(cfg.Hub)
|
||||||
//DownloadHubIdx()
|
// DownloadHubIdx()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to download index : %s", err)
|
t.Fatalf("failed to download index : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := GetHubIdx(cfg.Hub); err != nil {
|
if err := GetHubIdx(cfg.Hub); err != nil {
|
||||||
t.Fatalf("failed to load hub index : %s", err)
|
t.Fatalf("failed to load hub index : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
//get non existing map
|
// get non existing map
|
||||||
empty := GetItemMap("ratata")
|
empty := GetItemMap("ratata")
|
||||||
if empty != nil {
|
if empty != nil {
|
||||||
t.Fatalf("expected nil result")
|
t.Fatalf("expected nil result")
|
||||||
}
|
}
|
||||||
//get existing map
|
|
||||||
|
// get existing map
|
||||||
x := GetItemMap(COLLECTIONS)
|
x := GetItemMap(COLLECTIONS)
|
||||||
if len(x) == 0 {
|
if len(x) == 0 {
|
||||||
t.Fatalf("expected non empty result")
|
t.Fatalf("expected non empty result")
|
||||||
}
|
}
|
||||||
|
|
||||||
//Get item : good and bad
|
// Get item : good and bad
|
||||||
for k := range x {
|
for k := range x {
|
||||||
empty := GetItem(COLLECTIONS, k+"nope")
|
empty := GetItem(COLLECTIONS, k+"nope")
|
||||||
if empty != nil {
|
if empty != nil {
|
||||||
|
@ -108,7 +116,7 @@ func TestGetters(t *testing.T) {
|
||||||
t.Fatalf("expected non empty item")
|
t.Fatalf("expected non empty item")
|
||||||
}
|
}
|
||||||
|
|
||||||
//Add item and get it
|
// Add item and get it
|
||||||
item.Name += "nope"
|
item.Name += "nope"
|
||||||
if err := AddItem(COLLECTIONS, *item); err != nil {
|
if err := AddItem(COLLECTIONS, *item); err != nil {
|
||||||
t.Fatalf("didn't expect error : %s", err)
|
t.Fatalf("didn't expect error : %s", err)
|
||||||
|
@ -119,7 +127,7 @@ func TestGetters(t *testing.T) {
|
||||||
t.Fatalf("expected non empty item")
|
t.Fatalf("expected non empty item")
|
||||||
}
|
}
|
||||||
|
|
||||||
//Add bad item
|
// Add bad item
|
||||||
if err := AddItem("ratata", *item); err != nil {
|
if err := AddItem("ratata", *item); err != nil {
|
||||||
if fmt.Sprintf("%s", err) != "ItemType ratata is unknown" {
|
if fmt.Sprintf("%s", err) != "ItemType ratata is unknown" {
|
||||||
t.Fatalf("unexpected error")
|
t.Fatalf("unexpected error")
|
||||||
|
@ -130,7 +138,6 @@ func TestGetters(t *testing.T) {
|
||||||
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestIndexDownload(t *testing.T) {
|
func TestIndexDownload(t *testing.T) {
|
||||||
|
@ -138,10 +145,11 @@ func TestIndexDownload(t *testing.T) {
|
||||||
defer envTearDown(cfg)
|
defer envTearDown(cfg)
|
||||||
|
|
||||||
err := UpdateHubIdx(cfg.Hub)
|
err := UpdateHubIdx(cfg.Hub)
|
||||||
//DownloadHubIdx()
|
// DownloadHubIdx()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to download index : %s", err)
|
t.Fatalf("failed to download index : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := GetHubIdx(cfg.Hub); err != nil {
|
if err := GetHubIdx(cfg.Hub); err != nil {
|
||||||
t.Fatalf("failed to load hub index : %s", err)
|
t.Fatalf("failed to load hub index : %s", err)
|
||||||
}
|
}
|
||||||
|
@ -152,20 +160,23 @@ func getTestCfg() (cfg *csconfig.Config) {
|
||||||
cfg.Hub.ConfigDir, _ = filepath.Abs("./install")
|
cfg.Hub.ConfigDir, _ = filepath.Abs("./install")
|
||||||
cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
|
cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
|
||||||
cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
|
cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func envSetup(t *testing.T) *csconfig.Config {
|
func envSetup(t *testing.T) *csconfig.Config {
|
||||||
resetResponseByPath()
|
resetResponseByPath()
|
||||||
log.SetLevel(log.DebugLevel)
|
log.SetLevel(log.DebugLevel)
|
||||||
|
|
||||||
cfg := getTestCfg()
|
cfg := getTestCfg()
|
||||||
|
|
||||||
defaultTransport := http.DefaultClient.Transport
|
defaultTransport := http.DefaultClient.Transport
|
||||||
|
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
http.DefaultClient.Transport = defaultTransport
|
http.DefaultClient.Transport = defaultTransport
|
||||||
})
|
})
|
||||||
|
|
||||||
//Mock the http client
|
// Mock the http client
|
||||||
http.DefaultClient.Transport = newMockTransport()
|
http.DefaultClient.Transport = newMockTransport()
|
||||||
|
|
||||||
if err := os.MkdirAll(cfg.Hub.ConfigDir, 0700); err != nil {
|
if err := os.MkdirAll(cfg.Hub.ConfigDir, 0700); err != nil {
|
||||||
|
@ -189,7 +200,6 @@ func envSetup(t *testing.T) *csconfig.Config {
|
||||||
return cfg
|
return cfg
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func envTearDown(cfg *csconfig.Config) {
|
func envTearDown(cfg *csconfig.Config) {
|
||||||
if err := os.RemoveAll(cfg.Hub.ConfigDir); err != nil {
|
if err := os.RemoveAll(cfg.Hub.ConfigDir); err != nil {
|
||||||
log.Fatalf("failed to remove %s : %s", cfg.Hub.ConfigDir, err)
|
log.Fatalf("failed to remove %s : %s", cfg.Hub.ConfigDir, err)
|
||||||
|
@ -200,23 +210,25 @@ func envTearDown(cfg *csconfig.Config) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
||||||
|
// Install the parser
|
||||||
//Install the parser
|
|
||||||
item, err := DownloadLatest(cfg, item, false, false)
|
item, err := DownloadLatest(cfg, item, false, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error while downloading %s : %v", item.Name, err)
|
t.Fatalf("error while downloading %s : %v", item.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err, _ := LocalSync(cfg); err != nil {
|
if err, _ := LocalSync(cfg); err != nil {
|
||||||
t.Fatalf("taint: failed to run localSync : %s", err)
|
t.Fatalf("taint: failed to run localSync : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !hubIdx[item.Type][item.Name].UpToDate {
|
if !hubIdx[item.Type][item.Name].UpToDate {
|
||||||
t.Fatalf("download: %s should be up-to-date", item.Name)
|
t.Fatalf("download: %s should be up-to-date", item.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if hubIdx[item.Type][item.Name].Installed {
|
if hubIdx[item.Type][item.Name].Installed {
|
||||||
t.Fatalf("download: %s should not be installed", item.Name)
|
t.Fatalf("download: %s should not be installed", item.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if hubIdx[item.Type][item.Name].Tainted {
|
if hubIdx[item.Type][item.Name].Tainted {
|
||||||
t.Fatalf("download: %s should not be tainted", item.Name)
|
t.Fatalf("download: %s should not be tainted", item.Name)
|
||||||
}
|
}
|
||||||
|
@ -225,9 +237,11 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error while enabling %s : %v.", item.Name, err)
|
t.Fatalf("error while enabling %s : %v.", item.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err, _ := LocalSync(cfg); err != nil {
|
if err, _ := LocalSync(cfg); err != nil {
|
||||||
t.Fatalf("taint: failed to run localSync : %s", err)
|
t.Fatalf("taint: failed to run localSync : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !hubIdx[item.Type][item.Name].Installed {
|
if !hubIdx[item.Type][item.Name].Installed {
|
||||||
t.Fatalf("install: %s should be installed", item.Name)
|
t.Fatalf("install: %s should be installed", item.Name)
|
||||||
}
|
}
|
||||||
|
@ -237,6 +251,7 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
||||||
if hubIdx[item.Type][item.Name].Tainted {
|
if hubIdx[item.Type][item.Name].Tainted {
|
||||||
t.Fatalf("pre-taint: %s should not be tainted", item.Name)
|
t.Fatalf("pre-taint: %s should not be tainted", item.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
|
f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("(taint) opening %s (%s) : %s", item.LocalPath, item.Name, err)
|
t.Fatalf("(taint) opening %s (%s) : %s", item.LocalPath, item.Name, err)
|
||||||
|
@ -246,32 +261,37 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
||||||
if _, err = f.WriteString("tainted"); err != nil {
|
if _, err = f.WriteString("tainted"); err != nil {
|
||||||
t.Fatalf("tainting %s : %s", item.Name, err)
|
t.Fatalf("tainting %s : %s", item.Name, err)
|
||||||
}
|
}
|
||||||
//Local sync and check status
|
|
||||||
|
// Local sync and check status
|
||||||
if err, _ := LocalSync(cfg); err != nil {
|
if err, _ := LocalSync(cfg); err != nil {
|
||||||
t.Fatalf("taint: failed to run localSync : %s", err)
|
t.Fatalf("taint: failed to run localSync : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !hubIdx[item.Type][item.Name].Tainted {
|
if !hubIdx[item.Type][item.Name].Tainted {
|
||||||
t.Fatalf("taint: %s should be tainted", item.Name)
|
t.Fatalf("taint: %s should be tainted", item.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
||||||
|
|
||||||
if hubIdx[item.Type][item.Name].UpToDate {
|
if hubIdx[item.Type][item.Name].UpToDate {
|
||||||
t.Fatalf("update: %s should NOT be up-to-date", item.Name)
|
t.Fatalf("update: %s should NOT be up-to-date", item.Name)
|
||||||
}
|
}
|
||||||
//Update it + check status
|
|
||||||
|
// Update it + check status
|
||||||
item, err := DownloadLatest(cfg, item, true, true)
|
item, err := DownloadLatest(cfg, item, true, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to update %s : %s", item.Name, err)
|
t.Fatalf("failed to update %s : %s", item.Name, err)
|
||||||
}
|
}
|
||||||
//Local sync and check status
|
|
||||||
|
// Local sync and check status
|
||||||
if err, _ := LocalSync(cfg); err != nil {
|
if err, _ := LocalSync(cfg); err != nil {
|
||||||
t.Fatalf("failed to run localSync : %s", err)
|
t.Fatalf("failed to run localSync : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !hubIdx[item.Type][item.Name].UpToDate {
|
if !hubIdx[item.Type][item.Name].UpToDate {
|
||||||
t.Fatalf("update: %s should be up-to-date", item.Name)
|
t.Fatalf("update: %s should be up-to-date", item.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if hubIdx[item.Type][item.Name].Tainted {
|
if hubIdx[item.Type][item.Name].Tainted {
|
||||||
t.Fatalf("update: %s should not be tainted anymore", item.Name)
|
t.Fatalf("update: %s should not be tainted anymore", item.Name)
|
||||||
}
|
}
|
||||||
|
@ -281,43 +301,51 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
||||||
if !item.Installed {
|
if !item.Installed {
|
||||||
t.Fatalf("disable: %s should be installed", item.Name)
|
t.Fatalf("disable: %s should be installed", item.Name)
|
||||||
}
|
}
|
||||||
//Remove
|
|
||||||
|
// Remove
|
||||||
item, err := DisableItem(cfg, item, false, false)
|
item, err := DisableItem(cfg, item, false, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to disable item : %v", err)
|
t.Fatalf("failed to disable item : %v", err)
|
||||||
}
|
}
|
||||||
//Local sync and check status
|
|
||||||
|
// Local sync and check status
|
||||||
if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 {
|
if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 {
|
||||||
t.Fatalf("failed to run localSync : %s (%+v)", err, warns)
|
t.Fatalf("failed to run localSync : %s (%+v)", err, warns)
|
||||||
}
|
}
|
||||||
|
|
||||||
if hubIdx[item.Type][item.Name].Tainted {
|
if hubIdx[item.Type][item.Name].Tainted {
|
||||||
t.Fatalf("disable: %s should not be tainted anymore", item.Name)
|
t.Fatalf("disable: %s should not be tainted anymore", item.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if hubIdx[item.Type][item.Name].Installed {
|
if hubIdx[item.Type][item.Name].Installed {
|
||||||
t.Fatalf("disable: %s should not be installed anymore", item.Name)
|
t.Fatalf("disable: %s should not be installed anymore", item.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !hubIdx[item.Type][item.Name].Downloaded {
|
if !hubIdx[item.Type][item.Name].Downloaded {
|
||||||
t.Fatalf("disable: %s should still be downloaded", item.Name)
|
t.Fatalf("disable: %s should still be downloaded", item.Name)
|
||||||
}
|
}
|
||||||
//Purge
|
|
||||||
|
// Purge
|
||||||
item, err = DisableItem(cfg, item, true, false)
|
item, err = DisableItem(cfg, item, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to purge item : %v", err)
|
t.Fatalf("failed to purge item : %v", err)
|
||||||
}
|
}
|
||||||
//Local sync and check status
|
|
||||||
|
// Local sync and check status
|
||||||
if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 {
|
if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 {
|
||||||
t.Fatalf("failed to run localSync : %s (%+v)", err, warns)
|
t.Fatalf("failed to run localSync : %s (%+v)", err, warns)
|
||||||
}
|
}
|
||||||
|
|
||||||
if hubIdx[item.Type][item.Name].Installed {
|
if hubIdx[item.Type][item.Name].Installed {
|
||||||
t.Fatalf("disable: %s should not be installed anymore", item.Name)
|
t.Fatalf("disable: %s should not be installed anymore", item.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if hubIdx[item.Type][item.Name].Downloaded {
|
if hubIdx[item.Type][item.Name].Downloaded {
|
||||||
t.Fatalf("disable: %s should not be downloaded", item.Name)
|
t.Fatalf("disable: %s should not be downloaded", item.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInstallParser(t *testing.T) {
|
func TestInstallParser(t *testing.T) {
|
||||||
|
|
||||||
/*
|
/*
|
||||||
- install a random parser
|
- install a random parser
|
||||||
- check its status
|
- check its status
|
||||||
|
@ -331,7 +359,7 @@ func TestInstallParser(t *testing.T) {
|
||||||
defer envTearDown(cfg)
|
defer envTearDown(cfg)
|
||||||
|
|
||||||
getHubIdxOrFail(t)
|
getHubIdxOrFail(t)
|
||||||
//map iteration is random by itself
|
// map iteration is random by itself
|
||||||
for _, it := range hubIdx[PARSERS] {
|
for _, it := range hubIdx[PARSERS] {
|
||||||
testInstallItem(cfg.Hub, t, it)
|
testInstallItem(cfg.Hub, t, it)
|
||||||
it = hubIdx[PARSERS][it.Name]
|
it = hubIdx[PARSERS][it.Name]
|
||||||
|
@ -349,7 +377,6 @@ func TestInstallParser(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInstallCollection(t *testing.T) {
|
func TestInstallCollection(t *testing.T) {
|
||||||
|
|
||||||
/*
|
/*
|
||||||
- install a random parser
|
- install a random parser
|
||||||
- check its status
|
- check its status
|
||||||
|
@ -363,7 +390,7 @@ func TestInstallCollection(t *testing.T) {
|
||||||
defer envTearDown(cfg)
|
defer envTearDown(cfg)
|
||||||
|
|
||||||
getHubIdxOrFail(t)
|
getHubIdxOrFail(t)
|
||||||
//map iteration is random by itself
|
// map iteration is random by itself
|
||||||
for _, it := range hubIdx[COLLECTIONS] {
|
for _, it := range hubIdx[COLLECTIONS] {
|
||||||
testInstallItem(cfg.Hub, t, it)
|
testInstallItem(cfg.Hub, t, it)
|
||||||
it = hubIdx[COLLECTIONS][it.Name]
|
it = hubIdx[COLLECTIONS][it.Name]
|
||||||
|
@ -376,6 +403,7 @@ func TestInstallCollection(t *testing.T) {
|
||||||
it = hubIdx[COLLECTIONS][it.Name]
|
it = hubIdx[COLLECTIONS][it.Name]
|
||||||
x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
|
x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
|
||||||
log.Printf("%+v", x)
|
log.Printf("%+v", x)
|
||||||
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -395,10 +423,12 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
}
|
}
|
||||||
response.Header.Set("Content-Type", "application/json")
|
response.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
responseBody := ""
|
responseBody := ""
|
||||||
|
|
||||||
log.Printf("---> %s", req.URL.Path)
|
log.Printf("---> %s", req.URL.Path)
|
||||||
|
|
||||||
/*FAKE PARSER*/
|
// FAKE PARSER
|
||||||
if resp, ok := responseByPath[req.URL.Path]; ok {
|
if resp, ok := responseByPath[req.URL.Path]; ok {
|
||||||
responseBody = resp
|
responseBody = resp
|
||||||
} else {
|
} else {
|
||||||
|
@ -406,12 +436,14 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
response.Body = io.NopCloser(strings.NewReader(responseBody))
|
response.Body = io.NopCloser(strings.NewReader(responseBody))
|
||||||
|
|
||||||
return response, nil
|
return response, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func fileToStringX(path string) string {
|
func fileToStringX(path string) string {
|
||||||
if f, err := os.Open(path); err == nil {
|
if f, err := os.Open(path); err == nil {
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
|
|
||||||
if data, err := io.ReadAll(f); err == nil {
|
if data, err := io.ReadAll(f); err == nil {
|
||||||
return strings.ReplaceAll(string(data), "\r\n", "\n")
|
return strings.ReplaceAll(string(data), "\r\n", "\n")
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -18,6 +18,7 @@ type DataSet struct {
|
||||||
|
|
||||||
func downloadFile(url string, destPath string) error {
|
func downloadFile(url string, destPath string) error {
|
||||||
log.Debugf("downloading %s in %s", url, destPath)
|
log.Debugf("downloading %s in %s", url, destPath)
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -60,6 +61,7 @@ func GetData(data []*types.DataSource, dataDir string) error {
|
||||||
for _, dataS := range data {
|
for _, dataS := range data {
|
||||||
destPath := filepath.Join(dataDir, dataS.DestPath)
|
destPath := filepath.Join(dataDir, dataS.DestPath)
|
||||||
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
|
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
|
||||||
|
|
||||||
err := downloadFile(dataS.SourceURL, destPath)
|
err := downloadFile(dataS.SourceURL, destPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
|
@ -4,9 +4,8 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
|
|
||||||
"github.com/jarcoal/httpmock"
|
"github.com/jarcoal/httpmock"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDownloadFile(t *testing.T) {
|
func TestDownloadFile(t *testing.T) {
|
||||||
|
@ -26,6 +25,7 @@ func TestDownloadFile(t *testing.T) {
|
||||||
"https://example.com/x",
|
"https://example.com/x",
|
||||||
httpmock.NewStringResponder(404, "not found"),
|
httpmock.NewStringResponder(404, "not found"),
|
||||||
)
|
)
|
||||||
|
|
||||||
err := downloadFile("https://example.com/xx", examplePath)
|
err := downloadFile("https://example.com/xx", examplePath)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
content, err := os.ReadFile(examplePath)
|
content, err := os.ReadFile(examplePath)
|
||||||
|
|
|
@ -24,36 +24,45 @@ func UpdateHubIdx(hub *csconfig.Hub) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to download index: %w", err)
|
return fmt.Errorf("failed to download index: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ret, err := LoadPkgIndex(bidx)
|
ret, err := LoadPkgIndex(bidx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, ReferenceMissingError) {
|
if !errors.Is(err, ReferenceMissingError) {
|
||||||
return fmt.Errorf("failed to read index: %w", err)
|
return fmt.Errorf("failed to read index: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
hubIdx = ret
|
hubIdx = ret
|
||||||
|
|
||||||
if err, _ := LocalSync(hub); err != nil {
|
if err, _ := LocalSync(hub); err != nil {
|
||||||
return fmt.Errorf("failed to sync: %w", err)
|
return fmt.Errorf("failed to sync: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
|
func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
|
||||||
log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
|
log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
|
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to build request for hub index: %w", err)
|
return nil, fmt.Errorf("failed to build request for hub index: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
resp, err := http.DefaultClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed http request for hub index: %w", err)
|
return nil, fmt.Errorf("failed http request for hub index: %w", err)
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
if resp.StatusCode == http.StatusNotFound {
|
if resp.StatusCode == http.StatusNotFound {
|
||||||
return nil, ErrIndexNotFound
|
return nil, ErrIndexNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String())
|
return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
body, err := io.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
|
return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
|
||||||
|
@ -80,7 +89,9 @@ func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("while writing hub index file: %w", err)
|
return nil, fmt.Errorf("while writing hub index file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
|
log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
|
||||||
|
|
||||||
return body, nil
|
return body, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -89,11 +100,13 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
log.Debugf("Downloading %s %s", target.Type, target.Name)
|
log.Debugf("Downloading %s %s", target.Type, target.Name)
|
||||||
|
|
||||||
if target.Type != COLLECTIONS {
|
if target.Type != COLLECTIONS {
|
||||||
if !target.Installed && updateOnly && target.Downloaded {
|
if !target.Installed && updateOnly && target.Downloaded {
|
||||||
log.Debugf("skipping upgrade of %s : not installed", target.Name)
|
log.Debugf("skipping upgrade of %s : not installed", target.Name)
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return DownloadItem(hub, target, overwrite)
|
return DownloadItem(hub, target, overwrite)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -116,11 +129,13 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b
|
||||||
//recurse as it's a collection
|
//recurse as it's a collection
|
||||||
if ptrtype == COLLECTIONS {
|
if ptrtype == COLLECTIONS {
|
||||||
log.Tracef("collection, recurse")
|
log.Tracef("collection, recurse")
|
||||||
|
|
||||||
hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly)
|
hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return target, fmt.Errorf("while downloading %s: %w", val.Name, err)
|
return target, fmt.Errorf("while downloading %s: %w", val.Name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
item, err := DownloadItem(hub, val, overwrite)
|
item, err := DownloadItem(hub, val, overwrite)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return target, fmt.Errorf("while downloading %s: %w", val.Name, err)
|
return target, fmt.Errorf("while downloading %s: %w", val.Name, err)
|
||||||
|
@ -133,77 +148,95 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b
|
||||||
return target, fmt.Errorf("enabling '%s': %w", item.Name, err)
|
return target, fmt.Errorf("enabling '%s': %w", item.Name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
hubIdx[ptrtype][p] = item
|
hubIdx[ptrtype][p] = item
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
target, err = DownloadItem(hub, target, overwrite)
|
target, err = DownloadItem(hub, target, overwrite)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return target, fmt.Errorf("failed to download item : %s", err)
|
return target, fmt.Errorf("failed to download item : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) {
|
func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) {
|
||||||
var tdir = hub.HubDir
|
tdir := hub.HubDir
|
||||||
var dataFolder = hub.DataDir
|
dataFolder := hub.DataDir
|
||||||
/*if user didn't --force, don't overwrite local, tainted, up-to-date files*/
|
|
||||||
|
// if user didn't --force, don't overwrite local, tainted, up-to-date files
|
||||||
if !overwrite {
|
if !overwrite {
|
||||||
if target.Tainted {
|
if target.Tainted {
|
||||||
log.Debugf("%s : tainted, not updated", target.Name)
|
log.Debugf("%s : tainted, not updated", target.Name)
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if target.UpToDate {
|
if target.UpToDate {
|
||||||
log.Debugf("%s : up-to-date, not updated", target.Name)
|
|
||||||
// We still have to check if data files are present
|
// We still have to check if data files are present
|
||||||
|
log.Debugf("%s : up-to-date, not updated", target.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
|
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
|
return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
resp, err := http.DefaultClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
|
return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return target, fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
|
return target, fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
body, err := io.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return target, fmt.Errorf("while reading %s: %w", req.URL.String(), err)
|
return target, fmt.Errorf("while reading %s: %w", req.URL.String(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
h := sha256.New()
|
h := sha256.New()
|
||||||
if _, err := h.Write(body); err != nil {
|
if _, err := h.Write(body); err != nil {
|
||||||
return target, fmt.Errorf("while hashing %s: %w", target.Name, err)
|
return target, fmt.Errorf("while hashing %s: %w", target.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
meow := fmt.Sprintf("%x", h.Sum(nil))
|
meow := fmt.Sprintf("%x", h.Sum(nil))
|
||||||
if meow != target.Versions[target.Version].Digest {
|
if meow != target.Versions[target.Version].Digest {
|
||||||
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
|
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
|
||||||
log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
|
log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
|
||||||
|
|
||||||
return target, fmt.Errorf("invalid download hash for %s", target.Name)
|
return target, fmt.Errorf("invalid download hash for %s", target.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
//all good, install
|
//all good, install
|
||||||
//check if parent dir exists
|
//check if parent dir exists
|
||||||
tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/")
|
tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/")
|
||||||
parent_dir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
|
parent_dir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
|
||||||
|
|
||||||
/*ensure that target file is within target dir*/
|
// ensure that target file is within target dir
|
||||||
finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath)
|
finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return target, fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err)
|
return target, fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.HasPrefix(finalPath, tdir) {
|
if !strings.HasPrefix(finalPath, tdir) {
|
||||||
return target, fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir)
|
return target, fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir)
|
||||||
}
|
}
|
||||||
/*check dir*/
|
|
||||||
|
// check dir
|
||||||
if _, err = os.Stat(parent_dir); os.IsNotExist(err) {
|
if _, err = os.Stat(parent_dir); os.IsNotExist(err) {
|
||||||
log.Debugf("%s doesn't exist, create", parent_dir)
|
log.Debugf("%s doesn't exist, create", parent_dir)
|
||||||
|
|
||||||
if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil {
|
if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil {
|
||||||
return target, fmt.Errorf("while creating parent directories: %w", err)
|
return target, fmt.Errorf("while creating parent directories: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/*check actual file*/
|
|
||||||
|
// check actual file
|
||||||
if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
|
if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
|
||||||
log.Warningf("%s : overwrite", target.Name)
|
log.Warningf("%s : overwrite", target.Name)
|
||||||
log.Debugf("target: %s/%s", tdir, target.RemotePath)
|
log.Debugf("target: %s/%s", tdir, target.RemotePath)
|
||||||
|
@ -215,11 +248,14 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return target, fmt.Errorf("while opening file: %w", err)
|
return target, fmt.Errorf("while opening file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
|
|
||||||
_, err = f.Write(body)
|
_, err = f.Write(body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return target, fmt.Errorf("while writing file: %w", err)
|
return target, fmt.Errorf("while writing file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
target.Downloaded = true
|
target.Downloaded = true
|
||||||
target.Tainted = false
|
target.Tainted = false
|
||||||
target.UpToDate = true
|
target.UpToDate = true
|
||||||
|
@ -229,6 +265,7 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
hubIdx[target.Type][target.Name] = target
|
hubIdx[target.Type][target.Name] = target
|
||||||
|
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -238,37 +275,47 @@ func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
|
||||||
itemFile *os.File
|
itemFile *os.File
|
||||||
err error
|
err error
|
||||||
)
|
)
|
||||||
|
|
||||||
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.ConfigDir, target.Type, target.Stage, target.FileName)
|
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.ConfigDir, target.Type, target.Stage, target.FileName)
|
||||||
|
|
||||||
if itemFile, err = os.Open(itemFilePath); err != nil {
|
if itemFile, err = os.Open(itemFilePath); err != nil {
|
||||||
return fmt.Errorf("while opening %s: %w", itemFilePath, err)
|
return fmt.Errorf("while opening %s: %w", itemFilePath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer itemFile.Close()
|
defer itemFile.Close()
|
||||||
|
|
||||||
if err = downloadData(dataFolder, force, itemFile); err != nil {
|
if err = downloadData(dataFolder, force, itemFile); err != nil {
|
||||||
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
|
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func downloadData(dataFolder string, force bool, reader io.Reader) error {
|
func downloadData(dataFolder string, force bool, reader io.Reader) error {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
dec := yaml.NewDecoder(reader)
|
dec := yaml.NewDecoder(reader)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
data := &DataSet{}
|
data := &DataSet{}
|
||||||
|
|
||||||
err = dec.Decode(data)
|
err = dec.Decode(data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, io.EOF) {
|
if errors.Is(err, io.EOF) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf("while reading file: %w", err)
|
return fmt.Errorf("while reading file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
download := false
|
download := false
|
||||||
|
|
||||||
for _, dataS := range data.Data {
|
for _, dataS := range data.Data {
|
||||||
if _, err := os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
|
if _, err := os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
|
||||||
download = true
|
download = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if download || force {
|
if download || force {
|
||||||
err = GetData(data.Data, dataFolder)
|
err = GetData(data.Data, dataFolder)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -276,5 +323,6 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,38 +5,48 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDownloadHubIdx(t *testing.T) {
|
func TestDownloadHubIdx(t *testing.T) {
|
||||||
back := RawFileURLTemplate
|
back := RawFileURLTemplate
|
||||||
//bad url template
|
// bad url template
|
||||||
fmt.Println("Test 'bad URL'")
|
fmt.Println("Test 'bad URL'")
|
||||||
|
|
||||||
RawFileURLTemplate = "x"
|
RawFileURLTemplate = "x"
|
||||||
|
|
||||||
ret, err := DownloadHubIdx(&csconfig.Hub{})
|
ret, err := DownloadHubIdx(&csconfig.Hub{})
|
||||||
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
|
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
|
||||||
log.Errorf("unexpected error %s", err)
|
log.Errorf("unexpected error %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("->%+v", ret)
|
fmt.Printf("->%+v", ret)
|
||||||
|
|
||||||
//bad domain
|
// bad domain
|
||||||
fmt.Println("Test 'bad domain'")
|
fmt.Println("Test 'bad domain'")
|
||||||
|
|
||||||
RawFileURLTemplate = "https://baddomain/%s/%s"
|
RawFileURLTemplate = "https://baddomain/%s/%s"
|
||||||
|
|
||||||
ret, err = DownloadHubIdx(&csconfig.Hub{})
|
ret, err = DownloadHubIdx(&csconfig.Hub{})
|
||||||
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
|
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
|
||||||
log.Errorf("unexpected error %s", err)
|
log.Errorf("unexpected error %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("->%+v", ret)
|
fmt.Printf("->%+v", ret)
|
||||||
|
|
||||||
//bad target path
|
// bad target path
|
||||||
fmt.Println("Test 'bad target path'")
|
fmt.Println("Test 'bad target path'")
|
||||||
|
|
||||||
RawFileURLTemplate = back
|
RawFileURLTemplate = back
|
||||||
|
|
||||||
ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
|
ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
|
||||||
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
|
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
|
||||||
log.Errorf("unexpected error %s", err)
|
log.Errorf("unexpected error %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
RawFileURLTemplate = back
|
RawFileURLTemplate = back
|
||||||
|
|
||||||
fmt.Printf("->%+v", ret)
|
fmt.Printf("->%+v", ret)
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,7 +17,7 @@ func chooseHubBranch() (string, error) {
|
||||||
latest, err := cwversion.Latest()
|
latest, err := cwversion.Latest()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
|
log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
|
||||||
//lint:ignore nilerr reason
|
//lint:ignore nilerr
|
||||||
return "master", nil // ignore
|
return "master", nil // ignore
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -41,8 +41,10 @@ func chooseHubBranch() (string, error) {
|
||||||
log.Warnf("Crowdsec is not the latest version. "+
|
log.Warnf("Crowdsec is not the latest version. "+
|
||||||
"Current version is '%s' and the latest stable version is '%s'. Please update it!",
|
"Current version is '%s' and the latest stable version is '%s'. Please update it!",
|
||||||
csVersion, latest)
|
csVersion, latest)
|
||||||
|
|
||||||
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
|
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
|
||||||
"added to Crowdsec Hub after CrowdSec %s", latest)
|
"added to Crowdsec Hub after CrowdSec %s", latest)
|
||||||
|
|
||||||
return csVersion, nil
|
return csVersion, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -58,8 +60,10 @@ func SetHubBranch() error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
HubBranch = branch
|
HubBranch = branch
|
||||||
log.Debugf("Using branch '%s' for the hub", HubBranch)
|
log.Debugf("Using branch '%s' for the hub", HubBranch)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -72,6 +76,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo
|
||||||
item := *it
|
item := *it
|
||||||
if downloadOnly && item.Downloaded && item.UpToDate {
|
if downloadOnly && item.Downloaded && item.UpToDate {
|
||||||
log.Warningf("%s is already downloaded and up-to-date", item.Name)
|
log.Warningf("%s is already downloaded and up-to-date", item.Name)
|
||||||
|
|
||||||
if !force {
|
if !force {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -120,6 +125,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
|
||||||
|
|
||||||
item := *it
|
item := *it
|
||||||
item, err = DisableItem(csConfig.Hub, item, purge, forceAction)
|
item, err = DisableItem(csConfig.Hub, item, purge, forceAction)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("unable to disable %s : %v", item.Name, err)
|
log.Fatalf("unable to disable %s : %v", item.Name, err)
|
||||||
}
|
}
|
||||||
|
@ -127,6 +133,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
|
||||||
if err := AddItem(itemType, item); err != nil {
|
if err := AddItem(itemType, item); err != nil {
|
||||||
log.Fatalf("unable to add %s: %v", item.Name, err)
|
log.Fatalf("unable to add %s: %v", item.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -139,6 +146,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
|
||||||
if !v.Installed {
|
if !v.Installed {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
v, err = DisableItem(csConfig.Hub, v, purge, forceAction)
|
v, err = DisableItem(csConfig.Hub, v, purge, forceAction)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("unable to disable %s : %v", v.Name, err)
|
log.Fatalf("unable to disable %s : %v", v.Name, err)
|
||||||
|
@ -149,6 +157,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
|
||||||
}
|
}
|
||||||
disabled++
|
disabled++
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("Disabled %d items", disabled)
|
log.Infof("Disabled %d items", disabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,7 @@ import (
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
//Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
|
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
|
||||||
// We expect the new scenario to be installed
|
// We expect the new scenario to be installed
|
||||||
func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
|
func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
|
||||||
cfg := envSetup(t)
|
cfg := envSetup(t)
|
||||||
|
@ -37,6 +37,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
|
||||||
if err := UpdateHubIdx(cfg.Hub); err != nil {
|
if err := UpdateHubIdx(cfg.Hub); err != nil {
|
||||||
t.Fatalf("failed to download index : %s", err)
|
t.Fatalf("failed to download index : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
getHubIdxOrFail(t)
|
getHubIdxOrFail(t)
|
||||||
|
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
||||||
|
@ -49,7 +50,6 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
|
||||||
|
|
||||||
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
|
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
|
||||||
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
|
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Install a collection, disable a scenario.
|
// Install a collection, disable a scenario.
|
||||||
|
@ -140,6 +140,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
|
||||||
if err := UpdateHubIdx(cfg.Hub); err != nil {
|
if err := UpdateHubIdx(cfg.Hub); err != nil {
|
||||||
t.Fatalf("failed to download index : %s", err)
|
t.Fatalf("failed to download index : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
||||||
getHubIdxOrFail(t)
|
getHubIdxOrFail(t)
|
||||||
|
|
||||||
|
@ -151,6 +152,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
|
||||||
|
|
||||||
func assertCollectionDepsInstalled(t *testing.T, collection string) {
|
func assertCollectionDepsInstalled(t *testing.T, collection string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
c := hubIdx[COLLECTIONS][collection]
|
c := hubIdx[COLLECTIONS][collection]
|
||||||
require.NoError(t, CollecDepsCheck(&c))
|
require.NoError(t, CollecDepsCheck(&c))
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,15 +22,17 @@ func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
|
||||||
target.Downloaded = false
|
target.Downloaded = false
|
||||||
log.Infof("Removed source file [%s] : %s", target.Name, hubpath)
|
log.Infof("Removed source file [%s] : %s", target.Name, hubpath)
|
||||||
hubIdx[target.Type][target.Name] = target
|
hubIdx[target.Type][target.Name] = target
|
||||||
|
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
//DisableItem to disable an item managed by the hub, removes the symlink if purge is true
|
// DisableItem to disable an item managed by the hub, removes the symlink if purge is true
|
||||||
func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, error) {
|
func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, error) {
|
||||||
var tdir = hub.ConfigDir
|
|
||||||
var hdir = hub.HubDir
|
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
|
tdir := hub.ConfigDir
|
||||||
|
hdir := hub.HubDir
|
||||||
|
|
||||||
if !target.Installed {
|
if !target.Installed {
|
||||||
if purge {
|
if purge {
|
||||||
target, err = purgeItem(hub, target)
|
target, err = purgeItem(hub, target)
|
||||||
|
@ -38,6 +40,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
||||||
return target, err
|
return target, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -54,7 +57,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
||||||
return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
|
return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
/*for a COLLECTIONS, disable sub-items*/
|
// for a COLLECTIONS, disable sub-items
|
||||||
if target.Type == COLLECTIONS {
|
if target.Type == COLLECTIONS {
|
||||||
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
|
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
|
||||||
for idx, ptr := range tmp {
|
for idx, ptr := range tmp {
|
||||||
|
@ -63,12 +66,14 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
||||||
if val, ok := hubIdx[ptrtype][p]; ok {
|
if val, ok := hubIdx[ptrtype][p]; ok {
|
||||||
// check if the item doesn't belong to another collection before removing it
|
// check if the item doesn't belong to another collection before removing it
|
||||||
toRemove := true
|
toRemove := true
|
||||||
|
|
||||||
for _, collection := range val.BelongsToCollections {
|
for _, collection := range val.BelongsToCollections {
|
||||||
if collection != target.Name {
|
if collection != target.Name {
|
||||||
toRemove = false
|
toRemove = false
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if toRemove {
|
if toRemove {
|
||||||
hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
|
hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -114,6 +119,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
||||||
}
|
}
|
||||||
log.Infof("Removed symlink [%s] : %s", target.Name, syml)
|
log.Infof("Removed symlink [%s] : %s", target.Name, syml)
|
||||||
}
|
}
|
||||||
|
|
||||||
target.Installed = false
|
target.Installed = false
|
||||||
|
|
||||||
if purge {
|
if purge {
|
||||||
|
@ -122,39 +128,46 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
||||||
return target, err
|
return target, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
hubIdx[target.Type][target.Name] = target
|
hubIdx[target.Type][target.Name] = target
|
||||||
|
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// creates symlink between actual config file at hub.HubDir and hub.ConfigDir
|
// creates symlink between actual config file at hub.HubDir and hub.ConfigDir
|
||||||
// Handles collections recursively
|
// Handles collections recursively
|
||||||
func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
|
func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
|
||||||
var tdir = hub.ConfigDir
|
|
||||||
var hdir = hub.HubDir
|
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
|
tdir := hub.ConfigDir
|
||||||
|
hdir := hub.HubDir
|
||||||
|
|
||||||
parent_dir := filepath.Clean(tdir + "/" + target.Type + "/" + target.Stage + "/")
|
parent_dir := filepath.Clean(tdir + "/" + target.Type + "/" + target.Stage + "/")
|
||||||
/*create directories if needed*/
|
// create directories if needed
|
||||||
if target.Installed {
|
if target.Installed {
|
||||||
if target.Tainted {
|
if target.Tainted {
|
||||||
return target, fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
|
return target, fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if target.Local {
|
if target.Local {
|
||||||
return target, fmt.Errorf("%s is local, won't enable", target.Name)
|
return target, fmt.Errorf("%s is local, won't enable", target.Name)
|
||||||
}
|
}
|
||||||
/* if it's a collection, check sub-items even if the collection file itself is up-to-date */
|
// if it's a collection, check sub-items even if the collection file itself is up-to-date
|
||||||
if target.UpToDate && target.Type != COLLECTIONS {
|
if target.UpToDate && target.Type != COLLECTIONS {
|
||||||
log.Tracef("%s is installed and up-to-date, skip.", target.Name)
|
log.Tracef("%s is installed and up-to-date, skip.", target.Name)
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := os.Stat(parent_dir); os.IsNotExist(err) {
|
if _, err := os.Stat(parent_dir); os.IsNotExist(err) {
|
||||||
log.Printf("%s doesn't exist, create", parent_dir)
|
log.Printf("%s doesn't exist, create", parent_dir)
|
||||||
|
|
||||||
if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil {
|
if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil {
|
||||||
return target, fmt.Errorf("while creating directory: %w", err)
|
return target, fmt.Errorf("while creating directory: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*install sub-items if it's a collection*/
|
// install sub-items if it's a collection
|
||||||
if target.Type == COLLECTIONS {
|
if target.Type == COLLECTIONS {
|
||||||
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
|
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
|
||||||
for idx, ptr := range tmp {
|
for idx, ptr := range tmp {
|
||||||
|
@ -179,7 +192,7 @@ func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
//tdir+target.RemotePath
|
// tdir+target.RemotePath
|
||||||
srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath)
|
srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return target, fmt.Errorf("while getting source path: %w", err)
|
return target, fmt.Errorf("while getting source path: %w", err)
|
||||||
|
@ -197,5 +210,6 @@ func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
|
||||||
log.Printf("Enabled %s : %s", target.Type, target.Name)
|
log.Printf("Enabled %s : %s", target.Type, target.Name)
|
||||||
target.Installed = true
|
target.Installed = true
|
||||||
hubIdx[target.Type][target.Name] = target
|
hubIdx[target.Type][target.Name] = target
|
||||||
|
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,19 +15,20 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
)
|
)
|
||||||
|
|
||||||
/*the walk/parser_visit function can't receive extra args*/
|
// the walk/parser_visit function can't receive extra args
|
||||||
var hubdir, installdir string
|
var hubdir, installdir string
|
||||||
|
|
||||||
func parser_visit(path string, f os.DirEntry, err error) error {
|
func parser_visit(path string, f os.DirEntry, err error) error {
|
||||||
|
var (
|
||||||
var target Item
|
target Item
|
||||||
var local bool
|
local bool
|
||||||
var hubpath string
|
hubpath string
|
||||||
var inhub bool
|
inhub bool
|
||||||
var fname string
|
fname string
|
||||||
var ftype string
|
ftype string
|
||||||
var fauthor string
|
fauthor string
|
||||||
var stage string
|
stage string
|
||||||
|
)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debugf("while syncing hub dir: %s", err)
|
log.Debugf("while syncing hub dir: %s", err)
|
||||||
|
@ -39,11 +40,11 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
//we only care about files
|
// we only care about files
|
||||||
if f == nil || f.IsDir() {
|
if f == nil || f.IsDir() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
//we only care about yaml files
|
// we only care about yaml files
|
||||||
if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") {
|
if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -52,9 +53,10 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
||||||
|
|
||||||
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubdir, installdir)
|
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubdir, installdir)
|
||||||
log.Tracef("subs:%v", subs)
|
log.Tracef("subs:%v", subs)
|
||||||
/*we're in hub (~/.hub/hub/)*/
|
// we're in hub (~/.hub/hub/)
|
||||||
if strings.HasPrefix(path, hubdir) {
|
if strings.HasPrefix(path, hubdir) {
|
||||||
log.Tracef("in hub dir")
|
log.Tracef("in hub dir")
|
||||||
|
|
||||||
inhub = true
|
inhub = true
|
||||||
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
|
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
|
||||||
//.../hub/scenarios/crowdsec/ssh_bf.yaml
|
//.../hub/scenarios/crowdsec/ssh_bf.yaml
|
||||||
|
@ -62,11 +64,12 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
||||||
if len(subs) < 4 {
|
if len(subs) < 4 {
|
||||||
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
||||||
}
|
}
|
||||||
|
|
||||||
fname = subs[len(subs)-1]
|
fname = subs[len(subs)-1]
|
||||||
fauthor = subs[len(subs)-2]
|
fauthor = subs[len(subs)-2]
|
||||||
stage = subs[len(subs)-3]
|
stage = subs[len(subs)-3]
|
||||||
ftype = subs[len(subs)-4]
|
ftype = subs[len(subs)-4]
|
||||||
} else if strings.HasPrefix(path, installdir) { /*we're in install /etc/crowdsec/<type>/... */
|
} else if strings.HasPrefix(path, installdir) { // we're in install /etc/crowdsec/<type>/...
|
||||||
log.Tracef("in install dir")
|
log.Tracef("in install dir")
|
||||||
if len(subs) < 3 {
|
if len(subs) < 3 {
|
||||||
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
||||||
|
@ -84,14 +87,15 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Tracef("stage:%s ftype:%s", stage, ftype)
|
log.Tracef("stage:%s ftype:%s", stage, ftype)
|
||||||
//log.Printf("%s -> name:%s stage:%s", path, fname, stage)
|
// log.Printf("%s -> name:%s stage:%s", path, fname, stage)
|
||||||
if stage == SCENARIOS {
|
if stage == SCENARIOS {
|
||||||
ftype = SCENARIOS
|
ftype = SCENARIOS
|
||||||
stage = ""
|
stage = ""
|
||||||
} else if stage == COLLECTIONS {
|
} else if stage == COLLECTIONS {
|
||||||
ftype = COLLECTIONS
|
ftype = COLLECTIONS
|
||||||
stage = ""
|
stage = ""
|
||||||
} else if ftype != PARSERS && ftype != PARSERS_OVFLW /*its a PARSER / PARSER_OVFLW with a stage */ {
|
} else if ftype != PARSERS && ftype != PARSERS_OVFLW {
|
||||||
|
// its a PARSER / PARSER_OVFLW with a stage
|
||||||
return fmt.Errorf("unknown configuration type for file '%s'", path)
|
return fmt.Errorf("unknown configuration type for file '%s'", path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -102,20 +106,21 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
||||||
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
|
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
|
||||||
when the collection is installed, both files are created
|
when the collection is installed, both files are created
|
||||||
*/
|
*/
|
||||||
//non symlinks are local user files or hub files
|
// non symlinks are local user files or hub files
|
||||||
if f.Type()&os.ModeSymlink == 0 {
|
if f.Type()&os.ModeSymlink == 0 {
|
||||||
local = true
|
local = true
|
||||||
|
|
||||||
log.Tracef("%s isn't a symlink", path)
|
log.Tracef("%s isn't a symlink", path)
|
||||||
} else {
|
} else {
|
||||||
hubpath, err = os.Readlink(path)
|
hubpath, err = os.Readlink(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to read symlink of %s", path)
|
return fmt.Errorf("unable to read symlink of %s", path)
|
||||||
}
|
}
|
||||||
//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
|
// the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
|
||||||
_, err := os.Lstat(hubpath)
|
_, err := os.Lstat(hubpath)
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
|
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
|
||||||
//remove the symlink
|
// remove the symlink
|
||||||
if err = os.Remove(path); err != nil {
|
if err = os.Remove(path); err != nil {
|
||||||
return fmt.Errorf("failed to unlink %s: %+v", path, err)
|
return fmt.Errorf("failed to unlink %s: %+v", path, err)
|
||||||
}
|
}
|
||||||
|
@ -124,7 +129,7 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
||||||
log.Tracef("%s points to %s", path, hubpath)
|
log.Tracef("%s points to %s", path, hubpath)
|
||||||
}
|
}
|
||||||
|
|
||||||
//if it's not a symlink and not in hub, it's a local file, don't bother
|
// if it's not a symlink and not in hub, it's a local file, don't bother
|
||||||
if local && !inhub {
|
if local && !inhub {
|
||||||
log.Tracef("%s is a local file, skip", path)
|
log.Tracef("%s is a local file, skip", path)
|
||||||
skippedLocal++
|
skippedLocal++
|
||||||
|
@ -139,29 +144,34 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
||||||
_, target.FileName = filepath.Split(path)
|
_, target.FileName = filepath.Split(path)
|
||||||
|
|
||||||
hubIdx[ftype][fname] = target
|
hubIdx[ftype][fname] = target
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
//try to find which configuration item it is
|
// try to find which configuration item it is
|
||||||
log.Tracef("check [%s] of %s", fname, ftype)
|
log.Tracef("check [%s] of %s", fname, ftype)
|
||||||
|
|
||||||
match := false
|
match := false
|
||||||
|
|
||||||
for k, v := range hubIdx[ftype] {
|
for k, v := range hubIdx[ftype] {
|
||||||
log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml")
|
log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml")
|
||||||
|
|
||||||
if fname != v.FileName {
|
if fname != v.FileName {
|
||||||
log.Tracef("%s != %s (filename)", fname, v.FileName)
|
log.Tracef("%s != %s (filename)", fname, v.FileName)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
//wrong stage
|
|
||||||
|
// wrong stage
|
||||||
if v.Stage != stage {
|
if v.Stage != stage {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
/*if we are walking hub dir, just mark present files as downloaded*/
|
|
||||||
|
// if we are walking hub dir, just mark present files as downloaded
|
||||||
if inhub {
|
if inhub {
|
||||||
//wrong author
|
// wrong author
|
||||||
if fauthor != v.Author {
|
if fauthor != v.Author {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
//wrong file
|
// wrong file
|
||||||
if CheckName(v.Name, fauthor, fname) {
|
if CheckName(v.Name, fauthor, fname) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -171,35 +181,38 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
||||||
v.Downloaded = true
|
v.Downloaded = true
|
||||||
}
|
}
|
||||||
} else if CheckSuffix(hubpath, v.RemotePath) {
|
} else if CheckSuffix(hubpath, v.RemotePath) {
|
||||||
//wrong file
|
// wrong file
|
||||||
//<type>/<stage>/<author>/<name>.yaml
|
// <type>/<stage>/<author>/<name>.yaml
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
sha, err := getSHA256(path)
|
sha, err := getSHA256(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Failed to get sha of %s : %v", path, err)
|
log.Fatalf("Failed to get sha of %s : %v", path, err)
|
||||||
}
|
}
|
||||||
//let's reverse sort the versions to deal with hash collisions (#154)
|
|
||||||
|
// let's reverse sort the versions to deal with hash collisions (#154)
|
||||||
versions := make([]string, 0, len(v.Versions))
|
versions := make([]string, 0, len(v.Versions))
|
||||||
for k := range v.Versions {
|
for k := range v.Versions {
|
||||||
versions = append(versions, k)
|
versions = append(versions, k)
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
|
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
|
||||||
|
|
||||||
for _, version := range versions {
|
for _, version := range versions {
|
||||||
val := v.Versions[version]
|
val := v.Versions[version]
|
||||||
if sha != val.Digest {
|
if sha != val.Digest {
|
||||||
//log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
|
// log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
/*we got an exact match, update struct*/
|
// we got an exact match, update struct
|
||||||
if !inhub {
|
if !inhub {
|
||||||
log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version)
|
log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version)
|
||||||
v.LocalPath = path
|
v.LocalPath = path
|
||||||
v.LocalVersion = version
|
v.LocalVersion = version
|
||||||
v.Tainted = false
|
v.Tainted = false
|
||||||
v.Downloaded = true
|
v.Downloaded = true
|
||||||
/*if we're walking the hub, present file doesn't means installed file*/
|
// if we're walking the hub, present file doesn't means installed file
|
||||||
v.Installed = true
|
v.Installed = true
|
||||||
v.LocalHash = sha
|
v.LocalHash = sha
|
||||||
_, target.FileName = filepath.Split(path)
|
_, target.FileName = filepath.Split(path)
|
||||||
|
@ -207,29 +220,34 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
||||||
v.Downloaded = true
|
v.Downloaded = true
|
||||||
v.LocalHash = sha
|
v.LocalHash = sha
|
||||||
}
|
}
|
||||||
|
|
||||||
if version == v.Version {
|
if version == v.Version {
|
||||||
log.Tracef("%s is up-to-date", v.Name)
|
log.Tracef("%s is up-to-date", v.Name)
|
||||||
v.UpToDate = true
|
v.UpToDate = true
|
||||||
}
|
}
|
||||||
|
|
||||||
match = true
|
match = true
|
||||||
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
if !match {
|
if !match {
|
||||||
log.Tracef("got tainted match for %s : %s", v.Name, path)
|
log.Tracef("got tainted match for %s : %s", v.Name, path)
|
||||||
|
|
||||||
skippedTainted += 1
|
skippedTainted += 1
|
||||||
//the file and the stage is right, but the hash is wrong, it has been tainted by user
|
// the file and the stage is right, but the hash is wrong, it has been tainted by user
|
||||||
if !inhub {
|
if !inhub {
|
||||||
v.LocalPath = path
|
v.LocalPath = path
|
||||||
v.Installed = true
|
v.Installed = true
|
||||||
}
|
}
|
||||||
|
|
||||||
v.UpToDate = false
|
v.UpToDate = false
|
||||||
v.LocalVersion = "?"
|
v.LocalVersion = "?"
|
||||||
v.Tainted = true
|
v.Tainted = true
|
||||||
v.LocalHash = sha
|
v.LocalHash = sha
|
||||||
_, target.FileName = filepath.Split(path)
|
_, target.FileName = filepath.Split(path)
|
||||||
|
|
||||||
}
|
}
|
||||||
//update the entry if appropriate
|
// update the entry if appropriate
|
||||||
// if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D {
|
// if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D {
|
||||||
// fmt.Printf("Updating %s", k)
|
// fmt.Printf("Updating %s", k)
|
||||||
// hubIdx[ftype][k] = v
|
// hubIdx[ftype][k] = v
|
||||||
|
@ -237,22 +255,25 @@ func parser_visit(path string, f os.DirEntry, err error) error {
|
||||||
|
|
||||||
// } else if
|
// } else if
|
||||||
hubIdx[ftype][k] = v
|
hubIdx[ftype][k] = v
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("Ignoring file %s of type %s", path, ftype)
|
log.Infof("Ignoring file %s of type %s", path, ftype)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func CollecDepsCheck(v *Item) error {
|
func CollecDepsCheck(v *Item) error {
|
||||||
|
if GetVersionStatus(v) != 0 { // not up-to-date
|
||||||
if GetVersionStatus(v) != 0 { //not up-to-date
|
|
||||||
log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
|
log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
/*if it's a collection, ensure all the items are installed, or tag it as tainted*/
|
// if it's a collection, ensure all the items are installed, or tag it as tainted
|
||||||
if v.Type == COLLECTIONS {
|
if v.Type == COLLECTIONS {
|
||||||
log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
|
log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
|
||||||
|
|
||||||
var tmp = [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections}
|
var tmp = [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections}
|
||||||
for idx, ptr := range tmp {
|
for idx, ptr := range tmp {
|
||||||
ptrtype := ItemTypes[idx]
|
ptrtype := ItemTypes[idx]
|
||||||
|
@ -261,48 +282,62 @@ func CollecDepsCheck(v *Item) error {
|
||||||
if !ok {
|
if !ok {
|
||||||
log.Fatalf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, v.Name)
|
log.Fatalf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, v.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Tracef("check %s installed:%t", val.Name, val.Installed)
|
log.Tracef("check %s installed:%t", val.Name, val.Installed)
|
||||||
|
|
||||||
if !v.Installed {
|
if !v.Installed {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if val.Type == COLLECTIONS {
|
if val.Type == COLLECTIONS {
|
||||||
log.Tracef("collec, recurse.")
|
log.Tracef("collec, recurse.")
|
||||||
|
|
||||||
if err := CollecDepsCheck(&val); err != nil {
|
if err := CollecDepsCheck(&val); err != nil {
|
||||||
if val.Tainted {
|
if val.Tainted {
|
||||||
v.Tainted = true
|
v.Tainted = true
|
||||||
}
|
}
|
||||||
return fmt.Errorf("sub collection %s is broken : %s", val.Name, err)
|
|
||||||
|
return fmt.Errorf("sub collection %s is broken: %s", val.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
hubIdx[ptrtype][p] = val
|
hubIdx[ptrtype][p] = val
|
||||||
}
|
}
|
||||||
|
|
||||||
//propagate the state of sub-items to set
|
// propagate the state of sub-items to set
|
||||||
if val.Tainted {
|
if val.Tainted {
|
||||||
v.Tainted = true
|
v.Tainted = true
|
||||||
return fmt.Errorf("tainted %s %s, tainted.", ptrtype, p)
|
return fmt.Errorf("tainted %s %s, tainted", ptrtype, p)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !val.Installed && v.Installed {
|
if !val.Installed && v.Installed {
|
||||||
v.Tainted = true
|
v.Tainted = true
|
||||||
return fmt.Errorf("missing %s %s, tainted.", ptrtype, p)
|
return fmt.Errorf("missing %s %s, tainted", ptrtype, p)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !val.UpToDate {
|
if !val.UpToDate {
|
||||||
v.UpToDate = false
|
v.UpToDate = false
|
||||||
return fmt.Errorf("outdated %s %s", ptrtype, p)
|
return fmt.Errorf("outdated %s %s", ptrtype, p)
|
||||||
}
|
}
|
||||||
|
|
||||||
skip := false
|
skip := false
|
||||||
|
|
||||||
for idx := range val.BelongsToCollections {
|
for idx := range val.BelongsToCollections {
|
||||||
if val.BelongsToCollections[idx] == v.Name {
|
if val.BelongsToCollections[idx] == v.Name {
|
||||||
skip = true
|
skip = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !skip {
|
if !skip {
|
||||||
val.BelongsToCollections = append(val.BelongsToCollections, v.Name)
|
val.BelongsToCollections = append(val.BelongsToCollections, v.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
hubIdx[ptrtype][p] = val
|
hubIdx[ptrtype][p] = val
|
||||||
|
|
||||||
log.Tracef("checking for %s - tainted:%t uptodate:%t", p, v.Tainted, v.UpToDate)
|
log.Tracef("checking for %s - tainted:%t uptodate:%t", p, v.Tainted, v.UpToDate)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -311,39 +346,41 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
|
||||||
installdir = hub.ConfigDir
|
installdir = hub.ConfigDir
|
||||||
warnings := []string{}
|
warnings := []string{}
|
||||||
|
|
||||||
/*For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last*/
|
// For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last
|
||||||
for _, scan := range ItemTypes {
|
for _, scan := range ItemTypes {
|
||||||
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
|
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("failed %s : %s", cpath, err)
|
log.Errorf("failed %s : %s", cpath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = filepath.WalkDir(cpath, parser_visit)
|
err = filepath.WalkDir(cpath, parser_visit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err, warnings
|
return err, warnings
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for k, v := range hubIdx[COLLECTIONS] {
|
for k, v := range hubIdx[COLLECTIONS] {
|
||||||
if v.Installed {
|
if v.Installed {
|
||||||
versStat := GetVersionStatus(&v)
|
versStat := GetVersionStatus(&v)
|
||||||
if versStat == 0 { //latest
|
if versStat == 0 { // latest
|
||||||
if err := CollecDepsCheck(&v); err != nil {
|
if err := CollecDepsCheck(&v); err != nil {
|
||||||
warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", v.Name, err))
|
warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", v.Name, err))
|
||||||
hubIdx[COLLECTIONS][k] = v
|
hubIdx[COLLECTIONS][k] = v
|
||||||
}
|
}
|
||||||
} else if versStat == 1 { //not up-to-date
|
} else if versStat == 1 { // not up-to-date
|
||||||
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version))
|
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version))
|
||||||
} else { //version is higher than the highest available from hub?
|
} else { // version is higher than the highest available from hub?
|
||||||
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version))
|
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version))
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", v.Name, semver.Compare("v"+v.Version, "v"+v.LocalVersion), v.LocalVersion, v.Version, v.Versions)
|
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", v.Name, semver.Compare("v"+v.Version, "v"+v.LocalVersion), v.LocalVersion, v.Version, v.Versions)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, warnings
|
return nil, warnings
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Updates the infos from HubInit() with the local state */
|
// Updates the infos from HubInit() with the local state
|
||||||
func LocalSync(hub *csconfig.Hub) (error, []string) {
|
func LocalSync(hub *csconfig.Hub) (error, []string) {
|
||||||
skippedLocal = 0
|
skippedLocal = 0
|
||||||
skippedTainted = 0
|
skippedTainted = 0
|
||||||
|
@ -352,10 +389,12 @@ func LocalSync(hub *csconfig.Hub) (error, []string) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to scan %s : %s", hub.ConfigDir, err), warnings
|
return fmt.Errorf("failed to scan %s : %s", hub.ConfigDir, err), warnings
|
||||||
}
|
}
|
||||||
|
|
||||||
err, _ = SyncDir(hub, hub.HubDir)
|
err, _ = SyncDir(hub, hub.HubDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings
|
return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, warnings
|
return nil, warnings
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -363,49 +402,61 @@ func GetHubIdx(hub *csconfig.Hub) error {
|
||||||
if hub == nil {
|
if hub == nil {
|
||||||
return fmt.Errorf("no configuration found for hub")
|
return fmt.Errorf("no configuration found for hub")
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debugf("loading hub idx %s", hub.HubIndexFile)
|
log.Debugf("loading hub idx %s", hub.HubIndexFile)
|
||||||
|
|
||||||
bidx, err := os.ReadFile(hub.HubIndexFile)
|
bidx, err := os.ReadFile(hub.HubIndexFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to read index file: %w", err)
|
return fmt.Errorf("unable to read index file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ret, err := LoadPkgIndex(bidx)
|
ret, err := LoadPkgIndex(bidx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, ReferenceMissingError) {
|
if !errors.Is(err, ReferenceMissingError) {
|
||||||
log.Fatalf("Unable to load existing index : %v.", err)
|
log.Fatalf("Unable to load existing index : %v.", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
hubIdx = ret
|
hubIdx = ret
|
||||||
|
|
||||||
err, _ = LocalSync(hub)
|
err, _ = LocalSync(hub)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Failed to sync Hub index with local deployment : %v", err)
|
log.Fatalf("Failed to sync Hub index with local deployment : %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
/*LoadPkgIndex loads a local .index.json file and returns the map of parsers/scenarios/collections associated*/
|
// LoadPkgIndex loads a local .index.json file and returns the map of parsers/scenarios/collections associated
|
||||||
func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
|
func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
|
||||||
var err error
|
var (
|
||||||
var RawIndex map[string]map[string]Item
|
err error
|
||||||
var missingItems []string
|
RawIndex map[string]map[string]Item
|
||||||
|
missingItems []string
|
||||||
|
)
|
||||||
|
|
||||||
if err = json.Unmarshal(buff, &RawIndex); err != nil {
|
if err = json.Unmarshal(buff, &RawIndex); err != nil {
|
||||||
return nil, fmt.Errorf("failed to unmarshal index : %v", err)
|
return nil, fmt.Errorf("failed to unmarshal index : %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debugf("%d item types in hub index", len(ItemTypes))
|
log.Debugf("%d item types in hub index", len(ItemTypes))
|
||||||
/*Iterate over the different types to complete struct */
|
|
||||||
|
// Iterate over the different types to complete struct
|
||||||
for _, itemType := range ItemTypes {
|
for _, itemType := range ItemTypes {
|
||||||
/*complete struct*/
|
// complete struct
|
||||||
log.Tracef("%d item", len(RawIndex[itemType]))
|
log.Tracef("%d item", len(RawIndex[itemType]))
|
||||||
|
|
||||||
for idx, item := range RawIndex[itemType] {
|
for idx, item := range RawIndex[itemType] {
|
||||||
item.Name = idx
|
item.Name = idx
|
||||||
item.Type = itemType
|
item.Type = itemType
|
||||||
x := strings.Split(item.RemotePath, "/")
|
x := strings.Split(item.RemotePath, "/")
|
||||||
item.FileName = x[len(x)-1]
|
item.FileName = x[len(x)-1]
|
||||||
RawIndex[itemType][idx] = item
|
RawIndex[itemType][idx] = item
|
||||||
/*if it's a collection, check its sub-items are present*/
|
|
||||||
//XX should be done later
|
// if it's a collection, check its sub-items are present
|
||||||
|
// XXX should be done later
|
||||||
if itemType == COLLECTIONS {
|
if itemType == COLLECTIONS {
|
||||||
var tmp = [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections}
|
var tmp = [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections}
|
||||||
for idx, ptr := range tmp {
|
for idx, ptr := range tmp {
|
||||||
|
@ -420,8 +471,9 @@ func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(missingItems) > 0 {
|
if len(missingItems) > 0 {
|
||||||
return RawIndex, fmt.Errorf("%q : %w", missingItems, ReferenceMissingError)
|
return RawIndex, fmt.Errorf("%q: %w", missingItems, ReferenceMissingError)
|
||||||
}
|
}
|
||||||
|
|
||||||
return RawIndex, nil
|
return RawIndex, nil
|
||||||
|
|
Loading…
Reference in a new issue