lint: pkg/cwhub (#2510)

no functional changes
 
 - reformat
 - comments
 - whitespace
 - removed a dot or two in log messages
 - some "var x=y" -> x:=y
This commit is contained in:
mmetc 2023-10-03 11:20:56 +02:00 committed by GitHub
parent 6dadfcb2ef
commit 8b5ad6990d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 360 additions and 157 deletions

View file

@ -15,7 +15,7 @@ import (
"golang.org/x/mod/semver" "golang.org/x/mod/semver"
) )
/*managed configuration types*/ // managed configuration types
var PARSERS = "parsers" var PARSERS = "parsers"
var PARSERS_OVFLW = "postoverflows" var PARSERS_OVFLW = "postoverflows"
var SCENARIOS = "scenarios" var SCENARIOS = "scenarios"
@ -42,37 +42,37 @@ type ItemHubStatus struct {
Status string `json:"status"` Status string `json:"status"`
} }
//Item can be : parsed, scenario, collection // Item can be: parsed, scenario, collection
type Item struct { type Item struct {
/*descriptive info*/ // descriptive info
Type string `yaml:"type,omitempty" json:"type,omitempty"` //parser|postoverflows|scenario|collection(|enrich) Type string `yaml:"type,omitempty" json:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich)
Stage string `json:"stage,omitempty" yaml:"stage,omitempty,omitempty"` //Stage for parser|postoverflow : s00-raw/s01-... Stage string `json:"stage,omitempty" yaml:"stage,omitempty,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
Name string `json:"name,omitempty"` //as seen in .config.json, usually "author/name" Name string `json:"name,omitempty"` // as seen in .config.json, usually "author/name"
FileName string `json:"file_name,omitempty"` //the filename, ie. apache2-logs.yaml FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml
Description string `yaml:"description,omitempty" json:"description,omitempty"` //as seen in .config.json Description string `yaml:"description,omitempty" json:"description,omitempty"` // as seen in .config.json
Author string `json:"author,omitempty"` //as seen in .config.json Author string `json:"author,omitempty"` // as seen in .config.json
References []string `yaml:"references,omitempty" json:"references,omitempty"` //as seen in .config.json References []string `yaml:"references,omitempty" json:"references,omitempty"` // as seen in .config.json
BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` /*if it's part of collections, track name here*/ BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` // if it's part of collections, track name here
/*remote (hub) infos*/ // remote (hub) infos
RemoteURL string `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` //the full remote uri of file in http RemoteURL string `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` // the full remote uri of file in http
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` //the path relative to git ie. /parsers/stage/author/file.yaml RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to git ie. /parsers/stage/author/file.yaml
RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` //the meow RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` // the meow
Version string `json:"version,omitempty"` //the last version Version string `json:"version,omitempty"` // the last version
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` //the list of existing versions Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions
/*local (deployed) infos*/ // local (deployed) infos
LocalPath string `yaml:"local_path,omitempty" json:"local_path,omitempty"` //the local path relative to ${CFG_DIR} LocalPath string `yaml:"local_path,omitempty" json:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
//LocalHubPath string // LocalHubPath string
LocalVersion string `json:"local_version,omitempty"` LocalVersion string `json:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty"` //the local meow LocalHash string `json:"local_hash,omitempty"` // the local meow
Installed bool `json:"installed,omitempty"` Installed bool `json:"installed,omitempty"`
Downloaded bool `json:"downloaded,omitempty"` Downloaded bool `json:"downloaded,omitempty"`
UpToDate bool `json:"up_to_date,omitempty"` UpToDate bool `json:"up_to_date,omitempty"`
Tainted bool `json:"tainted,omitempty"` //has it been locally modified Tainted bool `json:"tainted,omitempty"` // has it been locally modified
Local bool `json:"local,omitempty"` //if it's a non versioned control one Local bool `json:"local,omitempty"` // if it's a non versioned control one
/*if it's a collection, it not a single file*/ // if it's a collection, it not a single file
Parsers []string `yaml:"parsers,omitempty" json:"parsers,omitempty"` Parsers []string `yaml:"parsers,omitempty" json:"parsers,omitempty"`
PostOverflows []string `yaml:"postoverflows,omitempty" json:"postoverflows,omitempty"` PostOverflows []string `yaml:"postoverflows,omitempty" json:"postoverflows,omitempty"`
Scenarios []string `yaml:"scenarios,omitempty" json:"scenarios,omitempty"` Scenarios []string `yaml:"scenarios,omitempty" json:"scenarios,omitempty"`
@ -88,6 +88,7 @@ func (i *Item) toHubStatus() ItemHubStatus {
status, ok, warning, managed := ItemStatus(*i) status, ok, warning, managed := ItemStatus(*i)
hubStatus.Status = status hubStatus.Status = status
if !managed { if !managed {
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.House, status) hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.House, status)
} else if !i.Installed { } else if !i.Installed {
@ -97,27 +98,28 @@ func (i *Item) toHubStatus() ItemHubStatus {
} else if ok { } else if ok {
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.CheckMark, status) hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.CheckMark, status)
} }
return hubStatus return hubStatus
} }
var skippedLocal = 0 var skippedLocal = 0
var skippedTainted = 0 var skippedTainted = 0
/*To be used when reference(s) (is/are) missing in a collection*/ // To be used when reference(s) (is/are) missing in a collection
var ReferenceMissingError = errors.New("Reference(s) missing in collection") var ReferenceMissingError = errors.New("Reference(s) missing in collection")
var MissingHubIndex = errors.New("hub index can't be found") var MissingHubIndex = errors.New("hub index can't be found")
//GetVersionStatus : semver requires 'v' prefix // GetVersionStatus: semver requires 'v' prefix
func GetVersionStatus(v *Item) int { func GetVersionStatus(v *Item) int {
return semver.Compare("v"+v.Version, "v"+v.LocalVersion) return semver.Compare("v"+v.Version, "v"+v.LocalVersion)
} }
// calculate sha256 of a file // calculate sha256 of a file
func getSHA256(filepath string) (string, error) { func getSHA256(filepath string) (string, error) {
/* Digest of file */ // Digest of file
f, err := os.Open(filepath) f, err := os.Open(filepath)
if err != nil { if err != nil {
return "", fmt.Errorf("unable to open '%s' : %s", filepath, err) return "", fmt.Errorf("unable to open '%s': %s", filepath, err)
} }
defer f.Close() defer f.Close()
@ -131,49 +133,55 @@ func getSHA256(filepath string) (string, error) {
} }
func GetItemMap(itemType string) map[string]Item { func GetItemMap(itemType string) map[string]Item {
var m map[string]Item var (
var ok bool m map[string]Item
ok bool
)
if m, ok = hubIdx[itemType]; !ok { if m, ok = hubIdx[itemType]; !ok {
return nil return nil
} }
return m return m
} }
//GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. // GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
func GetItemByPath(itemType string, itemPath string) (*Item, error) { func GetItemByPath(itemType string, itemPath string) (*Item, error) {
/*try to resolve symlink*/ // try to resolve symlink
finalName := "" finalName := ""
f, err := os.Lstat(itemPath) f, err := os.Lstat(itemPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("while performing lstat on %s: %w", itemPath, err) return nil, fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
} }
if f.Mode()&os.ModeSymlink == 0 { if f.Mode()&os.ModeSymlink == 0 {
/*it's not a symlink, it should be the filename itsef the key*/ // it's not a symlink, it should be the filename itsef the key
finalName = filepath.Base(itemPath) finalName = filepath.Base(itemPath)
} else { } else {
/*resolve the symlink to hub file*/ // resolve the symlink to hub file
pathInHub, err := os.Readlink(itemPath) pathInHub, err := os.Readlink(itemPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("while reading symlink of %s: %w", itemPath, err) return nil, fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
} }
//extract author from path // extract author from path
fname := filepath.Base(pathInHub) fname := filepath.Base(pathInHub)
author := filepath.Base(filepath.Dir(pathInHub)) author := filepath.Base(filepath.Dir(pathInHub))
//trim yaml suffix // trim yaml suffix
fname = strings.TrimSuffix(fname, ".yaml") fname = strings.TrimSuffix(fname, ".yaml")
fname = strings.TrimSuffix(fname, ".yml") fname = strings.TrimSuffix(fname, ".yml")
finalName = fmt.Sprintf("%s/%s", author, fname) finalName = fmt.Sprintf("%s/%s", author, fname)
} }
/*it's not a symlink, it should be the filename itsef the key*/ // it's not a symlink, it should be the filename itsef the key
if m := GetItemMap(itemType); m != nil { if m := GetItemMap(itemType); m != nil {
if v, ok := m[finalName]; ok { if v, ok := m[finalName]; ok {
return &v, nil return &v, nil
} }
return nil, fmt.Errorf("%s not found in %s", finalName, itemType) return nil, fmt.Errorf("%s not found in %s", finalName, itemType)
} }
return nil, fmt.Errorf("item type %s doesn't exist", itemType) return nil, fmt.Errorf("item type %s doesn't exist", itemType)
} }
@ -181,35 +189,42 @@ func GetItem(itemType string, itemName string) *Item {
if m, ok := GetItemMap(itemType)[itemName]; ok { if m, ok := GetItemMap(itemType)[itemName]; ok {
return &m return &m
} }
return nil return nil
} }
func AddItem(itemType string, item Item) error { func AddItem(itemType string, item Item) error {
in := false in := false
for _, itype := range ItemTypes { for _, itype := range ItemTypes {
if itype == itemType { if itype == itemType {
in = true in = true
} }
} }
if !in { if !in {
return fmt.Errorf("ItemType %s is unknown", itemType) return fmt.Errorf("ItemType %s is unknown", itemType)
} }
hubIdx[itemType][item.Name] = item hubIdx[itemType][item.Name] = item
return nil return nil
} }
func DisplaySummary() { func DisplaySummary() {
log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]), log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW])) len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
if skippedLocal > 0 || skippedTainted > 0 { if skippedLocal > 0 || skippedTainted > 0 {
log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted) log.Printf("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted)
} }
} }
//returns: human-text, Enabled, Warning, Unmanaged // returns: human-text, Enabled, Warning, Unmanaged
func ItemStatus(v Item) (string, bool, bool, bool) { func ItemStatus(v Item) (string, bool, bool, bool) {
strret := "disabled" strret := "disabled"
Ok := false Ok := false
if v.Installed { if v.Installed {
Ok = true Ok = true
strret = "enabled" strret = "enabled"
@ -221,7 +236,7 @@ func ItemStatus(v Item) (string, bool, bool, bool) {
strret += ",local" strret += ",local"
} }
//tainted or out of date // tainted or out of date
Warning := false Warning := false
if v.Tainted { if v.Tainted {
Warning = true Warning = true
@ -230,6 +245,7 @@ func ItemStatus(v Item) (string, bool, bool, bool) {
strret += ",update-available" strret += ",update-available"
Warning = true Warning = true
} }
return strret, Ok, Warning, Managed return strret, Ok, Warning, Managed
} }
@ -240,9 +256,11 @@ func GetInstalledScenariosAsString() ([]string, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("while fetching scenarios: %w", err) return nil, fmt.Errorf("while fetching scenarios: %w", err)
} }
for _, it := range items { for _, it := range items {
retStr = append(retStr, it.Name) retStr = append(retStr, it.Name)
} }
return retStr, nil return retStr, nil
} }
@ -252,11 +270,13 @@ func GetInstalledScenarios() ([]Item, error) {
if _, ok := hubIdx[SCENARIOS]; !ok { if _, ok := hubIdx[SCENARIOS]; !ok {
return nil, fmt.Errorf("no scenarios in hubIdx") return nil, fmt.Errorf("no scenarios in hubIdx")
} }
for _, item := range hubIdx[SCENARIOS] { for _, item := range hubIdx[SCENARIOS] {
if item.Installed { if item.Installed {
retItems = append(retItems, item) retItems = append(retItems, item)
} }
} }
return retItems, nil return retItems, nil
} }
@ -266,11 +286,13 @@ func GetInstalledParsers() ([]Item, error) {
if _, ok := hubIdx[PARSERS]; !ok { if _, ok := hubIdx[PARSERS]; !ok {
return nil, fmt.Errorf("no parsers in hubIdx") return nil, fmt.Errorf("no parsers in hubIdx")
} }
for _, item := range hubIdx[PARSERS] { for _, item := range hubIdx[PARSERS] {
if item.Installed { if item.Installed {
retItems = append(retItems, item) retItems = append(retItems, item)
} }
} }
return retItems, nil return retItems, nil
} }
@ -281,9 +303,11 @@ func GetInstalledParsersAsString() ([]string, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("while fetching parsers: %w", err) return nil, fmt.Errorf("while fetching parsers: %w", err)
} }
for _, it := range items { for _, it := range items {
retStr = append(retStr, it.Name) retStr = append(retStr, it.Name)
} }
return retStr, nil return retStr, nil
} }
@ -293,11 +317,13 @@ func GetInstalledPostOverflows() ([]Item, error) {
if _, ok := hubIdx[PARSERS_OVFLW]; !ok { if _, ok := hubIdx[PARSERS_OVFLW]; !ok {
return nil, fmt.Errorf("no post overflows in hubIdx") return nil, fmt.Errorf("no post overflows in hubIdx")
} }
for _, item := range hubIdx[PARSERS_OVFLW] { for _, item := range hubIdx[PARSERS_OVFLW] {
if item.Installed { if item.Installed {
retItems = append(retItems, item) retItems = append(retItems, item)
} }
} }
return retItems, nil return retItems, nil
} }
@ -308,9 +334,11 @@ func GetInstalledPostOverflowsAsString() ([]string, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("while fetching post overflows: %w", err) return nil, fmt.Errorf("while fetching post overflows: %w", err)
} }
for _, it := range items { for _, it := range items {
retStr = append(retStr, it.Name) retStr = append(retStr, it.Name)
} }
return retStr, nil return retStr, nil
} }
@ -325,6 +353,7 @@ func GetInstalledCollectionsAsString() ([]string, error) {
for _, it := range items { for _, it := range items {
retStr = append(retStr, it.Name) retStr = append(retStr, it.Name)
} }
return retStr, nil return retStr, nil
} }
@ -334,15 +363,17 @@ func GetInstalledCollections() ([]Item, error) {
if _, ok := hubIdx[COLLECTIONS]; !ok { if _, ok := hubIdx[COLLECTIONS]; !ok {
return nil, fmt.Errorf("no collection in hubIdx") return nil, fmt.Errorf("no collection in hubIdx")
} }
for _, item := range hubIdx[COLLECTIONS] { for _, item := range hubIdx[COLLECTIONS] {
if item.Installed { if item.Installed {
retItems = append(retItems, item) retItems = append(retItems, item)
} }
} }
return retItems, nil return retItems, nil
} }
//Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy) // Returns a list of entries for packages: name, status, local_path, local_version, utf8_status (fancy)
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus { func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
if _, ok := hubIdx[itemType]; !ok { if _, ok := hubIdx[itemType]; !ok {
log.Errorf("type %s doesn't exist", itemType) log.Errorf("type %s doesn't exist", itemType)
@ -351,19 +382,22 @@ func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubSt
} }
var ret = make([]ItemHubStatus, 0) var ret = make([]ItemHubStatus, 0)
/*remember, you do it for the user :)*/
// remember, you do it for the user :)
for _, item := range hubIdx[itemType] { for _, item := range hubIdx[itemType] {
if name != "" && name != item.Name { if name != "" && name != item.Name {
//user has requested a specific name // user has requested a specific name
continue continue
} }
//Only enabled items ? // Only enabled items ?
if !all && !item.Installed { if !all && !item.Installed {
continue continue
} }
//Check the item status // Check the item status
ret = append(ret, item.toHubStatus()) ret = append(ret, item.toHubStatus())
} }
sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
return ret return ret
} }

View file

@ -9,8 +9,9 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
) )
/* /*
@ -29,30 +30,33 @@ func TestItemStatus(t *testing.T) {
defer envTearDown(cfg) defer envTearDown(cfg)
err := UpdateHubIdx(cfg.Hub) err := UpdateHubIdx(cfg.Hub)
//DownloadHubIdx() // DownloadHubIdx()
if err != nil { if err != nil {
t.Fatalf("failed to download index : %s", err) t.Fatalf("failed to download index : %s", err)
} }
if err := GetHubIdx(cfg.Hub); err != nil { if err := GetHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to load hub index : %s", err) t.Fatalf("failed to load hub index : %s", err)
} }
//get existing map // get existing map
x := GetItemMap(COLLECTIONS) x := GetItemMap(COLLECTIONS)
if len(x) == 0 { if len(x) == 0 {
t.Fatalf("expected non empty result") t.Fatalf("expected non empty result")
} }
//Get item : good and bad // Get item : good and bad
for k := range x { for k := range x {
item := GetItem(COLLECTIONS, k) item := GetItem(COLLECTIONS, k)
if item == nil { if item == nil {
t.Fatalf("expected item") t.Fatalf("expected item")
} }
item.Installed = true item.Installed = true
item.UpToDate = false item.UpToDate = false
item.Local = false item.Local = false
item.Tainted = false item.Tainted = false
txt, _, _, _ := ItemStatus(*item) txt, _, _, _ := ItemStatus(*item)
if txt != "enabled,update-available" { if txt != "enabled,update-available" {
t.Fatalf("got '%s'", txt) t.Fatalf("got '%s'", txt)
@ -62,6 +66,7 @@ func TestItemStatus(t *testing.T) {
item.UpToDate = false item.UpToDate = false
item.Local = true item.Local = true
item.Tainted = false item.Tainted = false
txt, _, _, _ = ItemStatus(*item) txt, _, _, _ = ItemStatus(*item)
if txt != "disabled,local" { if txt != "disabled,local" {
t.Fatalf("got '%s'", txt) t.Fatalf("got '%s'", txt)
@ -69,6 +74,7 @@ func TestItemStatus(t *testing.T) {
break break
} }
DisplaySummary() DisplaySummary()
} }
@ -77,26 +83,28 @@ func TestGetters(t *testing.T) {
defer envTearDown(cfg) defer envTearDown(cfg)
err := UpdateHubIdx(cfg.Hub) err := UpdateHubIdx(cfg.Hub)
//DownloadHubIdx() // DownloadHubIdx()
if err != nil { if err != nil {
t.Fatalf("failed to download index : %s", err) t.Fatalf("failed to download index : %s", err)
} }
if err := GetHubIdx(cfg.Hub); err != nil { if err := GetHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to load hub index : %s", err) t.Fatalf("failed to load hub index : %s", err)
} }
//get non existing map // get non existing map
empty := GetItemMap("ratata") empty := GetItemMap("ratata")
if empty != nil { if empty != nil {
t.Fatalf("expected nil result") t.Fatalf("expected nil result")
} }
//get existing map
// get existing map
x := GetItemMap(COLLECTIONS) x := GetItemMap(COLLECTIONS)
if len(x) == 0 { if len(x) == 0 {
t.Fatalf("expected non empty result") t.Fatalf("expected non empty result")
} }
//Get item : good and bad // Get item : good and bad
for k := range x { for k := range x {
empty := GetItem(COLLECTIONS, k+"nope") empty := GetItem(COLLECTIONS, k+"nope")
if empty != nil { if empty != nil {
@ -108,7 +116,7 @@ func TestGetters(t *testing.T) {
t.Fatalf("expected non empty item") t.Fatalf("expected non empty item")
} }
//Add item and get it // Add item and get it
item.Name += "nope" item.Name += "nope"
if err := AddItem(COLLECTIONS, *item); err != nil { if err := AddItem(COLLECTIONS, *item); err != nil {
t.Fatalf("didn't expect error : %s", err) t.Fatalf("didn't expect error : %s", err)
@ -119,7 +127,7 @@ func TestGetters(t *testing.T) {
t.Fatalf("expected non empty item") t.Fatalf("expected non empty item")
} }
//Add bad item // Add bad item
if err := AddItem("ratata", *item); err != nil { if err := AddItem("ratata", *item); err != nil {
if fmt.Sprintf("%s", err) != "ItemType ratata is unknown" { if fmt.Sprintf("%s", err) != "ItemType ratata is unknown" {
t.Fatalf("unexpected error") t.Fatalf("unexpected error")
@ -130,7 +138,6 @@ func TestGetters(t *testing.T) {
break break
} }
} }
func TestIndexDownload(t *testing.T) { func TestIndexDownload(t *testing.T) {
@ -138,10 +145,11 @@ func TestIndexDownload(t *testing.T) {
defer envTearDown(cfg) defer envTearDown(cfg)
err := UpdateHubIdx(cfg.Hub) err := UpdateHubIdx(cfg.Hub)
//DownloadHubIdx() // DownloadHubIdx()
if err != nil { if err != nil {
t.Fatalf("failed to download index : %s", err) t.Fatalf("failed to download index : %s", err)
} }
if err := GetHubIdx(cfg.Hub); err != nil { if err := GetHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to load hub index : %s", err) t.Fatalf("failed to load hub index : %s", err)
} }
@ -152,20 +160,23 @@ func getTestCfg() (cfg *csconfig.Config) {
cfg.Hub.ConfigDir, _ = filepath.Abs("./install") cfg.Hub.ConfigDir, _ = filepath.Abs("./install")
cfg.Hub.HubDir, _ = filepath.Abs("./hubdir") cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json") cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
return return
} }
func envSetup(t *testing.T) *csconfig.Config { func envSetup(t *testing.T) *csconfig.Config {
resetResponseByPath() resetResponseByPath()
log.SetLevel(log.DebugLevel) log.SetLevel(log.DebugLevel)
cfg := getTestCfg() cfg := getTestCfg()
defaultTransport := http.DefaultClient.Transport defaultTransport := http.DefaultClient.Transport
t.Cleanup(func() { t.Cleanup(func() {
http.DefaultClient.Transport = defaultTransport http.DefaultClient.Transport = defaultTransport
}) })
//Mock the http client // Mock the http client
http.DefaultClient.Transport = newMockTransport() http.DefaultClient.Transport = newMockTransport()
if err := os.MkdirAll(cfg.Hub.ConfigDir, 0700); err != nil { if err := os.MkdirAll(cfg.Hub.ConfigDir, 0700); err != nil {
@ -189,7 +200,6 @@ func envSetup(t *testing.T) *csconfig.Config {
return cfg return cfg
} }
func envTearDown(cfg *csconfig.Config) { func envTearDown(cfg *csconfig.Config) {
if err := os.RemoveAll(cfg.Hub.ConfigDir); err != nil { if err := os.RemoveAll(cfg.Hub.ConfigDir); err != nil {
log.Fatalf("failed to remove %s : %s", cfg.Hub.ConfigDir, err) log.Fatalf("failed to remove %s : %s", cfg.Hub.ConfigDir, err)
@ -200,23 +210,25 @@ func envTearDown(cfg *csconfig.Config) {
} }
} }
func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
// Install the parser
//Install the parser
item, err := DownloadLatest(cfg, item, false, false) item, err := DownloadLatest(cfg, item, false, false)
if err != nil { if err != nil {
t.Fatalf("error while downloading %s : %v", item.Name, err) t.Fatalf("error while downloading %s : %v", item.Name, err)
} }
if err, _ := LocalSync(cfg); err != nil { if err, _ := LocalSync(cfg); err != nil {
t.Fatalf("taint: failed to run localSync : %s", err) t.Fatalf("taint: failed to run localSync : %s", err)
} }
if !hubIdx[item.Type][item.Name].UpToDate { if !hubIdx[item.Type][item.Name].UpToDate {
t.Fatalf("download: %s should be up-to-date", item.Name) t.Fatalf("download: %s should be up-to-date", item.Name)
} }
if hubIdx[item.Type][item.Name].Installed { if hubIdx[item.Type][item.Name].Installed {
t.Fatalf("download: %s should not be installed", item.Name) t.Fatalf("download: %s should not be installed", item.Name)
} }
if hubIdx[item.Type][item.Name].Tainted { if hubIdx[item.Type][item.Name].Tainted {
t.Fatalf("download: %s should not be tainted", item.Name) t.Fatalf("download: %s should not be tainted", item.Name)
} }
@ -225,9 +237,11 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
if err != nil { if err != nil {
t.Fatalf("error while enabling %s : %v.", item.Name, err) t.Fatalf("error while enabling %s : %v.", item.Name, err)
} }
if err, _ := LocalSync(cfg); err != nil { if err, _ := LocalSync(cfg); err != nil {
t.Fatalf("taint: failed to run localSync : %s", err) t.Fatalf("taint: failed to run localSync : %s", err)
} }
if !hubIdx[item.Type][item.Name].Installed { if !hubIdx[item.Type][item.Name].Installed {
t.Fatalf("install: %s should be installed", item.Name) t.Fatalf("install: %s should be installed", item.Name)
} }
@ -237,6 +251,7 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
if hubIdx[item.Type][item.Name].Tainted { if hubIdx[item.Type][item.Name].Tainted {
t.Fatalf("pre-taint: %s should not be tainted", item.Name) t.Fatalf("pre-taint: %s should not be tainted", item.Name)
} }
f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
if err != nil { if err != nil {
t.Fatalf("(taint) opening %s (%s) : %s", item.LocalPath, item.Name, err) t.Fatalf("(taint) opening %s (%s) : %s", item.LocalPath, item.Name, err)
@ -246,32 +261,37 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
if _, err = f.WriteString("tainted"); err != nil { if _, err = f.WriteString("tainted"); err != nil {
t.Fatalf("tainting %s : %s", item.Name, err) t.Fatalf("tainting %s : %s", item.Name, err)
} }
//Local sync and check status
// Local sync and check status
if err, _ := LocalSync(cfg); err != nil { if err, _ := LocalSync(cfg); err != nil {
t.Fatalf("taint: failed to run localSync : %s", err) t.Fatalf("taint: failed to run localSync : %s", err)
} }
if !hubIdx[item.Type][item.Name].Tainted { if !hubIdx[item.Type][item.Name].Tainted {
t.Fatalf("taint: %s should be tainted", item.Name) t.Fatalf("taint: %s should be tainted", item.Name)
} }
} }
func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) { func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
if hubIdx[item.Type][item.Name].UpToDate { if hubIdx[item.Type][item.Name].UpToDate {
t.Fatalf("update: %s should NOT be up-to-date", item.Name) t.Fatalf("update: %s should NOT be up-to-date", item.Name)
} }
//Update it + check status
// Update it + check status
item, err := DownloadLatest(cfg, item, true, true) item, err := DownloadLatest(cfg, item, true, true)
if err != nil { if err != nil {
t.Fatalf("failed to update %s : %s", item.Name, err) t.Fatalf("failed to update %s : %s", item.Name, err)
} }
//Local sync and check status
// Local sync and check status
if err, _ := LocalSync(cfg); err != nil { if err, _ := LocalSync(cfg); err != nil {
t.Fatalf("failed to run localSync : %s", err) t.Fatalf("failed to run localSync : %s", err)
} }
if !hubIdx[item.Type][item.Name].UpToDate { if !hubIdx[item.Type][item.Name].UpToDate {
t.Fatalf("update: %s should be up-to-date", item.Name) t.Fatalf("update: %s should be up-to-date", item.Name)
} }
if hubIdx[item.Type][item.Name].Tainted { if hubIdx[item.Type][item.Name].Tainted {
t.Fatalf("update: %s should not be tainted anymore", item.Name) t.Fatalf("update: %s should not be tainted anymore", item.Name)
} }
@ -281,43 +301,51 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
if !item.Installed { if !item.Installed {
t.Fatalf("disable: %s should be installed", item.Name) t.Fatalf("disable: %s should be installed", item.Name)
} }
//Remove
// Remove
item, err := DisableItem(cfg, item, false, false) item, err := DisableItem(cfg, item, false, false)
if err != nil { if err != nil {
t.Fatalf("failed to disable item : %v", err) t.Fatalf("failed to disable item : %v", err)
} }
//Local sync and check status
// Local sync and check status
if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 { if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 {
t.Fatalf("failed to run localSync : %s (%+v)", err, warns) t.Fatalf("failed to run localSync : %s (%+v)", err, warns)
} }
if hubIdx[item.Type][item.Name].Tainted { if hubIdx[item.Type][item.Name].Tainted {
t.Fatalf("disable: %s should not be tainted anymore", item.Name) t.Fatalf("disable: %s should not be tainted anymore", item.Name)
} }
if hubIdx[item.Type][item.Name].Installed { if hubIdx[item.Type][item.Name].Installed {
t.Fatalf("disable: %s should not be installed anymore", item.Name) t.Fatalf("disable: %s should not be installed anymore", item.Name)
} }
if !hubIdx[item.Type][item.Name].Downloaded { if !hubIdx[item.Type][item.Name].Downloaded {
t.Fatalf("disable: %s should still be downloaded", item.Name) t.Fatalf("disable: %s should still be downloaded", item.Name)
} }
//Purge
// Purge
item, err = DisableItem(cfg, item, true, false) item, err = DisableItem(cfg, item, true, false)
if err != nil { if err != nil {
t.Fatalf("failed to purge item : %v", err) t.Fatalf("failed to purge item : %v", err)
} }
//Local sync and check status
// Local sync and check status
if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 { if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 {
t.Fatalf("failed to run localSync : %s (%+v)", err, warns) t.Fatalf("failed to run localSync : %s (%+v)", err, warns)
} }
if hubIdx[item.Type][item.Name].Installed { if hubIdx[item.Type][item.Name].Installed {
t.Fatalf("disable: %s should not be installed anymore", item.Name) t.Fatalf("disable: %s should not be installed anymore", item.Name)
} }
if hubIdx[item.Type][item.Name].Downloaded { if hubIdx[item.Type][item.Name].Downloaded {
t.Fatalf("disable: %s should not be downloaded", item.Name) t.Fatalf("disable: %s should not be downloaded", item.Name)
} }
} }
func TestInstallParser(t *testing.T) { func TestInstallParser(t *testing.T) {
/* /*
- install a random parser - install a random parser
- check its status - check its status
@ -331,7 +359,7 @@ func TestInstallParser(t *testing.T) {
defer envTearDown(cfg) defer envTearDown(cfg)
getHubIdxOrFail(t) getHubIdxOrFail(t)
//map iteration is random by itself // map iteration is random by itself
for _, it := range hubIdx[PARSERS] { for _, it := range hubIdx[PARSERS] {
testInstallItem(cfg.Hub, t, it) testInstallItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name] it = hubIdx[PARSERS][it.Name]
@ -349,7 +377,6 @@ func TestInstallParser(t *testing.T) {
} }
func TestInstallCollection(t *testing.T) { func TestInstallCollection(t *testing.T) {
/* /*
- install a random parser - install a random parser
- check its status - check its status
@ -363,7 +390,7 @@ func TestInstallCollection(t *testing.T) {
defer envTearDown(cfg) defer envTearDown(cfg)
getHubIdxOrFail(t) getHubIdxOrFail(t)
//map iteration is random by itself // map iteration is random by itself
for _, it := range hubIdx[COLLECTIONS] { for _, it := range hubIdx[COLLECTIONS] {
testInstallItem(cfg.Hub, t, it) testInstallItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name] it = hubIdx[COLLECTIONS][it.Name]
@ -376,6 +403,7 @@ func TestInstallCollection(t *testing.T) {
it = hubIdx[COLLECTIONS][it.Name] it = hubIdx[COLLECTIONS][it.Name]
x := GetHubStatusForItemType(COLLECTIONS, it.Name, false) x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
log.Printf("%+v", x) log.Printf("%+v", x)
break break
} }
} }
@ -395,10 +423,12 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
StatusCode: http.StatusOK, StatusCode: http.StatusOK,
} }
response.Header.Set("Content-Type", "application/json") response.Header.Set("Content-Type", "application/json")
responseBody := "" responseBody := ""
log.Printf("---> %s", req.URL.Path) log.Printf("---> %s", req.URL.Path)
/*FAKE PARSER*/ // FAKE PARSER
if resp, ok := responseByPath[req.URL.Path]; ok { if resp, ok := responseByPath[req.URL.Path]; ok {
responseBody = resp responseBody = resp
} else { } else {
@ -406,12 +436,14 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
} }
response.Body = io.NopCloser(strings.NewReader(responseBody)) response.Body = io.NopCloser(strings.NewReader(responseBody))
return response, nil return response, nil
} }
func fileToStringX(path string) string { func fileToStringX(path string) string {
if f, err := os.Open(path); err == nil { if f, err := os.Open(path); err == nil {
defer f.Close() defer f.Close()
if data, err := io.ReadAll(f); err == nil { if data, err := io.ReadAll(f); err == nil {
return strings.ReplaceAll(string(data), "\r\n", "\n") return strings.ReplaceAll(string(data), "\r\n", "\n")
} else { } else {

View file

@ -18,6 +18,7 @@ type DataSet struct {
func downloadFile(url string, destPath string) error { func downloadFile(url string, destPath string) error {
log.Debugf("downloading %s in %s", url, destPath) log.Debugf("downloading %s in %s", url, destPath)
req, err := http.NewRequest(http.MethodGet, url, nil) req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil { if err != nil {
return err return err
@ -60,6 +61,7 @@ func GetData(data []*types.DataSource, dataDir string) error {
for _, dataS := range data { for _, dataS := range data {
destPath := filepath.Join(dataDir, dataS.DestPath) destPath := filepath.Join(dataDir, dataS.DestPath)
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
err := downloadFile(dataS.SourceURL, destPath) err := downloadFile(dataS.SourceURL, destPath)
if err != nil { if err != nil {
return err return err

View file

@ -4,9 +4,8 @@ import (
"os" "os"
"testing" "testing"
"github.com/stretchr/testify/assert"
"github.com/jarcoal/httpmock" "github.com/jarcoal/httpmock"
"github.com/stretchr/testify/assert"
) )
func TestDownloadFile(t *testing.T) { func TestDownloadFile(t *testing.T) {
@ -26,6 +25,7 @@ func TestDownloadFile(t *testing.T) {
"https://example.com/x", "https://example.com/x",
httpmock.NewStringResponder(404, "not found"), httpmock.NewStringResponder(404, "not found"),
) )
err := downloadFile("https://example.com/xx", examplePath) err := downloadFile("https://example.com/xx", examplePath)
assert.NoError(t, err) assert.NoError(t, err)
content, err := os.ReadFile(examplePath) content, err := os.ReadFile(examplePath)

View file

@ -24,36 +24,45 @@ func UpdateHubIdx(hub *csconfig.Hub) error {
if err != nil { if err != nil {
return fmt.Errorf("failed to download index: %w", err) return fmt.Errorf("failed to download index: %w", err)
} }
ret, err := LoadPkgIndex(bidx) ret, err := LoadPkgIndex(bidx)
if err != nil { if err != nil {
if !errors.Is(err, ReferenceMissingError) { if !errors.Is(err, ReferenceMissingError) {
return fmt.Errorf("failed to read index: %w", err) return fmt.Errorf("failed to read index: %w", err)
} }
} }
hubIdx = ret hubIdx = ret
if err, _ := LocalSync(hub); err != nil { if err, _ := LocalSync(hub); err != nil {
return fmt.Errorf("failed to sync: %w", err) return fmt.Errorf("failed to sync: %w", err)
} }
return nil return nil
} }
func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) { func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to build request for hub index: %w", err) return nil, fmt.Errorf("failed to build request for hub index: %w", err)
} }
resp, err := http.DefaultClient.Do(req) resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed http request for hub index: %w", err) return nil, fmt.Errorf("failed http request for hub index: %w", err)
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusNotFound { if resp.StatusCode == http.StatusNotFound {
return nil, ErrIndexNotFound return nil, ErrIndexNotFound
} }
return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String()) return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String())
} }
body, err := io.ReadAll(resp.Body) body, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
@ -80,7 +89,9 @@ func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("while writing hub index file: %w", err) return nil, fmt.Errorf("while writing hub index file: %w", err)
} }
log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile) log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
return body, nil return body, nil
} }
@ -89,11 +100,13 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b
var err error var err error
log.Debugf("Downloading %s %s", target.Type, target.Name) log.Debugf("Downloading %s %s", target.Type, target.Name)
if target.Type != COLLECTIONS { if target.Type != COLLECTIONS {
if !target.Installed && updateOnly && target.Downloaded { if !target.Installed && updateOnly && target.Downloaded {
log.Debugf("skipping upgrade of %s : not installed", target.Name) log.Debugf("skipping upgrade of %s : not installed", target.Name)
return target, nil return target, nil
} }
return DownloadItem(hub, target, overwrite) return DownloadItem(hub, target, overwrite)
} }
@ -116,11 +129,13 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b
//recurse as it's a collection //recurse as it's a collection
if ptrtype == COLLECTIONS { if ptrtype == COLLECTIONS {
log.Tracef("collection, recurse") log.Tracef("collection, recurse")
hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly) hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly)
if err != nil { if err != nil {
return target, fmt.Errorf("while downloading %s: %w", val.Name, err) return target, fmt.Errorf("while downloading %s: %w", val.Name, err)
} }
} }
item, err := DownloadItem(hub, val, overwrite) item, err := DownloadItem(hub, val, overwrite)
if err != nil { if err != nil {
return target, fmt.Errorf("while downloading %s: %w", val.Name, err) return target, fmt.Errorf("while downloading %s: %w", val.Name, err)
@ -133,77 +148,95 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b
return target, fmt.Errorf("enabling '%s': %w", item.Name, err) return target, fmt.Errorf("enabling '%s': %w", item.Name, err)
} }
} }
hubIdx[ptrtype][p] = item hubIdx[ptrtype][p] = item
} }
} }
target, err = DownloadItem(hub, target, overwrite) target, err = DownloadItem(hub, target, overwrite)
if err != nil { if err != nil {
return target, fmt.Errorf("failed to download item : %s", err) return target, fmt.Errorf("failed to download item : %s", err)
} }
return target, nil return target, nil
} }
func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) { func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) {
var tdir = hub.HubDir tdir := hub.HubDir
var dataFolder = hub.DataDir dataFolder := hub.DataDir
/*if user didn't --force, don't overwrite local, tainted, up-to-date files*/
// if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite { if !overwrite {
if target.Tainted { if target.Tainted {
log.Debugf("%s : tainted, not updated", target.Name) log.Debugf("%s : tainted, not updated", target.Name)
return target, nil return target, nil
} }
if target.UpToDate { if target.UpToDate {
log.Debugf("%s : up-to-date, not updated", target.Name)
// We still have to check if data files are present // We still have to check if data files are present
log.Debugf("%s : up-to-date, not updated", target.Name)
} }
} }
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
if err != nil { if err != nil {
return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err) return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
} }
resp, err := http.DefaultClient.Do(req) resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err) return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
} }
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
return target, fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) return target, fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
} }
defer resp.Body.Close() defer resp.Body.Close()
body, err := io.ReadAll(resp.Body) body, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return target, fmt.Errorf("while reading %s: %w", req.URL.String(), err) return target, fmt.Errorf("while reading %s: %w", req.URL.String(), err)
} }
h := sha256.New() h := sha256.New()
if _, err := h.Write(body); err != nil { if _, err := h.Write(body); err != nil {
return target, fmt.Errorf("while hashing %s: %w", target.Name, err) return target, fmt.Errorf("while hashing %s: %w", target.Name, err)
} }
meow := fmt.Sprintf("%x", h.Sum(nil)) meow := fmt.Sprintf("%x", h.Sum(nil))
if meow != target.Versions[target.Version].Digest { if meow != target.Versions[target.Version].Digest {
log.Errorf("Downloaded version doesn't match index, please 'hub update'") log.Errorf("Downloaded version doesn't match index, please 'hub update'")
log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
return target, fmt.Errorf("invalid download hash for %s", target.Name) return target, fmt.Errorf("invalid download hash for %s", target.Name)
} }
//all good, install //all good, install
//check if parent dir exists //check if parent dir exists
tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/")
parent_dir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") parent_dir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
/*ensure that target file is within target dir*/ // ensure that target file is within target dir
finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath) finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath)
if err != nil { if err != nil {
return target, fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err) return target, fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err)
} }
if !strings.HasPrefix(finalPath, tdir) { if !strings.HasPrefix(finalPath, tdir) {
return target, fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir) return target, fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir)
} }
/*check dir*/
// check dir
if _, err = os.Stat(parent_dir); os.IsNotExist(err) { if _, err = os.Stat(parent_dir); os.IsNotExist(err) {
log.Debugf("%s doesn't exist, create", parent_dir) log.Debugf("%s doesn't exist, create", parent_dir)
if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil { if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil {
return target, fmt.Errorf("while creating parent directories: %w", err) return target, fmt.Errorf("while creating parent directories: %w", err)
} }
} }
/*check actual file*/
// check actual file
if _, err = os.Stat(finalPath); !os.IsNotExist(err) { if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
log.Warningf("%s : overwrite", target.Name) log.Warningf("%s : overwrite", target.Name)
log.Debugf("target: %s/%s", tdir, target.RemotePath) log.Debugf("target: %s/%s", tdir, target.RemotePath)
@ -215,11 +248,14 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error)
if err != nil { if err != nil {
return target, fmt.Errorf("while opening file: %w", err) return target, fmt.Errorf("while opening file: %w", err)
} }
defer f.Close() defer f.Close()
_, err = f.Write(body) _, err = f.Write(body)
if err != nil { if err != nil {
return target, fmt.Errorf("while writing file: %w", err) return target, fmt.Errorf("while writing file: %w", err)
} }
target.Downloaded = true target.Downloaded = true
target.Tainted = false target.Tainted = false
target.UpToDate = true target.UpToDate = true
@ -229,6 +265,7 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error)
} }
hubIdx[target.Type][target.Name] = target hubIdx[target.Type][target.Name] = target
return target, nil return target, nil
} }
@ -238,37 +275,47 @@ func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
itemFile *os.File itemFile *os.File
err error err error
) )
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.ConfigDir, target.Type, target.Stage, target.FileName) itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.ConfigDir, target.Type, target.Stage, target.FileName)
if itemFile, err = os.Open(itemFilePath); err != nil { if itemFile, err = os.Open(itemFilePath); err != nil {
return fmt.Errorf("while opening %s: %w", itemFilePath, err) return fmt.Errorf("while opening %s: %w", itemFilePath, err)
} }
defer itemFile.Close() defer itemFile.Close()
if err = downloadData(dataFolder, force, itemFile); err != nil { if err = downloadData(dataFolder, force, itemFile); err != nil {
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
} }
return nil return nil
} }
func downloadData(dataFolder string, force bool, reader io.Reader) error { func downloadData(dataFolder string, force bool, reader io.Reader) error {
var err error var err error
dec := yaml.NewDecoder(reader) dec := yaml.NewDecoder(reader)
for { for {
data := &DataSet{} data := &DataSet{}
err = dec.Decode(data) err = dec.Decode(data)
if err != nil { if err != nil {
if errors.Is(err, io.EOF) { if errors.Is(err, io.EOF) {
break break
} }
return fmt.Errorf("while reading file: %w", err) return fmt.Errorf("while reading file: %w", err)
} }
download := false download := false
for _, dataS := range data.Data { for _, dataS := range data.Data {
if _, err := os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { if _, err := os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
download = true download = true
} }
} }
if download || force { if download || force {
err = GetData(data.Data, dataFolder) err = GetData(data.Data, dataFolder)
if err != nil { if err != nil {
@ -276,5 +323,6 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error {
} }
} }
} }
return nil return nil
} }

View file

@ -5,38 +5,48 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
) )
func TestDownloadHubIdx(t *testing.T) { func TestDownloadHubIdx(t *testing.T) {
back := RawFileURLTemplate back := RawFileURLTemplate
//bad url template // bad url template
fmt.Println("Test 'bad URL'") fmt.Println("Test 'bad URL'")
RawFileURLTemplate = "x" RawFileURLTemplate = "x"
ret, err := DownloadHubIdx(&csconfig.Hub{}) ret, err := DownloadHubIdx(&csconfig.Hub{})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") { if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
log.Errorf("unexpected error %s", err) log.Errorf("unexpected error %s", err)
} }
fmt.Printf("->%+v", ret) fmt.Printf("->%+v", ret)
//bad domain // bad domain
fmt.Println("Test 'bad domain'") fmt.Println("Test 'bad domain'")
RawFileURLTemplate = "https://baddomain/%s/%s" RawFileURLTemplate = "https://baddomain/%s/%s"
ret, err = DownloadHubIdx(&csconfig.Hub{}) ret, err = DownloadHubIdx(&csconfig.Hub{})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") { if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
log.Errorf("unexpected error %s", err) log.Errorf("unexpected error %s", err)
} }
fmt.Printf("->%+v", ret) fmt.Printf("->%+v", ret)
//bad target path // bad target path
fmt.Println("Test 'bad target path'") fmt.Println("Test 'bad target path'")
RawFileURLTemplate = back RawFileURLTemplate = back
ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"}) ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") { if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
log.Errorf("unexpected error %s", err) log.Errorf("unexpected error %s", err)
} }
RawFileURLTemplate = back RawFileURLTemplate = back
fmt.Printf("->%+v", ret) fmt.Printf("->%+v", ret)
} }

View file

@ -17,7 +17,7 @@ func chooseHubBranch() (string, error) {
latest, err := cwversion.Latest() latest, err := cwversion.Latest()
if err != nil { if err != nil {
log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err) log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
//lint:ignore nilerr reason //lint:ignore nilerr
return "master", nil // ignore return "master", nil // ignore
} }
@ -41,8 +41,10 @@ func chooseHubBranch() (string, error) {
log.Warnf("Crowdsec is not the latest version. "+ log.Warnf("Crowdsec is not the latest version. "+
"Current version is '%s' and the latest stable version is '%s'. Please update it!", "Current version is '%s' and the latest stable version is '%s'. Please update it!",
csVersion, latest) csVersion, latest)
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+ log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
"added to Crowdsec Hub after CrowdSec %s", latest) "added to Crowdsec Hub after CrowdSec %s", latest)
return csVersion, nil return csVersion, nil
} }
@ -58,8 +60,10 @@ func SetHubBranch() error {
if err != nil { if err != nil {
return err return err
} }
HubBranch = branch HubBranch = branch
log.Debugf("Using branch '%s' for the hub", HubBranch) log.Debugf("Using branch '%s' for the hub", HubBranch)
return nil return nil
} }
@ -72,6 +76,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo
item := *it item := *it
if downloadOnly && item.Downloaded && item.UpToDate { if downloadOnly && item.Downloaded && item.UpToDate {
log.Warningf("%s is already downloaded and up-to-date", item.Name) log.Warningf("%s is already downloaded and up-to-date", item.Name)
if !force { if !force {
return nil return nil
} }
@ -120,6 +125,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
item := *it item := *it
item, err = DisableItem(csConfig.Hub, item, purge, forceAction) item, err = DisableItem(csConfig.Hub, item, purge, forceAction)
if err != nil { if err != nil {
log.Fatalf("unable to disable %s : %v", item.Name, err) log.Fatalf("unable to disable %s : %v", item.Name, err)
} }
@ -127,6 +133,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
if err := AddItem(itemType, item); err != nil { if err := AddItem(itemType, item); err != nil {
log.Fatalf("unable to add %s: %v", item.Name, err) log.Fatalf("unable to add %s: %v", item.Name, err)
} }
return return
} }
@ -139,6 +146,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
if !v.Installed { if !v.Installed {
continue continue
} }
v, err = DisableItem(csConfig.Hub, v, purge, forceAction) v, err = DisableItem(csConfig.Hub, v, purge, forceAction)
if err != nil { if err != nil {
log.Fatalf("unable to disable %s : %v", v.Name, err) log.Fatalf("unable to disable %s : %v", v.Name, err)
@ -149,6 +157,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
} }
disabled++ disabled++
} }
log.Infof("Disabled %d items", disabled) log.Infof("Disabled %d items", disabled)
} }

View file

@ -6,7 +6,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
//Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection // Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
// We expect the new scenario to be installed // We expect the new scenario to be installed
func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
cfg := envSetup(t) cfg := envSetup(t)
@ -37,6 +37,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
if err := UpdateHubIdx(cfg.Hub); err != nil { if err := UpdateHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to download index : %s", err) t.Fatalf("failed to download index : %s", err)
} }
getHubIdxOrFail(t) getHubIdxOrFail(t)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
@ -49,7 +50,6 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
} }
// Install a collection, disable a scenario. // Install a collection, disable a scenario.
@ -140,6 +140,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
if err := UpdateHubIdx(cfg.Hub); err != nil { if err := UpdateHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to download index : %s", err) t.Fatalf("failed to download index : %s", err)
} }
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
getHubIdxOrFail(t) getHubIdxOrFail(t)
@ -151,6 +152,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
func assertCollectionDepsInstalled(t *testing.T, collection string) { func assertCollectionDepsInstalled(t *testing.T, collection string) {
t.Helper() t.Helper()
c := hubIdx[COLLECTIONS][collection] c := hubIdx[COLLECTIONS][collection]
require.NoError(t, CollecDepsCheck(&c)) require.NoError(t, CollecDepsCheck(&c))
} }

View file

@ -22,15 +22,17 @@ func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
target.Downloaded = false target.Downloaded = false
log.Infof("Removed source file [%s] : %s", target.Name, hubpath) log.Infof("Removed source file [%s] : %s", target.Name, hubpath)
hubIdx[target.Type][target.Name] = target hubIdx[target.Type][target.Name] = target
return target, nil return target, nil
} }
//DisableItem to disable an item managed by the hub, removes the symlink if purge is true // DisableItem to disable an item managed by the hub, removes the symlink if purge is true
func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, error) { func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, error) {
var tdir = hub.ConfigDir
var hdir = hub.HubDir
var err error var err error
tdir := hub.ConfigDir
hdir := hub.HubDir
if !target.Installed { if !target.Installed {
if purge { if purge {
target, err = purgeItem(hub, target) target, err = purgeItem(hub, target)
@ -38,6 +40,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
return target, err return target, err
} }
} }
return target, nil return target, nil
} }
@ -54,7 +57,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name) return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
} }
/*for a COLLECTIONS, disable sub-items*/ // for a COLLECTIONS, disable sub-items
if target.Type == COLLECTIONS { if target.Type == COLLECTIONS {
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
for idx, ptr := range tmp { for idx, ptr := range tmp {
@ -63,12 +66,14 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
if val, ok := hubIdx[ptrtype][p]; ok { if val, ok := hubIdx[ptrtype][p]; ok {
// check if the item doesn't belong to another collection before removing it // check if the item doesn't belong to another collection before removing it
toRemove := true toRemove := true
for _, collection := range val.BelongsToCollections { for _, collection := range val.BelongsToCollections {
if collection != target.Name { if collection != target.Name {
toRemove = false toRemove = false
break break
} }
} }
if toRemove { if toRemove {
hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force) hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
if err != nil { if err != nil {
@ -114,6 +119,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
} }
log.Infof("Removed symlink [%s] : %s", target.Name, syml) log.Infof("Removed symlink [%s] : %s", target.Name, syml)
} }
target.Installed = false target.Installed = false
if purge { if purge {
@ -122,39 +128,46 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
return target, err return target, err
} }
} }
hubIdx[target.Type][target.Name] = target hubIdx[target.Type][target.Name] = target
return target, nil return target, nil
} }
// creates symlink between actual config file at hub.HubDir and hub.ConfigDir // creates symlink between actual config file at hub.HubDir and hub.ConfigDir
// Handles collections recursively // Handles collections recursively
func EnableItem(hub *csconfig.Hub, target Item) (Item, error) { func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
var tdir = hub.ConfigDir
var hdir = hub.HubDir
var err error var err error
tdir := hub.ConfigDir
hdir := hub.HubDir
parent_dir := filepath.Clean(tdir + "/" + target.Type + "/" + target.Stage + "/") parent_dir := filepath.Clean(tdir + "/" + target.Type + "/" + target.Stage + "/")
/*create directories if needed*/ // create directories if needed
if target.Installed { if target.Installed {
if target.Tainted { if target.Tainted {
return target, fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) return target, fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
} }
if target.Local { if target.Local {
return target, fmt.Errorf("%s is local, won't enable", target.Name) return target, fmt.Errorf("%s is local, won't enable", target.Name)
} }
/* if it's a collection, check sub-items even if the collection file itself is up-to-date */ // if it's a collection, check sub-items even if the collection file itself is up-to-date
if target.UpToDate && target.Type != COLLECTIONS { if target.UpToDate && target.Type != COLLECTIONS {
log.Tracef("%s is installed and up-to-date, skip.", target.Name) log.Tracef("%s is installed and up-to-date, skip.", target.Name)
return target, nil return target, nil
} }
} }
if _, err := os.Stat(parent_dir); os.IsNotExist(err) { if _, err := os.Stat(parent_dir); os.IsNotExist(err) {
log.Printf("%s doesn't exist, create", parent_dir) log.Printf("%s doesn't exist, create", parent_dir)
if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil { if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil {
return target, fmt.Errorf("while creating directory: %w", err) return target, fmt.Errorf("while creating directory: %w", err)
} }
} }
/*install sub-items if it's a collection*/ // install sub-items if it's a collection
if target.Type == COLLECTIONS { if target.Type == COLLECTIONS {
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
for idx, ptr := range tmp { for idx, ptr := range tmp {
@ -179,7 +192,7 @@ func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
return target, nil return target, nil
} }
//tdir+target.RemotePath // tdir+target.RemotePath
srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath) srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath)
if err != nil { if err != nil {
return target, fmt.Errorf("while getting source path: %w", err) return target, fmt.Errorf("while getting source path: %w", err)
@ -197,5 +210,6 @@ func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
log.Printf("Enabled %s : %s", target.Type, target.Name) log.Printf("Enabled %s : %s", target.Type, target.Name)
target.Installed = true target.Installed = true
hubIdx[target.Type][target.Name] = target hubIdx[target.Type][target.Name] = target
return target, nil return target, nil
} }

View file

@ -15,19 +15,20 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
) )
/*the walk/parser_visit function can't receive extra args*/ // the walk/parser_visit function can't receive extra args
var hubdir, installdir string var hubdir, installdir string
func parser_visit(path string, f os.DirEntry, err error) error { func parser_visit(path string, f os.DirEntry, err error) error {
var (
var target Item target Item
var local bool local bool
var hubpath string hubpath string
var inhub bool inhub bool
var fname string fname string
var ftype string ftype string
var fauthor string fauthor string
var stage string stage string
)
if err != nil { if err != nil {
log.Debugf("while syncing hub dir: %s", err) log.Debugf("while syncing hub dir: %s", err)
@ -39,11 +40,11 @@ func parser_visit(path string, f os.DirEntry, err error) error {
if err != nil { if err != nil {
return err return err
} }
//we only care about files // we only care about files
if f == nil || f.IsDir() { if f == nil || f.IsDir() {
return nil return nil
} }
//we only care about yaml files // we only care about yaml files
if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") { if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") {
return nil return nil
} }
@ -52,9 +53,10 @@ func parser_visit(path string, f os.DirEntry, err error) error {
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubdir, installdir) log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubdir, installdir)
log.Tracef("subs:%v", subs) log.Tracef("subs:%v", subs)
/*we're in hub (~/.hub/hub/)*/ // we're in hub (~/.hub/hub/)
if strings.HasPrefix(path, hubdir) { if strings.HasPrefix(path, hubdir) {
log.Tracef("in hub dir") log.Tracef("in hub dir")
inhub = true inhub = true
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
//.../hub/scenarios/crowdsec/ssh_bf.yaml //.../hub/scenarios/crowdsec/ssh_bf.yaml
@ -62,11 +64,12 @@ func parser_visit(path string, f os.DirEntry, err error) error {
if len(subs) < 4 { if len(subs) < 4 {
log.Fatalf("path is too short : %s (%d)", path, len(subs)) log.Fatalf("path is too short : %s (%d)", path, len(subs))
} }
fname = subs[len(subs)-1] fname = subs[len(subs)-1]
fauthor = subs[len(subs)-2] fauthor = subs[len(subs)-2]
stage = subs[len(subs)-3] stage = subs[len(subs)-3]
ftype = subs[len(subs)-4] ftype = subs[len(subs)-4]
} else if strings.HasPrefix(path, installdir) { /*we're in install /etc/crowdsec/<type>/... */ } else if strings.HasPrefix(path, installdir) { // we're in install /etc/crowdsec/<type>/...
log.Tracef("in install dir") log.Tracef("in install dir")
if len(subs) < 3 { if len(subs) < 3 {
log.Fatalf("path is too short : %s (%d)", path, len(subs)) log.Fatalf("path is too short : %s (%d)", path, len(subs))
@ -84,14 +87,15 @@ func parser_visit(path string, f os.DirEntry, err error) error {
} }
log.Tracef("stage:%s ftype:%s", stage, ftype) log.Tracef("stage:%s ftype:%s", stage, ftype)
//log.Printf("%s -> name:%s stage:%s", path, fname, stage) // log.Printf("%s -> name:%s stage:%s", path, fname, stage)
if stage == SCENARIOS { if stage == SCENARIOS {
ftype = SCENARIOS ftype = SCENARIOS
stage = "" stage = ""
} else if stage == COLLECTIONS { } else if stage == COLLECTIONS {
ftype = COLLECTIONS ftype = COLLECTIONS
stage = "" stage = ""
} else if ftype != PARSERS && ftype != PARSERS_OVFLW /*its a PARSER / PARSER_OVFLW with a stage */ { } else if ftype != PARSERS && ftype != PARSERS_OVFLW {
// its a PARSER / PARSER_OVFLW with a stage
return fmt.Errorf("unknown configuration type for file '%s'", path) return fmt.Errorf("unknown configuration type for file '%s'", path)
} }
@ -102,20 +106,21 @@ func parser_visit(path string, f os.DirEntry, err error) error {
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml /etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
when the collection is installed, both files are created when the collection is installed, both files are created
*/ */
//non symlinks are local user files or hub files // non symlinks are local user files or hub files
if f.Type()&os.ModeSymlink == 0 { if f.Type()&os.ModeSymlink == 0 {
local = true local = true
log.Tracef("%s isn't a symlink", path) log.Tracef("%s isn't a symlink", path)
} else { } else {
hubpath, err = os.Readlink(path) hubpath, err = os.Readlink(path)
if err != nil { if err != nil {
return fmt.Errorf("unable to read symlink of %s", path) return fmt.Errorf("unable to read symlink of %s", path)
} }
//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml // the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
_, err := os.Lstat(hubpath) _, err := os.Lstat(hubpath)
if os.IsNotExist(err) { if os.IsNotExist(err) {
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath) log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
//remove the symlink // remove the symlink
if err = os.Remove(path); err != nil { if err = os.Remove(path); err != nil {
return fmt.Errorf("failed to unlink %s: %+v", path, err) return fmt.Errorf("failed to unlink %s: %+v", path, err)
} }
@ -124,7 +129,7 @@ func parser_visit(path string, f os.DirEntry, err error) error {
log.Tracef("%s points to %s", path, hubpath) log.Tracef("%s points to %s", path, hubpath)
} }
//if it's not a symlink and not in hub, it's a local file, don't bother // if it's not a symlink and not in hub, it's a local file, don't bother
if local && !inhub { if local && !inhub {
log.Tracef("%s is a local file, skip", path) log.Tracef("%s is a local file, skip", path)
skippedLocal++ skippedLocal++
@ -139,29 +144,34 @@ func parser_visit(path string, f os.DirEntry, err error) error {
_, target.FileName = filepath.Split(path) _, target.FileName = filepath.Split(path)
hubIdx[ftype][fname] = target hubIdx[ftype][fname] = target
return nil return nil
} }
//try to find which configuration item it is // try to find which configuration item it is
log.Tracef("check [%s] of %s", fname, ftype) log.Tracef("check [%s] of %s", fname, ftype)
match := false match := false
for k, v := range hubIdx[ftype] { for k, v := range hubIdx[ftype] {
log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml") log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml")
if fname != v.FileName { if fname != v.FileName {
log.Tracef("%s != %s (filename)", fname, v.FileName) log.Tracef("%s != %s (filename)", fname, v.FileName)
continue continue
} }
//wrong stage
// wrong stage
if v.Stage != stage { if v.Stage != stage {
continue continue
} }
/*if we are walking hub dir, just mark present files as downloaded*/
// if we are walking hub dir, just mark present files as downloaded
if inhub { if inhub {
//wrong author // wrong author
if fauthor != v.Author { if fauthor != v.Author {
continue continue
} }
//wrong file // wrong file
if CheckName(v.Name, fauthor, fname) { if CheckName(v.Name, fauthor, fname) {
continue continue
} }
@ -171,35 +181,38 @@ func parser_visit(path string, f os.DirEntry, err error) error {
v.Downloaded = true v.Downloaded = true
} }
} else if CheckSuffix(hubpath, v.RemotePath) { } else if CheckSuffix(hubpath, v.RemotePath) {
//wrong file // wrong file
//<type>/<stage>/<author>/<name>.yaml // <type>/<stage>/<author>/<name>.yaml
continue continue
} }
sha, err := getSHA256(path) sha, err := getSHA256(path)
if err != nil { if err != nil {
log.Fatalf("Failed to get sha of %s : %v", path, err) log.Fatalf("Failed to get sha of %s : %v", path, err)
} }
//let's reverse sort the versions to deal with hash collisions (#154)
// let's reverse sort the versions to deal with hash collisions (#154)
versions := make([]string, 0, len(v.Versions)) versions := make([]string, 0, len(v.Versions))
for k := range v.Versions { for k := range v.Versions {
versions = append(versions, k) versions = append(versions, k)
} }
sort.Sort(sort.Reverse(sort.StringSlice(versions))) sort.Sort(sort.Reverse(sort.StringSlice(versions)))
for _, version := range versions { for _, version := range versions {
val := v.Versions[version] val := v.Versions[version]
if sha != val.Digest { if sha != val.Digest {
//log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v)) // log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
continue continue
} }
/*we got an exact match, update struct*/ // we got an exact match, update struct
if !inhub { if !inhub {
log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version) log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version)
v.LocalPath = path v.LocalPath = path
v.LocalVersion = version v.LocalVersion = version
v.Tainted = false v.Tainted = false
v.Downloaded = true v.Downloaded = true
/*if we're walking the hub, present file doesn't means installed file*/ // if we're walking the hub, present file doesn't means installed file
v.Installed = true v.Installed = true
v.LocalHash = sha v.LocalHash = sha
_, target.FileName = filepath.Split(path) _, target.FileName = filepath.Split(path)
@ -207,29 +220,34 @@ func parser_visit(path string, f os.DirEntry, err error) error {
v.Downloaded = true v.Downloaded = true
v.LocalHash = sha v.LocalHash = sha
} }
if version == v.Version { if version == v.Version {
log.Tracef("%s is up-to-date", v.Name) log.Tracef("%s is up-to-date", v.Name)
v.UpToDate = true v.UpToDate = true
} }
match = true match = true
break break
} }
if !match { if !match {
log.Tracef("got tainted match for %s : %s", v.Name, path) log.Tracef("got tainted match for %s : %s", v.Name, path)
skippedTainted += 1 skippedTainted += 1
//the file and the stage is right, but the hash is wrong, it has been tainted by user // the file and the stage is right, but the hash is wrong, it has been tainted by user
if !inhub { if !inhub {
v.LocalPath = path v.LocalPath = path
v.Installed = true v.Installed = true
} }
v.UpToDate = false v.UpToDate = false
v.LocalVersion = "?" v.LocalVersion = "?"
v.Tainted = true v.Tainted = true
v.LocalHash = sha v.LocalHash = sha
_, target.FileName = filepath.Split(path) _, target.FileName = filepath.Split(path)
} }
//update the entry if appropriate // update the entry if appropriate
// if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D { // if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D {
// fmt.Printf("Updating %s", k) // fmt.Printf("Updating %s", k)
// hubIdx[ftype][k] = v // hubIdx[ftype][k] = v
@ -237,22 +255,25 @@ func parser_visit(path string, f os.DirEntry, err error) error {
// } else if // } else if
hubIdx[ftype][k] = v hubIdx[ftype][k] = v
return nil return nil
} }
log.Infof("Ignoring file %s of type %s", path, ftype) log.Infof("Ignoring file %s of type %s", path, ftype)
return nil return nil
} }
func CollecDepsCheck(v *Item) error { func CollecDepsCheck(v *Item) error {
if GetVersionStatus(v) != 0 { // not up-to-date
if GetVersionStatus(v) != 0 { //not up-to-date
log.Debugf("%s dependencies not checked : not up-to-date", v.Name) log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
return nil return nil
} }
/*if it's a collection, ensure all the items are installed, or tag it as tainted*/ // if it's a collection, ensure all the items are installed, or tag it as tainted
if v.Type == COLLECTIONS { if v.Type == COLLECTIONS {
log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
var tmp = [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} var tmp = [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections}
for idx, ptr := range tmp { for idx, ptr := range tmp {
ptrtype := ItemTypes[idx] ptrtype := ItemTypes[idx]
@ -261,48 +282,62 @@ func CollecDepsCheck(v *Item) error {
if !ok { if !ok {
log.Fatalf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, v.Name) log.Fatalf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, v.Name)
} }
log.Tracef("check %s installed:%t", val.Name, val.Installed) log.Tracef("check %s installed:%t", val.Name, val.Installed)
if !v.Installed { if !v.Installed {
continue continue
} }
if val.Type == COLLECTIONS { if val.Type == COLLECTIONS {
log.Tracef("collec, recurse.") log.Tracef("collec, recurse.")
if err := CollecDepsCheck(&val); err != nil { if err := CollecDepsCheck(&val); err != nil {
if val.Tainted { if val.Tainted {
v.Tainted = true v.Tainted = true
} }
return fmt.Errorf("sub collection %s is broken : %s", val.Name, err)
return fmt.Errorf("sub collection %s is broken: %s", val.Name, err)
} }
hubIdx[ptrtype][p] = val hubIdx[ptrtype][p] = val
} }
//propagate the state of sub-items to set // propagate the state of sub-items to set
if val.Tainted { if val.Tainted {
v.Tainted = true v.Tainted = true
return fmt.Errorf("tainted %s %s, tainted.", ptrtype, p) return fmt.Errorf("tainted %s %s, tainted", ptrtype, p)
} }
if !val.Installed && v.Installed { if !val.Installed && v.Installed {
v.Tainted = true v.Tainted = true
return fmt.Errorf("missing %s %s, tainted.", ptrtype, p) return fmt.Errorf("missing %s %s, tainted", ptrtype, p)
} }
if !val.UpToDate { if !val.UpToDate {
v.UpToDate = false v.UpToDate = false
return fmt.Errorf("outdated %s %s", ptrtype, p) return fmt.Errorf("outdated %s %s", ptrtype, p)
} }
skip := false skip := false
for idx := range val.BelongsToCollections { for idx := range val.BelongsToCollections {
if val.BelongsToCollections[idx] == v.Name { if val.BelongsToCollections[idx] == v.Name {
skip = true skip = true
} }
} }
if !skip { if !skip {
val.BelongsToCollections = append(val.BelongsToCollections, v.Name) val.BelongsToCollections = append(val.BelongsToCollections, v.Name)
} }
hubIdx[ptrtype][p] = val hubIdx[ptrtype][p] = val
log.Tracef("checking for %s - tainted:%t uptodate:%t", p, v.Tainted, v.UpToDate) log.Tracef("checking for %s - tainted:%t uptodate:%t", p, v.Tainted, v.UpToDate)
} }
} }
} }
return nil return nil
} }
@ -311,39 +346,41 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
installdir = hub.ConfigDir installdir = hub.ConfigDir
warnings := []string{} warnings := []string{}
/*For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last*/ // For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last
for _, scan := range ItemTypes { for _, scan := range ItemTypes {
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
if err != nil { if err != nil {
log.Errorf("failed %s : %s", cpath, err) log.Errorf("failed %s : %s", cpath, err)
} }
err = filepath.WalkDir(cpath, parser_visit) err = filepath.WalkDir(cpath, parser_visit)
if err != nil { if err != nil {
return err, warnings return err, warnings
} }
} }
for k, v := range hubIdx[COLLECTIONS] { for k, v := range hubIdx[COLLECTIONS] {
if v.Installed { if v.Installed {
versStat := GetVersionStatus(&v) versStat := GetVersionStatus(&v)
if versStat == 0 { //latest if versStat == 0 { // latest
if err := CollecDepsCheck(&v); err != nil { if err := CollecDepsCheck(&v); err != nil {
warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", v.Name, err)) warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", v.Name, err))
hubIdx[COLLECTIONS][k] = v hubIdx[COLLECTIONS][k] = v
} }
} else if versStat == 1 { //not up-to-date } else if versStat == 1 { // not up-to-date
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version)) warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version))
} else { //version is higher than the highest available from hub? } else { // version is higher than the highest available from hub?
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version)) warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version))
} }
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", v.Name, semver.Compare("v"+v.Version, "v"+v.LocalVersion), v.LocalVersion, v.Version, v.Versions) log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", v.Name, semver.Compare("v"+v.Version, "v"+v.LocalVersion), v.LocalVersion, v.Version, v.Versions)
} }
} }
return nil, warnings return nil, warnings
} }
/* Updates the infos from HubInit() with the local state */ // Updates the infos from HubInit() with the local state
func LocalSync(hub *csconfig.Hub) (error, []string) { func LocalSync(hub *csconfig.Hub) (error, []string) {
skippedLocal = 0 skippedLocal = 0
skippedTainted = 0 skippedTainted = 0
@ -352,10 +389,12 @@ func LocalSync(hub *csconfig.Hub) (error, []string) {
if err != nil { if err != nil {
return fmt.Errorf("failed to scan %s : %s", hub.ConfigDir, err), warnings return fmt.Errorf("failed to scan %s : %s", hub.ConfigDir, err), warnings
} }
err, _ = SyncDir(hub, hub.HubDir) err, _ = SyncDir(hub, hub.HubDir)
if err != nil { if err != nil {
return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings
} }
return nil, warnings return nil, warnings
} }
@ -363,49 +402,61 @@ func GetHubIdx(hub *csconfig.Hub) error {
if hub == nil { if hub == nil {
return fmt.Errorf("no configuration found for hub") return fmt.Errorf("no configuration found for hub")
} }
log.Debugf("loading hub idx %s", hub.HubIndexFile) log.Debugf("loading hub idx %s", hub.HubIndexFile)
bidx, err := os.ReadFile(hub.HubIndexFile) bidx, err := os.ReadFile(hub.HubIndexFile)
if err != nil { if err != nil {
return fmt.Errorf("unable to read index file: %w", err) return fmt.Errorf("unable to read index file: %w", err)
} }
ret, err := LoadPkgIndex(bidx) ret, err := LoadPkgIndex(bidx)
if err != nil { if err != nil {
if !errors.Is(err, ReferenceMissingError) { if !errors.Is(err, ReferenceMissingError) {
log.Fatalf("Unable to load existing index : %v.", err) log.Fatalf("Unable to load existing index : %v.", err)
} }
return err return err
} }
hubIdx = ret hubIdx = ret
err, _ = LocalSync(hub) err, _ = LocalSync(hub)
if err != nil { if err != nil {
log.Fatalf("Failed to sync Hub index with local deployment : %v", err) log.Fatalf("Failed to sync Hub index with local deployment : %v", err)
} }
return nil return nil
} }
/*LoadPkgIndex loads a local .index.json file and returns the map of parsers/scenarios/collections associated*/ // LoadPkgIndex loads a local .index.json file and returns the map of parsers/scenarios/collections associated
func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) { func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
var err error var (
var RawIndex map[string]map[string]Item err error
var missingItems []string RawIndex map[string]map[string]Item
missingItems []string
)
if err = json.Unmarshal(buff, &RawIndex); err != nil { if err = json.Unmarshal(buff, &RawIndex); err != nil {
return nil, fmt.Errorf("failed to unmarshal index : %v", err) return nil, fmt.Errorf("failed to unmarshal index : %v", err)
} }
log.Debugf("%d item types in hub index", len(ItemTypes)) log.Debugf("%d item types in hub index", len(ItemTypes))
/*Iterate over the different types to complete struct */
// Iterate over the different types to complete struct
for _, itemType := range ItemTypes { for _, itemType := range ItemTypes {
/*complete struct*/ // complete struct
log.Tracef("%d item", len(RawIndex[itemType])) log.Tracef("%d item", len(RawIndex[itemType]))
for idx, item := range RawIndex[itemType] { for idx, item := range RawIndex[itemType] {
item.Name = idx item.Name = idx
item.Type = itemType item.Type = itemType
x := strings.Split(item.RemotePath, "/") x := strings.Split(item.RemotePath, "/")
item.FileName = x[len(x)-1] item.FileName = x[len(x)-1]
RawIndex[itemType][idx] = item RawIndex[itemType][idx] = item
/*if it's a collection, check its sub-items are present*/
//XX should be done later // if it's a collection, check its sub-items are present
// XXX should be done later
if itemType == COLLECTIONS { if itemType == COLLECTIONS {
var tmp = [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} var tmp = [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections}
for idx, ptr := range tmp { for idx, ptr := range tmp {
@ -420,8 +471,9 @@ func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
} }
} }
} }
if len(missingItems) > 0 { if len(missingItems) > 0 {
return RawIndex, fmt.Errorf("%q : %w", missingItems, ReferenceMissingError) return RawIndex, fmt.Errorf("%q: %w", missingItems, ReferenceMissingError)
} }
return RawIndex, nil return RawIndex, nil