From 8b5ad6990d390664c9f4be0e93ddf61bbf02f80e Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 3 Oct 2023 11:20:56 +0200 Subject: [PATCH] lint: pkg/cwhub (#2510) no functional changes - reformat - comments - whitespace - removed a dot or two in log messages - some "var x=y" -> x:=y --- pkg/cwhub/cwhub.go | 124 ++++++++++++++++++---------- pkg/cwhub/cwhub_test.go | 92 ++++++++++++++------- pkg/cwhub/dataset.go | 2 + pkg/cwhub/dataset_test.go | 4 +- pkg/cwhub/download.go | 62 ++++++++++++-- pkg/cwhub/download_test.go | 18 +++- pkg/cwhub/helpers.go | 11 ++- pkg/cwhub/helpers_test.go | 6 +- pkg/cwhub/install.go | 34 +++++--- pkg/cwhub/loader.go | 164 ++++++++++++++++++++++++------------- 10 files changed, 360 insertions(+), 157 deletions(-) diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index bdd03c89a..fe521a4ff 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -15,7 +15,7 @@ import ( "golang.org/x/mod/semver" ) -/*managed configuration types*/ +// managed configuration types var PARSERS = "parsers" var PARSERS_OVFLW = "postoverflows" var SCENARIOS = "scenarios" @@ -42,37 +42,37 @@ type ItemHubStatus struct { Status string `json:"status"` } -//Item can be : parsed, scenario, collection +// Item can be: parsed, scenario, collection type Item struct { - /*descriptive info*/ - Type string `yaml:"type,omitempty" json:"type,omitempty"` //parser|postoverflows|scenario|collection(|enrich) - Stage string `json:"stage,omitempty" yaml:"stage,omitempty,omitempty"` //Stage for parser|postoverflow : s00-raw/s01-... - Name string `json:"name,omitempty"` //as seen in .config.json, usually "author/name" - FileName string `json:"file_name,omitempty"` //the filename, ie. apache2-logs.yaml - Description string `yaml:"description,omitempty" json:"description,omitempty"` //as seen in .config.json - Author string `json:"author,omitempty"` //as seen in .config.json - References []string `yaml:"references,omitempty" json:"references,omitempty"` //as seen in .config.json - BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` /*if it's part of collections, track name here*/ + // descriptive info + Type string `yaml:"type,omitempty" json:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich) + Stage string `json:"stage,omitempty" yaml:"stage,omitempty,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... + Name string `json:"name,omitempty"` // as seen in .config.json, usually "author/name" + FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml + Description string `yaml:"description,omitempty" json:"description,omitempty"` // as seen in .config.json + Author string `json:"author,omitempty"` // as seen in .config.json + References []string `yaml:"references,omitempty" json:"references,omitempty"` // as seen in .config.json + BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` // if it's part of collections, track name here - /*remote (hub) infos*/ - RemoteURL string `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` //the full remote uri of file in http - RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` //the path relative to git ie. /parsers/stage/author/file.yaml - RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` //the meow - Version string `json:"version,omitempty"` //the last version - Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` //the list of existing versions + // remote (hub) infos + RemoteURL string `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` // the full remote uri of file in http + RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to git ie. /parsers/stage/author/file.yaml + RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` // the meow + Version string `json:"version,omitempty"` // the last version + Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions - /*local (deployed) infos*/ - LocalPath string `yaml:"local_path,omitempty" json:"local_path,omitempty"` //the local path relative to ${CFG_DIR} - //LocalHubPath string + // local (deployed) infos + LocalPath string `yaml:"local_path,omitempty" json:"local_path,omitempty"` // the local path relative to ${CFG_DIR} + // LocalHubPath string LocalVersion string `json:"local_version,omitempty"` - LocalHash string `json:"local_hash,omitempty"` //the local meow + LocalHash string `json:"local_hash,omitempty"` // the local meow Installed bool `json:"installed,omitempty"` Downloaded bool `json:"downloaded,omitempty"` UpToDate bool `json:"up_to_date,omitempty"` - Tainted bool `json:"tainted,omitempty"` //has it been locally modified - Local bool `json:"local,omitempty"` //if it's a non versioned control one + Tainted bool `json:"tainted,omitempty"` // has it been locally modified + Local bool `json:"local,omitempty"` // if it's a non versioned control one - /*if it's a collection, it not a single file*/ + // if it's a collection, it not a single file Parsers []string `yaml:"parsers,omitempty" json:"parsers,omitempty"` PostOverflows []string `yaml:"postoverflows,omitempty" json:"postoverflows,omitempty"` Scenarios []string `yaml:"scenarios,omitempty" json:"scenarios,omitempty"` @@ -88,6 +88,7 @@ func (i *Item) toHubStatus() ItemHubStatus { status, ok, warning, managed := ItemStatus(*i) hubStatus.Status = status + if !managed { hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.House, status) } else if !i.Installed { @@ -97,27 +98,28 @@ func (i *Item) toHubStatus() ItemHubStatus { } else if ok { hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.CheckMark, status) } + return hubStatus } var skippedLocal = 0 var skippedTainted = 0 -/*To be used when reference(s) (is/are) missing in a collection*/ +// To be used when reference(s) (is/are) missing in a collection var ReferenceMissingError = errors.New("Reference(s) missing in collection") var MissingHubIndex = errors.New("hub index can't be found") -//GetVersionStatus : semver requires 'v' prefix +// GetVersionStatus: semver requires 'v' prefix func GetVersionStatus(v *Item) int { return semver.Compare("v"+v.Version, "v"+v.LocalVersion) } // calculate sha256 of a file func getSHA256(filepath string) (string, error) { - /* Digest of file */ + // Digest of file f, err := os.Open(filepath) if err != nil { - return "", fmt.Errorf("unable to open '%s' : %s", filepath, err) + return "", fmt.Errorf("unable to open '%s': %s", filepath, err) } defer f.Close() @@ -131,49 +133,55 @@ func getSHA256(filepath string) (string, error) { } func GetItemMap(itemType string) map[string]Item { - var m map[string]Item - var ok bool + var ( + m map[string]Item + ok bool + ) if m, ok = hubIdx[itemType]; !ok { return nil } + return m } -//GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. +// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. func GetItemByPath(itemType string, itemPath string) (*Item, error) { - /*try to resolve symlink*/ + // try to resolve symlink finalName := "" + f, err := os.Lstat(itemPath) if err != nil { return nil, fmt.Errorf("while performing lstat on %s: %w", itemPath, err) } if f.Mode()&os.ModeSymlink == 0 { - /*it's not a symlink, it should be the filename itsef the key*/ + // it's not a symlink, it should be the filename itsef the key finalName = filepath.Base(itemPath) } else { - /*resolve the symlink to hub file*/ + // resolve the symlink to hub file pathInHub, err := os.Readlink(itemPath) if err != nil { return nil, fmt.Errorf("while reading symlink of %s: %w", itemPath, err) } - //extract author from path + // extract author from path fname := filepath.Base(pathInHub) author := filepath.Base(filepath.Dir(pathInHub)) - //trim yaml suffix + // trim yaml suffix fname = strings.TrimSuffix(fname, ".yaml") fname = strings.TrimSuffix(fname, ".yml") finalName = fmt.Sprintf("%s/%s", author, fname) } - /*it's not a symlink, it should be the filename itsef the key*/ + // it's not a symlink, it should be the filename itsef the key if m := GetItemMap(itemType); m != nil { if v, ok := m[finalName]; ok { return &v, nil } + return nil, fmt.Errorf("%s not found in %s", finalName, itemType) } + return nil, fmt.Errorf("item type %s doesn't exist", itemType) } @@ -181,35 +189,42 @@ func GetItem(itemType string, itemName string) *Item { if m, ok := GetItemMap(itemType)[itemName]; ok { return &m } + return nil } func AddItem(itemType string, item Item) error { in := false + for _, itype := range ItemTypes { if itype == itemType { in = true } } + if !in { return fmt.Errorf("ItemType %s is unknown", itemType) } + hubIdx[itemType][item.Name] = item + return nil } func DisplaySummary() { log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]), len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW])) + if skippedLocal > 0 || skippedTainted > 0 { - log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted) + log.Printf("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted) } } -//returns: human-text, Enabled, Warning, Unmanaged +// returns: human-text, Enabled, Warning, Unmanaged func ItemStatus(v Item) (string, bool, bool, bool) { strret := "disabled" Ok := false + if v.Installed { Ok = true strret = "enabled" @@ -221,7 +236,7 @@ func ItemStatus(v Item) (string, bool, bool, bool) { strret += ",local" } - //tainted or out of date + // tainted or out of date Warning := false if v.Tainted { Warning = true @@ -230,6 +245,7 @@ func ItemStatus(v Item) (string, bool, bool, bool) { strret += ",update-available" Warning = true } + return strret, Ok, Warning, Managed } @@ -240,9 +256,11 @@ func GetInstalledScenariosAsString() ([]string, error) { if err != nil { return nil, fmt.Errorf("while fetching scenarios: %w", err) } + for _, it := range items { retStr = append(retStr, it.Name) } + return retStr, nil } @@ -252,11 +270,13 @@ func GetInstalledScenarios() ([]Item, error) { if _, ok := hubIdx[SCENARIOS]; !ok { return nil, fmt.Errorf("no scenarios in hubIdx") } + for _, item := range hubIdx[SCENARIOS] { if item.Installed { retItems = append(retItems, item) } } + return retItems, nil } @@ -266,11 +286,13 @@ func GetInstalledParsers() ([]Item, error) { if _, ok := hubIdx[PARSERS]; !ok { return nil, fmt.Errorf("no parsers in hubIdx") } + for _, item := range hubIdx[PARSERS] { if item.Installed { retItems = append(retItems, item) } } + return retItems, nil } @@ -281,9 +303,11 @@ func GetInstalledParsersAsString() ([]string, error) { if err != nil { return nil, fmt.Errorf("while fetching parsers: %w", err) } + for _, it := range items { retStr = append(retStr, it.Name) } + return retStr, nil } @@ -293,11 +317,13 @@ func GetInstalledPostOverflows() ([]Item, error) { if _, ok := hubIdx[PARSERS_OVFLW]; !ok { return nil, fmt.Errorf("no post overflows in hubIdx") } + for _, item := range hubIdx[PARSERS_OVFLW] { if item.Installed { retItems = append(retItems, item) } } + return retItems, nil } @@ -308,9 +334,11 @@ func GetInstalledPostOverflowsAsString() ([]string, error) { if err != nil { return nil, fmt.Errorf("while fetching post overflows: %w", err) } + for _, it := range items { retStr = append(retStr, it.Name) } + return retStr, nil } @@ -325,6 +353,7 @@ func GetInstalledCollectionsAsString() ([]string, error) { for _, it := range items { retStr = append(retStr, it.Name) } + return retStr, nil } @@ -334,15 +363,17 @@ func GetInstalledCollections() ([]Item, error) { if _, ok := hubIdx[COLLECTIONS]; !ok { return nil, fmt.Errorf("no collection in hubIdx") } + for _, item := range hubIdx[COLLECTIONS] { if item.Installed { retItems = append(retItems, item) } } + return retItems, nil } -//Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy) +// Returns a list of entries for packages: name, status, local_path, local_version, utf8_status (fancy) func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus { if _, ok := hubIdx[itemType]; !ok { log.Errorf("type %s doesn't exist", itemType) @@ -351,19 +382,22 @@ func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubSt } var ret = make([]ItemHubStatus, 0) - /*remember, you do it for the user :)*/ + + // remember, you do it for the user :) for _, item := range hubIdx[itemType] { if name != "" && name != item.Name { - //user has requested a specific name + // user has requested a specific name continue } - //Only enabled items ? + // Only enabled items ? if !all && !item.Installed { continue } - //Check the item status + // Check the item status ret = append(ret, item.toHubStatus()) } + sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) + return ret } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index f91b0dced..cf114eeb6 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -9,8 +9,9 @@ import ( "strings" "testing" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) /* @@ -29,30 +30,33 @@ func TestItemStatus(t *testing.T) { defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) - //DownloadHubIdx() + // DownloadHubIdx() if err != nil { t.Fatalf("failed to download index : %s", err) } + if err := GetHubIdx(cfg.Hub); err != nil { t.Fatalf("failed to load hub index : %s", err) } - //get existing map + // get existing map x := GetItemMap(COLLECTIONS) if len(x) == 0 { t.Fatalf("expected non empty result") } - //Get item : good and bad + // Get item : good and bad for k := range x { item := GetItem(COLLECTIONS, k) if item == nil { t.Fatalf("expected item") } + item.Installed = true item.UpToDate = false item.Local = false item.Tainted = false + txt, _, _, _ := ItemStatus(*item) if txt != "enabled,update-available" { t.Fatalf("got '%s'", txt) @@ -62,6 +66,7 @@ func TestItemStatus(t *testing.T) { item.UpToDate = false item.Local = true item.Tainted = false + txt, _, _, _ = ItemStatus(*item) if txt != "disabled,local" { t.Fatalf("got '%s'", txt) @@ -69,6 +74,7 @@ func TestItemStatus(t *testing.T) { break } + DisplaySummary() } @@ -77,26 +83,28 @@ func TestGetters(t *testing.T) { defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) - //DownloadHubIdx() + // DownloadHubIdx() if err != nil { t.Fatalf("failed to download index : %s", err) } + if err := GetHubIdx(cfg.Hub); err != nil { t.Fatalf("failed to load hub index : %s", err) } - //get non existing map + // get non existing map empty := GetItemMap("ratata") if empty != nil { t.Fatalf("expected nil result") } - //get existing map + + // get existing map x := GetItemMap(COLLECTIONS) if len(x) == 0 { t.Fatalf("expected non empty result") } - //Get item : good and bad + // Get item : good and bad for k := range x { empty := GetItem(COLLECTIONS, k+"nope") if empty != nil { @@ -108,7 +116,7 @@ func TestGetters(t *testing.T) { t.Fatalf("expected non empty item") } - //Add item and get it + // Add item and get it item.Name += "nope" if err := AddItem(COLLECTIONS, *item); err != nil { t.Fatalf("didn't expect error : %s", err) @@ -119,7 +127,7 @@ func TestGetters(t *testing.T) { t.Fatalf("expected non empty item") } - //Add bad item + // Add bad item if err := AddItem("ratata", *item); err != nil { if fmt.Sprintf("%s", err) != "ItemType ratata is unknown" { t.Fatalf("unexpected error") @@ -130,7 +138,6 @@ func TestGetters(t *testing.T) { break } - } func TestIndexDownload(t *testing.T) { @@ -138,10 +145,11 @@ func TestIndexDownload(t *testing.T) { defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) - //DownloadHubIdx() + // DownloadHubIdx() if err != nil { t.Fatalf("failed to download index : %s", err) } + if err := GetHubIdx(cfg.Hub); err != nil { t.Fatalf("failed to load hub index : %s", err) } @@ -152,20 +160,23 @@ func getTestCfg() (cfg *csconfig.Config) { cfg.Hub.ConfigDir, _ = filepath.Abs("./install") cfg.Hub.HubDir, _ = filepath.Abs("./hubdir") cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json") + return } func envSetup(t *testing.T) *csconfig.Config { resetResponseByPath() log.SetLevel(log.DebugLevel) + cfg := getTestCfg() defaultTransport := http.DefaultClient.Transport + t.Cleanup(func() { http.DefaultClient.Transport = defaultTransport }) - //Mock the http client + // Mock the http client http.DefaultClient.Transport = newMockTransport() if err := os.MkdirAll(cfg.Hub.ConfigDir, 0700); err != nil { @@ -189,7 +200,6 @@ func envSetup(t *testing.T) *csconfig.Config { return cfg } - func envTearDown(cfg *csconfig.Config) { if err := os.RemoveAll(cfg.Hub.ConfigDir); err != nil { log.Fatalf("failed to remove %s : %s", cfg.Hub.ConfigDir, err) @@ -200,23 +210,25 @@ func envTearDown(cfg *csconfig.Config) { } } - func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { - - //Install the parser + // Install the parser item, err := DownloadLatest(cfg, item, false, false) if err != nil { t.Fatalf("error while downloading %s : %v", item.Name, err) } + if err, _ := LocalSync(cfg); err != nil { t.Fatalf("taint: failed to run localSync : %s", err) } + if !hubIdx[item.Type][item.Name].UpToDate { t.Fatalf("download: %s should be up-to-date", item.Name) } + if hubIdx[item.Type][item.Name].Installed { t.Fatalf("download: %s should not be installed", item.Name) } + if hubIdx[item.Type][item.Name].Tainted { t.Fatalf("download: %s should not be tainted", item.Name) } @@ -225,9 +237,11 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { if err != nil { t.Fatalf("error while enabling %s : %v.", item.Name, err) } + if err, _ := LocalSync(cfg); err != nil { t.Fatalf("taint: failed to run localSync : %s", err) } + if !hubIdx[item.Type][item.Name].Installed { t.Fatalf("install: %s should be installed", item.Name) } @@ -237,6 +251,7 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) { if hubIdx[item.Type][item.Name].Tainted { t.Fatalf("pre-taint: %s should not be tainted", item.Name) } + f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) if err != nil { t.Fatalf("(taint) opening %s (%s) : %s", item.LocalPath, item.Name, err) @@ -246,32 +261,37 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) { if _, err = f.WriteString("tainted"); err != nil { t.Fatalf("tainting %s : %s", item.Name, err) } - //Local sync and check status + + // Local sync and check status if err, _ := LocalSync(cfg); err != nil { t.Fatalf("taint: failed to run localSync : %s", err) } + if !hubIdx[item.Type][item.Name].Tainted { t.Fatalf("taint: %s should be tainted", item.Name) } } func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) { - if hubIdx[item.Type][item.Name].UpToDate { t.Fatalf("update: %s should NOT be up-to-date", item.Name) } - //Update it + check status + + // Update it + check status item, err := DownloadLatest(cfg, item, true, true) if err != nil { t.Fatalf("failed to update %s : %s", item.Name, err) } - //Local sync and check status + + // Local sync and check status if err, _ := LocalSync(cfg); err != nil { t.Fatalf("failed to run localSync : %s", err) } + if !hubIdx[item.Type][item.Name].UpToDate { t.Fatalf("update: %s should be up-to-date", item.Name) } + if hubIdx[item.Type][item.Name].Tainted { t.Fatalf("update: %s should not be tainted anymore", item.Name) } @@ -281,43 +301,51 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) { if !item.Installed { t.Fatalf("disable: %s should be installed", item.Name) } - //Remove + + // Remove item, err := DisableItem(cfg, item, false, false) if err != nil { t.Fatalf("failed to disable item : %v", err) } - //Local sync and check status + + // Local sync and check status if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 { t.Fatalf("failed to run localSync : %s (%+v)", err, warns) } + if hubIdx[item.Type][item.Name].Tainted { t.Fatalf("disable: %s should not be tainted anymore", item.Name) } + if hubIdx[item.Type][item.Name].Installed { t.Fatalf("disable: %s should not be installed anymore", item.Name) } + if !hubIdx[item.Type][item.Name].Downloaded { t.Fatalf("disable: %s should still be downloaded", item.Name) } - //Purge + + // Purge item, err = DisableItem(cfg, item, true, false) if err != nil { t.Fatalf("failed to purge item : %v", err) } - //Local sync and check status + + // Local sync and check status if err, warns := LocalSync(cfg); err != nil || len(warns) > 0 { t.Fatalf("failed to run localSync : %s (%+v)", err, warns) } + if hubIdx[item.Type][item.Name].Installed { t.Fatalf("disable: %s should not be installed anymore", item.Name) } + if hubIdx[item.Type][item.Name].Downloaded { t.Fatalf("disable: %s should not be downloaded", item.Name) } } func TestInstallParser(t *testing.T) { - /* - install a random parser - check its status @@ -331,7 +359,7 @@ func TestInstallParser(t *testing.T) { defer envTearDown(cfg) getHubIdxOrFail(t) - //map iteration is random by itself + // map iteration is random by itself for _, it := range hubIdx[PARSERS] { testInstallItem(cfg.Hub, t, it) it = hubIdx[PARSERS][it.Name] @@ -349,7 +377,6 @@ func TestInstallParser(t *testing.T) { } func TestInstallCollection(t *testing.T) { - /* - install a random parser - check its status @@ -363,7 +390,7 @@ func TestInstallCollection(t *testing.T) { defer envTearDown(cfg) getHubIdxOrFail(t) - //map iteration is random by itself + // map iteration is random by itself for _, it := range hubIdx[COLLECTIONS] { testInstallItem(cfg.Hub, t, it) it = hubIdx[COLLECTIONS][it.Name] @@ -376,6 +403,7 @@ func TestInstallCollection(t *testing.T) { it = hubIdx[COLLECTIONS][it.Name] x := GetHubStatusForItemType(COLLECTIONS, it.Name, false) log.Printf("%+v", x) + break } } @@ -395,10 +423,12 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { StatusCode: http.StatusOK, } response.Header.Set("Content-Type", "application/json") + responseBody := "" + log.Printf("---> %s", req.URL.Path) - /*FAKE PARSER*/ + // FAKE PARSER if resp, ok := responseByPath[req.URL.Path]; ok { responseBody = resp } else { @@ -406,12 +436,14 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { } response.Body = io.NopCloser(strings.NewReader(responseBody)) + return response, nil } func fileToStringX(path string) string { if f, err := os.Open(path); err == nil { defer f.Close() + if data, err := io.ReadAll(f); err == nil { return strings.ReplaceAll(string(data), "\r\n", "\n") } else { diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index 3bfcddee8..2255d40a7 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -18,6 +18,7 @@ type DataSet struct { func downloadFile(url string, destPath string) error { log.Debugf("downloading %s in %s", url, destPath) + req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return err @@ -60,6 +61,7 @@ func GetData(data []*types.DataSource, dataDir string) error { for _, dataS := range data { destPath := filepath.Join(dataDir, dataS.DestPath) log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) + err := downloadFile(dataS.SourceURL, destPath) if err != nil { return err diff --git a/pkg/cwhub/dataset_test.go b/pkg/cwhub/dataset_test.go index 106268c01..40f6ba847 100644 --- a/pkg/cwhub/dataset_test.go +++ b/pkg/cwhub/dataset_test.go @@ -4,9 +4,8 @@ import ( "os" "testing" - "github.com/stretchr/testify/assert" - "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" ) func TestDownloadFile(t *testing.T) { @@ -26,6 +25,7 @@ func TestDownloadFile(t *testing.T) { "https://example.com/x", httpmock.NewStringResponder(404, "not found"), ) + err := downloadFile("https://example.com/xx", examplePath) assert.NoError(t, err) content, err := os.ReadFile(examplePath) diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 06596fe14..1d058572f 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -24,36 +24,45 @@ func UpdateHubIdx(hub *csconfig.Hub) error { if err != nil { return fmt.Errorf("failed to download index: %w", err) } + ret, err := LoadPkgIndex(bidx) if err != nil { if !errors.Is(err, ReferenceMissingError) { return fmt.Errorf("failed to read index: %w", err) } } + hubIdx = ret + if err, _ := LocalSync(hub); err != nil { return fmt.Errorf("failed to sync: %w", err) } + return nil } func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) { log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) if err != nil { return nil, fmt.Errorf("failed to build request for hub index: %w", err) } + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, fmt.Errorf("failed http request for hub index: %w", err) } defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusNotFound { return nil, ErrIndexNotFound } + return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String()) } + body, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) @@ -80,7 +89,9 @@ func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) { if err != nil { return nil, fmt.Errorf("while writing hub index file: %w", err) } + log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile) + return body, nil } @@ -89,11 +100,13 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b var err error log.Debugf("Downloading %s %s", target.Type, target.Name) + if target.Type != COLLECTIONS { if !target.Installed && updateOnly && target.Downloaded { log.Debugf("skipping upgrade of %s : not installed", target.Name) return target, nil } + return DownloadItem(hub, target, overwrite) } @@ -116,11 +129,13 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b //recurse as it's a collection if ptrtype == COLLECTIONS { log.Tracef("collection, recurse") + hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite, updateOnly) if err != nil { return target, fmt.Errorf("while downloading %s: %w", val.Name, err) } } + item, err := DownloadItem(hub, val, overwrite) if err != nil { return target, fmt.Errorf("while downloading %s: %w", val.Name, err) @@ -133,77 +148,95 @@ func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly b return target, fmt.Errorf("enabling '%s': %w", item.Name, err) } } + hubIdx[ptrtype][p] = item } } + target, err = DownloadItem(hub, target, overwrite) if err != nil { return target, fmt.Errorf("failed to download item : %s", err) } + return target, nil } func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) { - var tdir = hub.HubDir - var dataFolder = hub.DataDir - /*if user didn't --force, don't overwrite local, tainted, up-to-date files*/ + tdir := hub.HubDir + dataFolder := hub.DataDir + + // if user didn't --force, don't overwrite local, tainted, up-to-date files if !overwrite { if target.Tainted { log.Debugf("%s : tainted, not updated", target.Name) return target, nil } + if target.UpToDate { - log.Debugf("%s : up-to-date, not updated", target.Name) // We still have to check if data files are present + log.Debugf("%s : up-to-date, not updated", target.Name) } } + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) if err != nil { return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err) } + resp, err := http.DefaultClient.Do(req) if err != nil { return target, fmt.Errorf("while downloading %s: %w", req.URL.String(), err) } + if resp.StatusCode != http.StatusOK { return target, fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) if err != nil { return target, fmt.Errorf("while reading %s: %w", req.URL.String(), err) } + h := sha256.New() if _, err := h.Write(body); err != nil { return target, fmt.Errorf("while hashing %s: %w", target.Name, err) } + meow := fmt.Sprintf("%x", h.Sum(nil)) if meow != target.Versions[target.Version].Digest { log.Errorf("Downloaded version doesn't match index, please 'hub update'") log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) + return target, fmt.Errorf("invalid download hash for %s", target.Name) } + //all good, install //check if parent dir exists tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") parent_dir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") - /*ensure that target file is within target dir*/ + // ensure that target file is within target dir finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath) if err != nil { return target, fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err) } + if !strings.HasPrefix(finalPath, tdir) { return target, fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir) } - /*check dir*/ + + // check dir if _, err = os.Stat(parent_dir); os.IsNotExist(err) { log.Debugf("%s doesn't exist, create", parent_dir) + if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil { return target, fmt.Errorf("while creating parent directories: %w", err) } } - /*check actual file*/ + + // check actual file if _, err = os.Stat(finalPath); !os.IsNotExist(err) { log.Warningf("%s : overwrite", target.Name) log.Debugf("target: %s/%s", tdir, target.RemotePath) @@ -215,11 +248,14 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) if err != nil { return target, fmt.Errorf("while opening file: %w", err) } + defer f.Close() + _, err = f.Write(body) if err != nil { return target, fmt.Errorf("while writing file: %w", err) } + target.Downloaded = true target.Tainted = false target.UpToDate = true @@ -229,6 +265,7 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) } hubIdx[target.Type][target.Name] = target + return target, nil } @@ -238,37 +275,47 @@ func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error { itemFile *os.File err error ) + itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.ConfigDir, target.Type, target.Stage, target.FileName) + if itemFile, err = os.Open(itemFilePath); err != nil { return fmt.Errorf("while opening %s: %w", itemFilePath, err) } + defer itemFile.Close() + if err = downloadData(dataFolder, force, itemFile); err != nil { return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) } + return nil } func downloadData(dataFolder string, force bool, reader io.Reader) error { var err error + dec := yaml.NewDecoder(reader) for { data := &DataSet{} + err = dec.Decode(data) if err != nil { if errors.Is(err, io.EOF) { break } + return fmt.Errorf("while reading file: %w", err) } download := false + for _, dataS := range data.Data { if _, err := os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { download = true } } + if download || force { err = GetData(data.Data, dataFolder) if err != nil { @@ -276,5 +323,6 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { } } } + return nil } diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index 156c41322..351b08f8e 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -5,38 +5,48 @@ import ( "strings" "testing" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) func TestDownloadHubIdx(t *testing.T) { back := RawFileURLTemplate - //bad url template + // bad url template fmt.Println("Test 'bad URL'") + RawFileURLTemplate = "x" + ret, err := DownloadHubIdx(&csconfig.Hub{}) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") { log.Errorf("unexpected error %s", err) } + fmt.Printf("->%+v", ret) - //bad domain + // bad domain fmt.Println("Test 'bad domain'") + RawFileURLTemplate = "https://baddomain/%s/%s" + ret, err = DownloadHubIdx(&csconfig.Hub{}) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") { log.Errorf("unexpected error %s", err) } + fmt.Printf("->%+v", ret) - //bad target path + // bad target path fmt.Println("Test 'bad target path'") + RawFileURLTemplate = back + ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"}) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") { log.Errorf("unexpected error %s", err) } RawFileURLTemplate = back + fmt.Printf("->%+v", ret) } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 4133e2272..1061acd15 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -17,7 +17,7 @@ func chooseHubBranch() (string, error) { latest, err := cwversion.Latest() if err != nil { log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err) - //lint:ignore nilerr reason + //lint:ignore nilerr return "master", nil // ignore } @@ -41,8 +41,10 @@ func chooseHubBranch() (string, error) { log.Warnf("Crowdsec is not the latest version. "+ "Current version is '%s' and the latest stable version is '%s'. Please update it!", csVersion, latest) + log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+ "added to Crowdsec Hub after CrowdSec %s", latest) + return csVersion, nil } @@ -58,8 +60,10 @@ func SetHubBranch() error { if err != nil { return err } + HubBranch = branch log.Debugf("Using branch '%s' for the hub", HubBranch) + return nil } @@ -72,6 +76,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo item := *it if downloadOnly && item.Downloaded && item.UpToDate { log.Warningf("%s is already downloaded and up-to-date", item.Name) + if !force { return nil } @@ -120,6 +125,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo item := *it item, err = DisableItem(csConfig.Hub, item, purge, forceAction) + if err != nil { log.Fatalf("unable to disable %s : %v", item.Name, err) } @@ -127,6 +133,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo if err := AddItem(itemType, item); err != nil { log.Fatalf("unable to add %s: %v", item.Name, err) } + return } @@ -139,6 +146,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo if !v.Installed { continue } + v, err = DisableItem(csConfig.Hub, v, purge, forceAction) if err != nil { log.Fatalf("unable to disable %s : %v", v.Name, err) @@ -149,6 +157,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo } disabled++ } + log.Infof("Disabled %d items", disabled) } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index b8a15519d..d9930e7ec 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" ) -//Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection +// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection // We expect the new scenario to be installed func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { cfg := envSetup(t) @@ -37,6 +37,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { if err := UpdateHubIdx(cfg.Hub); err != nil { t.Fatalf("failed to download index : %s", err) } + getHubIdxOrFail(t) require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) @@ -49,7 +50,6 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) - } // Install a collection, disable a scenario. @@ -140,6 +140,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * if err := UpdateHubIdx(cfg.Hub); err != nil { t.Fatalf("failed to download index : %s", err) } + require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) getHubIdxOrFail(t) @@ -151,6 +152,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * func assertCollectionDepsInstalled(t *testing.T, collection string) { t.Helper() + c := hubIdx[COLLECTIONS][collection] require.NoError(t, CollecDepsCheck(&c)) } diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go index 505c36297..cc06e0595 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/install.go @@ -22,15 +22,17 @@ func purgeItem(hub *csconfig.Hub, target Item) (Item, error) { target.Downloaded = false log.Infof("Removed source file [%s] : %s", target.Name, hubpath) hubIdx[target.Type][target.Name] = target + return target, nil } -//DisableItem to disable an item managed by the hub, removes the symlink if purge is true +// DisableItem to disable an item managed by the hub, removes the symlink if purge is true func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, error) { - var tdir = hub.ConfigDir - var hdir = hub.HubDir var err error + tdir := hub.ConfigDir + hdir := hub.HubDir + if !target.Installed { if purge { target, err = purgeItem(hub, target) @@ -38,6 +40,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, return target, err } } + return target, nil } @@ -54,7 +57,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name) } - /*for a COLLECTIONS, disable sub-items*/ + // for a COLLECTIONS, disable sub-items if target.Type == COLLECTIONS { var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} for idx, ptr := range tmp { @@ -63,12 +66,14 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, if val, ok := hubIdx[ptrtype][p]; ok { // check if the item doesn't belong to another collection before removing it toRemove := true + for _, collection := range val.BelongsToCollections { if collection != target.Name { toRemove = false break } } + if toRemove { hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force) if err != nil { @@ -114,6 +119,7 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, } log.Infof("Removed symlink [%s] : %s", target.Name, syml) } + target.Installed = false if purge { @@ -122,39 +128,46 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, return target, err } } + hubIdx[target.Type][target.Name] = target + return target, nil } // creates symlink between actual config file at hub.HubDir and hub.ConfigDir // Handles collections recursively func EnableItem(hub *csconfig.Hub, target Item) (Item, error) { - var tdir = hub.ConfigDir - var hdir = hub.HubDir var err error + + tdir := hub.ConfigDir + hdir := hub.HubDir + parent_dir := filepath.Clean(tdir + "/" + target.Type + "/" + target.Stage + "/") - /*create directories if needed*/ + // create directories if needed if target.Installed { if target.Tainted { return target, fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) } + if target.Local { return target, fmt.Errorf("%s is local, won't enable", target.Name) } - /* if it's a collection, check sub-items even if the collection file itself is up-to-date */ + // if it's a collection, check sub-items even if the collection file itself is up-to-date if target.UpToDate && target.Type != COLLECTIONS { log.Tracef("%s is installed and up-to-date, skip.", target.Name) return target, nil } } + if _, err := os.Stat(parent_dir); os.IsNotExist(err) { log.Printf("%s doesn't exist, create", parent_dir) + if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil { return target, fmt.Errorf("while creating directory: %w", err) } } - /*install sub-items if it's a collection*/ + // install sub-items if it's a collection if target.Type == COLLECTIONS { var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} for idx, ptr := range tmp { @@ -179,7 +192,7 @@ func EnableItem(hub *csconfig.Hub, target Item) (Item, error) { return target, nil } - //tdir+target.RemotePath + // tdir+target.RemotePath srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath) if err != nil { return target, fmt.Errorf("while getting source path: %w", err) @@ -197,5 +210,6 @@ func EnableItem(hub *csconfig.Hub, target Item) (Item, error) { log.Printf("Enabled %s : %s", target.Type, target.Name) target.Installed = true hubIdx[target.Type][target.Name] = target + return target, nil } diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 1b2b8c832..d2827ffef 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -15,19 +15,20 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -/*the walk/parser_visit function can't receive extra args*/ +// the walk/parser_visit function can't receive extra args var hubdir, installdir string func parser_visit(path string, f os.DirEntry, err error) error { - - var target Item - var local bool - var hubpath string - var inhub bool - var fname string - var ftype string - var fauthor string - var stage string + var ( + target Item + local bool + hubpath string + inhub bool + fname string + ftype string + fauthor string + stage string + ) if err != nil { log.Debugf("while syncing hub dir: %s", err) @@ -39,11 +40,11 @@ func parser_visit(path string, f os.DirEntry, err error) error { if err != nil { return err } - //we only care about files + // we only care about files if f == nil || f.IsDir() { return nil } - //we only care about yaml files + // we only care about yaml files if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") { return nil } @@ -52,9 +53,10 @@ func parser_visit(path string, f os.DirEntry, err error) error { log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubdir, installdir) log.Tracef("subs:%v", subs) - /*we're in hub (~/.hub/hub/)*/ + // we're in hub (~/.hub/hub/) if strings.HasPrefix(path, hubdir) { log.Tracef("in hub dir") + inhub = true //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml //.../hub/scenarios/crowdsec/ssh_bf.yaml @@ -62,11 +64,12 @@ func parser_visit(path string, f os.DirEntry, err error) error { if len(subs) < 4 { log.Fatalf("path is too short : %s (%d)", path, len(subs)) } + fname = subs[len(subs)-1] fauthor = subs[len(subs)-2] stage = subs[len(subs)-3] ftype = subs[len(subs)-4] - } else if strings.HasPrefix(path, installdir) { /*we're in install /etc/crowdsec//... */ + } else if strings.HasPrefix(path, installdir) { // we're in install /etc/crowdsec//... log.Tracef("in install dir") if len(subs) < 3 { log.Fatalf("path is too short : %s (%d)", path, len(subs)) @@ -84,14 +87,15 @@ func parser_visit(path string, f os.DirEntry, err error) error { } log.Tracef("stage:%s ftype:%s", stage, ftype) - //log.Printf("%s -> name:%s stage:%s", path, fname, stage) + // log.Printf("%s -> name:%s stage:%s", path, fname, stage) if stage == SCENARIOS { ftype = SCENARIOS stage = "" } else if stage == COLLECTIONS { ftype = COLLECTIONS stage = "" - } else if ftype != PARSERS && ftype != PARSERS_OVFLW /*its a PARSER / PARSER_OVFLW with a stage */ { + } else if ftype != PARSERS && ftype != PARSERS_OVFLW { + // its a PARSER / PARSER_OVFLW with a stage return fmt.Errorf("unknown configuration type for file '%s'", path) } @@ -102,20 +106,21 @@ func parser_visit(path string, f os.DirEntry, err error) error { /etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml when the collection is installed, both files are created */ - //non symlinks are local user files or hub files + // non symlinks are local user files or hub files if f.Type()&os.ModeSymlink == 0 { local = true + log.Tracef("%s isn't a symlink", path) } else { hubpath, err = os.Readlink(path) if err != nil { return fmt.Errorf("unable to read symlink of %s", path) } - //the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml + // the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml _, err := os.Lstat(hubpath) if os.IsNotExist(err) { log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath) - //remove the symlink + // remove the symlink if err = os.Remove(path); err != nil { return fmt.Errorf("failed to unlink %s: %+v", path, err) } @@ -124,7 +129,7 @@ func parser_visit(path string, f os.DirEntry, err error) error { log.Tracef("%s points to %s", path, hubpath) } - //if it's not a symlink and not in hub, it's a local file, don't bother + // if it's not a symlink and not in hub, it's a local file, don't bother if local && !inhub { log.Tracef("%s is a local file, skip", path) skippedLocal++ @@ -139,29 +144,34 @@ func parser_visit(path string, f os.DirEntry, err error) error { _, target.FileName = filepath.Split(path) hubIdx[ftype][fname] = target + return nil } - //try to find which configuration item it is + // try to find which configuration item it is log.Tracef("check [%s] of %s", fname, ftype) match := false + for k, v := range hubIdx[ftype] { log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml") + if fname != v.FileName { log.Tracef("%s != %s (filename)", fname, v.FileName) continue } - //wrong stage + + // wrong stage if v.Stage != stage { continue } - /*if we are walking hub dir, just mark present files as downloaded*/ + + // if we are walking hub dir, just mark present files as downloaded if inhub { - //wrong author + // wrong author if fauthor != v.Author { continue } - //wrong file + // wrong file if CheckName(v.Name, fauthor, fname) { continue } @@ -171,35 +181,38 @@ func parser_visit(path string, f os.DirEntry, err error) error { v.Downloaded = true } } else if CheckSuffix(hubpath, v.RemotePath) { - //wrong file - /////.yaml + // wrong file + // ///.yaml continue } + sha, err := getSHA256(path) if err != nil { log.Fatalf("Failed to get sha of %s : %v", path, err) } - //let's reverse sort the versions to deal with hash collisions (#154) + + // let's reverse sort the versions to deal with hash collisions (#154) versions := make([]string, 0, len(v.Versions)) for k := range v.Versions { versions = append(versions, k) } + sort.Sort(sort.Reverse(sort.StringSlice(versions))) for _, version := range versions { val := v.Versions[version] if sha != val.Digest { - //log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v)) + // log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v)) continue } - /*we got an exact match, update struct*/ + // we got an exact match, update struct if !inhub { log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version) v.LocalPath = path v.LocalVersion = version v.Tainted = false v.Downloaded = true - /*if we're walking the hub, present file doesn't means installed file*/ + // if we're walking the hub, present file doesn't means installed file v.Installed = true v.LocalHash = sha _, target.FileName = filepath.Split(path) @@ -207,29 +220,34 @@ func parser_visit(path string, f os.DirEntry, err error) error { v.Downloaded = true v.LocalHash = sha } + if version == v.Version { log.Tracef("%s is up-to-date", v.Name) v.UpToDate = true } + match = true + break } + if !match { log.Tracef("got tainted match for %s : %s", v.Name, path) + skippedTainted += 1 - //the file and the stage is right, but the hash is wrong, it has been tainted by user + // the file and the stage is right, but the hash is wrong, it has been tainted by user if !inhub { v.LocalPath = path v.Installed = true } + v.UpToDate = false v.LocalVersion = "?" v.Tainted = true v.LocalHash = sha _, target.FileName = filepath.Split(path) - } - //update the entry if appropriate + // update the entry if appropriate // if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D { // fmt.Printf("Updating %s", k) // hubIdx[ftype][k] = v @@ -237,22 +255,25 @@ func parser_visit(path string, f os.DirEntry, err error) error { // } else if hubIdx[ftype][k] = v + return nil } + log.Infof("Ignoring file %s of type %s", path, ftype) + return nil } func CollecDepsCheck(v *Item) error { - - if GetVersionStatus(v) != 0 { //not up-to-date + if GetVersionStatus(v) != 0 { // not up-to-date log.Debugf("%s dependencies not checked : not up-to-date", v.Name) return nil } - /*if it's a collection, ensure all the items are installed, or tag it as tainted*/ + // if it's a collection, ensure all the items are installed, or tag it as tainted if v.Type == COLLECTIONS { log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) + var tmp = [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} for idx, ptr := range tmp { ptrtype := ItemTypes[idx] @@ -261,48 +282,62 @@ func CollecDepsCheck(v *Item) error { if !ok { log.Fatalf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, v.Name) } + log.Tracef("check %s installed:%t", val.Name, val.Installed) + if !v.Installed { continue } + if val.Type == COLLECTIONS { log.Tracef("collec, recurse.") + if err := CollecDepsCheck(&val); err != nil { if val.Tainted { v.Tainted = true } - return fmt.Errorf("sub collection %s is broken : %s", val.Name, err) + + return fmt.Errorf("sub collection %s is broken: %s", val.Name, err) } + hubIdx[ptrtype][p] = val } - //propagate the state of sub-items to set + // propagate the state of sub-items to set if val.Tainted { v.Tainted = true - return fmt.Errorf("tainted %s %s, tainted.", ptrtype, p) + return fmt.Errorf("tainted %s %s, tainted", ptrtype, p) } + if !val.Installed && v.Installed { v.Tainted = true - return fmt.Errorf("missing %s %s, tainted.", ptrtype, p) + return fmt.Errorf("missing %s %s, tainted", ptrtype, p) } + if !val.UpToDate { v.UpToDate = false return fmt.Errorf("outdated %s %s", ptrtype, p) } + skip := false + for idx := range val.BelongsToCollections { if val.BelongsToCollections[idx] == v.Name { skip = true } } + if !skip { val.BelongsToCollections = append(val.BelongsToCollections, v.Name) } + hubIdx[ptrtype][p] = val + log.Tracef("checking for %s - tainted:%t uptodate:%t", p, v.Tainted, v.UpToDate) } } } + return nil } @@ -311,39 +346,41 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) { installdir = hub.ConfigDir warnings := []string{} - /*For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last*/ + // For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last for _, scan := range ItemTypes { cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) if err != nil { log.Errorf("failed %s : %s", cpath, err) } + err = filepath.WalkDir(cpath, parser_visit) if err != nil { return err, warnings } - } for k, v := range hubIdx[COLLECTIONS] { if v.Installed { versStat := GetVersionStatus(&v) - if versStat == 0 { //latest + if versStat == 0 { // latest if err := CollecDepsCheck(&v); err != nil { warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", v.Name, err)) hubIdx[COLLECTIONS][k] = v } - } else if versStat == 1 { //not up-to-date + } else if versStat == 1 { // not up-to-date warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version)) - } else { //version is higher than the highest available from hub? + } else { // version is higher than the highest available from hub? warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version)) } + log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", v.Name, semver.Compare("v"+v.Version, "v"+v.LocalVersion), v.LocalVersion, v.Version, v.Versions) } } + return nil, warnings } -/* Updates the infos from HubInit() with the local state */ +// Updates the infos from HubInit() with the local state func LocalSync(hub *csconfig.Hub) (error, []string) { skippedLocal = 0 skippedTainted = 0 @@ -352,10 +389,12 @@ func LocalSync(hub *csconfig.Hub) (error, []string) { if err != nil { return fmt.Errorf("failed to scan %s : %s", hub.ConfigDir, err), warnings } + err, _ = SyncDir(hub, hub.HubDir) if err != nil { return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings } + return nil, warnings } @@ -363,49 +402,61 @@ func GetHubIdx(hub *csconfig.Hub) error { if hub == nil { return fmt.Errorf("no configuration found for hub") } + log.Debugf("loading hub idx %s", hub.HubIndexFile) + bidx, err := os.ReadFile(hub.HubIndexFile) if err != nil { return fmt.Errorf("unable to read index file: %w", err) } + ret, err := LoadPkgIndex(bidx) if err != nil { if !errors.Is(err, ReferenceMissingError) { log.Fatalf("Unable to load existing index : %v.", err) } + return err } + hubIdx = ret + err, _ = LocalSync(hub) if err != nil { log.Fatalf("Failed to sync Hub index with local deployment : %v", err) } + return nil } -/*LoadPkgIndex loads a local .index.json file and returns the map of parsers/scenarios/collections associated*/ +// LoadPkgIndex loads a local .index.json file and returns the map of parsers/scenarios/collections associated func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) { - var err error - var RawIndex map[string]map[string]Item - var missingItems []string + var ( + err error + RawIndex map[string]map[string]Item + missingItems []string + ) if err = json.Unmarshal(buff, &RawIndex); err != nil { return nil, fmt.Errorf("failed to unmarshal index : %v", err) } log.Debugf("%d item types in hub index", len(ItemTypes)) - /*Iterate over the different types to complete struct */ + + // Iterate over the different types to complete struct for _, itemType := range ItemTypes { - /*complete struct*/ + // complete struct log.Tracef("%d item", len(RawIndex[itemType])) + for idx, item := range RawIndex[itemType] { item.Name = idx item.Type = itemType x := strings.Split(item.RemotePath, "/") item.FileName = x[len(x)-1] RawIndex[itemType][idx] = item - /*if it's a collection, check its sub-items are present*/ - //XX should be done later + + // if it's a collection, check its sub-items are present + // XXX should be done later if itemType == COLLECTIONS { var tmp = [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} for idx, ptr := range tmp { @@ -420,8 +471,9 @@ func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) { } } } + if len(missingItems) > 0 { - return RawIndex, fmt.Errorf("%q : %w", missingItems, ReferenceMissingError) + return RawIndex, fmt.Errorf("%q: %w", missingItems, ReferenceMissingError) } return RawIndex, nil