From 056c7801c6a6d2e8b48cfef204d9c24b825578df Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Mon, 27 Jul 2020 13:47:32 +0200 Subject: [PATCH] add unitest in cwhub package (#144) --- pkg/cwhub/cwhub.go | 174 +++++++++ pkg/cwhub/cwhub_test.go | 420 ++++++++++++++++++++ pkg/cwhub/download.go | 232 +++++++++++ pkg/cwhub/errors.go | 8 - pkg/cwhub/hubMgmt.go | 846 ---------------------------------------- pkg/cwhub/install.go | 151 +++++++ pkg/cwhub/loader.go | 352 +++++++++++++++++ 7 files changed, 1329 insertions(+), 854 deletions(-) create mode 100644 pkg/cwhub/cwhub.go create mode 100644 pkg/cwhub/cwhub_test.go create mode 100644 pkg/cwhub/download.go delete mode 100644 pkg/cwhub/errors.go delete mode 100644 pkg/cwhub/hubMgmt.go create mode 100644 pkg/cwhub/install.go create mode 100644 pkg/cwhub/loader.go diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go new file mode 100644 index 000000000..93be0fb10 --- /dev/null +++ b/pkg/cwhub/cwhub.go @@ -0,0 +1,174 @@ +package cwhub + +import ( + "crypto/sha256" + "errors" + "fmt" + "io" + "os" + + "github.com/enescakir/emoji" + + log "github.com/sirupsen/logrus" +) + +var PARSERS = "parsers" +var PARSERS_OVFLW = "postoverflows" +var SCENARIOS = "scenarios" +var COLLECTIONS = "collections" + +var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS} + +var HubIdx map[string]map[string]Item + +var Installdir = "/etc/crowdsec/" +var Hubdir = "/etc/crowdsec/cscli/hub/" +var Cfgdir = "/etc/crowdsec/cscli/" + +var RawFileURLTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/%s/%s" +var HubIndexFile = ".index.json" +var HubBranch = "master" + +type ItemVersion struct { + Digest string + Deprecated bool +} + +//Item can be : parsed, scenario, collection +type Item struct { + /*descriptive info*/ + Type string `yaml:"type,omitempty"` //parser|postoverflows|scenario|collection(|enrich) + Stage string `json:"stage" yaml:"stage,omitempty,omitempty"` //Stage for parser|postoverflow : s00-raw/s01-... + Name string //as seen in .config.json, usually "author/name" + FileName string //the filename, ie. apache2-logs.yaml + Description string `yaml:"description,omitempty"` //as seen in .config.json + Author string `json:"author"` //as seen in .config.json + References []string `yaml:"references,omitempty"` //as seen in .config.json + BelongsToCollections []string `yaml:"belongs_to_collections,omitempty"` /*if it's part of collections, track name here*/ + + /*remote (hub) infos*/ + RemoteURL string `yaml:"remoteURL,omitempty"` //the full remote uri of file in http + RemotePath string `json:"path" yaml:"remote_path,omitempty"` //the path relative to git ie. /parsers/stage/author/file.yaml + RemoteHash string `yaml:"hash,omitempty"` //the meow + Version string `json:"version"` //the last version + Versions map[string]ItemVersion `json:"versions" yaml:"-"` //the list of existing versions + + /*local (deployed) infos*/ + LocalPath string `yaml:"local_path,omitempty"` //the local path relative to ${CFG_DIR} + //LocalHubPath string + LocalVersion string + LocalHash string //the local meow + Installed bool + Downloaded bool + UpToDate bool + Tainted bool //has it been locally modified + Local bool //if it's a non versioned control one + + /*if it's a collection, it not a single file*/ + Parsers []string `yaml:"parsers,omitempty"` + PostOverflows []string `yaml:"postoverflows,omitempty"` + Scenarios []string `yaml:"scenarios,omitempty"` + Collections []string `yaml:"collections,omitempty"` +} + +var skippedLocal = 0 +var skippedTainted = 0 + +/*To be used when reference(s) (is/are) missing in a collection*/ +var ReferenceMissingError = errors.New("Reference(s) missing in collection") +var MissingHubIndex = errors.New("hub index can't be found") + +// calculate sha256 of a file +func getSHA256(filepath string) (string, error) { + /* Digest of file */ + f, err := os.Open(filepath) + if err != nil { + return "", fmt.Errorf("unable to open '%s' : %s", filepath, err.Error()) + } + + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", fmt.Errorf("unable to calculate sha256 of '%s': %s", filepath, err.Error()) + } + + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +func DisplaySummary() { + log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(HubIdx[COLLECTIONS]), + len(HubIdx[PARSERS]), len(HubIdx[SCENARIOS]), len(HubIdx[PARSERS_OVFLW])) + if skippedLocal > 0 || skippedTainted > 0 { + log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted) + } +} + +//returns: human-text, Enabled, Warning, Unmanaged +func ItemStatus(v Item) (string, bool, bool, bool) { + var Ok, Warning, Managed bool + var strret string + + if !v.Installed { + strret = "disabled" + Ok = false + } else { + Ok = true + strret = "enabled" + } + + if v.Local { + Managed = false + strret += ",local" + } else { + Managed = true + } + + //tainted or out of date + if v.Tainted { + Warning = true + strret += ",tainted" + } else if !v.UpToDate { + strret += ",update-available" + Warning = true + } + return strret, Ok, Warning, Managed +} + +//Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy) +func HubStatus(itype string, name string, list_all bool) []map[string]string { + if _, ok := HubIdx[itype]; !ok { + log.Errorf("type %s doesn't exist", itype) + return nil + } + + var mli []map[string]string + /*remember, you do it for the user :)*/ + for _, v := range HubIdx[itype] { + if name != "" && name != v.Name { + //user has required a specific name + continue + } + //Only enabled items ? + if !list_all && !v.Installed { + continue + } + //Check the item status + st, ok, warning, managed := ItemStatus(v) + tmp := make(map[string]string) + tmp["name"] = v.Name + tmp["status"] = st + tmp["local_version"] = v.LocalVersion + tmp["local_path"] = v.LocalPath + tmp["description"] = v.Description + if !managed || !v.Installed { + tmp["utf8_status"] = fmt.Sprintf("%v %s", emoji.Prohibited, st) + } else if warning { + tmp["utf8_status"] = fmt.Sprintf("%v %s", emoji.Warning, st) + } else if ok { + tmp["utf8_status"] = fmt.Sprintf("%v %s", emoji.CheckMark, st) + } + mli = append(mli, tmp) + } + return mli +} diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go new file mode 100644 index 000000000..02e601c04 --- /dev/null +++ b/pkg/cwhub/cwhub_test.go @@ -0,0 +1,420 @@ +package cwhub + +import ( + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + log "github.com/sirupsen/logrus" +) + +/* + To test : + - Download 'first' hub index + - Update hub index + - Install collection + list content + - Taint existing parser + list + - Upgrade collection +*/ + +var testDataFolder = "." + +func TestIndexDownload(t *testing.T) { + + os.RemoveAll(Cfgdir) + test_prepenv() + + err := LoadHubIdx() + //DownloadHubIdx() + if err != nil { + t.Fatalf("failed to download index") + } + if err := GetHubIdx(); err != nil { + t.Fatalf("failed to load hub index") + } +} + +func test_prepenv() { + log.SetLevel(log.DebugLevel) + + Cfgdir = filepath.Clean("./cscli") + Installdir = filepath.Clean("./install") + Hubdir = filepath.Clean("./hubdir") + + //Mock the http client + http.DefaultClient.Transport = newMockTransport() + + if err := os.RemoveAll(Cfgdir); err != nil { + log.Fatalf("failed to remove %s : %s", Installdir, err) + } + + if err := os.MkdirAll(Cfgdir, 0700); err != nil { + log.Fatalf("mkdir : %s", err) + } + if err := UpdateHubIdx(); err != nil { + log.Fatalf("failed to download index : %s", err) + } + + if err := os.RemoveAll(Installdir); err != nil { + log.Fatalf("failed to remove %s : %s", Installdir, err) + } + if err := os.MkdirAll(Installdir, 0700); err != nil { + log.Fatalf("failed to mkdir %s : %s", Installdir, err) + } + if err := os.RemoveAll(Hubdir); err != nil { + log.Fatalf("failed to remove %s : %s", Hubdir, err) + } + if err := os.MkdirAll(Hubdir, 0700); err != nil { + log.Fatalf("failed to mkdir %s : %s", Hubdir, err) + } + +} + +func testInstallItem(t *testing.T, item Item) { + //Install the parser + item, err := DownloadLatest(item, Hubdir, false, testDataFolder) + if err != nil { + t.Fatalf("error while downloading %s : %v", item.Name, err) + } + if err := LocalSync(); err != nil { + t.Fatalf("taint: failed to run localSync : %s", err) + } + if !HubIdx[item.Type][item.Name].UpToDate { + t.Fatalf("download: %s should be up-to-date", item.Name) + } + if HubIdx[item.Type][item.Name].Installed { + t.Fatalf("download: %s should not be install", item.Name) + } + if HubIdx[item.Type][item.Name].Tainted { + t.Fatalf("download: %s should not be tainted", item.Name) + } + + item, err = EnableItem(item, Installdir, Hubdir) + if err != nil { + t.Fatalf("error while enabled %s : %v.", item.Name, err) + } + if err := LocalSync(); err != nil { + t.Fatalf("taint: failed to run localSync : %s", err) + } + if !HubIdx[item.Type][item.Name].Installed { + t.Fatalf("install: %s should be install", item.Name) + } +} + +func testTaintItem(t *testing.T, item Item) { + if HubIdx[item.Type][item.Name].Tainted { + t.Fatalf("pre-taint: %s should not be tainted", item.Name) + } + f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + t.Fatalf("(taint) opening %s (%s) : %s", item.LocalPath, item.Name, err) + } + + if _, err = f.WriteString("tainted"); err != nil { + t.Fatalf("tainting %s : %s", item.Name, err) + } + f.Close() + //Local sync and check status + if err := LocalSync(); err != nil { + t.Fatalf("taint: failed to run localSync : %s", err) + } + if !HubIdx[item.Type][item.Name].Tainted { + t.Fatalf("taint: %s should be tainted", item.Name) + } +} + +func testUpdateItem(t *testing.T, item Item) { + + if HubIdx[item.Type][item.Name].UpToDate { + t.Fatalf("update: %s should NOT be up-to-date", item.Name) + } + //Update it + check status + item, err := DownloadLatest(item, Hubdir, true, testDataFolder) + if err != nil { + t.Fatalf("failed to update %s : %s", item.Name, err) + } + //Local sync and check status + if err := LocalSync(); err != nil { + t.Fatalf("failed to run localSync : %s", err) + } + if !HubIdx[item.Type][item.Name].UpToDate { + t.Fatalf("update: %s should be up-to-date", item.Name) + } + if HubIdx[item.Type][item.Name].Tainted { + t.Fatalf("update: %s should not be tainted anymore", item.Name) + } +} + +func testDisableItem(t *testing.T, item Item) { + if !item.Installed { + t.Fatalf("disable: %s should be installed", item.Name) + } + //Remove + item, err := DisableItem(item, Installdir, Hubdir, false) + if err != nil { + t.Fatalf("failed to disable item : %v", err) + } + //Local sync and check status + if err := LocalSync(); err != nil { + t.Fatalf("failed to run localSync : %s", err) + } + if HubIdx[item.Type][item.Name].Tainted { + t.Fatalf("disable: %s should not be tainted anymore", item.Name) + } + if HubIdx[item.Type][item.Name].Installed { + t.Fatalf("disable: %s should not be installed anymore", item.Name) + } + if !HubIdx[item.Type][item.Name].Downloaded { + t.Fatalf("disable: %s should still be downloaded", item.Name) + } + //Purge + item, err = DisableItem(item, Installdir, Hubdir, true) + if err != nil { + t.Fatalf("failed to purge item : %v", err) + } + //Local sync and check status + if err := LocalSync(); err != nil { + t.Fatalf("failed to run localSync : %s", err) + } + if HubIdx[item.Type][item.Name].Installed { + t.Fatalf("disable: %s should not be installed anymore", item.Name) + } + if HubIdx[item.Type][item.Name].Downloaded { + t.Fatalf("disable: %s should not be downloaded", item.Name) + } +} + +func TestInstallParser(t *testing.T) { + + /* + - install a random parser + - check its status + - taint it + - check its status + - force update it + - check its status + - remove it + */ + test_prepenv() + + if err := GetHubIdx(); err != nil { + t.Fatalf("failed to load hub index") + } + //map iteration is random by itself + for _, it := range HubIdx[PARSERS] { + testInstallItem(t, it) + it = HubIdx[PARSERS][it.Name] + _ = HubStatus(PARSERS, it.Name, false) + testTaintItem(t, it) + it = HubIdx[PARSERS][it.Name] + _ = HubStatus(PARSERS, it.Name, false) + testUpdateItem(t, it) + it = HubIdx[PARSERS][it.Name] + testDisableItem(t, it) + it = HubIdx[PARSERS][it.Name] + + break + } +} + +func TestInstallCollection(t *testing.T) { + + /* + - install a random parser + - check its status + - taint it + - check its status + - force update it + - check its status + - remove it + */ + test_prepenv() + + if err := GetHubIdx(); err != nil { + t.Fatalf("failed to load hub index") + } + //map iteration is random by itself + for _, it := range HubIdx[COLLECTIONS] { + testInstallItem(t, it) + it = HubIdx[COLLECTIONS][it.Name] + testTaintItem(t, it) + it = HubIdx[COLLECTIONS][it.Name] + testUpdateItem(t, it) + it = HubIdx[COLLECTIONS][it.Name] + testDisableItem(t, it) + it = HubIdx[COLLECTIONS][it.Name] + x := HubStatus(COLLECTIONS, it.Name, false) + log.Printf("%+v", x) + break + } +} + +type mockTransport struct{} + +func newMockTransport() http.RoundTripper { + return &mockTransport{} +} + +// Implement http.RoundTripper +func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // Create mocked http.Response + response := &http.Response{ + Header: make(http.Header), + Request: req, + StatusCode: http.StatusOK, + } + response.Header.Set("Content-Type", "application/json") + responseBody := "" + log.Printf("---> %s", req.URL.Path) + + /*FAKE PARSER*/ + if req.URL.Path == "/crowdsecurity/hub/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml" { + responseBody = `onsuccess: next_stage +filter: evt.Parsed.program == 'foobar_parser' +name: crowdsecurity/foobar_parser +#debug: true +description: A parser for foobar_parser WAF +grok: + name: foobar_parser + apply_on: message +` + + } else if req.URL.Path == "/crowdsecurity/hub/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml" { + responseBody = `onsuccess: next_stage +filter: evt.Parsed.program == 'foobar_parser' +name: crowdsecurity/foobar_parser +#debug: true +description: A parser for foobar_parser WAF +grok: + name: foobar_parser + apply_on: message +` + /*FAKE SCENARIO*/ + + } else if req.URL.Path == "/crowdsecurity/hub/master/scenarios/crowdsecurity/foobar_scenario.yaml" { + responseBody = `filter: true +name: crowdsecurity/foobar_scenario` + /*FAKE COLLECTIONS*/ + } else if req.URL.Path == "/crowdsecurity/hub/master/collections/crowdsecurity/foobar.yaml" { + responseBody = ` +blah: blalala +qwe: jejwejejw` + } else if req.URL.Path == "/crowdsecurity/hub/master/collections/crowdsecurity/foobar_subcollection.yaml" { + responseBody = ` +blah: blalala +qwe: jejwejejw` + } else if req.URL.Path == "/crowdsecurity/hub/master/.index.json" { + responseBody = + `{ + "collections": { + "crowdsecurity/foobar": { + "path": "collections/crowdsecurity/foobar.yaml", + "version": "0.1", + "versions": { + "0.1": { + "digest": "786c9490e4dd234453e53aa9bb7d28c60668e31c3c0c71a7dd6d0abbfa60261a", + "deprecated": false + } + }, + "long_description": "bG9uZyBkZXNjcmlwdGlvbgo=", + "content": "bG9uZyBkZXNjcmlwdGlvbgo=", + "description": "foobar collection : foobar", + "author": "crowdsecurity", + "labels": null, + "collections" : ["crowdsecurity/foobar_subcollection"], + "parsers": [ + "crowdsecurity/foobar_parser" + ], + "scenarios": [ + "crowdsecurity/foobar_scenario" + ] + }, + "crowdsecurity/foobar_subcollection": { + "path": "collections/crowdsecurity/foobar_subcollection.yaml", + "version": "0.1", + "versions": { + "0.1": { + "digest": "786c9490e4dd234453e53aa9bb7d28c60668e31c3c0c71a7dd6d0abbfa60261a", + "deprecated": false + } + }, + "long_description": "bG9uZyBkZXNjcmlwdGlvbgo=", + "content": "bG9uZyBkZXNjcmlwdGlvbgo=", + "description": "foobar collection : foobar", + "author": "crowdsecurity", + "labels": null, + "parsers": [ + "crowdsecurity/foobar_subparser" + ] + } + }, + "parsers": { + "crowdsecurity/foobar_parser": { + "path": "parsers/s01-parse/crowdsecurity/foobar_parser.yaml", + "stage": "s01-parse", + "version": "0.1", + "versions": { + "0.1": { + "digest": "7d72765baa7227095d8e83803d81f2a8f383e5808f1a4d72deb425352afd59ae", + "deprecated": false + } + }, + "long_description": "bG9uZyBkZXNjcmlwdGlvbgo=", + "content": "bG9uZyBkZXNjcmlwdGlvbgo=", + "description": "A foobar parser", + "author": "crowdsecurity", + "labels": null + }, + "crowdsecurity/foobar_subparser": { + "path": "parsers/s01-parse/crowdsecurity/foobar_subparser.yaml", + "stage": "s01-parse", + "version": "0.1", + "versions": { + "0.1": { + "digest": "7d72765baa7227095d8e83803d81f2a8f383e5808f1a4d72deb425352afd59ae", + "deprecated": false + } + }, + "long_description": "bG9uZyBkZXNjcmlwdGlvbgo=", + "content": "bG9uZyBkZXNjcmlwdGlvbgo=", + "description": "A foobar parser", + "author": "crowdsecurity", + "labels": null + } + }, + "postoverflows": { + }, + "scenarios": { + "crowdsecurity/foobar_scenario": { + "path": "scenarios/crowdsecurity/foobar_scenario.yaml", + "version": "0.1", + "versions": { + "0.1": { + "digest": "a76b389db944ca7a9e5a3f3ae61ee2d4ee98167164ec9b971174b1d44f5a01c6", + "deprecated": false + } + }, + "long_description": "bG9uZyBkZXNjcmlwdGlvbgo=", + "content": "bG9uZyBkZXNjcmlwdGlvbgo=", + "description": "a foobar scenario", + "author": "crowdsecurity", + "labels": { + "remediation": "true", + "scope": "ip", + "service": "http", + "type": "web_attack" + } + } + } + } + ` + } else { + log.Fatalf("unexpected url :/") + } + + response.Body = ioutil.NopCloser(strings.NewReader(responseBody)) + return response, nil +} diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go new file mode 100644 index 000000000..194539998 --- /dev/null +++ b/pkg/cwhub/download.go @@ -0,0 +1,232 @@ +package cwhub + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "strings" + + "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +func LoadHubIdx() error { + bidx, err := ioutil.ReadFile(path.Join(Cfgdir, "/.index.json")) + if err != nil { + return err + } + ret, err := LoadPkgIndex(bidx) + if err != nil { + if !errors.Is(err, ReferenceMissingError) { + log.Fatalf("Unable to load freshly downloaded index : %v.", err) + } + } + HubIdx = ret + if err := LocalSync(); err != nil { + log.Fatalf("Failed to sync Hub index with local deployment : %v", err) + } + return nil +} + +func UpdateHubIdx() error { + + bidx, err := DownloadHubIdx() + if err != nil { + log.Fatalf("Unable to download index : %v.", err) + } + ret, err := LoadPkgIndex(bidx) + if err != nil { + if !errors.Is(err, ReferenceMissingError) { + log.Fatalf("Unable to load freshly downloaded index : %v.", err) + } + } + HubIdx = ret + if err := LocalSync(); err != nil { + log.Fatalf("Failed to sync Hub index with local deployment : %v", err) + } + return nil +} + +func DownloadHubIdx() ([]byte, error) { + req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) + if err != nil { + log.Errorf("failed request : %s", err) + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Errorf("failed request Do : %s", err) + return nil, err + } + if resp.StatusCode != 200 { + log.Errorf("got code %d while requesting %s, abort", resp.StatusCode, + fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) + return nil, fmt.Errorf("bad http code") + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Errorf("failed request reqd: %s", err) + return nil, err + } + //os.Remove(path.Join(configFolder, GitIndexFile)) + file, err := os.OpenFile(path.Join(Cfgdir, "/.index.json"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + + if err != nil { + log.Fatalf(err.Error()) + } + defer file.Close() + + wsize, err := file.WriteString(string(body)) + if err != nil { + log.Fatalf(err.Error()) + } + log.Infof("Wrote new %d bytes index to %s", wsize, path.Join(Cfgdir, "/.index.json")) + return body, nil +} + +//DownloadLatest will download the latest version of Item to the tdir directory +func DownloadLatest(target Item, tdir string, overwrite bool, dataFolder string) (Item, error) { + var err error + log.Debugf("Downloading %s %s", target.Type, target.Name) + if target.Type == COLLECTIONS { + var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} + for idx, ptr := range tmp { + ptrtype := ItemTypes[idx] + for _, p := range ptr { + if val, ok := HubIdx[ptrtype][p]; ok { + log.Debugf("Download %s sub-item : %s %s", target.Name, ptrtype, p) + //recurse as it's a collection + if ptrtype == COLLECTIONS { + log.Tracef("collection, recurse") + HubIdx[ptrtype][p], err = DownloadLatest(val, tdir, overwrite, dataFolder) + if err != nil { + log.Errorf("Encountered error while downloading sub-item %s %s : %s.", ptrtype, p, err) + return target, fmt.Errorf("encountered error while downloading %s for %s, abort", val.Name, target.Name) + } + } + HubIdx[ptrtype][p], err = DownloadItem(val, tdir, overwrite, dataFolder) + if err != nil { + log.Errorf("Encountered error while downloading sub-item %s %s : %s.", ptrtype, p, err) + return target, fmt.Errorf("encountered error while downloading %s for %s, abort", val.Name, target.Name) + } + } else { + //log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name) + return target, fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) + } + } + } + target, err = DownloadItem(target, tdir, overwrite, dataFolder) + if err != nil { + return target, fmt.Errorf("failed to download item : %s", err) + } + } else { + return DownloadItem(target, tdir, overwrite, dataFolder) + } + return target, nil +} + +func DownloadItem(target Item, tdir string, overwrite bool, dataFolder string) (Item, error) { + + /*if user didn't --force, don't overwrite local, tainted, up-to-date files*/ + if !overwrite { + if target.Tainted { + log.Debugf("%s : tainted, not updated", target.Name) + return target, nil + } + if target.UpToDate { + log.Debugf("%s : up-to-date, not updated", target.Name) + return target, nil + } + } + + //log.Infof("Downloading %s to %s", target.Name, tdir) + req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) + if err != nil { + log.Errorf("%s : request creation failed : %s", target.Name, err) + return target, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Errorf("%s : request failed : %s", target.Name, err) + return target, err + } + if resp.StatusCode != 200 { + log.Errorf("%s : non 200 response : %d", target.Name, resp.StatusCode) + return target, fmt.Errorf("bad http code") + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Errorf("%s : failed request read: %s", target.Name, err) + return target, err + } + h := sha256.New() + if _, err := h.Write([]byte(body)); err != nil { + return target, fmt.Errorf("%s : failed to write : %s", target.Name, err) + } + meow := fmt.Sprintf("%x", h.Sum(nil)) + if meow != target.Versions[target.Version].Digest { + log.Errorf("Downloaded version doesn't match index, please 'hub update'") + log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) + return target, fmt.Errorf("invalid download hash") + } + //all good, install + //check if parent dir exists + tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") + parent_dir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") + + /*check dir*/ + if _, err = os.Stat(parent_dir); os.IsNotExist(err) { + log.Debugf("%s doesn't exist, create", parent_dir) + if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil { + return target, fmt.Errorf("unable to create parent directories") + } + } + /*check actual file*/ + if _, err = os.Stat(tdir + "/" + target.RemotePath); !os.IsNotExist(err) { + log.Warningf("%s : overwrite", target.Name) + log.Debugf("target: %s/%s", tdir, target.RemotePath) + } else { + log.Infof("%s : OK", target.Name) + } + + f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return target, fmt.Errorf("failed to open destination file %s : %v", tdir+"/"+target.RemotePath, err) + } + defer f.Close() + _, err = f.WriteString(string(body)) + if err != nil { + return target, fmt.Errorf("failed to write destination file %s : %v", tdir+"/"+target.RemotePath, err) + } + target.Downloaded = true + target.Tainted = false + target.UpToDate = true + + dec := yaml.NewDecoder(bytes.NewReader(body)) + for { + data := &types.DataSet{} + err = dec.Decode(data) + if err != nil { + if err == io.EOF { + break + } else { + return target, fmt.Errorf("unable to read file %s data: %s", tdir+"/"+target.RemotePath, err) + } + } + err = types.GetData(data.Data, dataFolder) + if err != nil { + return target, fmt.Errorf("unable to get data: %s", err) + } + } + HubIdx[target.Type][target.Name] = target + return target, nil +} diff --git a/pkg/cwhub/errors.go b/pkg/cwhub/errors.go deleted file mode 100644 index 8a8f4423e..000000000 --- a/pkg/cwhub/errors.go +++ /dev/null @@ -1,8 +0,0 @@ -package cwhub - -import ( - "errors" -) - -/*To be used when reference(s) (is/are) missing in a collection*/ -var ReferenceMissingError = errors.New("Reference(s) missing in collection") diff --git a/pkg/cwhub/hubMgmt.go b/pkg/cwhub/hubMgmt.go deleted file mode 100644 index 62ba902c4..000000000 --- a/pkg/cwhub/hubMgmt.go +++ /dev/null @@ -1,846 +0,0 @@ -package cwhub - -import ( - "bytes" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "path" - - //"log" - "net/http" - "os" - "path/filepath" - "strings" - - "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/enescakir/emoji" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" -) - -var PARSERS = "parsers" -var PARSERS_OVFLW = "postoverflows" -var SCENARIOS = "scenarios" -var COLLECTIONS = "collections" - -var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS} - -var HubIdx map[string]map[string]Item - -var Installdir = "/etc/crowdsec/" -var Hubdir = "/etc/crowdsec/cscli/hub/" -var Cfgdir = "/etc/crowdsec/cscli/" - -var RawFileURLTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/%s/%s" -var HubIndexFile = ".index.json" -var HubBranch = "master" - -type ItemVersion struct { - Digest string - Deprecated bool -} - -//Item can be : parsed, scenario, collection -type Item struct { - /*descriptive info*/ - Type string `yaml:"type,omitempty"` //parser|postoverflows|scenario|collection(|enrich) - Stage string `json:"stage" yaml:"stage,omitempty,omitempty"` //Stage for parser|postoverflow : s00-raw/s01-... - Name string //as seen in .config.json, usually "author/name" - FileName string //the filename, ie. apache2-logs.yaml - Description string `yaml:"description,omitempty"` //as seen in .config.json - Author string `json:"author"` //as seen in .config.json - References []string `yaml:"references,omitempty"` //as seen in .config.json - BelongsToCollections []string `yaml:"belongs_to_collections,omitempty"` /*if it's part of collections, track name here*/ - - /*remote (hub) infos*/ - RemoteURL string `yaml:"remoteURL,omitempty"` //the full remote uri of file in http - RemotePath string `json:"path" yaml:"remote_path,omitempty"` //the path relative to git ie. /parsers/stage/author/file.yaml - RemoteHash string `yaml:"hash,omitempty"` //the meow - Version string `json:"version"` //the last version - Versions map[string]ItemVersion `json:"versions" yaml:"-"` //the list of existing versions - - /*local (deployed) infos*/ - LocalPath string `yaml:"local_path,omitempty"` //the local path relative to ${CFG_DIR} - //LocalHubPath string - LocalVersion string - LocalHash string //the local meow - Installed bool - Downloaded bool - UpToDate bool - Tainted bool //has it been locally modified - Local bool //if it's a non versioned control one - - /*if it's a collection, it not a single file*/ - Parsers []string `yaml:"parsers,omitempty"` - PostOverflows []string `yaml:"postoverflows,omitempty"` - Scenarios []string `yaml:"scenarios,omitempty"` - Collections []string `yaml:"collections,omitempty"` -} - -// calculate sha256 of a file -func getSHA256(filepath string) (string, error) { - /* Digest of file */ - f, err := os.Open(filepath) - if err != nil { - return "", fmt.Errorf("unable to open '%s' : %s", filepath, err.Error()) - } - - defer f.Close() - - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return "", fmt.Errorf("unable to calculate sha256 of '%s': %s", filepath, err.Error()) - } - - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - -var skippedLocal = 0 -var skippedTainted = 0 - -func parser_visit(path string, f os.FileInfo, err error) error { - - var target Item - var local bool - var hubpath string - var inhub bool - var fname string - var ftype string - var fauthor string - var stage string - //we only care about files - if f == nil || f.IsDir() { - return nil - } - //we only care about yaml files - if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") { - return nil - } - - subs := strings.Split(path, "/") - - log.Tracef("path:%s, hubdir:%s, installdir:%s", path, Hubdir, Installdir) - /*we're in hub (~/.cscli/hub/)*/ - if strings.HasPrefix(path, Hubdir) { - inhub = true - //~/.cscli/hub/parsers/s00-raw/crowdsec/skip-pretag.yaml - //~/.cscli/hub/scenarios/crowdsec/ssh_bf.yaml - //~/.cscli/hub/profiles/crowdsec/linux.yaml - if len(subs) < 4 { - log.Fatalf("path is too short : %s", path) - } - fname = subs[len(subs)-1] - fauthor = subs[len(subs)-2] - stage = subs[len(subs)-3] - ftype = subs[len(subs)-4] - log.Tracef("HUBB check [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype) - - } else if strings.HasPrefix(path, Installdir) { /*we're in install /etc/crowdsec//... */ - if len(subs) < 3 { - log.Fatalf("path is too short : %s", path) - } - ///etc/.../parser/stage/file.yaml - ///etc/.../postoverflow/stage/file.yaml - ///etc/.../scenarios/scenar.yaml - ///etc/.../collections/linux.yaml //file is empty - fname = subs[len(subs)-1] - stage = subs[len(subs)-2] - ftype = subs[len(subs)-3] - fauthor = "" - log.Tracef("INSTALL check [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype) - } - - //log.Printf("%s -> name:%s stage:%s", path, fname, stage) - if stage == SCENARIOS { - ftype = SCENARIOS - stage = "" - } else if stage == COLLECTIONS { - ftype = COLLECTIONS - stage = "" - } else if ftype != PARSERS && ftype != PARSERS_OVFLW /*its a PARSER / PARSER_OVFLW with a stage */ { - return fmt.Errorf("unknown prefix in %s : fname:%s, fauthor:%s, stage:%s, ftype:%s", path, fname, fauthor, stage, ftype) - } - - log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype) - - /* - we can encounter 'collections' in the form of a symlink : - /etc/crowdsec/.../collections/linux.yaml -> ~/.cscli/hub/collections/.../linux.yaml - when the collection is installed, both files are created - */ - //non symlinks are local user files or hub files - if f.Mode()&os.ModeSymlink == 0 { - local = true - skippedLocal++ - log.Tracef("%s isn't a symlink", path) - } else { - hubpath, err = os.Readlink(path) - if err != nil { - return fmt.Errorf("unable to read symlink of %s", path) - } - //the symlink target doesn't exist, user might have remove ~/.cscli/hub/...yaml without deleting /etc/crowdsec/....yaml - _, err := os.Lstat(hubpath) - if os.IsNotExist(err) { - log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath) - //remove the symlink - if err = os.Remove(path); err != nil { - return fmt.Errorf("failed to unlink %s: %+v", path, err) - } - return nil - } - log.Tracef("%s points to %s", path, hubpath) - } - - //if it's not a symlink and not in hub, it's a local file, don't bother - if local && !inhub { - log.Debugf("%s is a local file, skip", path) - skippedLocal++ - // log.Printf("local scenario, skip.") - target.Name = fname - target.Stage = stage - target.Installed = true - target.Type = ftype - target.Local = true - target.LocalPath = path - target.UpToDate = true - x := strings.Split(path, "/") - target.FileName = x[len(x)-1] - - HubIdx[ftype][fname] = target - return nil - } - //try to find which configuration item it is - log.Tracef("check [%s] of %s", fname, ftype) - - match := false - for k, v := range HubIdx[ftype] { - log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml") - if fname != v.FileName { - log.Tracef("%s != %s (filename)", fname, v.FileName) - continue - } - //wrong stage - if v.Stage != stage { - continue - } - /*if we are walking hub dir, just mark present files as downloaded*/ - if inhub { - //wrong author - if fauthor != v.Author { - continue - } - //wrong file - if v.Name+".yaml" != fauthor+"/"+fname { - continue - } - if path == Hubdir+"/"+v.RemotePath { - log.Tracef("marking %s as downloaded", v.Name) - v.Downloaded = true - } - } else { - //wrong file - /////.yaml - if !strings.HasSuffix(hubpath, v.RemotePath) { - //log.Printf("wrong file %s %s", hubpath, spew.Sdump(v)) - - continue - } - } - //wrong hash - sha, err := getSHA256(path) - if err != nil { - log.Fatalf("Failed to get sha of %s : %v", path, err) - } - for version, val := range v.Versions { - if sha != val.Digest { - //log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v)) - continue - } else { - /*we got an exact match, update struct*/ - if !inhub { - log.Debugf("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version) - v.LocalPath = path - v.LocalVersion = version - v.Tainted = false - v.Downloaded = true - /*if we're walking the hub, present file doesn't means installed file*/ - v.Installed = true - v.LocalHash = sha - x := strings.Split(path, "/") - target.FileName = x[len(x)-1] - } - if version == v.Version { - log.Tracef("%s is up-to-date", v.Name) - v.UpToDate = true - } - match = true - - } - } - if !match { - log.Debugf("got tainted match for %s : %s", v.Name, path) - skippedTainted += 1 - //the file and the stage is right, but the hash is wrong, it has been tainted by user - if !inhub { - v.LocalPath = path - v.Installed = true - } - v.UpToDate = false - v.LocalVersion = "?" - v.Tainted = true - v.LocalHash = sha - x := strings.Split(path, "/") - target.FileName = x[len(x)-1] - - } - //update the entry - HubIdx[ftype][k] = v - return nil - } - log.Infof("Ignoring file %s of type %s", path, ftype) - return nil -} - -func CollecDepsCheck(v *Item) error { - /*if it's a collection, ensure all the items are installed, or tag it as tainted*/ - if v.Type == COLLECTIONS { - log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) - var tmp = [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} - for idx, ptr := range tmp { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if val, ok := HubIdx[ptrtype][p]; ok { - log.Tracef("check %s installed:%t", val.Name, val.Installed) - if !v.Installed { - continue - } - if val.Type == COLLECTIONS { - log.Tracef("collec, recurse.") - if err := CollecDepsCheck(&val); err != nil { - return fmt.Errorf("sub collection %s is broken : %s", val.Name, err) - } - HubIdx[ptrtype][p] = val - } - - //propagate the state of sub-items to set - if val.Tainted { - v.Tainted = true - return fmt.Errorf("tainted %s %s, tainted.", ptrtype, p) - } else if !val.Installed && v.Installed { - v.Tainted = true - return fmt.Errorf("missing %s %s, tainted.", ptrtype, p) - } else if !val.UpToDate { - v.UpToDate = false - return fmt.Errorf("outdated %s %s", ptrtype, p) - } - val.BelongsToCollections = append(val.BelongsToCollections, v.Name) - HubIdx[ptrtype][p] = val - log.Tracef("checking for %s - tainted:%t uptodate:%t", p, v.Tainted, v.UpToDate) - } else { - log.Fatalf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, v.Name) - } - } - } - } - return nil -} - -/* Updates the infos from HubInit() with the local state */ -func LocalSync() error { - skippedLocal = 0 - skippedTainted = 0 - /*For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last*/ - for _, scan := range ItemTypes { - /*Scan install and Hubdir to get local status*/ - for _, dir := range []string{Installdir, Hubdir} { - //walk the user's directory - err := filepath.Walk(dir+"/"+scan, parser_visit) - if err != nil { - return err - } - } - } - - for k, v := range HubIdx[COLLECTIONS] { - if err := CollecDepsCheck(&v); err != nil { - log.Infof("dependency issue %s : %s", v.Name, err) - } - HubIdx[COLLECTIONS][k] = v - } - return nil -} - -func GetHubIdx() error { - - bidx, err := ioutil.ReadFile(Cfgdir + "/.index.json") - if err != nil { - log.Fatalf("Unable to read downloaded index : %v. Please run update", err) - } - ret, err := LoadPkgIndex(bidx) - if err != nil { - if !errors.Is(err, ReferenceMissingError) { - log.Fatalf("Unable to load existing index : %v.", err) - } - } - HubIdx = ret - if err := LocalSync(); err != nil { - log.Fatalf("Failed to sync Hub index with local deployment : %v", err) - } - return nil -} - -func UpdateHubIdx() error { - bidx, err := DownloadHubIdx() - if err != nil { - log.Fatalf("Unable to download index : %v.", err) - } - ret, err := LoadPkgIndex(bidx) - if err != nil { - if !errors.Is(err, ReferenceMissingError) { - log.Fatalf("Unable to load freshly downloaded index : %v.", err) - } - } - HubIdx = ret - if err := LocalSync(); err != nil { - log.Fatalf("Failed to sync Hub index with local deployment : %v", err) - } - return nil -} - -func DownloadHubIdx() ([]byte, error) { - req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) - if err != nil { - log.Errorf("failed request : %s", err) - return nil, err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - log.Errorf("failed request Do : %s", err) - return nil, err - } - if resp.StatusCode != 200 { - log.Errorf("got code %d while requesting %s, abort", resp.StatusCode, - fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) - return nil, fmt.Errorf("bad http code") - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Errorf("failed request reqd: %s", err) - return nil, err - } - //os.Remove(path.Join(configFolder, GitIndexFile)) - file, err := os.OpenFile(path.Join(Cfgdir, "/.index.json"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - - if err != nil { - log.Fatalf(err.Error()) - } - defer file.Close() - - wsize, err := file.WriteString(string(body)) - if err != nil { - log.Fatalf(err.Error()) - } - log.Infof("Wrote new %d bytes index to %s", wsize, path.Join(Cfgdir, "/.index.json")) - return body, nil -} - -func DisplaySummary() { - log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(HubIdx[COLLECTIONS]), - len(HubIdx[PARSERS]), len(HubIdx[SCENARIOS]), len(HubIdx[PARSERS_OVFLW])) - if skippedLocal > 0 || skippedTainted > 0 { - log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted) - } -} - -/*LoadPkgIndex loads a local .index.json file and returns the map of parsers/scenarios/collections associated*/ -func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) { - var err error - var RawIndex map[string]map[string]Item - var missingItems []string - - if err = json.Unmarshal(buff, &RawIndex); err != nil { - return nil, fmt.Errorf("failed to unmarshal index : %v", err) - } - - /*Iterate over the different types to complete struct */ - for _, itemType := range ItemTypes { - /*complete struct*/ - for idx, item := range RawIndex[itemType] { - item.Name = idx - item.Type = itemType - x := strings.Split(item.RemotePath, "/") - item.FileName = x[len(x)-1] - RawIndex[itemType][idx] = item - /*if it's a collection, check its sub-items are present*/ - //XX should be done later - if itemType == COLLECTIONS { - var tmp = [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} - for idx, ptr := range tmp { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if _, ok := RawIndex[ptrtype][p]; !ok { - log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name) - missingItems = append(missingItems, p) - } - } - } - } - } - } - if len(missingItems) > 0 { - return RawIndex, fmt.Errorf("%q : %w", missingItems, ReferenceMissingError) - } - - return RawIndex, nil -} - -//DisableItem to disable an item managed by the hub, removes the symlink -func DisableItem(target Item, tdir string, hdir string, purge bool) (Item, error) { - syml, err := filepath.Abs(tdir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName) - if err != nil { - return Item{}, err - } - if target.Local { - return target, fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name) - } - - /*for a COLLECTIONS, disable sub-items*/ - if target.Type == COLLECTIONS { - var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} - for idx, ptr := range tmp { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if val, ok := HubIdx[ptrtype][p]; ok { - HubIdx[ptrtype][p], err = DisableItem(val, Installdir, Hubdir, false) - if err != nil { - log.Errorf("Encountered error while disabling %s %s : %s.", ptrtype, p, err) - } - } else { - log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name) - } - } - } - - } - - stat, err := os.Lstat(syml) - if os.IsNotExist(err) { - log.Warningf("%s (%s) doesn't exist, can't disable", target.Name, syml) - //return target, nil //fmt.Errorf("'%s' doesn't exist", syml) - } else { - //if it's managed by hub, it's a symlink to Hubdir / ... - if stat.Mode()&os.ModeSymlink == 0 { - log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml) - return target, fmt.Errorf("%s isn't managed by hub", target.Name) - } - hubpath, err := os.Readlink(syml) - if err != nil { - return target, fmt.Errorf("unable to read symlink of %s (%s)", target.Name, syml) - } - absPath, err := filepath.Abs(hdir + "/" + target.RemotePath) - if err != nil { - return target, err - } - if hubpath != absPath { - log.Warningf("%s (%s) isn't a symlink to %s", target.Name, syml, absPath) - return target, fmt.Errorf("%s isn't managed by hub", target.Name) - } - - //remove the symlink - if err = os.Remove(syml); err != nil { - return target, fmt.Errorf("failed to unlink %s: %+v", syml, err) - } - log.Infof("Removed symlink [%s] : %s", target.Name, syml) - } - target.Installed = false - - if purge { - hubpath := hdir + "/" + target.RemotePath - //if purge, disable hub file - if err = os.Remove(hubpath); err != nil { - return target, fmt.Errorf("failed to purge hub file %s: %+v", hubpath, err) - } - target.Downloaded = false - log.Infof("Removed source file [%s] : %s", target.Name, hubpath) - } - return target, nil -} - -func EnableItem(target Item, tdir string, hdir string) (Item, error) { - var err error - parent_dir := filepath.Clean(tdir + "/" + target.Type + "/" + target.Stage + "/") - /*create directories if needed*/ - if target.Installed { - if target.Tainted { - return target, fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) - } - if target.Local { - return target, fmt.Errorf("%s is local, won't enable", target.Name) - } - /* if it's a collection, check sub-items even if the collection file itself is up-to-date */ - if target.UpToDate && target.Type != COLLECTIONS { - log.Debugf("%s is installed and up-to-date, skip.", target.Name) - return target, nil - } - } - if _, err := os.Stat(parent_dir); os.IsNotExist(err) { - log.Printf("%s doesn't exist, create", parent_dir) - if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil { - return target, fmt.Errorf("unable to create parent directories") - } - } - - /*install sub-items if it's a collection*/ - if target.Type == COLLECTIONS { - var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} - for idx, ptr := range tmp { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if val, ok := HubIdx[ptrtype][p]; ok { - HubIdx[ptrtype][p], err = EnableItem(val, Installdir, Hubdir) - if err != nil { - log.Errorf("Encountered error while installing sub-item %s %s : %s.", ptrtype, p, err) - return target, fmt.Errorf("encountered error while install %s for %s, abort.", val.Name, target.Name) - } - } else { - //log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name) - return target, fmt.Errorf("required %s %s of %s doesn't exist, abort.", ptrtype, p, target.Name) - } - } - } - } - - if _, err := os.Lstat(parent_dir + "/" + target.FileName); os.IsNotExist(err) { - //tdir+target.RemotePath - srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath) - if err != nil { - return target, fmt.Errorf("failed to resolve %s : %s", hdir+"/"+target.RemotePath, err) - } - dstPath, err := filepath.Abs(parent_dir + "/" + target.FileName) - if err != nil { - return target, fmt.Errorf("failed to resolve %s : %s", parent_dir+"/"+target.FileName, err) - } - err = os.Symlink(srcPath, dstPath) - if err != nil { - log.Fatalf("Failed to symlink %s to %s : %v", srcPath, dstPath, err) - return target, fmt.Errorf("failed to symlink %s to %s", srcPath, dstPath) - } - log.Printf("Enabled %s : %s", target.Type, target.Name) - } else { - log.Printf("%s already exists.", parent_dir+"/"+target.FileName) - return target, nil - } - target.Installed = true - return target, nil -} - -func DownloadLatest(target Item, tdir string, overwrite bool, dataFolder string) (Item, error) { - var err error - log.Debugf("Downloading %s %s", target.Type, target.Name) - if target.Type == COLLECTIONS { - var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} - for idx, ptr := range tmp { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if val, ok := HubIdx[ptrtype][p]; ok { - log.Debugf("Download %s sub-item : %s %s", target.Name, ptrtype, p) - //recurse as it's a collection - if ptrtype == COLLECTIONS { - log.Tracef("collection, recurse") - HubIdx[ptrtype][p], err = DownloadLatest(val, tdir, overwrite, dataFolder) - if err != nil { - log.Errorf("Encountered error while downloading sub-item %s %s : %s.", ptrtype, p, err) - return target, fmt.Errorf("encountered error while downloading %s for %s, abort", val.Name, target.Name) - } - } - HubIdx[ptrtype][p], err = DownloadItem(val, tdir, overwrite, dataFolder) - if err != nil { - log.Errorf("Encountered error while downloading sub-item %s %s : %s.", ptrtype, p, err) - return target, fmt.Errorf("encountered error while downloading %s for %s, abort", val.Name, target.Name) - } - } else { - //log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name) - return target, fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) - } - } - } - target, err = DownloadItem(target, tdir, overwrite, dataFolder) - if err != nil { - return target, fmt.Errorf("failed to download item : %s", err) - } - } else { - return DownloadItem(target, tdir, overwrite, dataFolder) - } - return target, nil -} - -func DownloadItem(target Item, tdir string, overwrite bool, dataFolder string) (Item, error) { - - /*if user didn't --force, don't overwrite local, tainted, up-to-date files*/ - if !overwrite { - if target.Tainted { - log.Debugf("%s : tainted, not updated", target.Name) - return target, nil - } - if target.UpToDate { - log.Debugf("%s : up-to-date, not updated", target.Name) - return target, nil - } - } - - //log.Infof("Downloading %s to %s", target.Name, tdir) - req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) - if err != nil { - log.Errorf("%s : request creation failed : %s", target.Name, err) - return target, err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - log.Errorf("%s : request failed : %s", target.Name, err) - return target, err - } - if resp.StatusCode != 200 { - log.Errorf("%s : non 200 response : %d", target.Name, resp.StatusCode) - return target, fmt.Errorf("bad http code") - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Errorf("%s : failed request read: %s", target.Name, err) - return target, err - } - h := sha256.New() - if _, err := h.Write([]byte(body)); err != nil { - return target, fmt.Errorf("%s : failed to write : %s", target.Name, err) - } - meow := fmt.Sprintf("%x", h.Sum(nil)) - if meow != target.Versions[target.Version].Digest { - log.Errorf("Downloaded version doesn't match index, please 'hub update'") - return target, fmt.Errorf("invalid download hash") - } - //all good, install - //check if parent dir exists - tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") - parent_dir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") - - /*check dir*/ - if _, err = os.Stat(parent_dir); os.IsNotExist(err) { - log.Debugf("%s doesn't exist, create", parent_dir) - if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil { - return target, fmt.Errorf("unable to create parent directories") - } - } - /*check actual file*/ - if _, err = os.Stat(tdir + "/" + target.RemotePath); !os.IsNotExist(err) { - log.Warningf("%s : overwrite", target.Name) - log.Debugf("target: %s/%s", tdir, target.RemotePath) - } else { - log.Infof("%s : OK", target.Name) - } - - f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return target, fmt.Errorf("failed to open destination file %s : %v", tdir+"/"+target.RemotePath, err) - } - defer f.Close() - _, err = f.WriteString(string(body)) - if err != nil { - return target, fmt.Errorf("failed to write destination file %s : %v", tdir+"/"+target.RemotePath, err) - } - target.Downloaded = true - target.Tainted = false - target.UpToDate = true - - dec := yaml.NewDecoder(bytes.NewReader(body)) - for { - data := &types.DataSet{} - err = dec.Decode(data) - if err != nil { - if err == io.EOF { - break - } else { - return target, fmt.Errorf("unable to read file %s data: %s", tdir+"/"+target.RemotePath, err) - } - } - err = types.GetData(data.Data, dataFolder) - if err != nil { - return target, fmt.Errorf("unable to get data: %s", err) - } - } - - return target, nil -} - -//returns: human-text, Enabled, Warning, Unmanaged -func ItemStatus(v Item) (string, bool, bool, bool) { - var Ok, Warning, Managed bool - var strret string - - if !v.Installed { - strret = "disabled" - Ok = false - } else { - Ok = true - strret = "enabled" - } - - if v.Local { - Managed = false - strret += ",local" - } else { - Managed = true - } - - //tainted or out of date - if v.Tainted { - Warning = true - strret += ",tainted" - } else if !v.UpToDate { - strret += ",update-available" - Warning = true - } - return strret, Ok, Warning, Managed -} - -//Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy) -func HubStatus(itype string, name string, list_all bool) []map[string]string { - if _, ok := HubIdx[itype]; !ok { - log.Errorf("type %s doesn't exist", itype) - return nil - } - - var mli []map[string]string - /*remember, you do it for the user :)*/ - for _, v := range HubIdx[itype] { - if name != "" && name != v.Name { - //user has required a specific name - continue - } - //Only enabled items ? - if !list_all && !v.Installed { - continue - } - //Check the item status - st, ok, warning, managed := ItemStatus(v) - tmp := make(map[string]string) - tmp["name"] = v.Name - tmp["status"] = st - tmp["local_version"] = v.LocalVersion - tmp["local_path"] = v.LocalPath - tmp["description"] = v.Description - if !managed || !v.Installed { - tmp["utf8_status"] = fmt.Sprintf("%v %s", emoji.Prohibited, st) - } else if warning { - tmp["utf8_status"] = fmt.Sprintf("%v %s", emoji.Warning, st) - } else if ok { - tmp["utf8_status"] = fmt.Sprintf("%v %s", emoji.CheckMark, st) - } - mli = append(mli, tmp) - } - return mli -} diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go new file mode 100644 index 000000000..68c57b031 --- /dev/null +++ b/pkg/cwhub/install.go @@ -0,0 +1,151 @@ +package cwhub + +import ( + "fmt" + "os" + "path/filepath" + + log "github.com/sirupsen/logrus" +) + +//DisableItem to disable an item managed by the hub, removes the symlink if purge is true +func DisableItem(target Item, tdir string, hdir string, purge bool) (Item, error) { + syml, err := filepath.Abs(tdir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName) + if err != nil { + return Item{}, err + } + if target.Local { + return target, fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name) + } + + /*for a COLLECTIONS, disable sub-items*/ + if target.Type == COLLECTIONS { + var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} + for idx, ptr := range tmp { + ptrtype := ItemTypes[idx] + for _, p := range ptr { + if val, ok := HubIdx[ptrtype][p]; ok { + HubIdx[ptrtype][p], err = DisableItem(val, Installdir, Hubdir, false) + if err != nil { + log.Errorf("Encountered error while disabling %s %s : %s.", ptrtype, p, err) + } + } else { + log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name) + } + } + } + + } + + stat, err := os.Lstat(syml) + if os.IsNotExist(err) { + log.Warningf("%s (%s) doesn't exist, can't disable", target.Name, syml) + //return target, nil //fmt.Errorf("'%s' doesn't exist", syml) + } else { + //if it's managed by hub, it's a symlink to Hubdir / ... + if stat.Mode()&os.ModeSymlink == 0 { + log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml) + return target, fmt.Errorf("%s isn't managed by hub", target.Name) + } + hubpath, err := os.Readlink(syml) + if err != nil { + return target, fmt.Errorf("unable to read symlink of %s (%s)", target.Name, syml) + } + absPath, err := filepath.Abs(hdir + "/" + target.RemotePath) + if err != nil { + return target, err + } + if hubpath != absPath { + log.Warningf("%s (%s) isn't a symlink to %s", target.Name, syml, absPath) + return target, fmt.Errorf("%s isn't managed by hub", target.Name) + } + + //remove the symlink + if err = os.Remove(syml); err != nil { + return target, fmt.Errorf("failed to unlink %s: %+v", syml, err) + } + log.Infof("Removed symlink [%s] : %s", target.Name, syml) + } + target.Installed = false + + if purge { + hubpath := hdir + "/" + target.RemotePath + //if purge, disable hub file + if err = os.Remove(hubpath); err != nil { + return target, fmt.Errorf("failed to purge hub file %s: %+v", hubpath, err) + } + target.Downloaded = false + log.Infof("Removed source file [%s] : %s", target.Name, hubpath) + } + HubIdx[target.Type][target.Name] = target + return target, nil +} + +func EnableItem(target Item, tdir string, hdir string) (Item, error) { + var err error + parent_dir := filepath.Clean(tdir + "/" + target.Type + "/" + target.Stage + "/") + /*create directories if needed*/ + if target.Installed { + if target.Tainted { + return target, fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) + } + if target.Local { + return target, fmt.Errorf("%s is local, won't enable", target.Name) + } + /* if it's a collection, check sub-items even if the collection file itself is up-to-date */ + if target.UpToDate && target.Type != COLLECTIONS { + log.Debugf("%s is installed and up-to-date, skip.", target.Name) + return target, nil + } + } + if _, err := os.Stat(parent_dir); os.IsNotExist(err) { + log.Printf("%s doesn't exist, create", parent_dir) + if err := os.MkdirAll(parent_dir, os.ModePerm); err != nil { + return target, fmt.Errorf("unable to create parent directories") + } + } + + /*install sub-items if it's a collection*/ + if target.Type == COLLECTIONS { + var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} + for idx, ptr := range tmp { + ptrtype := ItemTypes[idx] + for _, p := range ptr { + if val, ok := HubIdx[ptrtype][p]; ok { + HubIdx[ptrtype][p], err = EnableItem(val, Installdir, Hubdir) + if err != nil { + log.Errorf("Encountered error while installing sub-item %s %s : %s.", ptrtype, p, err) + return target, fmt.Errorf("encountered error while install %s for %s, abort.", val.Name, target.Name) + } + } else { + //log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name) + return target, fmt.Errorf("required %s %s of %s doesn't exist, abort.", ptrtype, p, target.Name) + } + } + } + } + + if _, err := os.Lstat(parent_dir + "/" + target.FileName); os.IsNotExist(err) { + //tdir+target.RemotePath + srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath) + if err != nil { + return target, fmt.Errorf("failed to resolve %s : %s", hdir+"/"+target.RemotePath, err) + } + dstPath, err := filepath.Abs(parent_dir + "/" + target.FileName) + if err != nil { + return target, fmt.Errorf("failed to resolve %s : %s", parent_dir+"/"+target.FileName, err) + } + err = os.Symlink(srcPath, dstPath) + if err != nil { + log.Fatalf("Failed to symlink %s to %s : %v", srcPath, dstPath, err) + return target, fmt.Errorf("failed to symlink %s to %s", srcPath, dstPath) + } + log.Printf("Enabled %s : %s", target.Type, target.Name) + } else { + log.Printf("%s already exists.", parent_dir+"/"+target.FileName) + return target, nil + } + target.Installed = true + HubIdx[target.Type][target.Name] = target + return target, nil +} diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go new file mode 100644 index 000000000..4137fcd42 --- /dev/null +++ b/pkg/cwhub/loader.go @@ -0,0 +1,352 @@ +package cwhub + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + + //"log" + + "os" + "path/filepath" + "strings" + + log "github.com/sirupsen/logrus" +) + +func parser_visit(path string, f os.FileInfo, err error) error { + + var target Item + var local bool + var hubpath string + var inhub bool + var fname string + var ftype string + var fauthor string + var stage string + //we only care about files + if f == nil || f.IsDir() { + return nil + } + //we only care about yaml files + if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") { + return nil + } + + subs := strings.Split(path, "/") + + log.Tracef("path:%s, hubdir:%s, installdir:%s", path, Hubdir, Installdir) + /*we're in hub (~/.cscli/hub/)*/ + if strings.HasPrefix(path, Hubdir) { + inhub = true + //~/.cscli/hub/parsers/s00-raw/crowdsec/skip-pretag.yaml + //~/.cscli/hub/scenarios/crowdsec/ssh_bf.yaml + //~/.cscli/hub/profiles/crowdsec/linux.yaml + if len(subs) < 4 { + log.Fatalf("path is too short : %s", path) + } + fname = subs[len(subs)-1] + fauthor = subs[len(subs)-2] + stage = subs[len(subs)-3] + ftype = subs[len(subs)-4] + log.Tracef("HUBB check [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype) + + } else if strings.HasPrefix(path, Installdir) { /*we're in install /etc/crowdsec//... */ + if len(subs) < 3 { + log.Fatalf("path is too short : %s", path) + } + ///etc/.../parser/stage/file.yaml + ///etc/.../postoverflow/stage/file.yaml + ///etc/.../scenarios/scenar.yaml + ///etc/.../collections/linux.yaml //file is empty + fname = subs[len(subs)-1] + stage = subs[len(subs)-2] + ftype = subs[len(subs)-3] + fauthor = "" + log.Tracef("INSTALL check [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype) + } else { + log.Errorf("unknown prefix in %s (not install:%s and not hub:%s)", path, Installdir, Hubdir) + } + + //log.Printf("%s -> name:%s stage:%s", path, fname, stage) + if stage == SCENARIOS { + ftype = SCENARIOS + stage = "" + } else if stage == COLLECTIONS { + ftype = COLLECTIONS + stage = "" + } else if ftype != PARSERS && ftype != PARSERS_OVFLW /*its a PARSER / PARSER_OVFLW with a stage */ { + return fmt.Errorf("unknown prefix in %s : fname:%s, fauthor:%s, stage:%s, ftype:%s", path, fname, fauthor, stage, ftype) + } + + log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype) + + /* + we can encounter 'collections' in the form of a symlink : + /etc/crowdsec/.../collections/linux.yaml -> ~/.cscli/hub/collections/.../linux.yaml + when the collection is installed, both files are created + */ + //non symlinks are local user files or hub files + if f.Mode()&os.ModeSymlink == 0 { + local = true + skippedLocal++ + log.Tracef("%s isn't a symlink", path) + } else { + hubpath, err = os.Readlink(path) + if err != nil { + return fmt.Errorf("unable to read symlink of %s", path) + } + //the symlink target doesn't exist, user might have remove ~/.cscli/hub/...yaml without deleting /etc/crowdsec/....yaml + _, err := os.Lstat(hubpath) + if os.IsNotExist(err) { + log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath) + //remove the symlink + if err = os.Remove(path); err != nil { + return fmt.Errorf("failed to unlink %s: %+v", path, err) + } + return nil + } + log.Tracef("%s points to %s", path, hubpath) + } + + //if it's not a symlink and not in hub, it's a local file, don't bother + if local && !inhub { + log.Debugf("%s is a local file, skip", path) + skippedLocal++ + // log.Printf("local scenario, skip.") + target.Name = fname + target.Stage = stage + target.Installed = true + target.Type = ftype + target.Local = true + target.LocalPath = path + target.UpToDate = true + x := strings.Split(path, "/") + target.FileName = x[len(x)-1] + + HubIdx[ftype][fname] = target + return nil + } + //try to find which configuration item it is + log.Tracef("check [%s] of %s", fname, ftype) + + match := false + for k, v := range HubIdx[ftype] { + log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml") + if fname != v.FileName { + log.Tracef("%s != %s (filename)", fname, v.FileName) + continue + } + //wrong stage + if v.Stage != stage { + continue + } + /*if we are walking hub dir, just mark present files as downloaded*/ + if inhub { + //wrong author + if fauthor != v.Author { + continue + } + //wrong file + if v.Name+".yaml" != fauthor+"/"+fname { + continue + } + if path == Hubdir+"/"+v.RemotePath { + log.Tracef("marking %s as downloaded", v.Name) + v.Downloaded = true + } + } else { + //wrong file + /////.yaml + if !strings.HasSuffix(hubpath, v.RemotePath) { + //log.Printf("wrong file %s %s", hubpath, spew.Sdump(v)) + + continue + } + } + //wrong hash + sha, err := getSHA256(path) + if err != nil { + log.Fatalf("Failed to get sha of %s : %v", path, err) + } + for version, val := range v.Versions { + if sha != val.Digest { + //log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v)) + continue + } else { + /*we got an exact match, update struct*/ + if !inhub { + log.Debugf("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version) + v.LocalPath = path + v.LocalVersion = version + v.Tainted = false + v.Downloaded = true + /*if we're walking the hub, present file doesn't means installed file*/ + v.Installed = true + v.LocalHash = sha + x := strings.Split(path, "/") + target.FileName = x[len(x)-1] + } + if version == v.Version { + log.Tracef("%s is up-to-date", v.Name) + v.UpToDate = true + } + match = true + break + } + } + if !match { + log.Debugf("got tainted match for %s : %s", v.Name, path) + skippedTainted += 1 + //the file and the stage is right, but the hash is wrong, it has been tainted by user + if !inhub { + v.LocalPath = path + v.Installed = true + } + v.UpToDate = false + v.LocalVersion = "?" + v.Tainted = true + v.LocalHash = sha + x := strings.Split(path, "/") + target.FileName = x[len(x)-1] + + } + //update the entry + HubIdx[ftype][k] = v + return nil + } + log.Infof("Ignoring file %s of type %s", path, ftype) + return nil +} + +func CollecDepsCheck(v *Item) error { + /*if it's a collection, ensure all the items are installed, or tag it as tainted*/ + if v.Type == COLLECTIONS { + log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) + var tmp = [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} + for idx, ptr := range tmp { + ptrtype := ItemTypes[idx] + for _, p := range ptr { + if val, ok := HubIdx[ptrtype][p]; ok { + log.Tracef("check %s installed:%t", val.Name, val.Installed) + if !v.Installed { + continue + } + if val.Type == COLLECTIONS { + log.Tracef("collec, recurse.") + if err := CollecDepsCheck(&val); err != nil { + return fmt.Errorf("sub collection %s is broken : %s", val.Name, err) + } + HubIdx[ptrtype][p] = val + } + + //propagate the state of sub-items to set + if val.Tainted { + v.Tainted = true + return fmt.Errorf("tainted %s %s, tainted.", ptrtype, p) + } else if !val.Installed && v.Installed { + v.Tainted = true + return fmt.Errorf("missing %s %s, tainted.", ptrtype, p) + } else if !val.UpToDate { + v.UpToDate = false + return fmt.Errorf("outdated %s %s", ptrtype, p) + } + val.BelongsToCollections = append(val.BelongsToCollections, v.Name) + HubIdx[ptrtype][p] = val + log.Tracef("checking for %s - tainted:%t uptodate:%t", p, v.Tainted, v.UpToDate) + } else { + log.Fatalf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, v.Name) + } + } + } + } + return nil +} + +/* Updates the infos from HubInit() with the local state */ +func LocalSync() error { + skippedLocal = 0 + skippedTainted = 0 + /*For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last*/ + for _, scan := range ItemTypes { + /*Scan install and Hubdir to get local status*/ + for _, dir := range []string{Installdir, Hubdir} { + //walk the user's directory + err := filepath.Walk(dir+"/"+scan, parser_visit) + if err != nil { + return err + } + } + } + + for k, v := range HubIdx[COLLECTIONS] { + if err := CollecDepsCheck(&v); err != nil { + log.Infof("dependency issue %s : %s", v.Name, err) + } + HubIdx[COLLECTIONS][k] = v + } + return nil +} + +func GetHubIdx() error { + + bidx, err := ioutil.ReadFile(Cfgdir + "/.index.json") + if err != nil { + log.Fatalf("Unable to read downloaded index : %v. Please run update", err) + } + ret, err := LoadPkgIndex(bidx) + if err != nil { + if !errors.Is(err, ReferenceMissingError) { + log.Fatalf("Unable to load existing index : %v.", err) + } + return err + } + HubIdx = ret + if err := LocalSync(); err != nil { + log.Fatalf("Failed to sync Hub index with local deployment : %v", err) + } + return nil +} + +/*LoadPkgIndex loads a local .index.json file and returns the map of parsers/scenarios/collections associated*/ +func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) { + var err error + var RawIndex map[string]map[string]Item + var missingItems []string + + if err = json.Unmarshal(buff, &RawIndex); err != nil { + return nil, fmt.Errorf("failed to unmarshal index : %v", err) + } + + /*Iterate over the different types to complete struct */ + for _, itemType := range ItemTypes { + /*complete struct*/ + for idx, item := range RawIndex[itemType] { + item.Name = idx + item.Type = itemType + x := strings.Split(item.RemotePath, "/") + item.FileName = x[len(x)-1] + RawIndex[itemType][idx] = item + /*if it's a collection, check its sub-items are present*/ + //XX should be done later + if itemType == COLLECTIONS { + var tmp = [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} + for idx, ptr := range tmp { + ptrtype := ItemTypes[idx] + for _, p := range ptr { + if _, ok := RawIndex[ptrtype][p]; !ok { + log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name) + missingItems = append(missingItems, p) + } + } + } + } + } + } + if len(missingItems) > 0 { + return RawIndex, fmt.Errorf("%q : %w", missingItems, ReferenceMissingError) + } + + return RawIndex, nil +}