From 5dbcb18cc82501d946035ff406d21c4a35f330be Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Tue, 8 Feb 2022 16:47:07 +0100 Subject: [PATCH] Parser visit split (#1242) * break/cleanup this parser_visit function * do not delete sub-collections if they belong to > 1 collection, fix #1118 --- pkg/cwhub/cwhub.go | 55 ++++- pkg/cwhub/cwhub_test.go | 46 +++- pkg/cwhub/install.go | 17 +- pkg/cwhub/loader.go | 491 +++++++++++++++++++++++----------------- 4 files changed, 394 insertions(+), 215 deletions(-) diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index c698c901f..fd1e718ee 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -3,6 +3,7 @@ package cwhub import ( "crypto/sha256" "path/filepath" + "sort" "strings" //"errors" @@ -81,6 +82,55 @@ type Item struct { Collections []string `yaml:"collections,omitempty" json:"collections,omitempty"` } +func (i *Item) compareFile(fname, fstage, fauthor, path string) bool { + //wrong filename + if fname != i.FileName { + log.Tracef("%s != %s (filename)", fname, i.FileName) + return false + } + //wrong stage + if fstage != "" && fstage != i.Stage { + log.Tracef("%s != %s (stage)", fstage, i.Stage) + return false + } + //wrong author + if fauthor != "" && fauthor != i.Author { + log.Tracef("%s != %s (author)", fauthor, i.Author) + return false + } + return true +} + +func (i *Item) getVersion(path string) (string, string, bool, error) { + //returns version, hash, up-to-date flag, error + sha, err := getSHA256(path) + if err != nil { + return "", "", false, err + } + + //let's reverse sort the versions to deal with hash collisions (#154) + versions := make([]string, 0, len(i.Versions)) + for k := range i.Versions { + versions = append(versions, k) + } + sort.Sort(sort.Reverse(sort.StringSlice(versions))) + + for _, version := range versions { + val := i.Versions[version] + if sha != val.Digest { + continue + } + //if the version is == item.version, it means it's the last available version + if version == i.Version { + return version, sha, true, nil + } + //otherwise, return the version nontheless + return version, sha, false, nil + } + //the version is unknown + return "", sha, false, nil +} + func (i *Item) toHubStatus() ItemHubStatus { hubStatus := ItemHubStatus{} hubStatus.Name = i.Name @@ -201,8 +251,8 @@ func AddItem(itemType string, item Item) error { } func DisplaySummary() { - log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]), - len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW])) + log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers, %d data files", len(hubIdx[COLLECTIONS]), + len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]), len(hubIdx[DATA_FILES])) if skippedLocal > 0 || skippedTainted > 0 { log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted) } @@ -266,7 +316,6 @@ func GetUpstreamInstalledScenarios() ([]Item, error) { func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus { if _, ok := hubIdx[itemType]; !ok { log.Errorf("type %s doesn't exist", itemType) - return nil } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index d9ea6f930..26bb0756d 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -214,6 +214,7 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { if !hubIdx[item.Type][item.Name].Installed { t.Fatalf("install: %s should be install", item.Name) } + } func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) { @@ -332,6 +333,49 @@ func TestInstallParser(t *testing.T) { } } +func TestGetItemByPath(t *testing.T) { + cfg := test_prepenv() + + if err := GetHubIdx(cfg.Hub); err != nil { + t.Fatalf("failed to load hub index") + } + for _, it := range hubIdx[PARSERS] { + //Install the parser + item, err := DownloadLatest(cfg.Hub, it, false, false) + if err != nil { + t.Fatalf("error while downloading %s : %v", item.Name, err) + } + //Enable the item + EnableItem(cfg.Hub, item) + } + //Sync + err, warns := LocalSync(cfg.Hub) + if err != nil || len(warns) > 0 { + t.Fatalf("unexpected err/warnings : %+v / %+v", err, warns) + } + + for _, item := range hubIdx[PARSERS] { + //Check that we get the same thing if we get it by the path + itemcopy, err := GetItemByPath(item.Type, item.LocalPath) + if err != nil { + t.Fatalf("error while getting item by path : %v -> %s", err, item.LocalPath) + } + if itemcopy.Name != item.Name { + t.Fatalf("GetItemByPath: %s != %s", itemcopy.Name, item.Name) + } + if itemcopy.UpToDate != item.UpToDate { + t.Fatalf("GetItemByPath: %v != %v", itemcopy.UpToDate, item.UpToDate) + } + if itemcopy.Installed != item.Installed { + t.Fatalf("GetItemByPath: %v != %v", itemcopy.Installed, item.Installed) + } + if itemcopy.Tainted != item.Tainted { + t.Fatalf("GetItemByPath: %v != %v", itemcopy.Tainted, item.Tainted) + } + } + +} + func TestInstallCollection(t *testing.T) { /* @@ -350,6 +394,7 @@ func TestInstallCollection(t *testing.T) { } //map iteration is random by itself for _, it := range hubIdx[COLLECTIONS] { + testInstallItem(cfg.Hub, t, it) it = hubIdx[COLLECTIONS][it.Name] testTaintItem(cfg.Hub, t, it) @@ -383,7 +428,6 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { responseBody := "" log.Printf("---> %s", req.URL.Path) - /*FAKE PARSER*/ if strings.HasSuffix(req.URL.Path, "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml") { responseBody = `onsuccess: next_stage diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go index 580300df1..3f140d4fd 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/install.go @@ -27,17 +27,26 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name) } - /*for a COLLECTIONS, disable sub-items*/ + /*for a COLLECTIONS, disable sub-items, if they are used _only_ by this item :)*/ if target.Type == COLLECTIONS { var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} for idx, ptr := range tmp { ptrtype := ItemTypes[idx] for _, p := range ptr { if val, ok := hubIdx[ptrtype][p]; ok { - hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force) - if err != nil { - return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p)) + + if len(val.BelongsToCollections) > 1 { + log.Infof("%s belongs to more than one collection, won't disable (%v)", p, val.BelongsToCollections) + continue } + if len(val.BelongsToCollections) == 1 && val.BelongsToCollections[0] == target.Name { + log.Infof("%s belongs exclusively to %s, disabling", p, target.Name) + hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force) + if err != nil { + return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p)) + } + } + } else { log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name) } diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 788437db5..31888d14d 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -5,7 +5,6 @@ import ( //"errors" "fmt" "io/ioutil" - "sort" "github.com/pkg/errors" "golang.org/x/mod/semver" @@ -20,253 +19,310 @@ import ( log "github.com/sirupsen/logrus" ) -/*the walk/parser_visit function can't receive extra args*/ +/*the walk function can't receive extra args*/ var hubdir, installdir, datadir string -// TODO: Break this function into smaller functions. -func parser_visit(path string, f os.FileInfo, err error) error { - var target Item - var local bool - var hubpath string - var inhub bool - var fname string - var ftype string - var fauthor string - var stage string - - path, err = filepath.Abs(path) +func visitDiscard(path string, f os.FileInfo) (string, bool, error) { + //return path, false, nil + path, err := filepath.Abs(path) if err != nil { - return err + return path, true, err } //we only care about files if f == nil || f.IsDir() { + return path, true, nil + } + return path, false, nil +} + +func hubdirVisit(path string, f os.FileInfo, err error) error { + + if err != nil { + log.Warningf("error visiting %s", err) + } + allowed_extensions := map[string]bool{".yaml": true, ".yml": true} + /*only interested by yaml files */ + path, discard, err := visitDiscard(path, f) + if err != nil { + return err + } + if discard { return nil } - // yamls -> collections, parsers, overflows etc. txt, mmdb -> data files - if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") && !strings.HasSuffix(f.Name(), ".txt") && !strings.HasSuffix(f.Name(), ".mmdb") { + if !allowed_extensions[filepath.Ext(path)] { + log.Debugf("discarding %s : not a yaml file", path) return nil } + //extract components from path : + //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml + //.../hub/scenarios/crowdsec/ssh_bf.yaml + //.../hub/profiles/crowdsec/linux.yaml + path_components := strings.Split(path, string(filepath.Separator)) - subs := strings.Split(path, "/") - - log.Tracef("path:%s, hubdir:%s, installdir:%s datadir%s", path, hubdir, installdir, datadir) - /*we're in hub (~/.hub/hub/)*/ - - hubDirSetter := func() { - log.Tracef("in hub dir") - inhub = true - //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml - //.../hub/scenarios/crowdsec/ssh_bf.yaml - //.../hub/profiles/crowdsec/linux.yaml - if len(subs) < 4 { - log.Fatalf("path is too short : %s (%d)", path, len(subs)) - } - fname = subs[len(subs)-1] - fauthor = subs[len(subs)-2] - stage = subs[len(subs)-3] - ftype = subs[len(subs)-4] + if len(path_components) < 4 { + log.Fatalf("path is too short : %s (%d)", path, len(path_components)) } + fname := path_components[len(path_components)-1] + fauthor := path_components[len(path_components)-2] + fstage := path_components[len(path_components)-3] + ftype := path_components[len(path_components)-4] - dataDirSetter := func() { - log.Tracef("in data dir") - fauthor = "" - fname = subs[len(subs)-1] - stage = "" - ftype = DATA_FILES - fauthor = "" - } + log.Tracef("%s : stage:%s ftype:%s", path, fstage, ftype) - installDirSetter := func() { - log.Tracef("in install dir") - if len(subs) < 3 { - log.Fatalf("path is too short : %s (%d)", path, len(subs)) - } - ///.../config/parser/stage/file.yaml - ///.../config/postoverflow/stage/file.yaml - ///.../config/scenarios/scenar.yaml - ///.../config/collections/linux.yaml //file is empty - fname = subs[len(subs)-1] - stage = subs[len(subs)-2] - ftype = subs[len(subs)-3] + if ftype == DATA_FILES { + return fmt.Errorf("unexpected data file in hub : %s", path) } - setterByPath := map[string]func(){ - installdir: installDirSetter, - hubdir: hubDirSetter, - datadir: dataDirSetter, - } - - paths := []string{installdir, hubdir, datadir} - sort.Slice(paths, func(i, j int) bool { - return len(paths[i]) > len(paths[j]) - }) - foundMatch := false - for _, p := range paths { - if strings.HasPrefix(path, p) { - setterByPath[p]() - foundMatch = true - break - } - } - if !foundMatch { - return fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubdir, installdir) - } - log.Tracef("stage:%s ftype:%s", stage, ftype) - //log.Printf("%s -> name:%s stage:%s", path, fname, stage) - // correct the stage and type for non-stage stuff. - if stage == SCENARIOS { + if fstage == SCENARIOS { ftype = SCENARIOS - stage = "" - } else if stage == COLLECTIONS { + fstage = "" + } else if fstage == COLLECTIONS { ftype = COLLECTIONS - stage = "" - } else if ftype != PARSERS && ftype != PARSERS_OVFLW && ftype != DATA_FILES { /*its a PARSER / PARSER_OVFLW with a stage */ + fstage = "" + } else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */ return fmt.Errorf("unknown configuration type for file '%s'", path) } - log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype) - /* - we can encounter 'collections' in the form of a symlink : - /etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml - when the collection is installed, both files are created - */ + log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, fstage, ftype) + + //in the hub, we don't expect symlinks + if f.Mode()&os.ModeSymlink != 0 { + log.Warningf("%s in the hub is a symlink, this isn't expected", path) + } + //try to find which configuration item it is + log.Tracef("check [%s] of %s", fname, ftype) + + for itemName, item := range hubIdx[ftype] { + log.Tracef("check [%s] vs [%s] : %s/%s/%s", fname, item.RemotePath, ftype, fstage, fname) + if !item.compareFile(fname, fstage, fauthor, path) { + continue + } + //we're in the hub, mark the file as present and downloaded + if path == hubdir+"/"+item.RemotePath { // + log.Tracef("marking %s as downloaded", item.Name) + item.Downloaded = true + } + + version, sha, uptodate, err := item.getVersion(path) + if err != nil { + return errors.Wrapf(err, "while getting version of %s", path) + } + + if version == "" { + log.Debugf("got tainted match for %s : %s", item.Name, path) + skippedTainted += 1 + item.UpToDate = uptodate + item.LocalVersion = "?" + item.Tainted = true + item.LocalHash = sha + } else { + item.UpToDate = uptodate + } + //if it was not present, update the index (it's the first time we're seeing this item. Might be downloaded and not installed) + if _, ok := hubIdx[ftype][itemName]; !ok { + hubIdx[ftype][itemName] = item + } + return nil + } + log.Infof("File %s found in hub directory wasn't found in the hub index, ignoring it", path) + return nil +} + +func configdirVisit(path string, f os.FileInfo, err error) error { + + if err != nil { + log.Warningf("error visiting %s", err) + } + allowed_extensions := map[string]bool{".yaml": true, ".yml": true} + /*only interested by yaml files */ + path, discard, err := visitDiscard(path, f) + if err != nil { + return err + } + if discard { + return nil + } + if !allowed_extensions[filepath.Ext(path)] { + log.Debugf("discarding %s : not a yaml file", path) + return nil + } + path_components := strings.Split(path, string(filepath.Separator)) + + if len(path_components) < 3 { + log.Fatalf("path is too short : %s (%d)", path, len(path_components)) + } + ///.../config/parser/stage/file.yaml + ///.../config/postoverflow/stage/file.yaml + ///.../config/scenarios/scenar.yaml + ///.../config/collections/linux.yaml //file is empty + fname := path_components[len(path_components)-1] + fstage := path_components[len(path_components)-2] + ftype := path_components[len(path_components)-3] + + if ftype == DATA_FILES { + return fmt.Errorf("unexpected data file in install directory : %s", path) + } + + log.Tracef("stage:%s ftype:%s", fstage, ftype) + + // correct the stage and type for non-stage stuff. + if fstage == SCENARIOS { + ftype = SCENARIOS + fstage = "" + } else if fstage == COLLECTIONS { + ftype = COLLECTIONS + fstage = "" + } else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */ + return fmt.Errorf("unknown configuration type for file '%s'", path) + } + log.Tracef("CORRECTED [%s] in stage [%s] of type [%s]", fname, fstage, ftype) + //non symlinks are local user files or hub files or data files if f.Mode()&os.ModeSymlink == 0 { - local = true log.Tracef("%s isn't a symlink", path) - } else { - hubpath, err = os.Readlink(path) + local_item := Item{} + log.Tracef("%s is a local file, skip", path) + skippedLocal++ + local_item.Name = fname + local_item.Stage = fstage + local_item.Installed = true + local_item.Type = ftype + local_item.Local = true + local_item.LocalPath = path + local_item.UpToDate = true + x := strings.Split(path, string(filepath.Separator)) + local_item.FileName = x[len(x)-1] + hubIdx[ftype][fname] = local_item + return nil + + } + hubpath, err := os.Readlink(path) + if err != nil { + return fmt.Errorf("unable to read symlink of %s", path) + } + //the symlink target doesn't exist, user might have removed hub directory without deleting /etc/crowdsec/....yaml + _, err = os.Lstat(hubpath) + if os.IsNotExist(err) { + log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath) + //remove the symlink + if err = os.Remove(path); err != nil { + return fmt.Errorf("failed to unlink %s: %+v", path, err) + } + return nil + } + + //try to get the matching item version + for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct + if !item.compareFile(fname, fstage, "", path) { + continue + } + log.Tracef("check [%s] vs [%s] : %s", fname, item.RemotePath, ftype+"/"+fstage+"/"+fname+".yaml") + version, sha, uptodate, err := item.getVersion(path) + if err != nil { + return errors.Wrapf(err, "while getting version of %s", path) + } + item.LocalPath = path + item.Installed = true + item.LocalHash = sha + log.Debugf("found exact match for %s : version is %s (up-to-date:%t)", path, version, uptodate) + /*we found the matching item, update it*/ + if version != "" { + item.LocalVersion = version + item.Tainted = false + item.Downloaded = true + item.UpToDate = uptodate + } else { + skippedTainted += 1 + //the file and the stage is right, but the hash is wrong, it has been tainted by user + item.UpToDate = false + item.LocalVersion = "?" + item.Tainted = true + } + hubIdx[ftype][itemName] = item + return nil + } + log.Warningf("File %s found in install directory wasn't accounted for : not a symlink to hub, not a local file", path) + return nil +} + +func datadirVisit(path string, f os.FileInfo, err error) error { + + if err != nil { + log.Warningf("error visiting %s", err) + } + + /*only interested by yaml files */ + path, discard, err := visitDiscard(path, f) + if err != nil { + return err + } + if discard { + return nil + } + + path_components := strings.Split(path, string(filepath.Separator)) + + if len(path_components) < 2 { + log.Fatalf("path is too short : %s (%d)", path, len(path_components)) + } + fname := path_components[len(path_components)-1] + fauthor := path_components[len(path_components)-2] + ftype := DATA_FILES + + log.Tracef("CORRECTED [%s] by [%s] of type [%s]", fname, fauthor, ftype) + + //non symlinks are local user files or hub files or data files + if f.Mode()&os.ModeSymlink != 0 { + log.Warningf("%s is a symlink, that's unexpected (but can be ok)", path) + final_path, err := os.Readlink(path) if err != nil { return fmt.Errorf("unable to read symlink of %s", path) } //the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml - _, err := os.Lstat(hubpath) + _, err = os.Lstat(path) if os.IsNotExist(err) { - log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath) - //remove the symlink + log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, final_path) if err = os.Remove(path); err != nil { return fmt.Errorf("failed to unlink %s: %+v", path, err) } return nil } - log.Tracef("%s points to %s", path, hubpath) } - //if it's not a symlink and not in hub nor it is a data file. Don't bother checking this with index - if local && !inhub && ftype != DATA_FILES { - log.Tracef("%s is a local file, skip", path) - skippedLocal++ - // log.Printf("local scenario, skip.") - target.Name = fname - target.Stage = stage - target.Installed = true - target.Type = ftype - target.Local = true - target.LocalPath = path - target.UpToDate = true - x := strings.Split(path, "/") - target.FileName = x[len(x)-1] - - hubIdx[ftype][fname] = target - return nil - } //try to find which configuration item it is log.Tracef("check [%s] of %s", fname, ftype) - match := false - for k, v := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct - log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml") - if fname != v.FileName { - log.Tracef("%s != %s (filename)", fname, v.FileName) + for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct + log.Tracef("check [%s] vs [%s]", fname, item.RemotePath) + if !item.compareFile(fname, "", fauthor, path) { continue } - //wrong stage - if v.Stage != stage { - log.Tracef("%s != %s (stage)", v.Stage, stage) - continue - } - /*if we are walking hub dir, just mark present files as downloaded*/ - if inhub { - //wrong author - if fauthor != v.Author { - continue - } - //wrong file - if v.Name+".yaml" != fauthor+"/"+fname { - continue - } - if path == hubdir+"/"+v.RemotePath { - log.Tracef("marking %s as downloaded", v.Name) - v.Downloaded = true - } - } - sha, err := getSHA256(path) + + version, sha, uptodate, err := item.getVersion(path) if err != nil { - log.Fatalf("Failed to get sha of %s : %v", path, err) + return errors.Wrapf(err, "while getting version of %s", path) } - //let's reverse sort the versions to deal with hash collisions (#154) - versions := make([]string, 0, len(v.Versions)) - for k := range v.Versions { - versions = append(versions, k) - } - sort.Sort(sort.Reverse(sort.StringSlice(versions))) - - for _, version := range versions { - val := v.Versions[version] - if sha != val.Digest { - //log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v)) - continue - } - /*we got an exact match, update struct*/ - if !inhub { - log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version) - v.LocalPath = path - v.LocalVersion = version - v.Tainted = false - v.Downloaded = true - /*if we're walking the hub, present file doesn't means installed file*/ - v.Installed = true - v.LocalHash = sha - x := strings.Split(path, "/") - target.FileName = x[len(x)-1] - } - if version == v.Version { - log.Tracef("%s is up-to-date", v.Name) - v.UpToDate = true - } - match = true - break - } - if !match { - log.Tracef("got tainted match for %s : %s", v.Name, path) + item.LocalPath = path + item.Installed = true + item.LocalHash = sha + log.Debugf("DATA [%s] found exact match for %s : version is %s (up-to-date:%t)", itemName, path, version, uptodate) + if version == "" { skippedTainted += 1 - //the file and the stage is right, but the hash is wrong, it has been tainted by user - if !inhub { - v.LocalPath = path - v.Installed = true - } - v.UpToDate = false - v.LocalVersion = "?" - v.Tainted = true - v.LocalHash = sha - x := strings.Split(path, "/") - target.FileName = x[len(x)-1] - - } - //update the entry if appropriate - if _, ok := hubIdx[ftype][k]; !ok { - hubIdx[ftype][k] = v - } else if !inhub { - hubIdx[ftype][k] = v + item.UpToDate = uptodate + item.LocalVersion = "?" + item.Tainted = true + item.LocalHash = sha + } else { + item.UpToDate = uptodate + item.LocalVersion = version + item.LocalHash = sha } + hubIdx[ftype][itemName] = item return nil } - if ftype != DATA_FILES { - log.Infof("Ignoring file %s of type %s", path, ftype) - } else { - log.Debugf("Ignoring file %s of type %s", path, ftype) - } + log.Debugf("Ignoring file %s of type %s", path, ftype) + return nil } @@ -295,7 +351,7 @@ func CollecDepsCheck(v *Item) error { if val.Type == COLLECTIONS { log.Tracef("collec, recurse.") if err := CollecDepsCheck(&val); err != nil { - return fmt.Errorf("sub collection %s is broken : %s", val.Name, err) + return fmt.Errorf("sub collection %s warning : %s", val.Name, err) } hubIdx[ptrtype][p] = val } @@ -336,19 +392,36 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) { datadir = hub.DataDir warnings := []string{} + //data_dir is quite simple : there is no collections and such + if dir == hub.DataDir { + var cpath string + var err error + cpath, err = filepath.Abs(hub.DataDir) + if err != nil { + log.Errorf("failed %s : %s", cpath, err) + } + err = filepath.Walk(cpath, datadirVisit) + return err, warnings + } + /*For each, scan PARSERS, PARSERS_OVFLW, DATA_FILES, SCENARIOS and COLLECTIONS last*/ for _, scan := range ItemTypes { var cpath string var err error - if scan == DATA_FILES { - cpath, err = filepath.Abs(hub.DataDir) - } else { - cpath, err = filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) - } + + cpath, err = filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) if err != nil { log.Errorf("failed %s : %s", cpath, err) } - err = filepath.Walk(cpath, parser_visit) + + switch dir { + case hub.HubDir: + err = filepath.Walk(cpath, hubdirVisit) + case hub.ConfigDir: + err = filepath.Walk(cpath, configdirVisit) + default: + log.Fatalf("unexpected dir %s", dir) + } if err != nil { return err, warnings } @@ -387,6 +460,10 @@ func LocalSync(hub *csconfig.Hub) (error, []string) { if err != nil { return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings } + err, _ = SyncDir(hub, hub.DataDir) + if err != nil { + return fmt.Errorf("failed to scan %s : %s", hub.DataDir, err), warnings + } return nil, warnings }