Parser visit split (#1242)

* break/cleanup this parser_visit function

* do not delete sub-collections if they belong to > 1 collection, fix #1118
This commit is contained in:
Thibault "bui" Koechlin 2022-02-08 16:47:07 +01:00 committed by GitHub
parent c5885bfaf3
commit 5dbcb18cc8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 394 additions and 215 deletions

View file

@ -3,6 +3,7 @@ package cwhub
import ( import (
"crypto/sha256" "crypto/sha256"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
//"errors" //"errors"
@ -81,6 +82,55 @@ type Item struct {
Collections []string `yaml:"collections,omitempty" json:"collections,omitempty"` Collections []string `yaml:"collections,omitempty" json:"collections,omitempty"`
} }
func (i *Item) compareFile(fname, fstage, fauthor, path string) bool {
//wrong filename
if fname != i.FileName {
log.Tracef("%s != %s (filename)", fname, i.FileName)
return false
}
//wrong stage
if fstage != "" && fstage != i.Stage {
log.Tracef("%s != %s (stage)", fstage, i.Stage)
return false
}
//wrong author
if fauthor != "" && fauthor != i.Author {
log.Tracef("%s != %s (author)", fauthor, i.Author)
return false
}
return true
}
func (i *Item) getVersion(path string) (string, string, bool, error) {
//returns version, hash, up-to-date flag, error
sha, err := getSHA256(path)
if err != nil {
return "", "", false, err
}
//let's reverse sort the versions to deal with hash collisions (#154)
versions := make([]string, 0, len(i.Versions))
for k := range i.Versions {
versions = append(versions, k)
}
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
for _, version := range versions {
val := i.Versions[version]
if sha != val.Digest {
continue
}
//if the version is == item.version, it means it's the last available version
if version == i.Version {
return version, sha, true, nil
}
//otherwise, return the version nontheless
return version, sha, false, nil
}
//the version is unknown
return "", sha, false, nil
}
func (i *Item) toHubStatus() ItemHubStatus { func (i *Item) toHubStatus() ItemHubStatus {
hubStatus := ItemHubStatus{} hubStatus := ItemHubStatus{}
hubStatus.Name = i.Name hubStatus.Name = i.Name
@ -201,8 +251,8 @@ func AddItem(itemType string, item Item) error {
} }
func DisplaySummary() { func DisplaySummary() {
log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]), log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers, %d data files", len(hubIdx[COLLECTIONS]),
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW])) len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]), len(hubIdx[DATA_FILES]))
if skippedLocal > 0 || skippedTainted > 0 { if skippedLocal > 0 || skippedTainted > 0 {
log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted) log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted)
} }
@ -266,7 +316,6 @@ func GetUpstreamInstalledScenarios() ([]Item, error) {
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus { func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
if _, ok := hubIdx[itemType]; !ok { if _, ok := hubIdx[itemType]; !ok {
log.Errorf("type %s doesn't exist", itemType) log.Errorf("type %s doesn't exist", itemType)
return nil return nil
} }

View file

@ -214,6 +214,7 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
if !hubIdx[item.Type][item.Name].Installed { if !hubIdx[item.Type][item.Name].Installed {
t.Fatalf("install: %s should be install", item.Name) t.Fatalf("install: %s should be install", item.Name)
} }
} }
func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) { func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
@ -332,6 +333,49 @@ func TestInstallParser(t *testing.T) {
} }
} }
func TestGetItemByPath(t *testing.T) {
cfg := test_prepenv()
if err := GetHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to load hub index")
}
for _, it := range hubIdx[PARSERS] {
//Install the parser
item, err := DownloadLatest(cfg.Hub, it, false, false)
if err != nil {
t.Fatalf("error while downloading %s : %v", item.Name, err)
}
//Enable the item
EnableItem(cfg.Hub, item)
}
//Sync
err, warns := LocalSync(cfg.Hub)
if err != nil || len(warns) > 0 {
t.Fatalf("unexpected err/warnings : %+v / %+v", err, warns)
}
for _, item := range hubIdx[PARSERS] {
//Check that we get the same thing if we get it by the path
itemcopy, err := GetItemByPath(item.Type, item.LocalPath)
if err != nil {
t.Fatalf("error while getting item by path : %v -> %s", err, item.LocalPath)
}
if itemcopy.Name != item.Name {
t.Fatalf("GetItemByPath: %s != %s", itemcopy.Name, item.Name)
}
if itemcopy.UpToDate != item.UpToDate {
t.Fatalf("GetItemByPath: %v != %v", itemcopy.UpToDate, item.UpToDate)
}
if itemcopy.Installed != item.Installed {
t.Fatalf("GetItemByPath: %v != %v", itemcopy.Installed, item.Installed)
}
if itemcopy.Tainted != item.Tainted {
t.Fatalf("GetItemByPath: %v != %v", itemcopy.Tainted, item.Tainted)
}
}
}
func TestInstallCollection(t *testing.T) { func TestInstallCollection(t *testing.T) {
/* /*
@ -350,6 +394,7 @@ func TestInstallCollection(t *testing.T) {
} }
//map iteration is random by itself //map iteration is random by itself
for _, it := range hubIdx[COLLECTIONS] { for _, it := range hubIdx[COLLECTIONS] {
testInstallItem(cfg.Hub, t, it) testInstallItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name] it = hubIdx[COLLECTIONS][it.Name]
testTaintItem(cfg.Hub, t, it) testTaintItem(cfg.Hub, t, it)
@ -383,7 +428,6 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
responseBody := "" responseBody := ""
log.Printf("---> %s", req.URL.Path) log.Printf("---> %s", req.URL.Path)
/*FAKE PARSER*/ /*FAKE PARSER*/
if strings.HasSuffix(req.URL.Path, "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml") { if strings.HasSuffix(req.URL.Path, "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml") {
responseBody = `onsuccess: next_stage responseBody = `onsuccess: next_stage

View file

@ -27,17 +27,26 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name) return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
} }
/*for a COLLECTIONS, disable sub-items*/ /*for a COLLECTIONS, disable sub-items, if they are used _only_ by this item :)*/
if target.Type == COLLECTIONS { if target.Type == COLLECTIONS {
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
for idx, ptr := range tmp { for idx, ptr := range tmp {
ptrtype := ItemTypes[idx] ptrtype := ItemTypes[idx]
for _, p := range ptr { for _, p := range ptr {
if val, ok := hubIdx[ptrtype][p]; ok { if val, ok := hubIdx[ptrtype][p]; ok {
if len(val.BelongsToCollections) > 1 {
log.Infof("%s belongs to more than one collection, won't disable (%v)", p, val.BelongsToCollections)
continue
}
if len(val.BelongsToCollections) == 1 && val.BelongsToCollections[0] == target.Name {
log.Infof("%s belongs exclusively to %s, disabling", p, target.Name)
hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force) hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
if err != nil { if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p)) return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p))
} }
}
} else { } else {
log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name) log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
} }

View file

@ -5,7 +5,6 @@ import (
//"errors" //"errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"sort"
"github.com/pkg/errors" "github.com/pkg/errors"
"golang.org/x/mod/semver" "golang.org/x/mod/semver"
@ -20,128 +19,190 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
/*the walk/parser_visit function can't receive extra args*/ /*the walk function can't receive extra args*/
var hubdir, installdir, datadir string var hubdir, installdir, datadir string
// TODO: Break this function into smaller functions. func visitDiscard(path string, f os.FileInfo) (string, bool, error) {
func parser_visit(path string, f os.FileInfo, err error) error { //return path, false, nil
var target Item path, err := filepath.Abs(path)
var local bool
var hubpath string
var inhub bool
var fname string
var ftype string
var fauthor string
var stage string
path, err = filepath.Abs(path)
if err != nil { if err != nil {
return err return path, true, err
} }
//we only care about files //we only care about files
if f == nil || f.IsDir() { if f == nil || f.IsDir() {
return path, true, nil
}
return path, false, nil
}
func hubdirVisit(path string, f os.FileInfo, err error) error {
if err != nil {
log.Warningf("error visiting %s", err)
}
allowed_extensions := map[string]bool{".yaml": true, ".yml": true}
/*only interested by yaml files */
path, discard, err := visitDiscard(path, f)
if err != nil {
return err
}
if discard {
return nil return nil
} }
// yamls -> collections, parsers, overflows etc. txt, mmdb -> data files if !allowed_extensions[filepath.Ext(path)] {
if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") && !strings.HasSuffix(f.Name(), ".txt") && !strings.HasSuffix(f.Name(), ".mmdb") { log.Debugf("discarding %s : not a yaml file", path)
return nil return nil
} }
//extract components from path :
subs := strings.Split(path, "/")
log.Tracef("path:%s, hubdir:%s, installdir:%s datadir%s", path, hubdir, installdir, datadir)
/*we're in hub (~/.hub/hub/)*/
hubDirSetter := func() {
log.Tracef("in hub dir")
inhub = true
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
//.../hub/scenarios/crowdsec/ssh_bf.yaml //.../hub/scenarios/crowdsec/ssh_bf.yaml
//.../hub/profiles/crowdsec/linux.yaml //.../hub/profiles/crowdsec/linux.yaml
if len(subs) < 4 { path_components := strings.Split(path, string(filepath.Separator))
log.Fatalf("path is too short : %s (%d)", path, len(subs))
if len(path_components) < 4 {
log.Fatalf("path is too short : %s (%d)", path, len(path_components))
} }
fname = subs[len(subs)-1] fname := path_components[len(path_components)-1]
fauthor = subs[len(subs)-2] fauthor := path_components[len(path_components)-2]
stage = subs[len(subs)-3] fstage := path_components[len(path_components)-3]
ftype = subs[len(subs)-4] ftype := path_components[len(path_components)-4]
log.Tracef("%s : stage:%s ftype:%s", path, fstage, ftype)
if ftype == DATA_FILES {
return fmt.Errorf("unexpected data file in hub : %s", path)
} }
dataDirSetter := func() { // correct the stage and type for non-stage stuff.
log.Tracef("in data dir") if fstage == SCENARIOS {
fauthor = "" ftype = SCENARIOS
fname = subs[len(subs)-1] fstage = ""
stage = "" } else if fstage == COLLECTIONS {
ftype = DATA_FILES ftype = COLLECTIONS
fauthor = "" fstage = ""
} else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */
return fmt.Errorf("unknown configuration type for file '%s'", path)
} }
installDirSetter := func() { log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, fstage, ftype)
log.Tracef("in install dir")
if len(subs) < 3 { //in the hub, we don't expect symlinks
log.Fatalf("path is too short : %s (%d)", path, len(subs)) if f.Mode()&os.ModeSymlink != 0 {
log.Warningf("%s in the hub is a symlink, this isn't expected", path)
}
//try to find which configuration item it is
log.Tracef("check [%s] of %s", fname, ftype)
for itemName, item := range hubIdx[ftype] {
log.Tracef("check [%s] vs [%s] : %s/%s/%s", fname, item.RemotePath, ftype, fstage, fname)
if !item.compareFile(fname, fstage, fauthor, path) {
continue
}
//we're in the hub, mark the file as present and downloaded
if path == hubdir+"/"+item.RemotePath { //
log.Tracef("marking %s as downloaded", item.Name)
item.Downloaded = true
}
version, sha, uptodate, err := item.getVersion(path)
if err != nil {
return errors.Wrapf(err, "while getting version of %s", path)
}
if version == "" {
log.Debugf("got tainted match for %s : %s", item.Name, path)
skippedTainted += 1
item.UpToDate = uptodate
item.LocalVersion = "?"
item.Tainted = true
item.LocalHash = sha
} else {
item.UpToDate = uptodate
}
//if it was not present, update the index (it's the first time we're seeing this item. Might be downloaded and not installed)
if _, ok := hubIdx[ftype][itemName]; !ok {
hubIdx[ftype][itemName] = item
}
return nil
}
log.Infof("File %s found in hub directory wasn't found in the hub index, ignoring it", path)
return nil
}
func configdirVisit(path string, f os.FileInfo, err error) error {
if err != nil {
log.Warningf("error visiting %s", err)
}
allowed_extensions := map[string]bool{".yaml": true, ".yml": true}
/*only interested by yaml files */
path, discard, err := visitDiscard(path, f)
if err != nil {
return err
}
if discard {
return nil
}
if !allowed_extensions[filepath.Ext(path)] {
log.Debugf("discarding %s : not a yaml file", path)
return nil
}
path_components := strings.Split(path, string(filepath.Separator))
if len(path_components) < 3 {
log.Fatalf("path is too short : %s (%d)", path, len(path_components))
} }
///.../config/parser/stage/file.yaml ///.../config/parser/stage/file.yaml
///.../config/postoverflow/stage/file.yaml ///.../config/postoverflow/stage/file.yaml
///.../config/scenarios/scenar.yaml ///.../config/scenarios/scenar.yaml
///.../config/collections/linux.yaml //file is empty ///.../config/collections/linux.yaml //file is empty
fname = subs[len(subs)-1] fname := path_components[len(path_components)-1]
stage = subs[len(subs)-2] fstage := path_components[len(path_components)-2]
ftype = subs[len(subs)-3] ftype := path_components[len(path_components)-3]
if ftype == DATA_FILES {
return fmt.Errorf("unexpected data file in install directory : %s", path)
} }
setterByPath := map[string]func(){ log.Tracef("stage:%s ftype:%s", fstage, ftype)
installdir: installDirSetter,
hubdir: hubDirSetter,
datadir: dataDirSetter,
}
paths := []string{installdir, hubdir, datadir}
sort.Slice(paths, func(i, j int) bool {
return len(paths[i]) > len(paths[j])
})
foundMatch := false
for _, p := range paths {
if strings.HasPrefix(path, p) {
setterByPath[p]()
foundMatch = true
break
}
}
if !foundMatch {
return fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubdir, installdir)
}
log.Tracef("stage:%s ftype:%s", stage, ftype)
//log.Printf("%s -> name:%s stage:%s", path, fname, stage)
// correct the stage and type for non-stage stuff. // correct the stage and type for non-stage stuff.
if stage == SCENARIOS { if fstage == SCENARIOS {
ftype = SCENARIOS ftype = SCENARIOS
stage = "" fstage = ""
} else if stage == COLLECTIONS { } else if fstage == COLLECTIONS {
ftype = COLLECTIONS ftype = COLLECTIONS
stage = "" fstage = ""
} else if ftype != PARSERS && ftype != PARSERS_OVFLW && ftype != DATA_FILES { /*its a PARSER / PARSER_OVFLW with a stage */ } else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */
return fmt.Errorf("unknown configuration type for file '%s'", path) return fmt.Errorf("unknown configuration type for file '%s'", path)
} }
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype) log.Tracef("CORRECTED [%s] in stage [%s] of type [%s]", fname, fstage, ftype)
/*
we can encounter 'collections' in the form of a symlink :
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
when the collection is installed, both files are created
*/
//non symlinks are local user files or hub files or data files //non symlinks are local user files or hub files or data files
if f.Mode()&os.ModeSymlink == 0 { if f.Mode()&os.ModeSymlink == 0 {
local = true
log.Tracef("%s isn't a symlink", path) log.Tracef("%s isn't a symlink", path)
} else { local_item := Item{}
hubpath, err = os.Readlink(path) log.Tracef("%s is a local file, skip", path)
skippedLocal++
local_item.Name = fname
local_item.Stage = fstage
local_item.Installed = true
local_item.Type = ftype
local_item.Local = true
local_item.LocalPath = path
local_item.UpToDate = true
x := strings.Split(path, string(filepath.Separator))
local_item.FileName = x[len(x)-1]
hubIdx[ftype][fname] = local_item
return nil
}
hubpath, err := os.Readlink(path)
if err != nil { if err != nil {
return fmt.Errorf("unable to read symlink of %s", path) return fmt.Errorf("unable to read symlink of %s", path)
} }
//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml //the symlink target doesn't exist, user might have removed hub directory without deleting /etc/crowdsec/....yaml
_, err := os.Lstat(hubpath) _, err = os.Lstat(hubpath)
if os.IsNotExist(err) { if os.IsNotExist(err) {
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath) log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
//remove the symlink //remove the symlink
@ -150,123 +211,118 @@ func parser_visit(path string, f os.FileInfo, err error) error {
} }
return nil return nil
} }
log.Tracef("%s points to %s", path, hubpath)
//try to get the matching item version
for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
if !item.compareFile(fname, fstage, "", path) {
continue
} }
log.Tracef("check [%s] vs [%s] : %s", fname, item.RemotePath, ftype+"/"+fstage+"/"+fname+".yaml")
//if it's not a symlink and not in hub nor it is a data file. Don't bother checking this with index version, sha, uptodate, err := item.getVersion(path)
if local && !inhub && ftype != DATA_FILES { if err != nil {
log.Tracef("%s is a local file, skip", path) return errors.Wrapf(err, "while getting version of %s", path)
skippedLocal++ }
// log.Printf("local scenario, skip.") item.LocalPath = path
target.Name = fname item.Installed = true
target.Stage = stage item.LocalHash = sha
target.Installed = true log.Debugf("found exact match for %s : version is %s (up-to-date:%t)", path, version, uptodate)
target.Type = ftype /*we found the matching item, update it*/
target.Local = true if version != "" {
target.LocalPath = path item.LocalVersion = version
target.UpToDate = true item.Tainted = false
x := strings.Split(path, "/") item.Downloaded = true
target.FileName = x[len(x)-1] item.UpToDate = uptodate
} else {
hubIdx[ftype][fname] = target skippedTainted += 1
//the file and the stage is right, but the hash is wrong, it has been tainted by user
item.UpToDate = false
item.LocalVersion = "?"
item.Tainted = true
}
hubIdx[ftype][itemName] = item
return nil return nil
} }
log.Warningf("File %s found in install directory wasn't accounted for : not a symlink to hub, not a local file", path)
return nil
}
func datadirVisit(path string, f os.FileInfo, err error) error {
if err != nil {
log.Warningf("error visiting %s", err)
}
/*only interested by yaml files */
path, discard, err := visitDiscard(path, f)
if err != nil {
return err
}
if discard {
return nil
}
path_components := strings.Split(path, string(filepath.Separator))
if len(path_components) < 2 {
log.Fatalf("path is too short : %s (%d)", path, len(path_components))
}
fname := path_components[len(path_components)-1]
fauthor := path_components[len(path_components)-2]
ftype := DATA_FILES
log.Tracef("CORRECTED [%s] by [%s] of type [%s]", fname, fauthor, ftype)
//non symlinks are local user files or hub files or data files
if f.Mode()&os.ModeSymlink != 0 {
log.Warningf("%s is a symlink, that's unexpected (but can be ok)", path)
final_path, err := os.Readlink(path)
if err != nil {
return fmt.Errorf("unable to read symlink of %s", path)
}
//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
_, err = os.Lstat(path)
if os.IsNotExist(err) {
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, final_path)
if err = os.Remove(path); err != nil {
return fmt.Errorf("failed to unlink %s: %+v", path, err)
}
return nil
}
}
//try to find which configuration item it is //try to find which configuration item it is
log.Tracef("check [%s] of %s", fname, ftype) log.Tracef("check [%s] of %s", fname, ftype)
match := false for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
for k, v := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct log.Tracef("check [%s] vs [%s]", fname, item.RemotePath)
log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml") if !item.compareFile(fname, "", fauthor, path) {
if fname != v.FileName {
log.Tracef("%s != %s (filename)", fname, v.FileName)
continue continue
} }
//wrong stage
if v.Stage != stage { version, sha, uptodate, err := item.getVersion(path)
log.Tracef("%s != %s (stage)", v.Stage, stage)
continue
}
/*if we are walking hub dir, just mark present files as downloaded*/
if inhub {
//wrong author
if fauthor != v.Author {
continue
}
//wrong file
if v.Name+".yaml" != fauthor+"/"+fname {
continue
}
if path == hubdir+"/"+v.RemotePath {
log.Tracef("marking %s as downloaded", v.Name)
v.Downloaded = true
}
}
sha, err := getSHA256(path)
if err != nil { if err != nil {
log.Fatalf("Failed to get sha of %s : %v", path, err) return errors.Wrapf(err, "while getting version of %s", path)
} }
//let's reverse sort the versions to deal with hash collisions (#154) item.LocalPath = path
versions := make([]string, 0, len(v.Versions)) item.Installed = true
for k := range v.Versions { item.LocalHash = sha
versions = append(versions, k) log.Debugf("DATA [%s] found exact match for %s : version is %s (up-to-date:%t)", itemName, path, version, uptodate)
} if version == "" {
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
for _, version := range versions {
val := v.Versions[version]
if sha != val.Digest {
//log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
continue
}
/*we got an exact match, update struct*/
if !inhub {
log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version)
v.LocalPath = path
v.LocalVersion = version
v.Tainted = false
v.Downloaded = true
/*if we're walking the hub, present file doesn't means installed file*/
v.Installed = true
v.LocalHash = sha
x := strings.Split(path, "/")
target.FileName = x[len(x)-1]
}
if version == v.Version {
log.Tracef("%s is up-to-date", v.Name)
v.UpToDate = true
}
match = true
break
}
if !match {
log.Tracef("got tainted match for %s : %s", v.Name, path)
skippedTainted += 1 skippedTainted += 1
//the file and the stage is right, but the hash is wrong, it has been tainted by user item.UpToDate = uptodate
if !inhub { item.LocalVersion = "?"
v.LocalPath = path item.Tainted = true
v.Installed = true item.LocalHash = sha
} } else {
v.UpToDate = false item.UpToDate = uptodate
v.LocalVersion = "?" item.LocalVersion = version
v.Tainted = true item.LocalHash = sha
v.LocalHash = sha
x := strings.Split(path, "/")
target.FileName = x[len(x)-1]
}
//update the entry if appropriate
if _, ok := hubIdx[ftype][k]; !ok {
hubIdx[ftype][k] = v
} else if !inhub {
hubIdx[ftype][k] = v
} }
hubIdx[ftype][itemName] = item
return nil return nil
} }
if ftype != DATA_FILES {
log.Infof("Ignoring file %s of type %s", path, ftype)
} else {
log.Debugf("Ignoring file %s of type %s", path, ftype) log.Debugf("Ignoring file %s of type %s", path, ftype)
}
return nil return nil
} }
@ -295,7 +351,7 @@ func CollecDepsCheck(v *Item) error {
if val.Type == COLLECTIONS { if val.Type == COLLECTIONS {
log.Tracef("collec, recurse.") log.Tracef("collec, recurse.")
if err := CollecDepsCheck(&val); err != nil { if err := CollecDepsCheck(&val); err != nil {
return fmt.Errorf("sub collection %s is broken : %s", val.Name, err) return fmt.Errorf("sub collection %s warning : %s", val.Name, err)
} }
hubIdx[ptrtype][p] = val hubIdx[ptrtype][p] = val
} }
@ -336,19 +392,36 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
datadir = hub.DataDir datadir = hub.DataDir
warnings := []string{} warnings := []string{}
//data_dir is quite simple : there is no collections and such
if dir == hub.DataDir {
var cpath string
var err error
cpath, err = filepath.Abs(hub.DataDir)
if err != nil {
log.Errorf("failed %s : %s", cpath, err)
}
err = filepath.Walk(cpath, datadirVisit)
return err, warnings
}
/*For each, scan PARSERS, PARSERS_OVFLW, DATA_FILES, SCENARIOS and COLLECTIONS last*/ /*For each, scan PARSERS, PARSERS_OVFLW, DATA_FILES, SCENARIOS and COLLECTIONS last*/
for _, scan := range ItemTypes { for _, scan := range ItemTypes {
var cpath string var cpath string
var err error var err error
if scan == DATA_FILES {
cpath, err = filepath.Abs(hub.DataDir)
} else {
cpath, err = filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) cpath, err = filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
}
if err != nil { if err != nil {
log.Errorf("failed %s : %s", cpath, err) log.Errorf("failed %s : %s", cpath, err)
} }
err = filepath.Walk(cpath, parser_visit)
switch dir {
case hub.HubDir:
err = filepath.Walk(cpath, hubdirVisit)
case hub.ConfigDir:
err = filepath.Walk(cpath, configdirVisit)
default:
log.Fatalf("unexpected dir %s", dir)
}
if err != nil { if err != nil {
return err, warnings return err, warnings
} }
@ -387,6 +460,10 @@ func LocalSync(hub *csconfig.Hub) (error, []string) {
if err != nil { if err != nil {
return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings
} }
err, _ = SyncDir(hub, hub.DataDir)
if err != nil {
return fmt.Errorf("failed to scan %s : %s", hub.DataDir, err), warnings
}
return nil, warnings return nil, warnings
} }