Parser visit split (#1242)
* break/cleanup this parser_visit function * do not delete sub-collections if they belong to > 1 collection, fix #1118
This commit is contained in:
Thibault "bui" Koechlin
GitHub
parent
c5885bfaf3
commit
5dbcb18cc8
|
|
@ -3,6 +3,7 @@ package cwhub
|
|||
import (
|
||||
"crypto/sha256"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
//"errors"
|
||||
|
|
@ -81,6 +82,55 @@ type Item struct {
|
|||
Collections []string `yaml:"collections,omitempty" json:"collections,omitempty"`
|
||||
}
|
||||
|
||||
func (i *Item) compareFile(fname, fstage, fauthor, path string) bool {
|
||||
//wrong filename
|
||||
if fname != i.FileName {
|
||||
log.Tracef("%s != %s (filename)", fname, i.FileName)
|
||||
return false
|
||||
}
|
||||
//wrong stage
|
||||
if fstage != "" && fstage != i.Stage {
|
||||
log.Tracef("%s != %s (stage)", fstage, i.Stage)
|
||||
return false
|
||||
}
|
||||
//wrong author
|
||||
if fauthor != "" && fauthor != i.Author {
|
||||
log.Tracef("%s != %s (author)", fauthor, i.Author)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *Item) getVersion(path string) (string, string, bool, error) {
|
||||
//returns version, hash, up-to-date flag, error
|
||||
sha, err := getSHA256(path)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
|
||||
//let's reverse sort the versions to deal with hash collisions (#154)
|
||||
versions := make([]string, 0, len(i.Versions))
|
||||
for k := range i.Versions {
|
||||
versions = append(versions, k)
|
||||
}
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
|
||||
|
||||
for _, version := range versions {
|
||||
val := i.Versions[version]
|
||||
if sha != val.Digest {
|
||||
continue
|
||||
}
|
||||
//if the version is == item.version, it means it's the last available version
|
||||
if version == i.Version {
|
||||
return version, sha, true, nil
|
||||
}
|
||||
//otherwise, return the version nontheless
|
||||
return version, sha, false, nil
|
||||
}
|
||||
//the version is unknown
|
||||
return "", sha, false, nil
|
||||
}
|
||||
|
||||
func (i *Item) toHubStatus() ItemHubStatus {
|
||||
hubStatus := ItemHubStatus{}
|
||||
hubStatus.Name = i.Name
|
||||
|
|
@ -201,8 +251,8 @@ func AddItem(itemType string, item Item) error {
|
|||
}
|
||||
|
||||
func DisplaySummary() {
|
||||
log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
|
||||
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
|
||||
log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers, %d data files", len(hubIdx[COLLECTIONS]),
|
||||
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]), len(hubIdx[DATA_FILES]))
|
||||
if skippedLocal > 0 || skippedTainted > 0 {
|
||||
log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted)
|
||||
}
|
||||
|
|
@ -266,7 +316,6 @@ func GetUpstreamInstalledScenarios() ([]Item, error) {
|
|||
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
|
||||
if _, ok := hubIdx[itemType]; !ok {
|
||||
log.Errorf("type %s doesn't exist", itemType)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -214,6 +214,7 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
|||
if !hubIdx[item.Type][item.Name].Installed {
|
||||
t.Fatalf("install: %s should be install", item.Name)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
||||
|
|
@ -332,6 +333,49 @@ func TestInstallParser(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestGetItemByPath(t *testing.T) {
|
||||
cfg := test_prepenv()
|
||||
|
||||
if err := GetHubIdx(cfg.Hub); err != nil {
|
||||
t.Fatalf("failed to load hub index")
|
||||
}
|
||||
for _, it := range hubIdx[PARSERS] {
|
||||
//Install the parser
|
||||
item, err := DownloadLatest(cfg.Hub, it, false, false)
|
||||
if err != nil {
|
||||
t.Fatalf("error while downloading %s : %v", item.Name, err)
|
||||
}
|
||||
//Enable the item
|
||||
EnableItem(cfg.Hub, item)
|
||||
}
|
||||
//Sync
|
||||
err, warns := LocalSync(cfg.Hub)
|
||||
if err != nil || len(warns) > 0 {
|
||||
t.Fatalf("unexpected err/warnings : %+v / %+v", err, warns)
|
||||
}
|
||||
|
||||
for _, item := range hubIdx[PARSERS] {
|
||||
//Check that we get the same thing if we get it by the path
|
||||
itemcopy, err := GetItemByPath(item.Type, item.LocalPath)
|
||||
if err != nil {
|
||||
t.Fatalf("error while getting item by path : %v -> %s", err, item.LocalPath)
|
||||
}
|
||||
if itemcopy.Name != item.Name {
|
||||
t.Fatalf("GetItemByPath: %s != %s", itemcopy.Name, item.Name)
|
||||
}
|
||||
if itemcopy.UpToDate != item.UpToDate {
|
||||
t.Fatalf("GetItemByPath: %v != %v", itemcopy.UpToDate, item.UpToDate)
|
||||
}
|
||||
if itemcopy.Installed != item.Installed {
|
||||
t.Fatalf("GetItemByPath: %v != %v", itemcopy.Installed, item.Installed)
|
||||
}
|
||||
if itemcopy.Tainted != item.Tainted {
|
||||
t.Fatalf("GetItemByPath: %v != %v", itemcopy.Tainted, item.Tainted)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestInstallCollection(t *testing.T) {
|
||||
|
||||
/*
|
||||
|
|
@ -350,6 +394,7 @@ func TestInstallCollection(t *testing.T) {
|
|||
}
|
||||
//map iteration is random by itself
|
||||
for _, it := range hubIdx[COLLECTIONS] {
|
||||
|
||||
testInstallItem(cfg.Hub, t, it)
|
||||
it = hubIdx[COLLECTIONS][it.Name]
|
||||
testTaintItem(cfg.Hub, t, it)
|
||||
|
|
@ -383,7 +428,6 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
|||
responseBody := ""
|
||||
log.Printf("---> %s", req.URL.Path)
|
||||
|
||||
|
||||
/*FAKE PARSER*/
|
||||
if strings.HasSuffix(req.URL.Path, "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml") {
|
||||
responseBody = `onsuccess: next_stage
|
||||
|
|
|
|||
|
|
@ -27,17 +27,26 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
|||
return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
|
||||
}
|
||||
|
||||
/*for a COLLECTIONS, disable sub-items*/
|
||||
/*for a COLLECTIONS, disable sub-items, if they are used _only_ by this item :)*/
|
||||
if target.Type == COLLECTIONS {
|
||||
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
|
||||
for idx, ptr := range tmp {
|
||||
ptrtype := ItemTypes[idx]
|
||||
for _, p := range ptr {
|
||||
if val, ok := hubIdx[ptrtype][p]; ok {
|
||||
hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
|
||||
if err != nil {
|
||||
return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p))
|
||||
|
||||
if len(val.BelongsToCollections) > 1 {
|
||||
log.Infof("%s belongs to more than one collection, won't disable (%v)", p, val.BelongsToCollections)
|
||||
continue
|
||||
}
|
||||
if len(val.BelongsToCollections) == 1 && val.BelongsToCollections[0] == target.Name {
|
||||
log.Infof("%s belongs exclusively to %s, disabling", p, target.Name)
|
||||
hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
|
||||
if err != nil {
|
||||
return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p))
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import (
|
|||
//"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"sort"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/mod/semver"
|
||||
|
|
@ -20,253 +19,310 @@ import (
|
|||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
/*the walk/parser_visit function can't receive extra args*/
|
||||
/*the walk function can't receive extra args*/
|
||||
var hubdir, installdir, datadir string
|
||||
|
||||
// TODO: Break this function into smaller functions.
|
||||
func parser_visit(path string, f os.FileInfo, err error) error {
|
||||
var target Item
|
||||
var local bool
|
||||
var hubpath string
|
||||
var inhub bool
|
||||
var fname string
|
||||
var ftype string
|
||||
var fauthor string
|
||||
var stage string
|
||||
|
||||
path, err = filepath.Abs(path)
|
||||
func visitDiscard(path string, f os.FileInfo) (string, bool, error) {
|
||||
//return path, false, nil
|
||||
path, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return err
|
||||
return path, true, err
|
||||
}
|
||||
//we only care about files
|
||||
if f == nil || f.IsDir() {
|
||||
return path, true, nil
|
||||
}
|
||||
return path, false, nil
|
||||
}
|
||||
|
||||
func hubdirVisit(path string, f os.FileInfo, err error) error {
|
||||
|
||||
if err != nil {
|
||||
log.Warningf("error visiting %s", err)
|
||||
}
|
||||
allowed_extensions := map[string]bool{".yaml": true, ".yml": true}
|
||||
/*only interested by yaml files */
|
||||
path, discard, err := visitDiscard(path, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if discard {
|
||||
return nil
|
||||
}
|
||||
// yamls -> collections, parsers, overflows etc. txt, mmdb -> data files
|
||||
if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") && !strings.HasSuffix(f.Name(), ".txt") && !strings.HasSuffix(f.Name(), ".mmdb") {
|
||||
if !allowed_extensions[filepath.Ext(path)] {
|
||||
log.Debugf("discarding %s : not a yaml file", path)
|
||||
return nil
|
||||
}
|
||||
//extract components from path :
|
||||
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
|
||||
//.../hub/scenarios/crowdsec/ssh_bf.yaml
|
||||
//.../hub/profiles/crowdsec/linux.yaml
|
||||
path_components := strings.Split(path, string(filepath.Separator))
|
||||
|
||||
subs := strings.Split(path, "/")
|
||||
|
||||
log.Tracef("path:%s, hubdir:%s, installdir:%s datadir%s", path, hubdir, installdir, datadir)
|
||||
/*we're in hub (~/.hub/hub/)*/
|
||||
|
||||
hubDirSetter := func() {
|
||||
log.Tracef("in hub dir")
|
||||
inhub = true
|
||||
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
|
||||
//.../hub/scenarios/crowdsec/ssh_bf.yaml
|
||||
//.../hub/profiles/crowdsec/linux.yaml
|
||||
if len(subs) < 4 {
|
||||
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
||||
}
|
||||
fname = subs[len(subs)-1]
|
||||
fauthor = subs[len(subs)-2]
|
||||
stage = subs[len(subs)-3]
|
||||
ftype = subs[len(subs)-4]
|
||||
if len(path_components) < 4 {
|
||||
log.Fatalf("path is too short : %s (%d)", path, len(path_components))
|
||||
}
|
||||
fname := path_components[len(path_components)-1]
|
||||
fauthor := path_components[len(path_components)-2]
|
||||
fstage := path_components[len(path_components)-3]
|
||||
ftype := path_components[len(path_components)-4]
|
||||
|
||||
dataDirSetter := func() {
|
||||
log.Tracef("in data dir")
|
||||
fauthor = ""
|
||||
fname = subs[len(subs)-1]
|
||||
stage = ""
|
||||
ftype = DATA_FILES
|
||||
fauthor = ""
|
||||
}
|
||||
log.Tracef("%s : stage:%s ftype:%s", path, fstage, ftype)
|
||||
|
||||
installDirSetter := func() {
|
||||
log.Tracef("in install dir")
|
||||
if len(subs) < 3 {
|
||||
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
||||
}
|
||||
///.../config/parser/stage/file.yaml
|
||||
///.../config/postoverflow/stage/file.yaml
|
||||
///.../config/scenarios/scenar.yaml
|
||||
///.../config/collections/linux.yaml //file is empty
|
||||
fname = subs[len(subs)-1]
|
||||
stage = subs[len(subs)-2]
|
||||
ftype = subs[len(subs)-3]
|
||||
if ftype == DATA_FILES {
|
||||
return fmt.Errorf("unexpected data file in hub : %s", path)
|
||||
}
|
||||
|
||||
setterByPath := map[string]func(){
|
||||
installdir: installDirSetter,
|
||||
hubdir: hubDirSetter,
|
||||
datadir: dataDirSetter,
|
||||
}
|
||||
|
||||
paths := []string{installdir, hubdir, datadir}
|
||||
sort.Slice(paths, func(i, j int) bool {
|
||||
return len(paths[i]) > len(paths[j])
|
||||
})
|
||||
foundMatch := false
|
||||
for _, p := range paths {
|
||||
if strings.HasPrefix(path, p) {
|
||||
setterByPath[p]()
|
||||
foundMatch = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !foundMatch {
|
||||
return fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubdir, installdir)
|
||||
}
|
||||
log.Tracef("stage:%s ftype:%s", stage, ftype)
|
||||
//log.Printf("%s -> name:%s stage:%s", path, fname, stage)
|
||||
|
||||
// correct the stage and type for non-stage stuff.
|
||||
if stage == SCENARIOS {
|
||||
if fstage == SCENARIOS {
|
||||
ftype = SCENARIOS
|
||||
stage = ""
|
||||
} else if stage == COLLECTIONS {
|
||||
fstage = ""
|
||||
} else if fstage == COLLECTIONS {
|
||||
ftype = COLLECTIONS
|
||||
stage = ""
|
||||
} else if ftype != PARSERS && ftype != PARSERS_OVFLW && ftype != DATA_FILES { /*its a PARSER / PARSER_OVFLW with a stage */
|
||||
fstage = ""
|
||||
} else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */
|
||||
return fmt.Errorf("unknown configuration type for file '%s'", path)
|
||||
}
|
||||
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype)
|
||||
|
||||
/*
|
||||
we can encounter 'collections' in the form of a symlink :
|
||||
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
|
||||
when the collection is installed, both files are created
|
||||
*/
|
||||
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, fstage, ftype)
|
||||
|
||||
//in the hub, we don't expect symlinks
|
||||
if f.Mode()&os.ModeSymlink != 0 {
|
||||
log.Warningf("%s in the hub is a symlink, this isn't expected", path)
|
||||
}
|
||||
//try to find which configuration item it is
|
||||
log.Tracef("check [%s] of %s", fname, ftype)
|
||||
|
||||
for itemName, item := range hubIdx[ftype] {
|
||||
log.Tracef("check [%s] vs [%s] : %s/%s/%s", fname, item.RemotePath, ftype, fstage, fname)
|
||||
if !item.compareFile(fname, fstage, fauthor, path) {
|
||||
continue
|
||||
}
|
||||
//we're in the hub, mark the file as present and downloaded
|
||||
if path == hubdir+"/"+item.RemotePath { //
|
||||
log.Tracef("marking %s as downloaded", item.Name)
|
||||
item.Downloaded = true
|
||||
}
|
||||
|
||||
version, sha, uptodate, err := item.getVersion(path)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "while getting version of %s", path)
|
||||
}
|
||||
|
||||
if version == "" {
|
||||
log.Debugf("got tainted match for %s : %s", item.Name, path)
|
||||
skippedTainted += 1
|
||||
item.UpToDate = uptodate
|
||||
item.LocalVersion = "?"
|
||||
item.Tainted = true
|
||||
item.LocalHash = sha
|
||||
} else {
|
||||
item.UpToDate = uptodate
|
||||
}
|
||||
//if it was not present, update the index (it's the first time we're seeing this item. Might be downloaded and not installed)
|
||||
if _, ok := hubIdx[ftype][itemName]; !ok {
|
||||
hubIdx[ftype][itemName] = item
|
||||
}
|
||||
return nil
|
||||
}
|
||||
log.Infof("File %s found in hub directory wasn't found in the hub index, ignoring it", path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func configdirVisit(path string, f os.FileInfo, err error) error {
|
||||
|
||||
if err != nil {
|
||||
log.Warningf("error visiting %s", err)
|
||||
}
|
||||
allowed_extensions := map[string]bool{".yaml": true, ".yml": true}
|
||||
/*only interested by yaml files */
|
||||
path, discard, err := visitDiscard(path, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if discard {
|
||||
return nil
|
||||
}
|
||||
if !allowed_extensions[filepath.Ext(path)] {
|
||||
log.Debugf("discarding %s : not a yaml file", path)
|
||||
return nil
|
||||
}
|
||||
path_components := strings.Split(path, string(filepath.Separator))
|
||||
|
||||
if len(path_components) < 3 {
|
||||
log.Fatalf("path is too short : %s (%d)", path, len(path_components))
|
||||
}
|
||||
///.../config/parser/stage/file.yaml
|
||||
///.../config/postoverflow/stage/file.yaml
|
||||
///.../config/scenarios/scenar.yaml
|
||||
///.../config/collections/linux.yaml //file is empty
|
||||
fname := path_components[len(path_components)-1]
|
||||
fstage := path_components[len(path_components)-2]
|
||||
ftype := path_components[len(path_components)-3]
|
||||
|
||||
if ftype == DATA_FILES {
|
||||
return fmt.Errorf("unexpected data file in install directory : %s", path)
|
||||
}
|
||||
|
||||
log.Tracef("stage:%s ftype:%s", fstage, ftype)
|
||||
|
||||
// correct the stage and type for non-stage stuff.
|
||||
if fstage == SCENARIOS {
|
||||
ftype = SCENARIOS
|
||||
fstage = ""
|
||||
} else if fstage == COLLECTIONS {
|
||||
ftype = COLLECTIONS
|
||||
fstage = ""
|
||||
} else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */
|
||||
return fmt.Errorf("unknown configuration type for file '%s'", path)
|
||||
}
|
||||
log.Tracef("CORRECTED [%s] in stage [%s] of type [%s]", fname, fstage, ftype)
|
||||
|
||||
//non symlinks are local user files or hub files or data files
|
||||
if f.Mode()&os.ModeSymlink == 0 {
|
||||
local = true
|
||||
log.Tracef("%s isn't a symlink", path)
|
||||
} else {
|
||||
hubpath, err = os.Readlink(path)
|
||||
local_item := Item{}
|
||||
log.Tracef("%s is a local file, skip", path)
|
||||
skippedLocal++
|
||||
local_item.Name = fname
|
||||
local_item.Stage = fstage
|
||||
local_item.Installed = true
|
||||
local_item.Type = ftype
|
||||
local_item.Local = true
|
||||
local_item.LocalPath = path
|
||||
local_item.UpToDate = true
|
||||
x := strings.Split(path, string(filepath.Separator))
|
||||
local_item.FileName = x[len(x)-1]
|
||||
hubIdx[ftype][fname] = local_item
|
||||
return nil
|
||||
|
||||
}
|
||||
hubpath, err := os.Readlink(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read symlink of %s", path)
|
||||
}
|
||||
//the symlink target doesn't exist, user might have removed hub directory without deleting /etc/crowdsec/....yaml
|
||||
_, err = os.Lstat(hubpath)
|
||||
if os.IsNotExist(err) {
|
||||
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
|
||||
//remove the symlink
|
||||
if err = os.Remove(path); err != nil {
|
||||
return fmt.Errorf("failed to unlink %s: %+v", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
//try to get the matching item version
|
||||
for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
|
||||
if !item.compareFile(fname, fstage, "", path) {
|
||||
continue
|
||||
}
|
||||
log.Tracef("check [%s] vs [%s] : %s", fname, item.RemotePath, ftype+"/"+fstage+"/"+fname+".yaml")
|
||||
version, sha, uptodate, err := item.getVersion(path)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "while getting version of %s", path)
|
||||
}
|
||||
item.LocalPath = path
|
||||
item.Installed = true
|
||||
item.LocalHash = sha
|
||||
log.Debugf("found exact match for %s : version is %s (up-to-date:%t)", path, version, uptodate)
|
||||
/*we found the matching item, update it*/
|
||||
if version != "" {
|
||||
item.LocalVersion = version
|
||||
item.Tainted = false
|
||||
item.Downloaded = true
|
||||
item.UpToDate = uptodate
|
||||
} else {
|
||||
skippedTainted += 1
|
||||
//the file and the stage is right, but the hash is wrong, it has been tainted by user
|
||||
item.UpToDate = false
|
||||
item.LocalVersion = "?"
|
||||
item.Tainted = true
|
||||
}
|
||||
hubIdx[ftype][itemName] = item
|
||||
return nil
|
||||
}
|
||||
log.Warningf("File %s found in install directory wasn't accounted for : not a symlink to hub, not a local file", path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func datadirVisit(path string, f os.FileInfo, err error) error {
|
||||
|
||||
if err != nil {
|
||||
log.Warningf("error visiting %s", err)
|
||||
}
|
||||
|
||||
/*only interested by yaml files */
|
||||
path, discard, err := visitDiscard(path, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if discard {
|
||||
return nil
|
||||
}
|
||||
|
||||
path_components := strings.Split(path, string(filepath.Separator))
|
||||
|
||||
if len(path_components) < 2 {
|
||||
log.Fatalf("path is too short : %s (%d)", path, len(path_components))
|
||||
}
|
||||
fname := path_components[len(path_components)-1]
|
||||
fauthor := path_components[len(path_components)-2]
|
||||
ftype := DATA_FILES
|
||||
|
||||
log.Tracef("CORRECTED [%s] by [%s] of type [%s]", fname, fauthor, ftype)
|
||||
|
||||
//non symlinks are local user files or hub files or data files
|
||||
if f.Mode()&os.ModeSymlink != 0 {
|
||||
log.Warningf("%s is a symlink, that's unexpected (but can be ok)", path)
|
||||
final_path, err := os.Readlink(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read symlink of %s", path)
|
||||
}
|
||||
//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
|
||||
_, err := os.Lstat(hubpath)
|
||||
_, err = os.Lstat(path)
|
||||
if os.IsNotExist(err) {
|
||||
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
|
||||
//remove the symlink
|
||||
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, final_path)
|
||||
if err = os.Remove(path); err != nil {
|
||||
return fmt.Errorf("failed to unlink %s: %+v", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
log.Tracef("%s points to %s", path, hubpath)
|
||||
}
|
||||
|
||||
//if it's not a symlink and not in hub nor it is a data file. Don't bother checking this with index
|
||||
if local && !inhub && ftype != DATA_FILES {
|
||||
log.Tracef("%s is a local file, skip", path)
|
||||
skippedLocal++
|
||||
// log.Printf("local scenario, skip.")
|
||||
target.Name = fname
|
||||
target.Stage = stage
|
||||
target.Installed = true
|
||||
target.Type = ftype
|
||||
target.Local = true
|
||||
target.LocalPath = path
|
||||
target.UpToDate = true
|
||||
x := strings.Split(path, "/")
|
||||
target.FileName = x[len(x)-1]
|
||||
|
||||
hubIdx[ftype][fname] = target
|
||||
return nil
|
||||
}
|
||||
//try to find which configuration item it is
|
||||
log.Tracef("check [%s] of %s", fname, ftype)
|
||||
|
||||
match := false
|
||||
for k, v := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
|
||||
log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml")
|
||||
if fname != v.FileName {
|
||||
log.Tracef("%s != %s (filename)", fname, v.FileName)
|
||||
for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
|
||||
log.Tracef("check [%s] vs [%s]", fname, item.RemotePath)
|
||||
if !item.compareFile(fname, "", fauthor, path) {
|
||||
continue
|
||||
}
|
||||
//wrong stage
|
||||
if v.Stage != stage {
|
||||
log.Tracef("%s != %s (stage)", v.Stage, stage)
|
||||
continue
|
||||
}
|
||||
/*if we are walking hub dir, just mark present files as downloaded*/
|
||||
if inhub {
|
||||
//wrong author
|
||||
if fauthor != v.Author {
|
||||
continue
|
||||
}
|
||||
//wrong file
|
||||
if v.Name+".yaml" != fauthor+"/"+fname {
|
||||
continue
|
||||
}
|
||||
if path == hubdir+"/"+v.RemotePath {
|
||||
log.Tracef("marking %s as downloaded", v.Name)
|
||||
v.Downloaded = true
|
||||
}
|
||||
}
|
||||
sha, err := getSHA256(path)
|
||||
|
||||
version, sha, uptodate, err := item.getVersion(path)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get sha of %s : %v", path, err)
|
||||
return errors.Wrapf(err, "while getting version of %s", path)
|
||||
}
|
||||
//let's reverse sort the versions to deal with hash collisions (#154)
|
||||
versions := make([]string, 0, len(v.Versions))
|
||||
for k := range v.Versions {
|
||||
versions = append(versions, k)
|
||||
}
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
|
||||
|
||||
for _, version := range versions {
|
||||
val := v.Versions[version]
|
||||
if sha != val.Digest {
|
||||
//log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
|
||||
continue
|
||||
}
|
||||
/*we got an exact match, update struct*/
|
||||
if !inhub {
|
||||
log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version)
|
||||
v.LocalPath = path
|
||||
v.LocalVersion = version
|
||||
v.Tainted = false
|
||||
v.Downloaded = true
|
||||
/*if we're walking the hub, present file doesn't means installed file*/
|
||||
v.Installed = true
|
||||
v.LocalHash = sha
|
||||
x := strings.Split(path, "/")
|
||||
target.FileName = x[len(x)-1]
|
||||
}
|
||||
if version == v.Version {
|
||||
log.Tracef("%s is up-to-date", v.Name)
|
||||
v.UpToDate = true
|
||||
}
|
||||
match = true
|
||||
break
|
||||
}
|
||||
if !match {
|
||||
log.Tracef("got tainted match for %s : %s", v.Name, path)
|
||||
item.LocalPath = path
|
||||
item.Installed = true
|
||||
item.LocalHash = sha
|
||||
log.Debugf("DATA [%s] found exact match for %s : version is %s (up-to-date:%t)", itemName, path, version, uptodate)
|
||||
if version == "" {
|
||||
skippedTainted += 1
|
||||
//the file and the stage is right, but the hash is wrong, it has been tainted by user
|
||||
if !inhub {
|
||||
v.LocalPath = path
|
||||
v.Installed = true
|
||||
}
|
||||
v.UpToDate = false
|
||||
v.LocalVersion = "?"
|
||||
v.Tainted = true
|
||||
v.LocalHash = sha
|
||||
x := strings.Split(path, "/")
|
||||
target.FileName = x[len(x)-1]
|
||||
|
||||
}
|
||||
//update the entry if appropriate
|
||||
if _, ok := hubIdx[ftype][k]; !ok {
|
||||
hubIdx[ftype][k] = v
|
||||
} else if !inhub {
|
||||
hubIdx[ftype][k] = v
|
||||
item.UpToDate = uptodate
|
||||
item.LocalVersion = "?"
|
||||
item.Tainted = true
|
||||
item.LocalHash = sha
|
||||
} else {
|
||||
item.UpToDate = uptodate
|
||||
item.LocalVersion = version
|
||||
item.LocalHash = sha
|
||||
}
|
||||
hubIdx[ftype][itemName] = item
|
||||
return nil
|
||||
}
|
||||
if ftype != DATA_FILES {
|
||||
log.Infof("Ignoring file %s of type %s", path, ftype)
|
||||
} else {
|
||||
log.Debugf("Ignoring file %s of type %s", path, ftype)
|
||||
}
|
||||
log.Debugf("Ignoring file %s of type %s", path, ftype)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -295,7 +351,7 @@ func CollecDepsCheck(v *Item) error {
|
|||
if val.Type == COLLECTIONS {
|
||||
log.Tracef("collec, recurse.")
|
||||
if err := CollecDepsCheck(&val); err != nil {
|
||||
return fmt.Errorf("sub collection %s is broken : %s", val.Name, err)
|
||||
return fmt.Errorf("sub collection %s warning : %s", val.Name, err)
|
||||
}
|
||||
hubIdx[ptrtype][p] = val
|
||||
}
|
||||
|
|
@ -336,19 +392,36 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
|
|||
datadir = hub.DataDir
|
||||
warnings := []string{}
|
||||
|
||||
//data_dir is quite simple : there is no collections and such
|
||||
if dir == hub.DataDir {
|
||||
var cpath string
|
||||
var err error
|
||||
cpath, err = filepath.Abs(hub.DataDir)
|
||||
if err != nil {
|
||||
log.Errorf("failed %s : %s", cpath, err)
|
||||
}
|
||||
err = filepath.Walk(cpath, datadirVisit)
|
||||
return err, warnings
|
||||
}
|
||||
|
||||
/*For each, scan PARSERS, PARSERS_OVFLW, DATA_FILES, SCENARIOS and COLLECTIONS last*/
|
||||
for _, scan := range ItemTypes {
|
||||
var cpath string
|
||||
var err error
|
||||
if scan == DATA_FILES {
|
||||
cpath, err = filepath.Abs(hub.DataDir)
|
||||
} else {
|
||||
cpath, err = filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
|
||||
}
|
||||
|
||||
cpath, err = filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
|
||||
if err != nil {
|
||||
log.Errorf("failed %s : %s", cpath, err)
|
||||
}
|
||||
err = filepath.Walk(cpath, parser_visit)
|
||||
|
||||
switch dir {
|
||||
case hub.HubDir:
|
||||
err = filepath.Walk(cpath, hubdirVisit)
|
||||
case hub.ConfigDir:
|
||||
err = filepath.Walk(cpath, configdirVisit)
|
||||
default:
|
||||
log.Fatalf("unexpected dir %s", dir)
|
||||
}
|
||||
if err != nil {
|
||||
return err, warnings
|
||||
}
|
||||
|
|
@ -387,6 +460,10 @@ func LocalSync(hub *csconfig.Hub) (error, []string) {
|
|||
if err != nil {
|
||||
return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings
|
||||
}
|
||||
err, _ = SyncDir(hub, hub.DataDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to scan %s : %s", hub.DataDir, err), warnings
|
||||
}
|
||||
return nil, warnings
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue