Parser visit split (#1242)
* break/cleanup this parser_visit function * do not delete sub-collections if they belong to > 1 collection, fix #1118
This commit is contained in:
parent
c5885bfaf3
commit
5dbcb18cc8
|
@ -3,6 +3,7 @@ package cwhub
|
||||||
import (
|
import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
//"errors"
|
//"errors"
|
||||||
|
@ -81,6 +82,55 @@ type Item struct {
|
||||||
Collections []string `yaml:"collections,omitempty" json:"collections,omitempty"`
|
Collections []string `yaml:"collections,omitempty" json:"collections,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (i *Item) compareFile(fname, fstage, fauthor, path string) bool {
|
||||||
|
//wrong filename
|
||||||
|
if fname != i.FileName {
|
||||||
|
log.Tracef("%s != %s (filename)", fname, i.FileName)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
//wrong stage
|
||||||
|
if fstage != "" && fstage != i.Stage {
|
||||||
|
log.Tracef("%s != %s (stage)", fstage, i.Stage)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
//wrong author
|
||||||
|
if fauthor != "" && fauthor != i.Author {
|
||||||
|
log.Tracef("%s != %s (author)", fauthor, i.Author)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Item) getVersion(path string) (string, string, bool, error) {
|
||||||
|
//returns version, hash, up-to-date flag, error
|
||||||
|
sha, err := getSHA256(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
//let's reverse sort the versions to deal with hash collisions (#154)
|
||||||
|
versions := make([]string, 0, len(i.Versions))
|
||||||
|
for k := range i.Versions {
|
||||||
|
versions = append(versions, k)
|
||||||
|
}
|
||||||
|
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
|
||||||
|
|
||||||
|
for _, version := range versions {
|
||||||
|
val := i.Versions[version]
|
||||||
|
if sha != val.Digest {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
//if the version is == item.version, it means it's the last available version
|
||||||
|
if version == i.Version {
|
||||||
|
return version, sha, true, nil
|
||||||
|
}
|
||||||
|
//otherwise, return the version nontheless
|
||||||
|
return version, sha, false, nil
|
||||||
|
}
|
||||||
|
//the version is unknown
|
||||||
|
return "", sha, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (i *Item) toHubStatus() ItemHubStatus {
|
func (i *Item) toHubStatus() ItemHubStatus {
|
||||||
hubStatus := ItemHubStatus{}
|
hubStatus := ItemHubStatus{}
|
||||||
hubStatus.Name = i.Name
|
hubStatus.Name = i.Name
|
||||||
|
@ -201,8 +251,8 @@ func AddItem(itemType string, item Item) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func DisplaySummary() {
|
func DisplaySummary() {
|
||||||
log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
|
log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers, %d data files", len(hubIdx[COLLECTIONS]),
|
||||||
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
|
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]), len(hubIdx[DATA_FILES]))
|
||||||
if skippedLocal > 0 || skippedTainted > 0 {
|
if skippedLocal > 0 || skippedTainted > 0 {
|
||||||
log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted)
|
log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted)
|
||||||
}
|
}
|
||||||
|
@ -266,7 +316,6 @@ func GetUpstreamInstalledScenarios() ([]Item, error) {
|
||||||
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
|
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
|
||||||
if _, ok := hubIdx[itemType]; !ok {
|
if _, ok := hubIdx[itemType]; !ok {
|
||||||
log.Errorf("type %s doesn't exist", itemType)
|
log.Errorf("type %s doesn't exist", itemType)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -214,6 +214,7 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
||||||
if !hubIdx[item.Type][item.Name].Installed {
|
if !hubIdx[item.Type][item.Name].Installed {
|
||||||
t.Fatalf("install: %s should be install", item.Name)
|
t.Fatalf("install: %s should be install", item.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
||||||
|
@ -332,6 +333,49 @@ func TestInstallParser(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGetItemByPath(t *testing.T) {
|
||||||
|
cfg := test_prepenv()
|
||||||
|
|
||||||
|
if err := GetHubIdx(cfg.Hub); err != nil {
|
||||||
|
t.Fatalf("failed to load hub index")
|
||||||
|
}
|
||||||
|
for _, it := range hubIdx[PARSERS] {
|
||||||
|
//Install the parser
|
||||||
|
item, err := DownloadLatest(cfg.Hub, it, false, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("error while downloading %s : %v", item.Name, err)
|
||||||
|
}
|
||||||
|
//Enable the item
|
||||||
|
EnableItem(cfg.Hub, item)
|
||||||
|
}
|
||||||
|
//Sync
|
||||||
|
err, warns := LocalSync(cfg.Hub)
|
||||||
|
if err != nil || len(warns) > 0 {
|
||||||
|
t.Fatalf("unexpected err/warnings : %+v / %+v", err, warns)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range hubIdx[PARSERS] {
|
||||||
|
//Check that we get the same thing if we get it by the path
|
||||||
|
itemcopy, err := GetItemByPath(item.Type, item.LocalPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("error while getting item by path : %v -> %s", err, item.LocalPath)
|
||||||
|
}
|
||||||
|
if itemcopy.Name != item.Name {
|
||||||
|
t.Fatalf("GetItemByPath: %s != %s", itemcopy.Name, item.Name)
|
||||||
|
}
|
||||||
|
if itemcopy.UpToDate != item.UpToDate {
|
||||||
|
t.Fatalf("GetItemByPath: %v != %v", itemcopy.UpToDate, item.UpToDate)
|
||||||
|
}
|
||||||
|
if itemcopy.Installed != item.Installed {
|
||||||
|
t.Fatalf("GetItemByPath: %v != %v", itemcopy.Installed, item.Installed)
|
||||||
|
}
|
||||||
|
if itemcopy.Tainted != item.Tainted {
|
||||||
|
t.Fatalf("GetItemByPath: %v != %v", itemcopy.Tainted, item.Tainted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
func TestInstallCollection(t *testing.T) {
|
func TestInstallCollection(t *testing.T) {
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -350,6 +394,7 @@ func TestInstallCollection(t *testing.T) {
|
||||||
}
|
}
|
||||||
//map iteration is random by itself
|
//map iteration is random by itself
|
||||||
for _, it := range hubIdx[COLLECTIONS] {
|
for _, it := range hubIdx[COLLECTIONS] {
|
||||||
|
|
||||||
testInstallItem(cfg.Hub, t, it)
|
testInstallItem(cfg.Hub, t, it)
|
||||||
it = hubIdx[COLLECTIONS][it.Name]
|
it = hubIdx[COLLECTIONS][it.Name]
|
||||||
testTaintItem(cfg.Hub, t, it)
|
testTaintItem(cfg.Hub, t, it)
|
||||||
|
@ -383,7 +428,6 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
responseBody := ""
|
responseBody := ""
|
||||||
log.Printf("---> %s", req.URL.Path)
|
log.Printf("---> %s", req.URL.Path)
|
||||||
|
|
||||||
|
|
||||||
/*FAKE PARSER*/
|
/*FAKE PARSER*/
|
||||||
if strings.HasSuffix(req.URL.Path, "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml") {
|
if strings.HasSuffix(req.URL.Path, "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml") {
|
||||||
responseBody = `onsuccess: next_stage
|
responseBody = `onsuccess: next_stage
|
||||||
|
|
|
@ -27,17 +27,26 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
|
||||||
return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
|
return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
/*for a COLLECTIONS, disable sub-items*/
|
/*for a COLLECTIONS, disable sub-items, if they are used _only_ by this item :)*/
|
||||||
if target.Type == COLLECTIONS {
|
if target.Type == COLLECTIONS {
|
||||||
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
|
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
|
||||||
for idx, ptr := range tmp {
|
for idx, ptr := range tmp {
|
||||||
ptrtype := ItemTypes[idx]
|
ptrtype := ItemTypes[idx]
|
||||||
for _, p := range ptr {
|
for _, p := range ptr {
|
||||||
if val, ok := hubIdx[ptrtype][p]; ok {
|
if val, ok := hubIdx[ptrtype][p]; ok {
|
||||||
|
|
||||||
|
if len(val.BelongsToCollections) > 1 {
|
||||||
|
log.Infof("%s belongs to more than one collection, won't disable (%v)", p, val.BelongsToCollections)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(val.BelongsToCollections) == 1 && val.BelongsToCollections[0] == target.Name {
|
||||||
|
log.Infof("%s belongs exclusively to %s, disabling", p, target.Name)
|
||||||
hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
|
hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p))
|
return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p))
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
|
log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,7 +5,6 @@ import (
|
||||||
//"errors"
|
//"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"sort"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"golang.org/x/mod/semver"
|
"golang.org/x/mod/semver"
|
||||||
|
@ -20,128 +19,190 @@ import (
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
/*the walk/parser_visit function can't receive extra args*/
|
/*the walk function can't receive extra args*/
|
||||||
var hubdir, installdir, datadir string
|
var hubdir, installdir, datadir string
|
||||||
|
|
||||||
// TODO: Break this function into smaller functions.
|
func visitDiscard(path string, f os.FileInfo) (string, bool, error) {
|
||||||
func parser_visit(path string, f os.FileInfo, err error) error {
|
//return path, false, nil
|
||||||
var target Item
|
path, err := filepath.Abs(path)
|
||||||
var local bool
|
|
||||||
var hubpath string
|
|
||||||
var inhub bool
|
|
||||||
var fname string
|
|
||||||
var ftype string
|
|
||||||
var fauthor string
|
|
||||||
var stage string
|
|
||||||
|
|
||||||
path, err = filepath.Abs(path)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return path, true, err
|
||||||
}
|
}
|
||||||
//we only care about files
|
//we only care about files
|
||||||
if f == nil || f.IsDir() {
|
if f == nil || f.IsDir() {
|
||||||
return nil
|
return path, true, nil
|
||||||
}
|
}
|
||||||
// yamls -> collections, parsers, overflows etc. txt, mmdb -> data files
|
return path, false, nil
|
||||||
if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") && !strings.HasSuffix(f.Name(), ".txt") && !strings.HasSuffix(f.Name(), ".mmdb") {
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
subs := strings.Split(path, "/")
|
func hubdirVisit(path string, f os.FileInfo, err error) error {
|
||||||
|
|
||||||
log.Tracef("path:%s, hubdir:%s, installdir:%s datadir%s", path, hubdir, installdir, datadir)
|
if err != nil {
|
||||||
/*we're in hub (~/.hub/hub/)*/
|
log.Warningf("error visiting %s", err)
|
||||||
|
}
|
||||||
hubDirSetter := func() {
|
allowed_extensions := map[string]bool{".yaml": true, ".yml": true}
|
||||||
log.Tracef("in hub dir")
|
/*only interested by yaml files */
|
||||||
inhub = true
|
path, discard, err := visitDiscard(path, f)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if discard {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !allowed_extensions[filepath.Ext(path)] {
|
||||||
|
log.Debugf("discarding %s : not a yaml file", path)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
//extract components from path :
|
||||||
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
|
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
|
||||||
//.../hub/scenarios/crowdsec/ssh_bf.yaml
|
//.../hub/scenarios/crowdsec/ssh_bf.yaml
|
||||||
//.../hub/profiles/crowdsec/linux.yaml
|
//.../hub/profiles/crowdsec/linux.yaml
|
||||||
if len(subs) < 4 {
|
path_components := strings.Split(path, string(filepath.Separator))
|
||||||
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
|
||||||
|
if len(path_components) < 4 {
|
||||||
|
log.Fatalf("path is too short : %s (%d)", path, len(path_components))
|
||||||
}
|
}
|
||||||
fname = subs[len(subs)-1]
|
fname := path_components[len(path_components)-1]
|
||||||
fauthor = subs[len(subs)-2]
|
fauthor := path_components[len(path_components)-2]
|
||||||
stage = subs[len(subs)-3]
|
fstage := path_components[len(path_components)-3]
|
||||||
ftype = subs[len(subs)-4]
|
ftype := path_components[len(path_components)-4]
|
||||||
|
|
||||||
|
log.Tracef("%s : stage:%s ftype:%s", path, fstage, ftype)
|
||||||
|
|
||||||
|
if ftype == DATA_FILES {
|
||||||
|
return fmt.Errorf("unexpected data file in hub : %s", path)
|
||||||
}
|
}
|
||||||
|
|
||||||
dataDirSetter := func() {
|
// correct the stage and type for non-stage stuff.
|
||||||
log.Tracef("in data dir")
|
if fstage == SCENARIOS {
|
||||||
fauthor = ""
|
ftype = SCENARIOS
|
||||||
fname = subs[len(subs)-1]
|
fstage = ""
|
||||||
stage = ""
|
} else if fstage == COLLECTIONS {
|
||||||
ftype = DATA_FILES
|
ftype = COLLECTIONS
|
||||||
fauthor = ""
|
fstage = ""
|
||||||
|
} else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */
|
||||||
|
return fmt.Errorf("unknown configuration type for file '%s'", path)
|
||||||
}
|
}
|
||||||
|
|
||||||
installDirSetter := func() {
|
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, fstage, ftype)
|
||||||
log.Tracef("in install dir")
|
|
||||||
if len(subs) < 3 {
|
//in the hub, we don't expect symlinks
|
||||||
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
if f.Mode()&os.ModeSymlink != 0 {
|
||||||
|
log.Warningf("%s in the hub is a symlink, this isn't expected", path)
|
||||||
|
}
|
||||||
|
//try to find which configuration item it is
|
||||||
|
log.Tracef("check [%s] of %s", fname, ftype)
|
||||||
|
|
||||||
|
for itemName, item := range hubIdx[ftype] {
|
||||||
|
log.Tracef("check [%s] vs [%s] : %s/%s/%s", fname, item.RemotePath, ftype, fstage, fname)
|
||||||
|
if !item.compareFile(fname, fstage, fauthor, path) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
//we're in the hub, mark the file as present and downloaded
|
||||||
|
if path == hubdir+"/"+item.RemotePath { //
|
||||||
|
log.Tracef("marking %s as downloaded", item.Name)
|
||||||
|
item.Downloaded = true
|
||||||
|
}
|
||||||
|
|
||||||
|
version, sha, uptodate, err := item.getVersion(path)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "while getting version of %s", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
if version == "" {
|
||||||
|
log.Debugf("got tainted match for %s : %s", item.Name, path)
|
||||||
|
skippedTainted += 1
|
||||||
|
item.UpToDate = uptodate
|
||||||
|
item.LocalVersion = "?"
|
||||||
|
item.Tainted = true
|
||||||
|
item.LocalHash = sha
|
||||||
|
} else {
|
||||||
|
item.UpToDate = uptodate
|
||||||
|
}
|
||||||
|
//if it was not present, update the index (it's the first time we're seeing this item. Might be downloaded and not installed)
|
||||||
|
if _, ok := hubIdx[ftype][itemName]; !ok {
|
||||||
|
hubIdx[ftype][itemName] = item
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
log.Infof("File %s found in hub directory wasn't found in the hub index, ignoring it", path)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func configdirVisit(path string, f os.FileInfo, err error) error {
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Warningf("error visiting %s", err)
|
||||||
|
}
|
||||||
|
allowed_extensions := map[string]bool{".yaml": true, ".yml": true}
|
||||||
|
/*only interested by yaml files */
|
||||||
|
path, discard, err := visitDiscard(path, f)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if discard {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !allowed_extensions[filepath.Ext(path)] {
|
||||||
|
log.Debugf("discarding %s : not a yaml file", path)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
path_components := strings.Split(path, string(filepath.Separator))
|
||||||
|
|
||||||
|
if len(path_components) < 3 {
|
||||||
|
log.Fatalf("path is too short : %s (%d)", path, len(path_components))
|
||||||
}
|
}
|
||||||
///.../config/parser/stage/file.yaml
|
///.../config/parser/stage/file.yaml
|
||||||
///.../config/postoverflow/stage/file.yaml
|
///.../config/postoverflow/stage/file.yaml
|
||||||
///.../config/scenarios/scenar.yaml
|
///.../config/scenarios/scenar.yaml
|
||||||
///.../config/collections/linux.yaml //file is empty
|
///.../config/collections/linux.yaml //file is empty
|
||||||
fname = subs[len(subs)-1]
|
fname := path_components[len(path_components)-1]
|
||||||
stage = subs[len(subs)-2]
|
fstage := path_components[len(path_components)-2]
|
||||||
ftype = subs[len(subs)-3]
|
ftype := path_components[len(path_components)-3]
|
||||||
|
|
||||||
|
if ftype == DATA_FILES {
|
||||||
|
return fmt.Errorf("unexpected data file in install directory : %s", path)
|
||||||
}
|
}
|
||||||
|
|
||||||
setterByPath := map[string]func(){
|
log.Tracef("stage:%s ftype:%s", fstage, ftype)
|
||||||
installdir: installDirSetter,
|
|
||||||
hubdir: hubDirSetter,
|
|
||||||
datadir: dataDirSetter,
|
|
||||||
}
|
|
||||||
|
|
||||||
paths := []string{installdir, hubdir, datadir}
|
|
||||||
sort.Slice(paths, func(i, j int) bool {
|
|
||||||
return len(paths[i]) > len(paths[j])
|
|
||||||
})
|
|
||||||
foundMatch := false
|
|
||||||
for _, p := range paths {
|
|
||||||
if strings.HasPrefix(path, p) {
|
|
||||||
setterByPath[p]()
|
|
||||||
foundMatch = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !foundMatch {
|
|
||||||
return fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubdir, installdir)
|
|
||||||
}
|
|
||||||
log.Tracef("stage:%s ftype:%s", stage, ftype)
|
|
||||||
//log.Printf("%s -> name:%s stage:%s", path, fname, stage)
|
|
||||||
|
|
||||||
// correct the stage and type for non-stage stuff.
|
// correct the stage and type for non-stage stuff.
|
||||||
if stage == SCENARIOS {
|
if fstage == SCENARIOS {
|
||||||
ftype = SCENARIOS
|
ftype = SCENARIOS
|
||||||
stage = ""
|
fstage = ""
|
||||||
} else if stage == COLLECTIONS {
|
} else if fstage == COLLECTIONS {
|
||||||
ftype = COLLECTIONS
|
ftype = COLLECTIONS
|
||||||
stage = ""
|
fstage = ""
|
||||||
} else if ftype != PARSERS && ftype != PARSERS_OVFLW && ftype != DATA_FILES { /*its a PARSER / PARSER_OVFLW with a stage */
|
} else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */
|
||||||
return fmt.Errorf("unknown configuration type for file '%s'", path)
|
return fmt.Errorf("unknown configuration type for file '%s'", path)
|
||||||
}
|
}
|
||||||
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype)
|
log.Tracef("CORRECTED [%s] in stage [%s] of type [%s]", fname, fstage, ftype)
|
||||||
|
|
||||||
/*
|
|
||||||
we can encounter 'collections' in the form of a symlink :
|
|
||||||
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
|
|
||||||
when the collection is installed, both files are created
|
|
||||||
*/
|
|
||||||
//non symlinks are local user files or hub files or data files
|
//non symlinks are local user files or hub files or data files
|
||||||
if f.Mode()&os.ModeSymlink == 0 {
|
if f.Mode()&os.ModeSymlink == 0 {
|
||||||
local = true
|
|
||||||
log.Tracef("%s isn't a symlink", path)
|
log.Tracef("%s isn't a symlink", path)
|
||||||
} else {
|
local_item := Item{}
|
||||||
hubpath, err = os.Readlink(path)
|
log.Tracef("%s is a local file, skip", path)
|
||||||
|
skippedLocal++
|
||||||
|
local_item.Name = fname
|
||||||
|
local_item.Stage = fstage
|
||||||
|
local_item.Installed = true
|
||||||
|
local_item.Type = ftype
|
||||||
|
local_item.Local = true
|
||||||
|
local_item.LocalPath = path
|
||||||
|
local_item.UpToDate = true
|
||||||
|
x := strings.Split(path, string(filepath.Separator))
|
||||||
|
local_item.FileName = x[len(x)-1]
|
||||||
|
hubIdx[ftype][fname] = local_item
|
||||||
|
return nil
|
||||||
|
|
||||||
|
}
|
||||||
|
hubpath, err := os.Readlink(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to read symlink of %s", path)
|
return fmt.Errorf("unable to read symlink of %s", path)
|
||||||
}
|
}
|
||||||
//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
|
//the symlink target doesn't exist, user might have removed hub directory without deleting /etc/crowdsec/....yaml
|
||||||
_, err := os.Lstat(hubpath)
|
_, err = os.Lstat(hubpath)
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
|
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
|
||||||
//remove the symlink
|
//remove the symlink
|
||||||
|
@ -150,123 +211,118 @@ func parser_visit(path string, f os.FileInfo, err error) error {
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
log.Tracef("%s points to %s", path, hubpath)
|
|
||||||
|
//try to get the matching item version
|
||||||
|
for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
|
||||||
|
if !item.compareFile(fname, fstage, "", path) {
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
log.Tracef("check [%s] vs [%s] : %s", fname, item.RemotePath, ftype+"/"+fstage+"/"+fname+".yaml")
|
||||||
//if it's not a symlink and not in hub nor it is a data file. Don't bother checking this with index
|
version, sha, uptodate, err := item.getVersion(path)
|
||||||
if local && !inhub && ftype != DATA_FILES {
|
if err != nil {
|
||||||
log.Tracef("%s is a local file, skip", path)
|
return errors.Wrapf(err, "while getting version of %s", path)
|
||||||
skippedLocal++
|
}
|
||||||
// log.Printf("local scenario, skip.")
|
item.LocalPath = path
|
||||||
target.Name = fname
|
item.Installed = true
|
||||||
target.Stage = stage
|
item.LocalHash = sha
|
||||||
target.Installed = true
|
log.Debugf("found exact match for %s : version is %s (up-to-date:%t)", path, version, uptodate)
|
||||||
target.Type = ftype
|
/*we found the matching item, update it*/
|
||||||
target.Local = true
|
if version != "" {
|
||||||
target.LocalPath = path
|
item.LocalVersion = version
|
||||||
target.UpToDate = true
|
item.Tainted = false
|
||||||
x := strings.Split(path, "/")
|
item.Downloaded = true
|
||||||
target.FileName = x[len(x)-1]
|
item.UpToDate = uptodate
|
||||||
|
} else {
|
||||||
hubIdx[ftype][fname] = target
|
skippedTainted += 1
|
||||||
|
//the file and the stage is right, but the hash is wrong, it has been tainted by user
|
||||||
|
item.UpToDate = false
|
||||||
|
item.LocalVersion = "?"
|
||||||
|
item.Tainted = true
|
||||||
|
}
|
||||||
|
hubIdx[ftype][itemName] = item
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
log.Warningf("File %s found in install directory wasn't accounted for : not a symlink to hub, not a local file", path)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func datadirVisit(path string, f os.FileInfo, err error) error {
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Warningf("error visiting %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
/*only interested by yaml files */
|
||||||
|
path, discard, err := visitDiscard(path, f)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if discard {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
path_components := strings.Split(path, string(filepath.Separator))
|
||||||
|
|
||||||
|
if len(path_components) < 2 {
|
||||||
|
log.Fatalf("path is too short : %s (%d)", path, len(path_components))
|
||||||
|
}
|
||||||
|
fname := path_components[len(path_components)-1]
|
||||||
|
fauthor := path_components[len(path_components)-2]
|
||||||
|
ftype := DATA_FILES
|
||||||
|
|
||||||
|
log.Tracef("CORRECTED [%s] by [%s] of type [%s]", fname, fauthor, ftype)
|
||||||
|
|
||||||
|
//non symlinks are local user files or hub files or data files
|
||||||
|
if f.Mode()&os.ModeSymlink != 0 {
|
||||||
|
log.Warningf("%s is a symlink, that's unexpected (but can be ok)", path)
|
||||||
|
final_path, err := os.Readlink(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("unable to read symlink of %s", path)
|
||||||
|
}
|
||||||
|
//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
|
||||||
|
_, err = os.Lstat(path)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, final_path)
|
||||||
|
if err = os.Remove(path); err != nil {
|
||||||
|
return fmt.Errorf("failed to unlink %s: %+v", path, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
//try to find which configuration item it is
|
//try to find which configuration item it is
|
||||||
log.Tracef("check [%s] of %s", fname, ftype)
|
log.Tracef("check [%s] of %s", fname, ftype)
|
||||||
|
|
||||||
match := false
|
for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
|
||||||
for k, v := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
|
log.Tracef("check [%s] vs [%s]", fname, item.RemotePath)
|
||||||
log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml")
|
if !item.compareFile(fname, "", fauthor, path) {
|
||||||
if fname != v.FileName {
|
|
||||||
log.Tracef("%s != %s (filename)", fname, v.FileName)
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
//wrong stage
|
|
||||||
if v.Stage != stage {
|
version, sha, uptodate, err := item.getVersion(path)
|
||||||
log.Tracef("%s != %s (stage)", v.Stage, stage)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
/*if we are walking hub dir, just mark present files as downloaded*/
|
|
||||||
if inhub {
|
|
||||||
//wrong author
|
|
||||||
if fauthor != v.Author {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
//wrong file
|
|
||||||
if v.Name+".yaml" != fauthor+"/"+fname {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if path == hubdir+"/"+v.RemotePath {
|
|
||||||
log.Tracef("marking %s as downloaded", v.Name)
|
|
||||||
v.Downloaded = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sha, err := getSHA256(path)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Failed to get sha of %s : %v", path, err)
|
return errors.Wrapf(err, "while getting version of %s", path)
|
||||||
}
|
}
|
||||||
//let's reverse sort the versions to deal with hash collisions (#154)
|
item.LocalPath = path
|
||||||
versions := make([]string, 0, len(v.Versions))
|
item.Installed = true
|
||||||
for k := range v.Versions {
|
item.LocalHash = sha
|
||||||
versions = append(versions, k)
|
log.Debugf("DATA [%s] found exact match for %s : version is %s (up-to-date:%t)", itemName, path, version, uptodate)
|
||||||
}
|
if version == "" {
|
||||||
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
|
|
||||||
|
|
||||||
for _, version := range versions {
|
|
||||||
val := v.Versions[version]
|
|
||||||
if sha != val.Digest {
|
|
||||||
//log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
/*we got an exact match, update struct*/
|
|
||||||
if !inhub {
|
|
||||||
log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version)
|
|
||||||
v.LocalPath = path
|
|
||||||
v.LocalVersion = version
|
|
||||||
v.Tainted = false
|
|
||||||
v.Downloaded = true
|
|
||||||
/*if we're walking the hub, present file doesn't means installed file*/
|
|
||||||
v.Installed = true
|
|
||||||
v.LocalHash = sha
|
|
||||||
x := strings.Split(path, "/")
|
|
||||||
target.FileName = x[len(x)-1]
|
|
||||||
}
|
|
||||||
if version == v.Version {
|
|
||||||
log.Tracef("%s is up-to-date", v.Name)
|
|
||||||
v.UpToDate = true
|
|
||||||
}
|
|
||||||
match = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if !match {
|
|
||||||
log.Tracef("got tainted match for %s : %s", v.Name, path)
|
|
||||||
skippedTainted += 1
|
skippedTainted += 1
|
||||||
//the file and the stage is right, but the hash is wrong, it has been tainted by user
|
item.UpToDate = uptodate
|
||||||
if !inhub {
|
item.LocalVersion = "?"
|
||||||
v.LocalPath = path
|
item.Tainted = true
|
||||||
v.Installed = true
|
item.LocalHash = sha
|
||||||
}
|
} else {
|
||||||
v.UpToDate = false
|
item.UpToDate = uptodate
|
||||||
v.LocalVersion = "?"
|
item.LocalVersion = version
|
||||||
v.Tainted = true
|
item.LocalHash = sha
|
||||||
v.LocalHash = sha
|
|
||||||
x := strings.Split(path, "/")
|
|
||||||
target.FileName = x[len(x)-1]
|
|
||||||
|
|
||||||
}
|
|
||||||
//update the entry if appropriate
|
|
||||||
if _, ok := hubIdx[ftype][k]; !ok {
|
|
||||||
hubIdx[ftype][k] = v
|
|
||||||
} else if !inhub {
|
|
||||||
hubIdx[ftype][k] = v
|
|
||||||
}
|
}
|
||||||
|
hubIdx[ftype][itemName] = item
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if ftype != DATA_FILES {
|
|
||||||
log.Infof("Ignoring file %s of type %s", path, ftype)
|
|
||||||
} else {
|
|
||||||
log.Debugf("Ignoring file %s of type %s", path, ftype)
|
log.Debugf("Ignoring file %s of type %s", path, ftype)
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -295,7 +351,7 @@ func CollecDepsCheck(v *Item) error {
|
||||||
if val.Type == COLLECTIONS {
|
if val.Type == COLLECTIONS {
|
||||||
log.Tracef("collec, recurse.")
|
log.Tracef("collec, recurse.")
|
||||||
if err := CollecDepsCheck(&val); err != nil {
|
if err := CollecDepsCheck(&val); err != nil {
|
||||||
return fmt.Errorf("sub collection %s is broken : %s", val.Name, err)
|
return fmt.Errorf("sub collection %s warning : %s", val.Name, err)
|
||||||
}
|
}
|
||||||
hubIdx[ptrtype][p] = val
|
hubIdx[ptrtype][p] = val
|
||||||
}
|
}
|
||||||
|
@ -336,19 +392,36 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
|
||||||
datadir = hub.DataDir
|
datadir = hub.DataDir
|
||||||
warnings := []string{}
|
warnings := []string{}
|
||||||
|
|
||||||
|
//data_dir is quite simple : there is no collections and such
|
||||||
|
if dir == hub.DataDir {
|
||||||
|
var cpath string
|
||||||
|
var err error
|
||||||
|
cpath, err = filepath.Abs(hub.DataDir)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed %s : %s", cpath, err)
|
||||||
|
}
|
||||||
|
err = filepath.Walk(cpath, datadirVisit)
|
||||||
|
return err, warnings
|
||||||
|
}
|
||||||
|
|
||||||
/*For each, scan PARSERS, PARSERS_OVFLW, DATA_FILES, SCENARIOS and COLLECTIONS last*/
|
/*For each, scan PARSERS, PARSERS_OVFLW, DATA_FILES, SCENARIOS and COLLECTIONS last*/
|
||||||
for _, scan := range ItemTypes {
|
for _, scan := range ItemTypes {
|
||||||
var cpath string
|
var cpath string
|
||||||
var err error
|
var err error
|
||||||
if scan == DATA_FILES {
|
|
||||||
cpath, err = filepath.Abs(hub.DataDir)
|
|
||||||
} else {
|
|
||||||
cpath, err = filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
|
cpath, err = filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("failed %s : %s", cpath, err)
|
log.Errorf("failed %s : %s", cpath, err)
|
||||||
}
|
}
|
||||||
err = filepath.Walk(cpath, parser_visit)
|
|
||||||
|
switch dir {
|
||||||
|
case hub.HubDir:
|
||||||
|
err = filepath.Walk(cpath, hubdirVisit)
|
||||||
|
case hub.ConfigDir:
|
||||||
|
err = filepath.Walk(cpath, configdirVisit)
|
||||||
|
default:
|
||||||
|
log.Fatalf("unexpected dir %s", dir)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err, warnings
|
return err, warnings
|
||||||
}
|
}
|
||||||
|
@ -387,6 +460,10 @@ func LocalSync(hub *csconfig.Hub) (error, []string) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings
|
return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings
|
||||||
}
|
}
|
||||||
|
err, _ = SyncDir(hub, hub.DataDir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to scan %s : %s", hub.DataDir, err), warnings
|
||||||
|
}
|
||||||
return nil, warnings
|
return nil, warnings
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue