2020-07-27 11:47:32 +00:00
|
|
|
package cwhub
|
|
|
|
|
|
|
|
import (
|
2023-10-06 11:59:51 +00:00
|
|
|
"crypto/sha256"
|
2023-10-30 16:23:50 +00:00
|
|
|
"encoding/hex"
|
2020-07-27 11:47:32 +00:00
|
|
|
"fmt"
|
2023-10-06 11:59:51 +00:00
|
|
|
"io"
|
2020-07-27 11:47:32 +00:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2022-05-24 13:46:48 +00:00
|
|
|
"sort"
|
2020-07-27 11:47:32 +00:00
|
|
|
"strings"
|
|
|
|
|
|
|
|
log "github.com/sirupsen/logrus"
|
|
|
|
)
|
|
|
|
|
2023-10-04 10:54:21 +00:00
|
|
|
func isYAMLFileName(path string) bool {
|
|
|
|
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
|
|
|
|
}
|
|
|
|
|
|
|
|
func handleSymlink(path string) (string, error) {
|
|
|
|
hubpath, err := os.Readlink(path)
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("unable to read symlink of %s", path)
|
|
|
|
}
|
|
|
|
// the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
|
|
|
|
_, err = os.Lstat(hubpath)
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
|
|
|
|
// remove the symlink
|
|
|
|
if err = os.Remove(path); err != nil {
|
|
|
|
return "", fmt.Errorf("failed to unlink %s: %w", path, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// XXX: is this correct?
|
|
|
|
return "", nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return hubpath, nil
|
|
|
|
}
|
|
|
|
|
2023-10-06 11:59:51 +00:00
|
|
|
func getSHA256(filepath string) (string, error) {
|
|
|
|
f, err := os.Open(filepath)
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
h := sha256.New()
|
|
|
|
if _, err := io.Copy(h, f); err != nil {
|
|
|
|
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
|
|
|
|
}
|
|
|
|
|
2023-10-30 16:23:50 +00:00
|
|
|
return hex.EncodeToString(h.Sum(nil)), nil
|
2023-10-06 11:59:51 +00:00
|
|
|
}
|
|
|
|
|
2023-10-04 09:17:35 +00:00
|
|
|
type itemFileInfo struct {
|
|
|
|
fname string
|
|
|
|
stage string
|
|
|
|
ftype string
|
|
|
|
fauthor string
|
|
|
|
}
|
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
|
2023-10-04 09:17:35 +00:00
|
|
|
ret := itemFileInfo{}
|
|
|
|
inhub := false
|
2020-07-27 11:47:32 +00:00
|
|
|
|
2023-10-30 16:23:50 +00:00
|
|
|
hubDir := h.local.HubDir
|
|
|
|
installDir := h.local.InstallDir
|
2023-10-19 10:04:29 +00:00
|
|
|
|
2022-05-17 10:14:59 +00:00
|
|
|
subs := strings.Split(path, string(os.PathSeparator))
|
2020-07-27 11:47:32 +00:00
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir)
|
2022-05-17 10:14:59 +00:00
|
|
|
log.Tracef("subs:%v", subs)
|
2023-10-03 09:20:56 +00:00
|
|
|
// we're in hub (~/.hub/hub/)
|
2023-10-19 10:04:29 +00:00
|
|
|
if strings.HasPrefix(path, hubDir) {
|
2020-11-30 09:37:17 +00:00
|
|
|
log.Tracef("in hub dir")
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2020-07-27 11:47:32 +00:00
|
|
|
inhub = true
|
2020-11-30 09:37:17 +00:00
|
|
|
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
|
|
|
|
//.../hub/scenarios/crowdsec/ssh_bf.yaml
|
|
|
|
//.../hub/profiles/crowdsec/linux.yaml
|
2020-07-27 11:47:32 +00:00
|
|
|
if len(subs) < 4 {
|
2023-10-12 12:53:17 +00:00
|
|
|
return itemFileInfo{}, false, fmt.Errorf("path is too short : %s (%d)", path, len(subs))
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-04 09:17:35 +00:00
|
|
|
ret.fname = subs[len(subs)-1]
|
|
|
|
ret.fauthor = subs[len(subs)-2]
|
|
|
|
ret.stage = subs[len(subs)-3]
|
|
|
|
ret.ftype = subs[len(subs)-4]
|
2023-10-19 10:04:29 +00:00
|
|
|
} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
|
2020-11-30 09:37:17 +00:00
|
|
|
log.Tracef("in install dir")
|
2020-07-27 11:47:32 +00:00
|
|
|
if len(subs) < 3 {
|
2023-10-12 12:53:17 +00:00
|
|
|
return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2020-11-30 09:37:17 +00:00
|
|
|
///.../config/parser/stage/file.yaml
|
|
|
|
///.../config/postoverflow/stage/file.yaml
|
|
|
|
///.../config/scenarios/scenar.yaml
|
|
|
|
///.../config/collections/linux.yaml //file is empty
|
2023-10-04 09:17:35 +00:00
|
|
|
ret.fname = subs[len(subs)-1]
|
|
|
|
ret.stage = subs[len(subs)-2]
|
|
|
|
ret.ftype = subs[len(subs)-3]
|
|
|
|
ret.fauthor = ""
|
2020-07-27 11:47:32 +00:00
|
|
|
} else {
|
2023-10-19 10:04:29 +00:00
|
|
|
return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir)
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
|
|
|
|
2023-10-04 09:17:35 +00:00
|
|
|
log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
|
2023-10-04 10:54:21 +00:00
|
|
|
// log.Infof("%s -> name:%s stage:%s", path, fname, stage)
|
2023-10-27 08:25:29 +00:00
|
|
|
|
2023-10-04 09:17:35 +00:00
|
|
|
if ret.stage == SCENARIOS {
|
|
|
|
ret.ftype = SCENARIOS
|
|
|
|
ret.stage = ""
|
|
|
|
} else if ret.stage == COLLECTIONS {
|
|
|
|
ret.ftype = COLLECTIONS
|
|
|
|
ret.stage = ""
|
2023-10-17 14:17:37 +00:00
|
|
|
} else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
|
|
|
|
// its a PARSER / POSTOVERFLOW with a stage
|
2023-10-04 09:17:35 +00:00
|
|
|
return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path)
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
|
|
|
|
|
|
|
|
return ret, inhub, nil
|
|
|
|
}
|
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
|
2023-10-04 09:17:35 +00:00
|
|
|
var (
|
|
|
|
local bool
|
|
|
|
hubpath string
|
|
|
|
)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
log.Debugf("while syncing hub dir: %s", err)
|
|
|
|
// there is a path error, we ignore the file
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
path, err = filepath.Abs(path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
|
|
|
|
2023-10-04 09:17:35 +00:00
|
|
|
// we only care about files
|
|
|
|
if f == nil || f.IsDir() {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-10-04 10:54:21 +00:00
|
|
|
if !isYAMLFileName(f.Name()) {
|
2023-10-04 09:17:35 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
info, inhub, err := h.getItemInfo(path)
|
2023-10-04 09:17:35 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-07-27 11:47:32 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
we can encounter 'collections' in the form of a symlink :
|
2021-03-24 17:16:17 +00:00
|
|
|
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
|
2020-07-27 11:47:32 +00:00
|
|
|
when the collection is installed, both files are created
|
|
|
|
*/
|
2023-10-03 09:20:56 +00:00
|
|
|
// non symlinks are local user files or hub files
|
2023-06-23 12:04:58 +00:00
|
|
|
if f.Type()&os.ModeSymlink == 0 {
|
2020-07-27 11:47:32 +00:00
|
|
|
local = true
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2020-07-27 11:47:32 +00:00
|
|
|
log.Tracef("%s isn't a symlink", path)
|
|
|
|
} else {
|
2023-10-04 10:54:21 +00:00
|
|
|
hubpath, err = handleSymlink(path)
|
2020-07-27 11:47:32 +00:00
|
|
|
if err != nil {
|
2023-10-04 10:54:21 +00:00
|
|
|
return err
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-04 10:54:21 +00:00
|
|
|
log.Tracef("%s points to %s", path, hubpath)
|
|
|
|
|
|
|
|
if hubpath == "" {
|
|
|
|
// XXX: is this correct?
|
2020-07-27 11:47:32 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-10-03 09:20:56 +00:00
|
|
|
// if it's not a symlink and not in hub, it's a local file, don't bother
|
2020-07-27 11:47:32 +00:00
|
|
|
if local && !inhub {
|
2020-11-30 09:37:17 +00:00
|
|
|
log.Tracef("%s is a local file, skip", path)
|
2023-10-19 10:04:29 +00:00
|
|
|
h.skippedLocal++
|
2023-10-04 10:54:21 +00:00
|
|
|
// log.Infof("local scenario, skip.")
|
2020-07-27 11:47:32 +00:00
|
|
|
|
2023-10-04 09:17:35 +00:00
|
|
|
_, fileName := filepath.Split(path)
|
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
h.Items[info.ftype][info.fname] = Item{
|
2023-10-04 09:17:35 +00:00
|
|
|
Name: info.fname,
|
|
|
|
Stage: info.stage,
|
|
|
|
Installed: true,
|
|
|
|
Type: info.ftype,
|
|
|
|
LocalPath: path,
|
|
|
|
UpToDate: true,
|
|
|
|
FileName: fileName,
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2020-07-27 11:47:32 +00:00
|
|
|
return nil
|
|
|
|
}
|
2023-10-04 09:17:35 +00:00
|
|
|
|
2023-10-03 09:20:56 +00:00
|
|
|
// try to find which configuration item it is
|
2023-10-04 09:17:35 +00:00
|
|
|
log.Tracef("check [%s] of %s", info.fname, info.ftype)
|
2020-07-27 11:47:32 +00:00
|
|
|
|
|
|
|
match := false
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
for name, item := range h.Items[info.ftype] {
|
2023-10-04 10:54:21 +00:00
|
|
|
log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml")
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-04 10:54:21 +00:00
|
|
|
if info.fname != item.FileName {
|
|
|
|
log.Tracef("%s != %s (filename)", info.fname, item.FileName)
|
2020-07-27 11:47:32 +00:00
|
|
|
continue
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
|
|
|
// wrong stage
|
2023-10-04 10:54:21 +00:00
|
|
|
if item.Stage != info.stage {
|
2020-07-27 11:47:32 +00:00
|
|
|
continue
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
|
|
|
// if we are walking hub dir, just mark present files as downloaded
|
2020-07-27 11:47:32 +00:00
|
|
|
if inhub {
|
2023-10-03 09:20:56 +00:00
|
|
|
// wrong author
|
2023-10-04 10:54:21 +00:00
|
|
|
if info.fauthor != item.Author {
|
2020-07-27 11:47:32 +00:00
|
|
|
continue
|
|
|
|
}
|
2023-10-04 08:34:10 +00:00
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
// not the item we're looking for
|
|
|
|
if !item.validPath(info.fauthor, info.fname) {
|
2020-07-27 11:47:32 +00:00
|
|
|
continue
|
|
|
|
}
|
2022-05-17 10:14:59 +00:00
|
|
|
|
2023-10-30 16:23:50 +00:00
|
|
|
if path == h.local.HubDir+"/"+item.RemotePath {
|
2023-10-04 10:54:21 +00:00
|
|
|
log.Tracef("marking %s as downloaded", item.Name)
|
|
|
|
item.Downloaded = true
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-04 10:54:21 +00:00
|
|
|
} else if !hasPathSuffix(hubpath, item.RemotePath) {
|
2023-10-03 09:20:56 +00:00
|
|
|
// wrong file
|
|
|
|
// <type>/<stage>/<author>/<name>.yaml
|
2022-11-07 09:36:50 +00:00
|
|
|
continue
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2020-07-27 11:47:32 +00:00
|
|
|
sha, err := getSHA256(path)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("Failed to get sha of %s : %v", path, err)
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
|
|
|
// let's reverse sort the versions to deal with hash collisions (#154)
|
2023-10-04 10:54:21 +00:00
|
|
|
versions := make([]string, 0, len(item.Versions))
|
|
|
|
for k := range item.Versions {
|
2020-07-29 13:02:52 +00:00
|
|
|
versions = append(versions, k)
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2020-07-29 13:02:52 +00:00
|
|
|
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
|
|
|
|
|
|
|
|
for _, version := range versions {
|
2023-10-04 10:54:21 +00:00
|
|
|
val := item.Versions[version]
|
2020-07-27 11:47:32 +00:00
|
|
|
if sha != val.Digest {
|
2023-10-04 10:54:21 +00:00
|
|
|
// log.Infof("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
|
2020-07-27 11:47:32 +00:00
|
|
|
continue
|
|
|
|
}
|
2023-10-04 09:17:35 +00:00
|
|
|
|
2023-10-03 09:20:56 +00:00
|
|
|
// we got an exact match, update struct
|
2023-10-04 10:54:21 +00:00
|
|
|
|
|
|
|
item.Downloaded = true
|
|
|
|
item.LocalHash = sha
|
|
|
|
|
2022-02-01 21:08:06 +00:00
|
|
|
if !inhub {
|
2023-10-04 10:54:21 +00:00
|
|
|
log.Tracef("found exact match for %s, version is %s, latest is %s", item.Name, version, item.Version)
|
|
|
|
item.LocalPath = path
|
|
|
|
item.LocalVersion = version
|
|
|
|
item.Tainted = false
|
2023-10-03 09:20:56 +00:00
|
|
|
// if we're walking the hub, present file doesn't means installed file
|
2023-10-04 10:54:21 +00:00
|
|
|
item.Installed = true
|
2022-02-01 21:08:06 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-04 10:54:21 +00:00
|
|
|
if version == item.Version {
|
|
|
|
log.Tracef("%s is up-to-date", item.Name)
|
|
|
|
item.UpToDate = true
|
2022-02-01 21:08:06 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2022-02-01 21:08:06 +00:00
|
|
|
match = true
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2022-02-01 21:08:06 +00:00
|
|
|
break
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2020-07-27 11:47:32 +00:00
|
|
|
if !match {
|
2023-10-04 10:54:21 +00:00
|
|
|
log.Tracef("got tainted match for %s: %s", item.Name, path)
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
h.skippedTainted++
|
2023-10-03 09:20:56 +00:00
|
|
|
// the file and the stage is right, but the hash is wrong, it has been tainted by user
|
2020-07-27 11:47:32 +00:00
|
|
|
if !inhub {
|
2023-10-04 10:54:21 +00:00
|
|
|
item.LocalPath = path
|
|
|
|
item.Installed = true
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-04 10:54:21 +00:00
|
|
|
item.UpToDate = false
|
|
|
|
item.LocalVersion = "?"
|
|
|
|
item.Tainted = true
|
|
|
|
item.LocalHash = sha
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-04 09:17:35 +00:00
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
h.Items[info.ftype][name] = item
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2020-07-27 11:47:32 +00:00
|
|
|
return nil
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-04 09:17:35 +00:00
|
|
|
log.Infof("Ignoring file %s of type %s", path, info.ftype)
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2020-07-27 11:47:32 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
func (h *Hub) CollectDepsCheck(v *Item) error {
|
|
|
|
if v.Type != COLLECTIONS {
|
2021-03-29 08:33:23 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
if v.versionStatus() != 0 { // not up-to-date
|
|
|
|
log.Debugf("%s dependencies not checked: not up-to-date", v.Name)
|
2023-10-04 10:54:21 +00:00
|
|
|
return nil
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-04 10:54:21 +00:00
|
|
|
// if it's a collection, ensure all the items are installed, or tag it as tainted
|
|
|
|
log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
|
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
for _, sub := range v.SubItems() {
|
|
|
|
subItem, ok := h.Items[sub.Type][sub.Name]
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("referred %s %s in collection %s doesn't exist", sub.Type, sub.Name, v.Name)
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed)
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
if !v.Installed {
|
|
|
|
continue
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
if subItem.Type == COLLECTIONS {
|
|
|
|
log.Tracef("collec, recurse.")
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
if err := h.CollectDepsCheck(&subItem); err != nil {
|
|
|
|
if subItem.Tainted {
|
|
|
|
v.Tainted = true
|
2022-02-01 21:08:06 +00:00
|
|
|
}
|
2020-07-27 11:47:32 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
|
2023-10-04 10:54:21 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
h.Items[sub.Type][sub.Name] = subItem
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
// propagate the state of sub-items to set
|
|
|
|
if subItem.Tainted {
|
|
|
|
v.Tainted = true
|
|
|
|
return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name)
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
if !subItem.Installed && v.Installed {
|
|
|
|
v.Tainted = true
|
|
|
|
return fmt.Errorf("missing %s %s, tainted", sub.Type, sub.Name)
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
if !subItem.UpToDate {
|
|
|
|
v.UpToDate = false
|
|
|
|
return fmt.Errorf("outdated %s %s", sub.Type, sub.Name)
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
skip := false
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
for idx := range subItem.BelongsToCollections {
|
|
|
|
if subItem.BelongsToCollections[idx] == v.Name {
|
|
|
|
skip = true
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-20 12:32:35 +00:00
|
|
|
}
|
2023-10-04 10:54:21 +00:00
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
if !skip {
|
|
|
|
subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-20 12:32:35 +00:00
|
|
|
|
|
|
|
h.Items[sub.Type][sub.Name] = subItem
|
|
|
|
|
|
|
|
log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate)
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2020-07-27 11:47:32 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
func (h *Hub) SyncDir(dir string) ([]string, error) {
|
2021-03-29 08:33:23 +00:00
|
|
|
warnings := []string{}
|
2020-11-30 09:37:17 +00:00
|
|
|
|
2023-10-17 14:17:37 +00:00
|
|
|
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
|
2020-07-27 11:47:32 +00:00
|
|
|
for _, scan := range ItemTypes {
|
2020-11-30 09:37:17 +00:00
|
|
|
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
|
|
|
|
if err != nil {
|
2023-10-30 16:23:50 +00:00
|
|
|
log.Errorf("failed %s: %s", cpath, err)
|
2020-11-30 09:37:17 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-27 08:25:29 +00:00
|
|
|
// explicit check for non existing directory, avoid spamming log.Debug
|
2023-10-30 16:23:50 +00:00
|
|
|
if _, err = os.Stat(cpath); os.IsNotExist(err) {
|
2023-10-27 08:25:29 +00:00
|
|
|
log.Tracef("directory %s doesn't exist, skipping", cpath)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
err = filepath.WalkDir(cpath, h.itemVisit)
|
2020-11-30 09:37:17 +00:00
|
|
|
if err != nil {
|
2023-10-09 19:33:35 +00:00
|
|
|
return warnings, err
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-10-19 10:04:29 +00:00
|
|
|
for name, item := range h.Items[COLLECTIONS] {
|
2023-10-04 10:54:21 +00:00
|
|
|
if !item.Installed {
|
2023-10-04 09:17:35 +00:00
|
|
|
continue
|
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-06 11:59:51 +00:00
|
|
|
vs := item.versionStatus()
|
|
|
|
switch vs {
|
2023-10-04 09:17:35 +00:00
|
|
|
case 0: // latest
|
2023-10-19 10:04:29 +00:00
|
|
|
if err := h.CollectDepsCheck(&item); err != nil {
|
2023-10-06 11:59:51 +00:00
|
|
|
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
|
2023-10-19 10:04:29 +00:00
|
|
|
h.Items[COLLECTIONS][name] = item
|
2023-10-04 09:17:35 +00:00
|
|
|
}
|
|
|
|
case 1: // not up-to-date
|
2023-10-04 10:54:21 +00:00
|
|
|
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
|
2023-10-04 09:17:35 +00:00
|
|
|
default: // version is higher than the highest available from hub?
|
2023-10-04 10:54:21 +00:00
|
|
|
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-04 09:17:35 +00:00
|
|
|
|
2023-10-06 11:59:51 +00:00
|
|
|
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions)
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-09 19:33:35 +00:00
|
|
|
return warnings, nil
|
2020-07-27 11:47:32 +00:00
|
|
|
}
|
|
|
|
|
2023-10-04 10:54:21 +00:00
|
|
|
// Updates the info from HubInit() with the local state
|
2023-10-19 10:04:29 +00:00
|
|
|
func (h *Hub) LocalSync() ([]string, error) {
|
|
|
|
h.skippedLocal = 0
|
|
|
|
h.skippedTainted = 0
|
2020-11-30 09:37:17 +00:00
|
|
|
|
2023-10-30 16:23:50 +00:00
|
|
|
warnings, err := h.SyncDir(h.local.InstallDir)
|
2021-03-29 08:33:23 +00:00
|
|
|
if err != nil {
|
2023-10-30 16:23:50 +00:00
|
|
|
return warnings, fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err)
|
2020-11-30 09:37:17 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-30 16:23:50 +00:00
|
|
|
_, err = h.SyncDir(h.local.HubDir)
|
2021-03-29 08:33:23 +00:00
|
|
|
if err != nil {
|
2023-10-30 16:23:50 +00:00
|
|
|
return warnings, fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err)
|
2021-03-29 08:33:23 +00:00
|
|
|
}
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2023-10-09 19:33:35 +00:00
|
|
|
return warnings, nil
|
2020-11-30 09:37:17 +00:00
|
|
|
}
|