526 lines
12 KiB
Go
526 lines
12 KiB
Go
package cwhub
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/Masterminds/semver/v3"
|
|
"github.com/sirupsen/logrus"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
func isYAMLFileName(path string) bool {
|
|
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
|
|
}
|
|
|
|
// linkTarget returns the target of a symlink, or empty string if it's dangling.
|
|
func linkTarget(path string, logger *logrus.Logger) (string, error) {
|
|
hubpath, err := os.Readlink(path)
|
|
if err != nil {
|
|
return "", fmt.Errorf("unable to read symlink: %s", path)
|
|
}
|
|
|
|
logger.Tracef("symlink %s -> %s", path, hubpath)
|
|
|
|
_, err = os.Lstat(hubpath)
|
|
if os.IsNotExist(err) {
|
|
logger.Warningf("link target does not exist: %s -> %s", path, hubpath)
|
|
return "", nil
|
|
}
|
|
|
|
return hubpath, nil
|
|
}
|
|
|
|
func getSHA256(filepath string) (string, error) {
|
|
f, err := os.Open(filepath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
h := sha256.New()
|
|
if _, err := io.Copy(h, f); err != nil {
|
|
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
|
|
}
|
|
|
|
return hex.EncodeToString(h.Sum(nil)), nil
|
|
}
|
|
|
|
// information used to create a new Item, from a file path.
|
|
type itemFileInfo struct {
|
|
inhub bool
|
|
fname string
|
|
stage string
|
|
ftype string
|
|
fauthor string
|
|
}
|
|
|
|
func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo, error) {
|
|
var ret *itemFileInfo
|
|
|
|
hubDir := h.local.HubDir
|
|
installDir := h.local.InstallDir
|
|
|
|
subs := strings.Split(path, string(os.PathSeparator))
|
|
|
|
logger.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir)
|
|
logger.Tracef("subs:%v", subs)
|
|
// we're in hub (~/.hub/hub/)
|
|
if strings.HasPrefix(path, hubDir) {
|
|
logger.Tracef("in hub dir")
|
|
|
|
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
|
|
//.../hub/scenarios/crowdsec/ssh_bf.yaml
|
|
//.../hub/profiles/crowdsec/linux.yaml
|
|
if len(subs) < 4 {
|
|
return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
|
|
}
|
|
|
|
ret = &itemFileInfo{
|
|
inhub: true,
|
|
fname: subs[len(subs)-1],
|
|
fauthor: subs[len(subs)-2],
|
|
stage: subs[len(subs)-3],
|
|
ftype: subs[len(subs)-4],
|
|
}
|
|
} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
|
|
logger.Tracef("in install dir")
|
|
if len(subs) < 3 {
|
|
return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
|
|
}
|
|
///.../config/parser/stage/file.yaml
|
|
///.../config/postoverflow/stage/file.yaml
|
|
///.../config/scenarios/scenar.yaml
|
|
///.../config/collections/linux.yaml //file is empty
|
|
ret = &itemFileInfo{
|
|
inhub: false,
|
|
fname: subs[len(subs)-1],
|
|
stage: subs[len(subs)-2],
|
|
ftype: subs[len(subs)-3],
|
|
fauthor: "",
|
|
}
|
|
} else {
|
|
return nil, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir)
|
|
}
|
|
|
|
logger.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
|
|
|
|
if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
|
|
if !slices.Contains(ItemTypes, ret.stage) {
|
|
return nil, fmt.Errorf("unknown configuration type for file '%s'", path)
|
|
}
|
|
|
|
ret.ftype = ret.stage
|
|
ret.stage = ""
|
|
}
|
|
|
|
logger.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
|
|
|
|
return ret, nil
|
|
}
|
|
|
|
// sortedVersions returns the input data, sorted in reverse order (new, old) by semver.
|
|
func sortedVersions(raw []string) ([]string, error) {
|
|
vs := make([]*semver.Version, len(raw))
|
|
|
|
for idx, r := range raw {
|
|
v, err := semver.NewVersion(r)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s: %w", r, err)
|
|
}
|
|
|
|
vs[idx] = v
|
|
}
|
|
|
|
sort.Sort(sort.Reverse(semver.Collection(vs)))
|
|
|
|
ret := make([]string, len(vs))
|
|
for idx, v := range vs {
|
|
ret[idx] = v.Original()
|
|
}
|
|
|
|
return ret, nil
|
|
}
|
|
|
|
func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) {
|
|
type localItemName struct {
|
|
Name string `yaml:"name"`
|
|
}
|
|
|
|
_, fileName := filepath.Split(path)
|
|
|
|
item := &Item{
|
|
hub: h,
|
|
Name: info.fname,
|
|
Stage: info.stage,
|
|
Type: info.ftype,
|
|
FileName: fileName,
|
|
State: ItemState{
|
|
LocalPath: path,
|
|
Installed: true,
|
|
UpToDate: true,
|
|
},
|
|
}
|
|
|
|
// try to read the name from the file
|
|
itemName := localItemName{}
|
|
|
|
itemContent, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read %s: %w", path, err)
|
|
}
|
|
|
|
err = yaml.Unmarshal(itemContent, &itemName)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal %s: %w", path, err)
|
|
}
|
|
|
|
if itemName.Name != "" {
|
|
item.Name = itemName.Name
|
|
}
|
|
|
|
return item, nil
|
|
}
|
|
|
|
func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
|
|
hubpath := ""
|
|
|
|
if err != nil {
|
|
h.logger.Debugf("while syncing hub dir: %s", err)
|
|
// there is a path error, we ignore the file
|
|
return nil
|
|
}
|
|
|
|
// only happens if the current working directory was removed (!)
|
|
path, err = filepath.Abs(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// we only care about YAML files
|
|
if f == nil || f.IsDir() || !isYAMLFileName(f.Name()) {
|
|
return nil
|
|
}
|
|
|
|
info, err := h.getItemFileInfo(path, h.logger)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// non symlinks are local user files or hub files
|
|
if f.Type()&os.ModeSymlink == 0 {
|
|
h.logger.Tracef("%s is not a symlink", path)
|
|
|
|
if !info.inhub {
|
|
h.logger.Tracef("%s is a local file, skip", path)
|
|
|
|
item, err := newLocalItem(h, path, info)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
h.addItem(item)
|
|
|
|
return nil
|
|
}
|
|
} else {
|
|
hubpath, err = linkTarget(path, h.logger)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if hubpath == "" {
|
|
// target does not exist, the user might have removed the file
|
|
// or switched to a hub branch without it
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// try to find which configuration item it is
|
|
h.logger.Tracef("check [%s] of %s", info.fname, info.ftype)
|
|
|
|
for _, item := range h.GetItemMap(info.ftype) {
|
|
if info.fname != item.FileName {
|
|
continue
|
|
}
|
|
|
|
if item.Stage != info.stage {
|
|
continue
|
|
}
|
|
|
|
// if we are walking hub dir, just mark present files as downloaded
|
|
if info.inhub {
|
|
// wrong author
|
|
if info.fauthor != item.Author {
|
|
continue
|
|
}
|
|
|
|
// not the item we're looking for
|
|
if !item.validPath(info.fauthor, info.fname) {
|
|
continue
|
|
}
|
|
|
|
src, err := item.downloadPath()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if path == src {
|
|
h.logger.Tracef("marking %s as downloaded", item.Name)
|
|
item.State.Downloaded = true
|
|
}
|
|
} else if !hasPathSuffix(hubpath, item.RemotePath) {
|
|
// wrong file
|
|
// <type>/<stage>/<author>/<name>.yaml
|
|
continue
|
|
}
|
|
|
|
err := item.setVersionState(path, info.inhub)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
h.logger.Infof("Ignoring file %s of type %s", path, info.ftype)
|
|
|
|
return nil
|
|
}
|
|
|
|
// checkSubItemVersions checks for the presence, taint and version state of sub-items.
|
|
func (i *Item) checkSubItemVersions() []string {
|
|
warn := make([]string, 0)
|
|
|
|
if !i.HasSubItems() {
|
|
return warn
|
|
}
|
|
|
|
if i.versionStatus() != versionUpToDate {
|
|
i.hub.logger.Debugf("%s dependencies not checked: not up-to-date", i.Name)
|
|
return warn
|
|
}
|
|
|
|
// ensure all the sub-items are installed, or tag the parent as tainted
|
|
i.hub.logger.Tracef("checking submembers of %s installed:%t", i.Name, i.State.Installed)
|
|
|
|
for _, sub := range i.SubItems() {
|
|
i.hub.logger.Tracef("check %s installed:%t", sub.Name, sub.State.Installed)
|
|
|
|
if !i.State.Installed {
|
|
continue
|
|
}
|
|
|
|
if w := sub.checkSubItemVersions(); len(w) > 0 {
|
|
if sub.State.Tainted {
|
|
i.addTaint(sub)
|
|
warn = append(warn, fmt.Sprintf("%s is tainted by %s", i.Name, sub.FQName()))
|
|
}
|
|
|
|
warn = append(warn, w...)
|
|
|
|
continue
|
|
}
|
|
|
|
if sub.State.Tainted {
|
|
i.addTaint(sub)
|
|
warn = append(warn, fmt.Sprintf("%s is tainted by %s", i.Name, sub.FQName()))
|
|
|
|
continue
|
|
}
|
|
|
|
if !sub.State.Installed && i.State.Installed {
|
|
i.addTaint(sub)
|
|
warn = append(warn, fmt.Sprintf("%s is tainted by missing %s", i.Name, sub.FQName()))
|
|
|
|
continue
|
|
}
|
|
|
|
if !sub.State.UpToDate {
|
|
i.State.UpToDate = false
|
|
warn = append(warn, fmt.Sprintf("%s is tainted by outdated %s", i.Name, sub.FQName()))
|
|
|
|
continue
|
|
}
|
|
|
|
i.hub.logger.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, i.State.Tainted, i.State.UpToDate)
|
|
}
|
|
|
|
return warn
|
|
}
|
|
|
|
// syncDir scans a directory for items, and updates the Hub state accordingly.
|
|
func (h *Hub) syncDir(dir string) error {
|
|
// For each, scan PARSERS, POSTOVERFLOWS... and COLLECTIONS last
|
|
for _, scan := range ItemTypes {
|
|
// cpath: top-level item directory, either downloaded or installed items.
|
|
// i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ...
|
|
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
|
|
if err != nil {
|
|
h.logger.Errorf("failed %s: %s", cpath, err)
|
|
continue
|
|
}
|
|
|
|
// explicit check for non existing directory, avoid spamming log.Debug
|
|
if _, err = os.Stat(cpath); os.IsNotExist(err) {
|
|
h.logger.Tracef("directory %s doesn't exist, skipping", cpath)
|
|
continue
|
|
}
|
|
|
|
if err = filepath.WalkDir(cpath, h.itemVisit); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// insert a string in a sorted slice, case insensitive, and return the new slice.
|
|
func insertInOrderNoCase(sl []string, value string) []string {
|
|
i := sort.Search(len(sl), func(i int) bool {
|
|
return strings.ToLower(sl[i]) >= strings.ToLower(value)
|
|
})
|
|
|
|
return append(sl[:i], append([]string{value}, sl[i:]...)...)
|
|
}
|
|
|
|
func removeDuplicates(sl []string) []string {
|
|
seen := make(map[string]struct{}, len(sl))
|
|
j := 0
|
|
|
|
for _, v := range sl {
|
|
if _, ok := seen[v]; ok {
|
|
continue
|
|
}
|
|
|
|
seen[v] = struct{}{}
|
|
sl[j] = v
|
|
j++
|
|
}
|
|
|
|
return sl[:j]
|
|
}
|
|
|
|
// localSync updates the hub state with downloaded, installed and local items.
|
|
func (h *Hub) localSync() error {
|
|
err := h.syncDir(h.local.InstallDir)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err)
|
|
}
|
|
|
|
if err = h.syncDir(h.local.HubDir); err != nil {
|
|
return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err)
|
|
}
|
|
|
|
warnings := make([]string, 0)
|
|
|
|
for _, item := range h.GetItemMap(COLLECTIONS) {
|
|
// check for cyclic dependencies
|
|
subs, err := item.descendants()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// populate the sub- and sub-sub-items with the collections they belong to
|
|
for _, sub := range subs {
|
|
sub.State.BelongsToCollections = insertInOrderNoCase(sub.State.BelongsToCollections, item.Name)
|
|
}
|
|
|
|
if !item.State.Installed {
|
|
continue
|
|
}
|
|
|
|
vs := item.versionStatus()
|
|
switch vs {
|
|
case versionUpToDate: // latest
|
|
if w := item.checkSubItemVersions(); len(w) > 0 {
|
|
warnings = append(warnings, w...)
|
|
}
|
|
case versionUpdateAvailable: // not up-to-date
|
|
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
|
|
case versionFuture:
|
|
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
|
|
case versionUnknown:
|
|
if !item.State.IsLocal() {
|
|
warnings = append(warnings, fmt.Sprintf("collection %s is tainted by local changes (latest:%s)", item.Name, item.Version))
|
|
}
|
|
}
|
|
|
|
h.logger.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions)
|
|
}
|
|
|
|
h.Warnings = removeDuplicates(warnings)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (i *Item) setVersionState(path string, inhub bool) error {
|
|
var err error
|
|
|
|
i.State.LocalHash, err = getSHA256(path)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get sha256 of %s: %w", path, err)
|
|
}
|
|
|
|
// let's reverse sort the versions to deal with hash collisions (#154)
|
|
versions := make([]string, 0, len(i.Versions))
|
|
for k := range i.Versions {
|
|
versions = append(versions, k)
|
|
}
|
|
|
|
versions, err = sortedVersions(versions)
|
|
if err != nil {
|
|
return fmt.Errorf("while syncing %s %s: %w", i.Type, i.FileName, err)
|
|
}
|
|
|
|
i.State.LocalVersion = "?"
|
|
|
|
for _, version := range versions {
|
|
if i.Versions[version].Digest == i.State.LocalHash {
|
|
i.State.LocalVersion = version
|
|
break
|
|
}
|
|
}
|
|
|
|
if i.State.LocalVersion == "?" {
|
|
i.hub.logger.Tracef("got tainted match for %s: %s", i.Name, path)
|
|
|
|
if !inhub {
|
|
i.State.LocalPath = path
|
|
i.State.Installed = true
|
|
}
|
|
|
|
i.State.UpToDate = false
|
|
i.addTaint(i)
|
|
|
|
return nil
|
|
}
|
|
|
|
// we got an exact match, update struct
|
|
|
|
i.State.Downloaded = true
|
|
|
|
if !inhub {
|
|
i.hub.logger.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.State.LocalVersion, i.Version)
|
|
i.State.LocalPath = path
|
|
i.State.Tainted = false
|
|
// if we're walking the hub, present file doesn't means installed file
|
|
i.State.Installed = true
|
|
}
|
|
|
|
if i.State.LocalVersion == i.Version {
|
|
i.hub.logger.Tracef("%s is up-to-date", i.Name)
|
|
i.State.UpToDate = true
|
|
}
|
|
|
|
return nil
|
|
}
|