package cwhub import ( "crypto/sha256" "fmt" "io" "os" "path/filepath" "sort" "strings" "github.com/enescakir/emoji" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" ) /*managed configuration types*/ var PARSERS = "parsers" var PARSERS_OVFLW = "postoverflows" var SCENARIOS = "scenarios" var COLLECTIONS = "collections" var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS} var hubIdx map[string]map[string]Item var RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" var HubBranch = "master" var HubIndexFile = ".index.json" type ItemVersion struct { Digest string `json:"digest,omitempty"` Deprecated bool `json:"deprecated,omitempty"` } type ItemHubStatus struct { Name string `json:"name"` LocalVersion string `json:"local_version"` LocalPath string `json:"local_path"` Description string `json:"description"` UTF8_Status string `json:"utf8_status"` Status string `json:"status"` } //Item can be : parsed, scenario, collection type Item struct { /*descriptive info*/ Type string `yaml:"type,omitempty" json:"type,omitempty"` //parser|postoverflows|scenario|collection(|enrich) Stage string `json:"stage,omitempty" yaml:"stage,omitempty,omitempty"` //Stage for parser|postoverflow : s00-raw/s01-... Name string `json:"name,omitempty"` //as seen in .config.json, usually "author/name" FileName string `json:"file_name,omitempty"` //the filename, ie. apache2-logs.yaml Description string `yaml:"description,omitempty" json:"description,omitempty"` //as seen in .config.json Author string `json:"author,omitempty"` //as seen in .config.json References []string `yaml:"references,omitempty" json:"references,omitempty"` //as seen in .config.json BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` /*if it's part of collections, track name here*/ /*remote (hub) infos*/ RemoteURL string `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` //the full remote uri of file in http RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` //the path relative to git ie. /parsers/stage/author/file.yaml RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` //the meow Version string `json:"version,omitempty"` //the last version Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` //the list of existing versions /*local (deployed) infos*/ LocalPath string `yaml:"local_path,omitempty" json:"local_path,omitempty"` //the local path relative to ${CFG_DIR} //LocalHubPath string LocalVersion string `json:"local_version,omitempty"` LocalHash string `json:"local_hash,omitempty"` //the local meow Installed bool `json:"installed,omitempty"` Downloaded bool `json:"downloaded,omitempty"` UpToDate bool `json:"up_to_date,omitempty"` Tainted bool `json:"tainted,omitempty"` //has it been locally modified Local bool `json:"local,omitempty"` //if it's a non versioned control one /*if it's a collection, it not a single file*/ Parsers []string `yaml:"parsers,omitempty" json:"parsers,omitempty"` PostOverflows []string `yaml:"postoverflows,omitempty" json:"postoverflows,omitempty"` Scenarios []string `yaml:"scenarios,omitempty" json:"scenarios,omitempty"` Collections []string `yaml:"collections,omitempty" json:"collections,omitempty"` } func (i *Item) toHubStatus() ItemHubStatus { hubStatus := ItemHubStatus{} hubStatus.Name = i.Name hubStatus.LocalVersion = i.LocalVersion hubStatus.LocalPath = i.LocalPath hubStatus.Description = i.Description status, ok, warning, managed := ItemStatus(*i) hubStatus.Status = status if !managed { hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.House, status) } else if !i.Installed { hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.Prohibited, status) } else if warning { hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.Warning, status) } else if ok { hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.CheckMark, status) } return hubStatus } var skippedLocal = 0 var skippedTainted = 0 /*To be used when reference(s) (is/are) missing in a collection*/ var ReferenceMissingError = errors.New("Reference(s) missing in collection") var MissingHubIndex = errors.New("hub index can't be found") //GetVersionStatus : semver requires 'v' prefix func GetVersionStatus(v *Item) int { return semver.Compare("v"+v.Version, "v"+v.LocalVersion) } // calculate sha256 of a file func getSHA256(filepath string) (string, error) { /* Digest of file */ f, err := os.Open(filepath) if err != nil { return "", fmt.Errorf("unable to open '%s' : %s", filepath, err) } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", fmt.Errorf("unable to calculate sha256 of '%s': %s", filepath, err) } return fmt.Sprintf("%x", h.Sum(nil)), nil } func GetItemMap(itemType string) map[string]Item { var m map[string]Item var ok bool if m, ok = hubIdx[itemType]; !ok { return nil } return m } //GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. func GetItemByPath(itemType string, itemPath string) (*Item, error) { /*try to resolve symlink*/ finalName := "" f, err := os.Lstat(itemPath) if err != nil { return nil, errors.Wrapf(err, "while performing lstat on %s", itemPath) } if f.Mode()&os.ModeSymlink == 0 { /*it's not a symlink, it should be the filename itsef the key*/ finalName = filepath.Base(itemPath) } else { /*resolve the symlink to hub file*/ pathInHub, err := os.Readlink(itemPath) if err != nil { return nil, errors.Wrapf(err, "while reading symlink of %s", itemPath) } //extract author from path fname := filepath.Base(pathInHub) author := filepath.Base(filepath.Dir(pathInHub)) //trim yaml suffix fname = strings.TrimSuffix(fname, ".yaml") fname = strings.TrimSuffix(fname, ".yml") finalName = fmt.Sprintf("%s/%s", author, fname) } /*it's not a symlink, it should be the filename itsef the key*/ if m := GetItemMap(itemType); m != nil { if v, ok := m[finalName]; ok { return &v, nil } return nil, fmt.Errorf("%s not found in %s", finalName, itemType) } return nil, fmt.Errorf("item type %s doesn't exist", itemType) } func GetItem(itemType string, itemName string) *Item { if m, ok := GetItemMap(itemType)[itemName]; ok { return &m } return nil } func AddItem(itemType string, item Item) error { in := false for _, itype := range ItemTypes { if itype == itemType { in = true } } if !in { return fmt.Errorf("ItemType %s is unknown", itemType) } hubIdx[itemType][item.Name] = item return nil } func DisplaySummary() { log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]), len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW])) if skippedLocal > 0 || skippedTainted > 0 { log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted) } } //returns: human-text, Enabled, Warning, Unmanaged func ItemStatus(v Item) (string, bool, bool, bool) { strret := "disabled" Ok := false if v.Installed { Ok = true strret = "enabled" } Managed := true if v.Local { Managed = false strret += ",local" } //tainted or out of date Warning := false if v.Tainted { Warning = true strret += ",tainted" } else if !v.UpToDate && !v.Local { strret += ",update-available" Warning = true } return strret, Ok, Warning, Managed } func GetInstalledScenariosAsString() ([]string, error) { var retStr []string items, err := GetInstalledScenarios() if err != nil { return nil, errors.Wrap(err, "while fetching scenarios") } for _, it := range items { retStr = append(retStr, it.Name) } return retStr, nil } func GetInstalledScenarios() ([]Item, error) { var retItems []Item if _, ok := hubIdx[SCENARIOS]; !ok { return nil, fmt.Errorf("no scenarios in hubIdx") } for _, item := range hubIdx[SCENARIOS] { if item.Installed { retItems = append(retItems, item) } } return retItems, nil } func GetInstalledParsers() ([]Item, error) { var retItems []Item if _, ok := hubIdx[PARSERS]; !ok { return nil, fmt.Errorf("no parsers in hubIdx") } for _, item := range hubIdx[PARSERS] { if item.Installed { retItems = append(retItems, item) } } return retItems, nil } func GetInstalledParsersAsString() ([]string, error) { var retStr []string items, err := GetInstalledParsers() if err != nil { return nil, errors.Wrap(err, "while fetching parsers") } for _, it := range items { retStr = append(retStr, it.Name) } return retStr, nil } func GetInstalledPostOverflows() ([]Item, error) { var retItems []Item if _, ok := hubIdx[PARSERS_OVFLW]; !ok { return nil, fmt.Errorf("no post overflows in hubIdx") } for _, item := range hubIdx[PARSERS_OVFLW] { if item.Installed { retItems = append(retItems, item) } } return retItems, nil } func GetInstalledPostOverflowsAsString() ([]string, error) { var retStr []string items, err := GetInstalledPostOverflows() if err != nil { return nil, errors.Wrap(err, "while fetching post overflows") } for _, it := range items { retStr = append(retStr, it.Name) } return retStr, nil } func GetInstalledCollectionsAsString() ([]string, error) { var retStr []string items, err := GetInstalledCollections() if err != nil { return nil, errors.Wrap(err, "while fetching collections") } for _, it := range items { retStr = append(retStr, it.Name) } return retStr, nil } func GetInstalledCollections() ([]Item, error) { var retItems []Item if _, ok := hubIdx[COLLECTIONS]; !ok { return nil, fmt.Errorf("no collection in hubIdx") } for _, item := range hubIdx[COLLECTIONS] { if item.Installed { retItems = append(retItems, item) } } return retItems, nil } //Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy) func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus { if _, ok := hubIdx[itemType]; !ok { log.Errorf("type %s doesn't exist", itemType) return nil } var ret = make([]ItemHubStatus, 0) /*remember, you do it for the user :)*/ for _, item := range hubIdx[itemType] { if name != "" && name != item.Name { //user has requested a specific name continue } //Only enabled items ? if !all && !item.Installed { continue } //Check the item status ret = append(ret, item.toHubStatus()) } sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) return ret }