crowdsec/pkg/cwhub/cwhub.go

370 lines
11 KiB
Go
Raw Normal View History

2020-07-27 11:47:32 +00:00
package cwhub
import (
"crypto/sha256"
"fmt"
"io"
"os"
2022-05-24 13:46:48 +00:00
"path/filepath"
"sort"
"strings"
2020-07-27 11:47:32 +00:00
"github.com/enescakir/emoji"
"github.com/pkg/errors"
2020-07-27 11:47:32 +00:00
log "github.com/sirupsen/logrus"
2022-05-24 13:46:48 +00:00
"golang.org/x/mod/semver"
2020-07-27 11:47:32 +00:00
)
/*managed configuration types*/
2020-07-27 11:47:32 +00:00
var PARSERS = "parsers"
var PARSERS_OVFLW = "postoverflows"
var SCENARIOS = "scenarios"
var COLLECTIONS = "collections"
var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS}
var hubIdx map[string]map[string]Item
2020-07-27 11:47:32 +00:00
var RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
2020-07-27 11:47:32 +00:00
var HubBranch = "master"
var HubIndexFile = ".index.json"
2020-07-27 11:47:32 +00:00
type ItemVersion struct {
Digest string `json:"digest,omitempty"`
Deprecated bool `json:"deprecated,omitempty"`
2020-07-27 11:47:32 +00:00
}
type ItemHubStatus struct {
Name string `json:"name"`
LocalVersion string `json:"local_version"`
LocalPath string `json:"local_path"`
Description string `json:"description"`
UTF8_Status string `json:"utf8_status"`
Status string `json:"status"`
}
2020-07-27 11:47:32 +00:00
//Item can be : parsed, scenario, collection
type Item struct {
/*descriptive info*/
Type string `yaml:"type,omitempty" json:"type,omitempty"` //parser|postoverflows|scenario|collection(|enrich)
Stage string `json:"stage,omitempty" yaml:"stage,omitempty,omitempty"` //Stage for parser|postoverflow : s00-raw/s01-...
Name string `json:"name,omitempty"` //as seen in .config.json, usually "author/name"
FileName string `json:"file_name,omitempty"` //the filename, ie. apache2-logs.yaml
Description string `yaml:"description,omitempty" json:"description,omitempty"` //as seen in .config.json
Author string `json:"author,omitempty"` //as seen in .config.json
References []string `yaml:"references,omitempty" json:"references,omitempty"` //as seen in .config.json
BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` /*if it's part of collections, track name here*/
2020-07-27 11:47:32 +00:00
/*remote (hub) infos*/
RemoteURL string `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` //the full remote uri of file in http
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` //the path relative to git ie. /parsers/stage/author/file.yaml
RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` //the meow
Version string `json:"version,omitempty"` //the last version
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` //the list of existing versions
2020-07-27 11:47:32 +00:00
/*local (deployed) infos*/
LocalPath string `yaml:"local_path,omitempty" json:"local_path,omitempty"` //the local path relative to ${CFG_DIR}
2020-07-27 11:47:32 +00:00
//LocalHubPath string
LocalVersion string `json:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty"` //the local meow
Installed bool `json:"installed,omitempty"`
Downloaded bool `json:"downloaded,omitempty"`
UpToDate bool `json:"up_to_date,omitempty"`
Tainted bool `json:"tainted,omitempty"` //has it been locally modified
Local bool `json:"local,omitempty"` //if it's a non versioned control one
2020-07-27 11:47:32 +00:00
/*if it's a collection, it not a single file*/
Parsers []string `yaml:"parsers,omitempty" json:"parsers,omitempty"`
PostOverflows []string `yaml:"postoverflows,omitempty" json:"postoverflows,omitempty"`
Scenarios []string `yaml:"scenarios,omitempty" json:"scenarios,omitempty"`
Collections []string `yaml:"collections,omitempty" json:"collections,omitempty"`
2020-07-27 11:47:32 +00:00
}
func (i *Item) toHubStatus() ItemHubStatus {
hubStatus := ItemHubStatus{}
hubStatus.Name = i.Name
hubStatus.LocalVersion = i.LocalVersion
hubStatus.LocalPath = i.LocalPath
hubStatus.Description = i.Description
status, ok, warning, managed := ItemStatus(*i)
hubStatus.Status = status
if !managed {
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.House, status)
} else if !i.Installed {
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.Prohibited, status)
} else if warning {
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.Warning, status)
} else if ok {
hubStatus.UTF8_Status = fmt.Sprintf("%v %s", emoji.CheckMark, status)
}
return hubStatus
}
2020-07-27 11:47:32 +00:00
var skippedLocal = 0
var skippedTainted = 0
/*To be used when reference(s) (is/are) missing in a collection*/
var ReferenceMissingError = errors.New("Reference(s) missing in collection")
var MissingHubIndex = errors.New("hub index can't be found")
//GetVersionStatus : semver requires 'v' prefix
func GetVersionStatus(v *Item) int {
return semver.Compare("v"+v.Version, "v"+v.LocalVersion)
}
2020-07-27 11:47:32 +00:00
// calculate sha256 of a file
func getSHA256(filepath string) (string, error) {
/* Digest of file */
f, err := os.Open(filepath)
if err != nil {
return "", fmt.Errorf("unable to open '%s' : %s", filepath, err)
2020-07-27 11:47:32 +00:00
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", fmt.Errorf("unable to calculate sha256 of '%s': %s", filepath, err)
2020-07-27 11:47:32 +00:00
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
func GetItemMap(itemType string) map[string]Item {
var m map[string]Item
var ok bool
if m, ok = hubIdx[itemType]; !ok {
return nil
}
return m
}
//GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
func GetItemByPath(itemType string, itemPath string) (*Item, error) {
/*try to resolve symlink*/
finalName := ""
f, err := os.Lstat(itemPath)
if err != nil {
2023-06-29 09:34:59 +00:00
return nil, fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
}
if f.Mode()&os.ModeSymlink == 0 {
/*it's not a symlink, it should be the filename itsef the key*/
finalName = filepath.Base(itemPath)
} else {
/*resolve the symlink to hub file*/
pathInHub, err := os.Readlink(itemPath)
if err != nil {
2023-06-29 09:34:59 +00:00
return nil, fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
}
//extract author from path
fname := filepath.Base(pathInHub)
author := filepath.Base(filepath.Dir(pathInHub))
//trim yaml suffix
fname = strings.TrimSuffix(fname, ".yaml")
fname = strings.TrimSuffix(fname, ".yml")
finalName = fmt.Sprintf("%s/%s", author, fname)
}
/*it's not a symlink, it should be the filename itsef the key*/
if m := GetItemMap(itemType); m != nil {
if v, ok := m[finalName]; ok {
return &v, nil
}
return nil, fmt.Errorf("%s not found in %s", finalName, itemType)
}
return nil, fmt.Errorf("item type %s doesn't exist", itemType)
}
func GetItem(itemType string, itemName string) *Item {
if m, ok := GetItemMap(itemType)[itemName]; ok {
return &m
}
return nil
}
func AddItem(itemType string, item Item) error {
in := false
for _, itype := range ItemTypes {
if itype == itemType {
in = true
}
}
if !in {
return fmt.Errorf("ItemType %s is unknown", itemType)
}
hubIdx[itemType][item.Name] = item
return nil
}
2020-07-27 11:47:32 +00:00
func DisplaySummary() {
log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
2020-07-27 11:47:32 +00:00
if skippedLocal > 0 || skippedTainted > 0 {
log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted)
}
}
//returns: human-text, Enabled, Warning, Unmanaged
func ItemStatus(v Item) (string, bool, bool, bool) {
strret := "disabled"
Ok := false
if v.Installed {
2020-07-27 11:47:32 +00:00
Ok = true
strret = "enabled"
}
Managed := true
2020-07-27 11:47:32 +00:00
if v.Local {
Managed = false
strret += ",local"
}
//tainted or out of date
Warning := false
2020-07-27 11:47:32 +00:00
if v.Tainted {
Warning = true
strret += ",tainted"
} else if !v.UpToDate && !v.Local {
2020-07-27 11:47:32 +00:00
strret += ",update-available"
Warning = true
}
return strret, Ok, Warning, Managed
}
func GetInstalledScenariosAsString() ([]string, error) {
var retStr []string
items, err := GetInstalledScenarios()
if err != nil {
2023-06-29 09:34:59 +00:00
return nil, fmt.Errorf("while fetching scenarios: %w", err)
}
for _, it := range items {
retStr = append(retStr, it.Name)
}
return retStr, nil
}
func GetInstalledScenarios() ([]Item, error) {
var retItems []Item
if _, ok := hubIdx[SCENARIOS]; !ok {
return nil, fmt.Errorf("no scenarios in hubIdx")
}
for _, item := range hubIdx[SCENARIOS] {
if item.Installed {
retItems = append(retItems, item)
}
}
return retItems, nil
}
func GetInstalledParsers() ([]Item, error) {
var retItems []Item
if _, ok := hubIdx[PARSERS]; !ok {
return nil, fmt.Errorf("no parsers in hubIdx")
}
for _, item := range hubIdx[PARSERS] {
if item.Installed {
retItems = append(retItems, item)
}
}
return retItems, nil
}
func GetInstalledParsersAsString() ([]string, error) {
var retStr []string
items, err := GetInstalledParsers()
if err != nil {
2023-06-29 09:34:59 +00:00
return nil, fmt.Errorf("while fetching parsers: %w", err)
}
for _, it := range items {
retStr = append(retStr, it.Name)
}
return retStr, nil
}
func GetInstalledPostOverflows() ([]Item, error) {
var retItems []Item
if _, ok := hubIdx[PARSERS_OVFLW]; !ok {
return nil, fmt.Errorf("no post overflows in hubIdx")
}
for _, item := range hubIdx[PARSERS_OVFLW] {
if item.Installed {
retItems = append(retItems, item)
}
}
return retItems, nil
}
func GetInstalledPostOverflowsAsString() ([]string, error) {
var retStr []string
items, err := GetInstalledPostOverflows()
if err != nil {
2023-06-29 09:34:59 +00:00
return nil, fmt.Errorf("while fetching post overflows: %w", err)
}
for _, it := range items {
retStr = append(retStr, it.Name)
}
return retStr, nil
}
func GetInstalledCollectionsAsString() ([]string, error) {
var retStr []string
items, err := GetInstalledCollections()
if err != nil {
2023-06-29 09:34:59 +00:00
return nil, fmt.Errorf("while fetching collections: %w", err)
}
2023-06-29 09:34:59 +00:00
for _, it := range items {
retStr = append(retStr, it.Name)
}
return retStr, nil
}
func GetInstalledCollections() ([]Item, error) {
var retItems []Item
if _, ok := hubIdx[COLLECTIONS]; !ok {
return nil, fmt.Errorf("no collection in hubIdx")
}
for _, item := range hubIdx[COLLECTIONS] {
if item.Installed {
retItems = append(retItems, item)
}
}
return retItems, nil
}
2020-07-27 11:47:32 +00:00
//Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy)
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
if _, ok := hubIdx[itemType]; !ok {
log.Errorf("type %s doesn't exist", itemType)
2020-07-27 11:47:32 +00:00
return nil
}
var ret = make([]ItemHubStatus, 0)
2020-07-27 11:47:32 +00:00
/*remember, you do it for the user :)*/
for _, item := range hubIdx[itemType] {
if name != "" && name != item.Name {
//user has requested a specific name
2020-07-27 11:47:32 +00:00
continue
}
//Only enabled items ?
if !all && !item.Installed {
2020-07-27 11:47:32 +00:00
continue
}
//Check the item status
ret = append(ret, item.toHubStatus())
2020-07-27 11:47:32 +00:00
}
sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
return ret
2020-07-27 11:47:32 +00:00
}