2020-07-27 11:47:32 +00:00
package cwhub
import (
"crypto/sha256"
"fmt"
"io"
"os"
2022-05-24 13:46:48 +00:00
"path/filepath"
"sort"
"strings"
2020-07-27 11:47:32 +00:00
"github.com/enescakir/emoji"
2020-11-30 09:37:17 +00:00
"github.com/pkg/errors"
2020-07-27 11:47:32 +00:00
log "github.com/sirupsen/logrus"
2022-05-24 13:46:48 +00:00
"golang.org/x/mod/semver"
2020-07-27 11:47:32 +00:00
)
2023-10-03 09:20:56 +00:00
// managed configuration types
2023-10-04 08:34:10 +00:00
const PARSERS = "parsers"
const PARSERS_OVFLW = "postoverflows"
const SCENARIOS = "scenarios"
const COLLECTIONS = "collections"
2023-10-04 09:17:35 +00:00
2020-07-27 11:47:32 +00:00
var ItemTypes = [ ] string { PARSERS , PARSERS_OVFLW , SCENARIOS , COLLECTIONS }
2020-11-30 09:37:17 +00:00
var hubIdx map [ string ] map [ string ] Item
2020-07-27 11:47:32 +00:00
2021-09-02 13:17:37 +00:00
var RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
2020-07-27 11:47:32 +00:00
var HubBranch = "master"
2023-10-04 09:17:35 +00:00
2023-10-04 08:34:10 +00:00
const HubIndexFile = ".index.json"
2020-07-27 11:47:32 +00:00
type ItemVersion struct {
2022-01-05 09:42:27 +00:00
Digest string ` json:"digest,omitempty" `
Deprecated bool ` json:"deprecated,omitempty" `
2020-07-27 11:47:32 +00:00
}
2022-01-04 10:49:23 +00:00
type ItemHubStatus struct {
Name string ` json:"name" `
LocalVersion string ` json:"local_version" `
LocalPath string ` json:"local_path" `
Description string ` json:"description" `
UTF8_Status string ` json:"utf8_status" `
Status string ` json:"status" `
}
2023-10-03 09:20:56 +00:00
// Item can be: parsed, scenario, collection
2020-07-27 11:47:32 +00:00
type Item struct {
2023-10-03 09:20:56 +00:00
// descriptive info
2023-10-04 10:54:21 +00:00
Type string ` json:"type,omitempty" yaml:"type,omitempty" ` // parser|postoverflows|scenario|collection(|enrich)
Stage string ` json:"stage,omitempty" yaml:"stage,omitempty" ` // Stage for parser|postoverflow: s00-raw/s01-...
2023-10-03 09:20:56 +00:00
Name string ` json:"name,omitempty" ` // as seen in .config.json, usually "author/name"
FileName string ` json:"file_name,omitempty" ` // the filename, ie. apache2-logs.yaml
2023-10-04 10:54:21 +00:00
Description string ` json:"description,omitempty" yaml:"description,omitempty" ` // as seen in .config.json
2023-10-03 09:20:56 +00:00
Author string ` json:"author,omitempty" ` // as seen in .config.json
2023-10-04 10:54:21 +00:00
References [ ] string ` json:"references,omitempty" yaml:"references,omitempty" ` // as seen in .config.json
BelongsToCollections [ ] string ` json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty" ` // if it's part of collections, track name here
// remote (hub) info
RemoteURL string ` json:"remoteURL,omitempty" yaml:"remoteURL,omitempty" ` // the full remote uri of file in http
RemotePath string ` json:"path,omitempty" yaml:"remote_path,omitempty" ` // the path relative to git ie. /parsers/stage/author/file.yaml
RemoteHash string ` json:"hash,omitempty" yaml:"hash,omitempty" ` // the meow
Version string ` json:"version,omitempty" ` // the last version
Versions map [ string ] ItemVersion ` json:"versions,omitempty" yaml:"-" ` // the list of existing versions
// local (deployed) info
LocalPath string ` json:"local_path,omitempty" yaml:"local_path,omitempty" ` // the local path relative to ${CFG_DIR}
2023-10-03 09:20:56 +00:00
// LocalHubPath string
2022-01-05 09:42:27 +00:00
LocalVersion string ` json:"local_version,omitempty" `
2023-10-03 09:20:56 +00:00
LocalHash string ` json:"local_hash,omitempty" ` // the local meow
2022-01-05 09:42:27 +00:00
Installed bool ` json:"installed,omitempty" `
Downloaded bool ` json:"downloaded,omitempty" `
UpToDate bool ` json:"up_to_date,omitempty" `
2023-10-03 09:20:56 +00:00
Tainted bool ` json:"tainted,omitempty" ` // has it been locally modified
Local bool ` json:"local,omitempty" ` // if it's a non versioned control one
2020-07-27 11:47:32 +00:00
2023-10-03 09:20:56 +00:00
// if it's a collection, it not a single file
2023-10-04 10:54:21 +00:00
Parsers [ ] string ` json:"parsers,omitempty" yaml:"parsers,omitempty" `
PostOverflows [ ] string ` json:"postoverflows,omitempty" yaml:"postoverflows,omitempty" `
Scenarios [ ] string ` json:"scenarios,omitempty" yaml:"scenarios,omitempty" `
Collections [ ] string ` json:"collections,omitempty" yaml:"collections,omitempty" `
2020-07-27 11:47:32 +00:00
}
2023-10-04 10:54:21 +00:00
func toEmoji ( managed bool , installed bool , warning bool , ok bool ) emoji . Emoji {
if ! managed {
return emoji . House
}
2022-01-04 10:49:23 +00:00
2023-10-04 10:54:21 +00:00
if ! installed {
return emoji . Prohibited
}
2023-10-03 09:20:56 +00:00
2023-10-04 10:54:21 +00:00
if warning {
return emoji . Warning
}
if ok {
return emoji . CheckMark
2022-01-04 10:49:23 +00:00
}
2023-10-03 09:20:56 +00:00
2023-10-04 10:54:21 +00:00
// XXX: this is new
return emoji . QuestionMark
}
func ( i * Item ) toHubStatus ( ) ItemHubStatus {
status , ok , warning , managed := ItemStatus ( * i )
return ItemHubStatus {
Name : i . Name ,
LocalVersion : i . LocalVersion ,
LocalPath : i . LocalPath ,
Description : i . Description ,
Status : status ,
UTF8_Status : fmt . Sprintf ( "%v %s" , toEmoji ( managed , i . Installed , warning , ok ) , status ) ,
}
2022-01-04 10:49:23 +00:00
}
2023-10-04 10:54:21 +00:00
// XXX: can we remove these globals?
2020-07-27 11:47:32 +00:00
var skippedLocal = 0
var skippedTainted = 0
2023-10-03 09:20:56 +00:00
// To be used when reference(s) (is/are) missing in a collection
2020-07-27 11:47:32 +00:00
var ReferenceMissingError = errors . New ( "Reference(s) missing in collection" )
2023-10-03 09:20:56 +00:00
// GetVersionStatus: semver requires 'v' prefix
2021-03-29 08:33:23 +00:00
func GetVersionStatus ( v * Item ) int {
return semver . Compare ( "v" + v . Version , "v" + v . LocalVersion )
}
2020-07-27 11:47:32 +00:00
// calculate sha256 of a file
func getSHA256 ( filepath string ) ( string , error ) {
2023-10-03 09:20:56 +00:00
// Digest of file
2020-07-27 11:47:32 +00:00
f , err := os . Open ( filepath )
if err != nil {
2023-10-04 08:34:10 +00:00
return "" , fmt . Errorf ( "unable to open '%s': %w" , filepath , err )
2020-07-27 11:47:32 +00:00
}
defer f . Close ( )
h := sha256 . New ( )
if _ , err := io . Copy ( h , f ) ; err != nil {
2023-10-04 08:34:10 +00:00
return "" , fmt . Errorf ( "unable to calculate sha256 of '%s': %w" , filepath , err )
2020-07-27 11:47:32 +00:00
}
return fmt . Sprintf ( "%x" , h . Sum ( nil ) ) , nil
}
2020-11-30 09:37:17 +00:00
func GetItemMap ( itemType string ) map [ string ] Item {
2023-10-04 08:34:10 +00:00
m , ok := hubIdx [ itemType ]
if ! ok {
2020-11-30 09:37:17 +00:00
return nil
}
2023-10-03 09:20:56 +00:00
2020-11-30 09:37:17 +00:00
return m
}
2023-10-03 09:20:56 +00:00
// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
2020-11-30 09:37:17 +00:00
func GetItemByPath ( itemType string , itemPath string ) ( * Item , error ) {
2023-10-03 09:20:56 +00:00
// try to resolve symlink
2020-11-30 09:37:17 +00:00
finalName := ""
2023-10-03 09:20:56 +00:00
2020-11-30 09:37:17 +00:00
f , err := os . Lstat ( itemPath )
if err != nil {
2023-06-29 09:34:59 +00:00
return nil , fmt . Errorf ( "while performing lstat on %s: %w" , itemPath , err )
2020-11-30 09:37:17 +00:00
}
if f . Mode ( ) & os . ModeSymlink == 0 {
2023-10-03 09:20:56 +00:00
// it's not a symlink, it should be the filename itsef the key
2020-11-30 09:37:17 +00:00
finalName = filepath . Base ( itemPath )
} else {
2023-10-03 09:20:56 +00:00
// resolve the symlink to hub file
2020-11-30 09:37:17 +00:00
pathInHub , err := os . Readlink ( itemPath )
if err != nil {
2023-06-29 09:34:59 +00:00
return nil , fmt . Errorf ( "while reading symlink of %s: %w" , itemPath , err )
2020-11-30 09:37:17 +00:00
}
2023-10-03 09:20:56 +00:00
// extract author from path
2020-11-30 09:37:17 +00:00
fname := filepath . Base ( pathInHub )
author := filepath . Base ( filepath . Dir ( pathInHub ) )
2023-10-03 09:20:56 +00:00
// trim yaml suffix
2020-11-30 09:37:17 +00:00
fname = strings . TrimSuffix ( fname , ".yaml" )
fname = strings . TrimSuffix ( fname , ".yml" )
finalName = fmt . Sprintf ( "%s/%s" , author , fname )
}
2023-10-03 09:20:56 +00:00
// it's not a symlink, it should be the filename itsef the key
2020-11-30 09:37:17 +00:00
if m := GetItemMap ( itemType ) ; m != nil {
if v , ok := m [ finalName ] ; ok {
return & v , nil
}
2023-10-03 09:20:56 +00:00
2022-02-01 21:08:06 +00:00
return nil , fmt . Errorf ( "%s not found in %s" , finalName , itemType )
2020-11-30 09:37:17 +00:00
}
2023-10-03 09:20:56 +00:00
2022-02-01 21:08:06 +00:00
return nil , fmt . Errorf ( "item type %s doesn't exist" , itemType )
2020-11-30 09:37:17 +00:00
}
func GetItem ( itemType string , itemName string ) * Item {
if m , ok := GetItemMap ( itemType ) [ itemName ] ; ok {
return & m
}
2023-10-03 09:20:56 +00:00
2020-11-30 09:37:17 +00:00
return nil
}
func AddItem ( itemType string , item Item ) error {
for _ , itype := range ItemTypes {
if itype == itemType {
2023-10-04 08:34:10 +00:00
hubIdx [ itemType ] [ item . Name ] = item
return nil
2020-11-30 09:37:17 +00:00
}
}
2023-10-03 09:20:56 +00:00
2023-10-04 08:34:10 +00:00
return fmt . Errorf ( "ItemType %s is unknown" , itemType )
2020-11-30 09:37:17 +00:00
}
2020-07-27 11:47:32 +00:00
func DisplaySummary ( ) {
2023-10-04 10:54:21 +00:00
log . Infof ( "Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers" , len ( hubIdx [ COLLECTIONS ] ) ,
2020-11-30 09:37:17 +00:00
len ( hubIdx [ PARSERS ] ) , len ( hubIdx [ SCENARIOS ] ) , len ( hubIdx [ PARSERS_OVFLW ] ) )
2023-10-03 09:20:56 +00:00
2020-07-27 11:47:32 +00:00
if skippedLocal > 0 || skippedTainted > 0 {
2023-10-04 10:54:21 +00:00
log . Infof ( "unmanaged items: %d local, %d tainted" , skippedLocal , skippedTainted )
2020-07-27 11:47:32 +00:00
}
}
2023-10-03 09:20:56 +00:00
// returns: human-text, Enabled, Warning, Unmanaged
2020-07-27 11:47:32 +00:00
func ItemStatus ( v Item ) ( string , bool , bool , bool ) {
2022-02-01 21:08:06 +00:00
strret := "disabled"
Ok := false
2023-10-03 09:20:56 +00:00
2022-02-01 21:08:06 +00:00
if v . Installed {
2020-07-27 11:47:32 +00:00
Ok = true
strret = "enabled"
}
2022-02-01 21:08:06 +00:00
Managed := true
2020-07-27 11:47:32 +00:00
if v . Local {
Managed = false
strret += ",local"
}
2023-10-03 09:20:56 +00:00
// tainted or out of date
2022-02-01 21:08:06 +00:00
Warning := false
2020-07-27 11:47:32 +00:00
if v . Tainted {
Warning = true
strret += ",tainted"
2020-11-30 09:37:17 +00:00
} else if ! v . UpToDate && ! v . Local {
2020-07-27 11:47:32 +00:00
Warning = true
2023-10-04 08:34:10 +00:00
strret += ",update-available"
2020-07-27 11:47:32 +00:00
}
2023-10-03 09:20:56 +00:00
2020-07-27 11:47:32 +00:00
return strret , Ok , Warning , Managed
}
2022-04-13 15:48:29 +00:00
func GetInstalledScenariosAsString ( ) ( [ ] string , error ) {
2020-11-30 09:37:17 +00:00
var retStr [ ] string
2022-04-13 15:48:29 +00:00
items , err := GetInstalledScenarios ( )
2020-11-30 09:37:17 +00:00
if err != nil {
2023-06-29 09:34:59 +00:00
return nil , fmt . Errorf ( "while fetching scenarios: %w" , err )
2020-11-30 09:37:17 +00:00
}
2023-10-03 09:20:56 +00:00
2020-11-30 09:37:17 +00:00
for _ , it := range items {
retStr = append ( retStr , it . Name )
}
2023-10-03 09:20:56 +00:00
2020-11-30 09:37:17 +00:00
return retStr , nil
}
2022-04-13 15:48:29 +00:00
func GetInstalledScenarios ( ) ( [ ] Item , error ) {
2020-11-30 09:37:17 +00:00
var retItems [ ] Item
if _ , ok := hubIdx [ SCENARIOS ] ; ! ok {
return nil , fmt . Errorf ( "no scenarios in hubIdx" )
}
2023-10-03 09:20:56 +00:00
2020-11-30 09:37:17 +00:00
for _ , item := range hubIdx [ SCENARIOS ] {
2022-04-13 15:48:29 +00:00
if item . Installed {
2020-11-30 09:37:17 +00:00
retItems = append ( retItems , item )
}
}
2023-10-03 09:20:56 +00:00
2020-11-30 09:37:17 +00:00
return retItems , nil
}
2022-04-20 13:44:48 +00:00
func GetInstalledParsers ( ) ( [ ] Item , error ) {
var retItems [ ] Item
if _ , ok := hubIdx [ PARSERS ] ; ! ok {
return nil , fmt . Errorf ( "no parsers in hubIdx" )
}
2023-10-03 09:20:56 +00:00
2022-04-20 13:44:48 +00:00
for _ , item := range hubIdx [ PARSERS ] {
if item . Installed {
retItems = append ( retItems , item )
}
}
2023-10-03 09:20:56 +00:00
2022-04-20 13:44:48 +00:00
return retItems , nil
}
func GetInstalledParsersAsString ( ) ( [ ] string , error ) {
var retStr [ ] string
items , err := GetInstalledParsers ( )
if err != nil {
2023-06-29 09:34:59 +00:00
return nil , fmt . Errorf ( "while fetching parsers: %w" , err )
2022-04-20 13:44:48 +00:00
}
2023-10-03 09:20:56 +00:00
2022-04-20 13:44:48 +00:00
for _ , it := range items {
retStr = append ( retStr , it . Name )
}
2023-10-03 09:20:56 +00:00
2022-04-20 13:44:48 +00:00
return retStr , nil
}
func GetInstalledPostOverflows ( ) ( [ ] Item , error ) {
var retItems [ ] Item
if _ , ok := hubIdx [ PARSERS_OVFLW ] ; ! ok {
return nil , fmt . Errorf ( "no post overflows in hubIdx" )
}
2023-10-03 09:20:56 +00:00
2022-04-20 13:44:48 +00:00
for _ , item := range hubIdx [ PARSERS_OVFLW ] {
if item . Installed {
retItems = append ( retItems , item )
}
}
2023-10-03 09:20:56 +00:00
2022-04-20 13:44:48 +00:00
return retItems , nil
}
func GetInstalledPostOverflowsAsString ( ) ( [ ] string , error ) {
var retStr [ ] string
items , err := GetInstalledPostOverflows ( )
if err != nil {
2023-06-29 09:34:59 +00:00
return nil , fmt . Errorf ( "while fetching post overflows: %w" , err )
2022-04-20 13:44:48 +00:00
}
2023-10-03 09:20:56 +00:00
2022-04-20 13:44:48 +00:00
for _ , it := range items {
retStr = append ( retStr , it . Name )
}
2023-10-03 09:20:56 +00:00
2022-04-20 13:44:48 +00:00
return retStr , nil
}
func GetInstalledCollectionsAsString ( ) ( [ ] string , error ) {
var retStr [ ] string
items , err := GetInstalledCollections ( )
if err != nil {
2023-06-29 09:34:59 +00:00
return nil , fmt . Errorf ( "while fetching collections: %w" , err )
2022-04-20 13:44:48 +00:00
}
2023-06-29 09:34:59 +00:00
2022-04-20 13:44:48 +00:00
for _ , it := range items {
retStr = append ( retStr , it . Name )
}
2023-10-03 09:20:56 +00:00
2022-04-20 13:44:48 +00:00
return retStr , nil
}
func GetInstalledCollections ( ) ( [ ] Item , error ) {
var retItems [ ] Item
if _ , ok := hubIdx [ COLLECTIONS ] ; ! ok {
return nil , fmt . Errorf ( "no collection in hubIdx" )
}
2023-10-03 09:20:56 +00:00
2022-04-20 13:44:48 +00:00
for _ , item := range hubIdx [ COLLECTIONS ] {
if item . Installed {
retItems = append ( retItems , item )
}
}
2023-10-03 09:20:56 +00:00
2022-04-20 13:44:48 +00:00
return retItems , nil
}
2023-10-03 09:20:56 +00:00
// Returns a list of entries for packages: name, status, local_path, local_version, utf8_status (fancy)
2022-01-04 10:49:23 +00:00
func GetHubStatusForItemType ( itemType string , name string , all bool ) [ ] ItemHubStatus {
2020-11-30 09:37:17 +00:00
if _ , ok := hubIdx [ itemType ] ; ! ok {
log . Errorf ( "type %s doesn't exist" , itemType )
2020-07-27 11:47:32 +00:00
return nil
}
2022-01-04 10:49:23 +00:00
var ret = make ( [ ] ItemHubStatus , 0 )
2023-10-03 09:20:56 +00:00
// remember, you do it for the user :)
2020-11-30 09:37:17 +00:00
for _ , item := range hubIdx [ itemType ] {
if name != "" && name != item . Name {
2023-10-03 09:20:56 +00:00
// user has requested a specific name
2020-07-27 11:47:32 +00:00
continue
}
2023-10-03 09:20:56 +00:00
// Only enabled items ?
2020-12-02 17:47:17 +00:00
if ! all && ! item . Installed {
2020-07-27 11:47:32 +00:00
continue
}
2023-10-03 09:20:56 +00:00
// Check the item status
2022-01-04 10:49:23 +00:00
ret = append ( ret , item . toHubStatus ( ) )
2020-07-27 11:47:32 +00:00
}
2023-10-03 09:20:56 +00:00
2022-03-21 11:13:36 +00:00
sort . Slice ( ret , func ( i , j int ) bool { return ret [ i ] . Name < ret [ j ] . Name } )
2023-10-03 09:20:56 +00:00
2020-11-30 09:37:17 +00:00
return ret
2020-07-27 11:47:32 +00:00
}