Compare commits
36 commits
master
...
local-hub-
Author | SHA1 | Date | |
---|---|---|---|
|
c01c5677d1 | ||
|
56ad2bbf98 | ||
|
65473d4e05 | ||
|
d9b0d440bf | ||
|
79d019f9a2 | ||
|
4a6fd338e0 | ||
|
f8c91d20b0 | ||
|
120f7cf578 | ||
|
042d316fab | ||
|
d5c7870826 | ||
|
9d7ed12950 | ||
|
ab8de19506 | ||
|
f80d841188 | ||
|
ec4b5bdc86 | ||
|
f4b5bcb865 | ||
|
ad54b99bf9 | ||
|
84be2b8c97 | ||
|
bfd94ceda7 | ||
|
41d19de092 | ||
|
450c263826 | ||
|
fcd6c468c4 | ||
|
590a19b768 | ||
|
17662e59a9 | ||
|
6b8ed0c9d0 | ||
|
ac98256602 | ||
|
b89c5652ca | ||
|
88e4f7c157 | ||
|
57d3ebba12 | ||
|
be6555e46c | ||
|
4eae40865e | ||
|
810a8adcf0 | ||
|
325003bb69 | ||
|
f496bd1692 | ||
|
a00bae6039 | ||
|
734ba46e6a | ||
|
7db5bf8979 |
2
.github/workflows/bats.yml
vendored
2
.github/workflows/bats.yml
vendored
|
@ -15,12 +15,14 @@ on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- hub-1.5.6
|
||||||
- releases/**
|
- releases/**
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- "README.md"
|
- "README.md"
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- hub-1.5.6
|
||||||
- releases/**
|
- releases/**
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- "README.md"
|
- "README.md"
|
||||||
|
|
2
.github/workflows/codeql-analysis.yml
vendored
2
.github/workflows/codeql-analysis.yml
vendored
|
@ -16,11 +16,13 @@ on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- hub-1.5.6
|
||||||
- releases/**
|
- releases/**
|
||||||
pull_request:
|
pull_request:
|
||||||
# The branches below must be a subset of the branches above
|
# The branches below must be a subset of the branches above
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- hub-1.5.6
|
||||||
- releases/**
|
- releases/**
|
||||||
schedule:
|
schedule:
|
||||||
- cron: '15 16 * * 2'
|
- cron: '15 16 * * 2'
|
||||||
|
|
2
.github/workflows/docker-tests.yml
vendored
2
.github/workflows/docker-tests.yml
vendored
|
@ -4,12 +4,14 @@ on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- hub-1.5.6
|
||||||
- releases/**
|
- releases/**
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- 'README.md'
|
- 'README.md'
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- hub-1.5.6
|
||||||
- releases/**
|
- releases/**
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- 'README.md'
|
- 'README.md'
|
||||||
|
|
2
.github/workflows/go-tests-windows.yml
vendored
2
.github/workflows/go-tests-windows.yml
vendored
|
@ -4,12 +4,14 @@ on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- hub-1.5.6
|
||||||
- releases/**
|
- releases/**
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- 'README.md'
|
- 'README.md'
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- hub-1.5.6
|
||||||
- releases/**
|
- releases/**
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- 'README.md'
|
- 'README.md'
|
||||||
|
|
2
.github/workflows/go-tests.yml
vendored
2
.github/workflows/go-tests.yml
vendored
|
@ -9,12 +9,14 @@ on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- hub-1.5.6
|
||||||
- releases/**
|
- releases/**
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- 'README.md'
|
- 'README.md'
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- hub-1.5.6
|
||||||
- releases/**
|
- releases/**
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- 'README.md'
|
- 'README.md'
|
||||||
|
|
|
@ -151,11 +151,12 @@ func NewCapiStatusCmd() *cobra.Command {
|
||||||
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
|
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
|
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to get scenarios: %w", err)
|
return fmt.Errorf("failed to get scenarios: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,176 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/fatih/color"
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
|
||||||
)
|
|
||||||
|
|
||||||
func NewCollectionsCmd() *cobra.Command {
|
|
||||||
var cmdCollections = &cobra.Command{
|
|
||||||
Use: "collections [action]",
|
|
||||||
Short: "Manage collections from hub",
|
|
||||||
Long: `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
|
|
||||||
/*TBD fix help*/
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
Aliases: []string{"collection"},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
PersistentPostRun: func(cmd *cobra.Command, args []string) {
|
|
||||||
if cmd.Name() == "inspect" || cmd.Name() == "list" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
log.Infof(ReloadMessage())
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
var ignoreError bool
|
|
||||||
|
|
||||||
var cmdCollectionsInstall = &cobra.Command{
|
|
||||||
Use: "install collection",
|
|
||||||
Short: "Install given collection(s)",
|
|
||||||
Long: `Fetch and install given collection(s) from hub`,
|
|
||||||
Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compAllItems(cwhub.COLLECTIONS, args, toComplete)
|
|
||||||
},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
for _, name := range args {
|
|
||||||
t := cwhub.GetItem(cwhub.COLLECTIONS, name)
|
|
||||||
if t == nil {
|
|
||||||
nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
|
|
||||||
Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
|
|
||||||
if !ignoreError {
|
|
||||||
return fmt.Errorf("error while installing '%s': %w", name, err)
|
|
||||||
}
|
|
||||||
log.Errorf("Error while installing '%s': %s", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
|
|
||||||
cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
|
|
||||||
cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections")
|
|
||||||
cmdCollections.AddCommand(cmdCollectionsInstall)
|
|
||||||
|
|
||||||
var cmdCollectionsRemove = &cobra.Command{
|
|
||||||
Use: "remove collection",
|
|
||||||
Short: "Remove given collection(s)",
|
|
||||||
Long: `Remove given collection(s) from hub`,
|
|
||||||
Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`,
|
|
||||||
Aliases: []string{"delete"},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
|
|
||||||
},
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if all {
|
|
||||||
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(args) == 0 {
|
|
||||||
return fmt.Errorf("specify at least one collection to remove or '--all'")
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range args {
|
|
||||||
if !forceAction {
|
|
||||||
item := cwhub.GetItem(cwhub.COLLECTIONS, name)
|
|
||||||
if item == nil {
|
|
||||||
return fmt.Errorf("unable to retrieve: %s", name)
|
|
||||||
}
|
|
||||||
if len(item.BelongsToCollections) > 0 {
|
|
||||||
log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
|
|
||||||
log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
|
|
||||||
cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
|
|
||||||
cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections")
|
|
||||||
cmdCollections.AddCommand(cmdCollectionsRemove)
|
|
||||||
|
|
||||||
var cmdCollectionsUpgrade = &cobra.Command{
|
|
||||||
Use: "upgrade collection",
|
|
||||||
Short: "Upgrade given collection(s)",
|
|
||||||
Long: `Fetch and upgrade given collection(s) from hub`,
|
|
||||||
Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
|
|
||||||
},
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if all {
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
|
|
||||||
} else {
|
|
||||||
if len(args) == 0 {
|
|
||||||
return fmt.Errorf("specify at least one collection to upgrade or '--all'")
|
|
||||||
}
|
|
||||||
for _, name := range args {
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")
|
|
||||||
cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
|
|
||||||
cmdCollections.AddCommand(cmdCollectionsUpgrade)
|
|
||||||
|
|
||||||
var cmdCollectionsInspect = &cobra.Command{
|
|
||||||
Use: "inspect collection",
|
|
||||||
Short: "Inspect given collection",
|
|
||||||
Long: `Inspect given collection`,
|
|
||||||
Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
|
|
||||||
},
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
for _, name := range args {
|
|
||||||
InspectItem(name, cwhub.COLLECTIONS)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
|
|
||||||
cmdCollections.AddCommand(cmdCollectionsInspect)
|
|
||||||
|
|
||||||
var cmdCollectionsList = &cobra.Command{
|
|
||||||
Use: "list collection [-a]",
|
|
||||||
Short: "List all collections",
|
|
||||||
Long: `List all collections`,
|
|
||||||
Example: `cscli collections list`,
|
|
||||||
Args: cobra.ExactArgs(0),
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
|
|
||||||
cmdCollections.AddCommand(cmdCollectionsList)
|
|
||||||
|
|
||||||
return cmdCollections
|
|
||||||
}
|
|
|
@ -14,21 +14,25 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func backupHub(dirPath string) error {
|
func backupHub(dirPath string) error {
|
||||||
var err error
|
|
||||||
var itemDirectory string
|
var itemDirectory string
|
||||||
var upstreamParsers []string
|
var upstreamParsers []string
|
||||||
|
|
||||||
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
for _, itemType := range cwhub.ItemTypes {
|
for _, itemType := range cwhub.ItemTypes {
|
||||||
clog := log.WithFields(log.Fields{
|
clog := log.WithFields(log.Fields{
|
||||||
"type": itemType,
|
"type": itemType,
|
||||||
})
|
})
|
||||||
itemMap := cwhub.GetItemMap(itemType)
|
itemMap := hub.GetItemMap(itemType)
|
||||||
if itemMap == nil {
|
if itemMap == nil {
|
||||||
clog.Infof("No %s to backup.", itemType)
|
clog.Infof("No %s to backup.", itemType)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
|
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
|
||||||
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
|
if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
|
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
|
||||||
}
|
}
|
||||||
upstreamParsers = []string{}
|
upstreamParsers = []string{}
|
||||||
|
@ -42,15 +46,15 @@ func backupHub(dirPath string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
//for the local/tainted ones, we back up the full file
|
//for the local/tainted ones, we back up the full file
|
||||||
if v.Tainted || v.Local || !v.UpToDate {
|
if v.Tainted || v.IsLocal() || !v.UpToDate {
|
||||||
//we need to back up stages for parsers
|
//we need to backup stages for parsers
|
||||||
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
|
if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
|
||||||
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
|
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
|
||||||
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
|
if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
|
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
|
clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.IsLocal(), v.UpToDate)
|
||||||
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
|
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
|
||||||
if err = CopyFile(v.LocalPath, tfile); err != nil {
|
if err = CopyFile(v.LocalPath, tfile); err != nil {
|
||||||
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
|
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
|
||||||
|
@ -100,7 +104,7 @@ func backupConfigToDirectory(dirPath string) error {
|
||||||
|
|
||||||
/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
|
/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
|
||||||
parentDir := filepath.Dir(dirPath)
|
parentDir := filepath.Dir(dirPath)
|
||||||
if _, err := os.Stat(parentDir); err != nil {
|
if _, err = os.Stat(parentDir); err != nil {
|
||||||
return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
|
return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -197,10 +201,6 @@ func backupConfigToDirectory(dirPath string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func runConfigBackup(cmd *cobra.Command, args []string) error {
|
func runConfigBackup(cmd *cobra.Command, args []string) error {
|
||||||
if err := require.Hub(csConfig); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := backupConfigToDirectory(args[0]); err != nil {
|
if err := backupConfigToDirectory(args[0]); err != nil {
|
||||||
return fmt.Errorf("failed to backup config: %w", err)
|
return fmt.Errorf("failed to backup config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,45 +21,12 @@ type OldAPICfg struct {
|
||||||
Password string `json:"password"`
|
Password string `json:"password"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// it's a rip of the cli version, but in silent-mode
|
|
||||||
func silentInstallItem(name string, obtype string) (string, error) {
|
|
||||||
var item = cwhub.GetItem(obtype, name)
|
|
||||||
if item == nil {
|
|
||||||
return "", fmt.Errorf("error retrieving item")
|
|
||||||
}
|
|
||||||
if downloadOnly && item.Downloaded && item.UpToDate {
|
|
||||||
return fmt.Sprintf("%s is already downloaded and up-to-date", item.Name), nil
|
|
||||||
}
|
|
||||||
err := cwhub.DownloadLatest(csConfig.Hub, item, forceAction, false)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
|
|
||||||
}
|
|
||||||
if err := cwhub.AddItem(obtype, *item); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
if downloadOnly {
|
|
||||||
return fmt.Sprintf("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath), nil
|
|
||||||
}
|
|
||||||
err = cwhub.EnableItem(csConfig.Hub, item)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
|
|
||||||
}
|
|
||||||
if err := cwhub.AddItem(obtype, *item); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("Enabled %s", item.Name), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func restoreHub(dirPath string) error {
|
func restoreHub(dirPath string) error {
|
||||||
var err error
|
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
|
||||||
|
if err != nil {
|
||||||
if err := csConfig.LoadHub(); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
cwhub.SetHubBranch()
|
|
||||||
|
|
||||||
for _, itype := range cwhub.ItemTypes {
|
for _, itype := range cwhub.ItemTypes {
|
||||||
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
|
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
|
||||||
if _, err = os.Stat(itemDirectory); err != nil {
|
if _, err = os.Stat(itemDirectory); err != nil {
|
||||||
|
@ -78,13 +45,14 @@ func restoreHub(dirPath string) error {
|
||||||
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
|
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
|
||||||
}
|
}
|
||||||
for _, toinstall := range upstreamList {
|
for _, toinstall := range upstreamList {
|
||||||
label, err := silentInstallItem(toinstall, itype)
|
item := hub.GetItem(itype, toinstall)
|
||||||
|
if item == nil {
|
||||||
|
log.Errorf("Item %s/%s not found in hub", itype, toinstall)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
err := item.Install(false, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("Error while installing %s : %s", toinstall, err)
|
log.Errorf("Error while installing %s : %s", toinstall, err)
|
||||||
} else if label != "" {
|
|
||||||
log.Infof("Installed %s : %s", toinstall, label)
|
|
||||||
} else {
|
|
||||||
log.Printf("Installed %s : ok", toinstall)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -98,7 +66,7 @@ func restoreHub(dirPath string) error {
|
||||||
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
|
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
|
if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
|
||||||
//we expect a stage here
|
//we expect a stage here
|
||||||
if !file.IsDir() {
|
if !file.IsDir() {
|
||||||
continue
|
continue
|
||||||
|
@ -302,10 +270,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
|
if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
|
||||||
return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
|
return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -82,7 +82,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled
|
||||||
cscli:
|
cscli:
|
||||||
- Output : {{.Cscli.Output}}
|
- Output : {{.Cscli.Output}}
|
||||||
- Hub Branch : {{.Cscli.HubBranch}}
|
- Hub Branch : {{.Cscli.HubBranch}}
|
||||||
- Hub Folder : {{.Cscli.HubDir}}
|
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
|
||||||
{{- if .API }}
|
{{- if .API }}
|
||||||
|
|
|
@ -71,11 +71,12 @@ After running this command your will need to validate the enrollment in the weba
|
||||||
return fmt.Errorf("could not parse CAPI URL: %s", err)
|
return fmt.Errorf("could not parse CAPI URL: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
|
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to get installed scenarios: %s", err)
|
return fmt.Errorf("failed to get installed scenarios: %s", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/fatih/color"
|
"github.com/fatih/color"
|
||||||
|
@ -13,30 +12,19 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func NewHubCmd() *cobra.Command {
|
func NewHubCmd() *cobra.Command {
|
||||||
var cmdHub = &cobra.Command{
|
cmdHub := &cobra.Command{
|
||||||
Use: "hub [action]",
|
Use: "hub [action]",
|
||||||
Short: "Manage Hub",
|
Short: "Manage hub index",
|
||||||
Long: `
|
Long: `Hub management
|
||||||
Hub management
|
|
||||||
|
|
||||||
List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
|
List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
|
||||||
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
|
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`,
|
||||||
`,
|
Example: `cscli hub list
|
||||||
Example: `
|
cscli hub update
|
||||||
cscli hub list # List all installed configurations
|
cscli hub upgrade`,
|
||||||
cscli hub update # Download list of available configurations from the hub
|
|
||||||
`,
|
|
||||||
Args: cobra.ExactArgs(0),
|
Args: cobra.ExactArgs(0),
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if csConfig.Cscli == nil {
|
|
||||||
return fmt.Errorf("you must configure cli before interacting with hub")
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
|
|
||||||
|
|
||||||
cmdHub.AddCommand(NewHubListCmd())
|
cmdHub.AddCommand(NewHubListCmd())
|
||||||
cmdHub.AddCommand(NewHubUpdateCmd())
|
cmdHub.AddCommand(NewHubUpdateCmd())
|
||||||
|
@ -45,116 +33,142 @@ cscli hub update # Download list of available configurations from the hub
|
||||||
return cmdHub
|
return cmdHub
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func runHubList(cmd *cobra.Command, args []string) error {
|
||||||
|
flags := cmd.Flags()
|
||||||
|
|
||||||
|
all, err := flags.GetBool("all")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, v := range hub.Warnings {
|
||||||
|
log.Info(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, line := range hub.ItemStats() {
|
||||||
|
log.Info(line)
|
||||||
|
}
|
||||||
|
|
||||||
|
items := make(map[string][]*cwhub.Item)
|
||||||
|
|
||||||
|
for _, itemType := range cwhub.ItemTypes {
|
||||||
|
items[itemType], err = selectItems(hub, itemType, nil, !all)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = listItems(color.Output, cwhub.ItemTypes, items)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func NewHubListCmd() *cobra.Command {
|
func NewHubListCmd() *cobra.Command {
|
||||||
var cmdHubList = &cobra.Command{
|
cmdHubList := &cobra.Command{
|
||||||
Use: "list [-a]",
|
Use: "list [-a]",
|
||||||
Short: "List installed configs",
|
Short: "List all installed configurations",
|
||||||
Args: cobra.ExactArgs(0),
|
Args: cobra.ExactArgs(0),
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: runHubList,
|
||||||
if err := require.Hub(csConfig); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// use LocalSync to get warnings about tainted / outdated items
|
|
||||||
warn, _ := cwhub.LocalSync(csConfig.Hub)
|
|
||||||
for _, v := range warn {
|
|
||||||
log.Info(v)
|
|
||||||
}
|
|
||||||
cwhub.DisplaySummary()
|
|
||||||
ListItems(color.Output, []string{
|
|
||||||
cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
|
|
||||||
}, args, true, false, all)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
|
|
||||||
|
flags := cmdHubList.Flags()
|
||||||
|
flags.BoolP("all", "a", false, "List disabled items as well")
|
||||||
|
|
||||||
return cmdHubList
|
return cmdHubList
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func runHubUpdate(cmd *cobra.Command, args []string) error {
|
||||||
|
local := csConfig.Hub
|
||||||
|
remote := require.RemoteHub(csConfig)
|
||||||
|
|
||||||
|
// don't use require.Hub because if there is no index file, it would fail
|
||||||
|
hub, err := cwhub.NewHub(local, remote, true)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to update hub: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, v := range hub.Warnings {
|
||||||
|
log.Info(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func NewHubUpdateCmd() *cobra.Command {
|
func NewHubUpdateCmd() *cobra.Command {
|
||||||
var cmdHubUpdate = &cobra.Command{
|
cmdHubUpdate := &cobra.Command{
|
||||||
Use: "update",
|
Use: "update",
|
||||||
Short: "Fetch available configs from hub",
|
Short: "Download the latest index (catalog of available configurations)",
|
||||||
Long: `
|
Long: `
|
||||||
Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
|
Fetches the .index.json file from the hub, containing the list of available configs.
|
||||||
`,
|
`,
|
||||||
Args: cobra.ExactArgs(0),
|
Args: cobra.ExactArgs(0),
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
RunE: runHubUpdate,
|
||||||
if csConfig.Cscli == nil {
|
|
||||||
return fmt.Errorf("you must configure cli before interacting with hub")
|
|
||||||
}
|
|
||||||
|
|
||||||
cwhub.SetHubBranch()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if err := csConfig.LoadHub(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
|
|
||||||
if !errors.Is(err, cwhub.ErrIndexNotFound) {
|
|
||||||
return fmt.Errorf("failed to get Hub index : %w", err)
|
|
||||||
}
|
|
||||||
log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
|
|
||||||
cwhub.HubBranch = "master"
|
|
||||||
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
|
|
||||||
return fmt.Errorf("failed to get Hub index after retry: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// use LocalSync to get warnings about tainted / outdated items
|
|
||||||
warn, _ := cwhub.LocalSync(csConfig.Hub)
|
|
||||||
for _, v := range warn {
|
|
||||||
log.Info(v)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return cmdHubUpdate
|
return cmdHubUpdate
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func runHubUpgrade(cmd *cobra.Command, args []string) error {
|
||||||
|
flags := cmd.Flags()
|
||||||
|
|
||||||
|
force, err := flags.GetBool("force")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, itemType := range cwhub.ItemTypes {
|
||||||
|
items, err := hub.GetInstalledItems(itemType)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
updated := 0
|
||||||
|
|
||||||
|
log.Infof("Upgrading %s", itemType)
|
||||||
|
for _, item := range items {
|
||||||
|
didUpdate, err := item.Upgrade(force)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if didUpdate {
|
||||||
|
updated++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.Infof("Upgraded %d %s", updated, itemType)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func NewHubUpgradeCmd() *cobra.Command {
|
func NewHubUpgradeCmd() *cobra.Command {
|
||||||
var cmdHubUpgrade = &cobra.Command{
|
cmdHubUpgrade := &cobra.Command{
|
||||||
Use: "upgrade",
|
Use: "upgrade",
|
||||||
Short: "Upgrade all configs installed from hub",
|
Short: "Upgrade all configurations to their latest version",
|
||||||
Long: `
|
Long: `
|
||||||
Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
|
Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
|
||||||
`,
|
`,
|
||||||
Args: cobra.ExactArgs(0),
|
Args: cobra.ExactArgs(0),
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
RunE: runHubUpgrade,
|
||||||
if csConfig.Cscli == nil {
|
|
||||||
return fmt.Errorf("you must configure cli before interacting with hub")
|
|
||||||
}
|
|
||||||
|
|
||||||
cwhub.SetHubBranch()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("Upgrading collections")
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
|
|
||||||
log.Infof("Upgrading parsers")
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
|
|
||||||
log.Infof("Upgrading scenarios")
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
|
|
||||||
log.Infof("Upgrading postoverflows")
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
|
|
||||||
|
flags := cmdHubUpgrade.Flags()
|
||||||
|
flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
|
||||||
|
|
||||||
return cmdHubUpgrade
|
return cmdHubUpgrade
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,9 +18,7 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
|
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var HubTest hubtest.HubTest
|
||||||
HubTest hubtest.HubTest
|
|
||||||
)
|
|
||||||
|
|
||||||
func NewHubTestCmd() *cobra.Command {
|
func NewHubTestCmd() *cobra.Command {
|
||||||
var hubPath string
|
var hubPath string
|
||||||
|
@ -43,6 +41,7 @@ func NewHubTestCmd() *cobra.Command {
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
|
cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
|
||||||
cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
|
cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
|
||||||
cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
|
cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
|
||||||
|
@ -59,7 +58,6 @@ func NewHubTestCmd() *cobra.Command {
|
||||||
return cmdHubTest
|
return cmdHubTest
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func NewHubTestCreateCmd() *cobra.Command {
|
func NewHubTestCreateCmd() *cobra.Command {
|
||||||
parsers := []string{}
|
parsers := []string{}
|
||||||
postoverflows := []string{}
|
postoverflows := []string{}
|
||||||
|
@ -138,7 +136,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
|
||||||
}
|
}
|
||||||
|
|
||||||
configFilePath := filepath.Join(testPath, "config.yaml")
|
configFilePath := filepath.Join(testPath, "config.yaml")
|
||||||
fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
|
fd, err := os.Create(configFilePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("open: %s", err)
|
return fmt.Errorf("open: %s", err)
|
||||||
}
|
}
|
||||||
|
@ -164,6 +162,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
|
cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
|
||||||
cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
|
cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
|
||||||
cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
|
cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
|
||||||
|
@ -173,7 +172,6 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
|
||||||
return cmdHubTestCreate
|
return cmdHubTestCreate
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func NewHubTestRunCmd() *cobra.Command {
|
func NewHubTestRunCmd() *cobra.Command {
|
||||||
var noClean bool
|
var noClean bool
|
||||||
var runAll bool
|
var runAll bool
|
||||||
|
@ -186,7 +184,7 @@ func NewHubTestRunCmd() *cobra.Command {
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
if !runAll && len(args) == 0 {
|
if !runAll && len(args) == 0 {
|
||||||
printHelp(cmd)
|
printHelp(cmd)
|
||||||
return fmt.Errorf("Please provide test to run or --all flag")
|
return fmt.Errorf("please provide test to run or --all flag")
|
||||||
}
|
}
|
||||||
|
|
||||||
if runAll {
|
if runAll {
|
||||||
|
@ -202,6 +200,9 @@ func NewHubTestRunCmd() *cobra.Command {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// set timezone to avoid DST issues
|
||||||
|
os.Setenv("TZ", "UTC")
|
||||||
|
|
||||||
for _, test := range HubTest.Tests {
|
for _, test := range HubTest.Tests {
|
||||||
if csConfig.Cscli.Output == "human" {
|
if csConfig.Cscli.Output == "human" {
|
||||||
log.Infof("Running test '%s'", test.Name)
|
log.Infof("Running test '%s'", test.Name)
|
||||||
|
@ -293,9 +294,11 @@ func NewHubTestRunCmd() *cobra.Command {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if csConfig.Cscli.Output == "human" {
|
|
||||||
|
switch csConfig.Cscli.Output {
|
||||||
|
case "human":
|
||||||
hubTestResultTable(color.Output, testResult)
|
hubTestResultTable(color.Output, testResult)
|
||||||
} else if csConfig.Cscli.Output == "json" {
|
case "json":
|
||||||
jsonResult := make(map[string][]string, 0)
|
jsonResult := make(map[string][]string, 0)
|
||||||
jsonResult["success"] = make([]string, 0)
|
jsonResult["success"] = make([]string, 0)
|
||||||
jsonResult["fail"] = make([]string, 0)
|
jsonResult["fail"] = make([]string, 0)
|
||||||
|
@ -311,6 +314,8 @@ func NewHubTestRunCmd() *cobra.Command {
|
||||||
return fmt.Errorf("unable to json test result: %s", err)
|
return fmt.Errorf("unable to json test result: %s", err)
|
||||||
}
|
}
|
||||||
fmt.Println(string(jsonStr))
|
fmt.Println(string(jsonStr))
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("only human/json output modes are supported")
|
||||||
}
|
}
|
||||||
|
|
||||||
if !success {
|
if !success {
|
||||||
|
@ -320,6 +325,7 @@ func NewHubTestRunCmd() *cobra.Command {
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
|
cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
|
||||||
cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
|
cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
|
||||||
cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
|
cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
|
||||||
|
@ -327,7 +333,6 @@ func NewHubTestRunCmd() *cobra.Command {
|
||||||
return cmdHubTestRun
|
return cmdHubTestRun
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func NewHubTestCleanCmd() *cobra.Command {
|
func NewHubTestCleanCmd() *cobra.Command {
|
||||||
var cmdHubTestClean = &cobra.Command{
|
var cmdHubTestClean = &cobra.Command{
|
||||||
Use: "clean",
|
Use: "clean",
|
||||||
|
@ -352,7 +357,6 @@ func NewHubTestCleanCmd() *cobra.Command {
|
||||||
return cmdHubTestClean
|
return cmdHubTestClean
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func NewHubTestInfoCmd() *cobra.Command {
|
func NewHubTestInfoCmd() *cobra.Command {
|
||||||
var cmdHubTestInfo = &cobra.Command{
|
var cmdHubTestInfo = &cobra.Command{
|
||||||
Use: "info",
|
Use: "info",
|
||||||
|
@ -381,7 +385,6 @@ func NewHubTestInfoCmd() *cobra.Command {
|
||||||
return cmdHubTestInfo
|
return cmdHubTestInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func NewHubTestListCmd() *cobra.Command {
|
func NewHubTestListCmd() *cobra.Command {
|
||||||
var cmdHubTestList = &cobra.Command{
|
var cmdHubTestList = &cobra.Command{
|
||||||
Use: "list",
|
Use: "list",
|
||||||
|
@ -412,7 +415,6 @@ func NewHubTestListCmd() *cobra.Command {
|
||||||
return cmdHubTestList
|
return cmdHubTestList
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func NewHubTestCoverageCmd() *cobra.Command {
|
func NewHubTestCoverageCmd() *cobra.Command {
|
||||||
var showParserCov bool
|
var showParserCov bool
|
||||||
var showScenarioCov bool
|
var showScenarioCov bool
|
||||||
|
@ -427,8 +429,8 @@ func NewHubTestCoverageCmd() *cobra.Command {
|
||||||
return fmt.Errorf("unable to load all tests: %+v", err)
|
return fmt.Errorf("unable to load all tests: %+v", err)
|
||||||
}
|
}
|
||||||
var err error
|
var err error
|
||||||
scenarioCoverage := []hubtest.ScenarioCoverage{}
|
scenarioCoverage := []hubtest.Coverage{}
|
||||||
parserCoverage := []hubtest.ParserCoverage{}
|
parserCoverage := []hubtest.Coverage{}
|
||||||
scenarioCoveragePercent := 0
|
scenarioCoveragePercent := 0
|
||||||
parserCoveragePercent := 0
|
parserCoveragePercent := 0
|
||||||
|
|
||||||
|
@ -443,7 +445,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
|
||||||
parserTested := 0
|
parserTested := 0
|
||||||
for _, test := range parserCoverage {
|
for _, test := range parserCoverage {
|
||||||
if test.TestsCount > 0 {
|
if test.TestsCount > 0 {
|
||||||
parserTested += 1
|
parserTested++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
|
parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
|
||||||
|
@ -454,12 +456,14 @@ func NewHubTestCoverageCmd() *cobra.Command {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("while getting scenario coverage: %s", err)
|
return fmt.Errorf("while getting scenario coverage: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
scenarioTested := 0
|
scenarioTested := 0
|
||||||
for _, test := range scenarioCoverage {
|
for _, test := range scenarioCoverage {
|
||||||
if test.TestsCount > 0 {
|
if test.TestsCount > 0 {
|
||||||
scenarioTested += 1
|
scenarioTested++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
|
scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -474,7 +478,8 @@ func NewHubTestCoverageCmd() *cobra.Command {
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
if csConfig.Cscli.Output == "human" {
|
switch csConfig.Cscli.Output {
|
||||||
|
case "human":
|
||||||
if showParserCov || showAll {
|
if showParserCov || showAll {
|
||||||
hubTestParserCoverageTable(color.Output, parserCoverage)
|
hubTestParserCoverageTable(color.Output, parserCoverage)
|
||||||
}
|
}
|
||||||
|
@ -489,7 +494,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
|
||||||
if showScenarioCov || showAll {
|
if showScenarioCov || showAll {
|
||||||
fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent)
|
fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent)
|
||||||
}
|
}
|
||||||
} else if csConfig.Cscli.Output == "json" {
|
case "json":
|
||||||
dump, err := json.MarshalIndent(parserCoverage, "", " ")
|
dump, err := json.MarshalIndent(parserCoverage, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -500,13 +505,14 @@ func NewHubTestCoverageCmd() *cobra.Command {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("%s", dump)
|
fmt.Printf("%s", dump)
|
||||||
} else {
|
default:
|
||||||
return fmt.Errorf("only human/json output modes are supported")
|
return fmt.Errorf("only human/json output modes are supported")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
|
cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
|
||||||
cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
|
cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
|
||||||
cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
|
cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
|
||||||
|
@ -514,7 +520,6 @@ func NewHubTestCoverageCmd() *cobra.Command {
|
||||||
return cmdHubTestCoverage
|
return cmdHubTestCoverage
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func NewHubTestEvalCmd() *cobra.Command {
|
func NewHubTestEvalCmd() *cobra.Command {
|
||||||
var evalExpression string
|
var evalExpression string
|
||||||
var cmdHubTestEval = &cobra.Command{
|
var cmdHubTestEval = &cobra.Command{
|
||||||
|
@ -528,26 +533,29 @@ func NewHubTestEvalCmd() *cobra.Command {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("can't load test: %+v", err)
|
return fmt.Errorf("can't load test: %+v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = test.ParserAssert.LoadTest(test.ParserResultFile)
|
err = test.ParserAssert.LoadTest(test.ParserResultFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
|
return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
output, err := test.ParserAssert.EvalExpression(evalExpression)
|
output, err := test.ParserAssert.EvalExpression(evalExpression)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Print(output)
|
fmt.Print(output)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
|
cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
|
||||||
|
|
||||||
return cmdHubTestEval
|
return cmdHubTestEval
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func NewHubTestExplainCmd() *cobra.Command {
|
func NewHubTestExplainCmd() *cobra.Command {
|
||||||
var cmdHubTestExplain = &cobra.Command{
|
var cmdHubTestExplain = &cobra.Command{
|
||||||
Use: "explain",
|
Use: "explain",
|
||||||
|
@ -562,24 +570,22 @@ func NewHubTestExplainCmd() *cobra.Command {
|
||||||
}
|
}
|
||||||
err = test.ParserAssert.LoadTest(test.ParserResultFile)
|
err = test.ParserAssert.LoadTest(test.ParserResultFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err := test.Run()
|
if err = test.Run(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
|
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
|
||||||
}
|
}
|
||||||
err = test.ParserAssert.LoadTest(test.ParserResultFile)
|
|
||||||
if err != nil {
|
if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil {
|
||||||
return fmt.Errorf("unable to load parser result after run: %s", err)
|
return fmt.Errorf("unable to load parser result after run: %s", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
|
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err := test.Run()
|
if err = test.Run(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
|
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
|
||||||
}
|
}
|
||||||
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
|
|
||||||
if err != nil {
|
if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil {
|
||||||
return fmt.Errorf("unable to load scenario result after run: %s", err)
|
return fmt.Errorf("unable to load scenario result after run: %s", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,39 +41,41 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) {
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
|
|
||||||
func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) {
|
func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
|
||||||
t := newLightTable(out)
|
t := newLightTable(out)
|
||||||
t.SetHeaders("Parser", "Status", "Number of tests")
|
t.SetHeaders("Parser", "Status", "Number of tests")
|
||||||
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
||||||
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
||||||
|
|
||||||
parserTested := 0
|
parserTested := 0
|
||||||
|
|
||||||
for _, test := range coverage {
|
for _, test := range coverage {
|
||||||
status := emoji.RedCircle.String()
|
status := emoji.RedCircle.String()
|
||||||
if test.TestsCount > 0 {
|
if test.TestsCount > 0 {
|
||||||
status = emoji.GreenCircle.String()
|
status = emoji.GreenCircle.String()
|
||||||
parserTested++
|
parserTested++
|
||||||
}
|
}
|
||||||
t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
|
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
|
|
||||||
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) {
|
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
|
||||||
t := newLightTable(out)
|
t := newLightTable(out)
|
||||||
t.SetHeaders("Scenario", "Status", "Number of tests")
|
t.SetHeaders("Scenario", "Status", "Number of tests")
|
||||||
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
||||||
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
||||||
|
|
||||||
parserTested := 0
|
parserTested := 0
|
||||||
|
|
||||||
for _, test := range coverage {
|
for _, test := range coverage {
|
||||||
status := emoji.RedCircle.String()
|
status := emoji.RedCircle.String()
|
||||||
if test.TestsCount > 0 {
|
if test.TestsCount > 0 {
|
||||||
status = emoji.GreenCircle.String()
|
status = emoji.GreenCircle.String()
|
||||||
parserTested++
|
parserTested++
|
||||||
}
|
}
|
||||||
t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
|
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Render()
|
t.Render()
|
||||||
|
|
236
cmd/crowdsec-cli/item_metrics.go
Normal file
236
cmd/crowdsec-cli/item_metrics.go
Normal file
|
@ -0,0 +1,236 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fatih/color"
|
||||||
|
dto "github.com/prometheus/client_model/go"
|
||||||
|
"github.com/prometheus/prom2json"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/go-cs-lib/trace"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ShowMetrics(hubItem *cwhub.Item) error {
|
||||||
|
switch hubItem.Type {
|
||||||
|
case cwhub.PARSERS:
|
||||||
|
metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
|
||||||
|
parserMetricsTable(color.Output, hubItem.Name, metrics)
|
||||||
|
case cwhub.SCENARIOS:
|
||||||
|
metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
|
||||||
|
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
|
||||||
|
case cwhub.COLLECTIONS:
|
||||||
|
for _, sub := range hubItem.SubItems() {
|
||||||
|
if err := ShowMetrics(sub); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
// no metrics for this item type
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetParserMetric is a complete rip from prom2json
|
||||||
|
func GetParserMetric(url string, itemName string) map[string]map[string]int {
|
||||||
|
stats := make(map[string]map[string]int)
|
||||||
|
|
||||||
|
result := GetPrometheusMetric(url)
|
||||||
|
for idx, fam := range result {
|
||||||
|
if !strings.HasPrefix(fam.Name, "cs_") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Tracef("round %d", idx)
|
||||||
|
for _, m := range fam.Metrics {
|
||||||
|
metric, ok := m.(prom2json.Metric)
|
||||||
|
if !ok {
|
||||||
|
log.Debugf("failed to convert metric to prom2json.Metric")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name, ok := metric.Labels["name"]
|
||||||
|
if !ok {
|
||||||
|
log.Debugf("no name in Metric %v", metric.Labels)
|
||||||
|
}
|
||||||
|
if name != itemName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
source, ok := metric.Labels["source"]
|
||||||
|
if !ok {
|
||||||
|
log.Debugf("no source in Metric %v", metric.Labels)
|
||||||
|
} else {
|
||||||
|
if srctype, ok := metric.Labels["type"]; ok {
|
||||||
|
source = srctype + ":" + source
|
||||||
|
}
|
||||||
|
}
|
||||||
|
value := m.(prom2json.Metric).Value
|
||||||
|
fval, err := strconv.ParseFloat(value, 32)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Unexpected int value %s : %s", value, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ival := int(fval)
|
||||||
|
|
||||||
|
switch fam.Name {
|
||||||
|
case "cs_reader_hits_total":
|
||||||
|
if _, ok := stats[source]; !ok {
|
||||||
|
stats[source] = make(map[string]int)
|
||||||
|
stats[source]["parsed"] = 0
|
||||||
|
stats[source]["reads"] = 0
|
||||||
|
stats[source]["unparsed"] = 0
|
||||||
|
stats[source]["hits"] = 0
|
||||||
|
}
|
||||||
|
stats[source]["reads"] += ival
|
||||||
|
case "cs_parser_hits_ok_total":
|
||||||
|
if _, ok := stats[source]; !ok {
|
||||||
|
stats[source] = make(map[string]int)
|
||||||
|
}
|
||||||
|
stats[source]["parsed"] += ival
|
||||||
|
case "cs_parser_hits_ko_total":
|
||||||
|
if _, ok := stats[source]; !ok {
|
||||||
|
stats[source] = make(map[string]int)
|
||||||
|
}
|
||||||
|
stats[source]["unparsed"] += ival
|
||||||
|
case "cs_node_hits_total":
|
||||||
|
if _, ok := stats[source]; !ok {
|
||||||
|
stats[source] = make(map[string]int)
|
||||||
|
}
|
||||||
|
stats[source]["hits"] += ival
|
||||||
|
case "cs_node_hits_ok_total":
|
||||||
|
if _, ok := stats[source]; !ok {
|
||||||
|
stats[source] = make(map[string]int)
|
||||||
|
}
|
||||||
|
stats[source]["parsed"] += ival
|
||||||
|
case "cs_node_hits_ko_total":
|
||||||
|
if _, ok := stats[source]; !ok {
|
||||||
|
stats[source] = make(map[string]int)
|
||||||
|
}
|
||||||
|
stats[source]["unparsed"] += ival
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return stats
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetScenarioMetric(url string, itemName string) map[string]int {
|
||||||
|
stats := make(map[string]int)
|
||||||
|
|
||||||
|
stats["instantiation"] = 0
|
||||||
|
stats["curr_count"] = 0
|
||||||
|
stats["overflow"] = 0
|
||||||
|
stats["pour"] = 0
|
||||||
|
stats["underflow"] = 0
|
||||||
|
|
||||||
|
result := GetPrometheusMetric(url)
|
||||||
|
for idx, fam := range result {
|
||||||
|
if !strings.HasPrefix(fam.Name, "cs_") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Tracef("round %d", idx)
|
||||||
|
for _, m := range fam.Metrics {
|
||||||
|
metric, ok := m.(prom2json.Metric)
|
||||||
|
if !ok {
|
||||||
|
log.Debugf("failed to convert metric to prom2json.Metric")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name, ok := metric.Labels["name"]
|
||||||
|
if !ok {
|
||||||
|
log.Debugf("no name in Metric %v", metric.Labels)
|
||||||
|
}
|
||||||
|
if name != itemName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
value := m.(prom2json.Metric).Value
|
||||||
|
fval, err := strconv.ParseFloat(value, 32)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Unexpected int value %s : %s", value, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ival := int(fval)
|
||||||
|
|
||||||
|
switch fam.Name {
|
||||||
|
case "cs_bucket_created_total":
|
||||||
|
stats["instantiation"] += ival
|
||||||
|
case "cs_buckets":
|
||||||
|
stats["curr_count"] += ival
|
||||||
|
case "cs_bucket_overflowed_total":
|
||||||
|
stats["overflow"] += ival
|
||||||
|
case "cs_bucket_poured_total":
|
||||||
|
stats["pour"] += ival
|
||||||
|
case "cs_bucket_underflowed_total":
|
||||||
|
stats["underflow"] += ival
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return stats
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetPrometheusMetric(url string) []*prom2json.Family {
|
||||||
|
mfChan := make(chan *dto.MetricFamily, 1024)
|
||||||
|
|
||||||
|
// Start with the DefaultTransport for sane defaults.
|
||||||
|
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||||
|
// Conservatively disable HTTP keep-alives as this program will only
|
||||||
|
// ever need a single HTTP request.
|
||||||
|
transport.DisableKeepAlives = true
|
||||||
|
// Timeout early if the server doesn't even return the headers.
|
||||||
|
transport.ResponseHeaderTimeout = time.Minute
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
|
||||||
|
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("failed to fetch prometheus metrics : %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
result := []*prom2json.Family{}
|
||||||
|
for mf := range mfChan {
|
||||||
|
result = append(result, prom2json.NewFamily(mf))
|
||||||
|
}
|
||||||
|
log.Debugf("Finished reading prometheus output, %d entries", len(result))
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
type unit struct {
|
||||||
|
value int64
|
||||||
|
symbol string
|
||||||
|
}
|
||||||
|
|
||||||
|
var ranges = []unit{
|
||||||
|
{value: 1e18, symbol: "E"},
|
||||||
|
{value: 1e15, symbol: "P"},
|
||||||
|
{value: 1e12, symbol: "T"},
|
||||||
|
{value: 1e9, symbol: "G"},
|
||||||
|
{value: 1e6, symbol: "M"},
|
||||||
|
{value: 1e3, symbol: "k"},
|
||||||
|
{value: 1, symbol: ""},
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatNumber(num int) string {
|
||||||
|
goodUnit := unit{}
|
||||||
|
for _, u := range ranges {
|
||||||
|
if int64(num) >= u.value {
|
||||||
|
goodUnit = u
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if goodUnit.value == 1 {
|
||||||
|
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
|
||||||
|
}
|
||||||
|
|
||||||
|
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
|
||||||
|
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
|
||||||
|
}
|
85
cmd/crowdsec-cli/item_suggest.go
Normal file
85
cmd/crowdsec-cli/item_suggest.go
Normal file
|
@ -0,0 +1,85 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/agext/levenshtein"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"slices"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
|
)
|
||||||
|
|
||||||
|
const MaxDistance = 7
|
||||||
|
|
||||||
|
// SuggestNearestMessage returns a message with the most similar item name, if one is found
|
||||||
|
func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
|
||||||
|
score := 100
|
||||||
|
nearest := ""
|
||||||
|
|
||||||
|
for _, item := range hub.GetItemMap(itemType) {
|
||||||
|
d := levenshtein.Distance(itemName, item.Name, nil)
|
||||||
|
if d < score {
|
||||||
|
score = d
|
||||||
|
nearest = item.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
|
||||||
|
|
||||||
|
if score < MaxDistance {
|
||||||
|
msg += fmt.Sprintf(", did you mean '%s'?", nearest)
|
||||||
|
}
|
||||||
|
|
||||||
|
return msg
|
||||||
|
}
|
||||||
|
|
||||||
|
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||||
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, cobra.ShellCompDirectiveDefault
|
||||||
|
}
|
||||||
|
|
||||||
|
comp := make([]string, 0)
|
||||||
|
|
||||||
|
for _, item := range hub.GetItemMap(itemType) {
|
||||||
|
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
|
||||||
|
comp = append(comp, item.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
|
||||||
|
|
||||||
|
return comp, cobra.ShellCompDirectiveNoFileComp
|
||||||
|
}
|
||||||
|
|
||||||
|
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||||
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, cobra.ShellCompDirectiveDefault
|
||||||
|
}
|
||||||
|
|
||||||
|
items, err := hub.GetInstalledItemsAsString(itemType)
|
||||||
|
if err != nil {
|
||||||
|
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
|
||||||
|
return nil, cobra.ShellCompDirectiveDefault
|
||||||
|
}
|
||||||
|
|
||||||
|
comp := make([]string, 0)
|
||||||
|
|
||||||
|
if toComplete != "" {
|
||||||
|
for _, item := range items {
|
||||||
|
if strings.Contains(item, toComplete) {
|
||||||
|
comp = append(comp, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
comp = items
|
||||||
|
}
|
||||||
|
|
||||||
|
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
|
||||||
|
|
||||||
|
return comp, cobra.ShellCompDirectiveNoFileComp
|
||||||
|
}
|
593
cmd/crowdsec-cli/itemcommands.go
Normal file
593
cmd/crowdsec-cli/itemcommands.go
Normal file
|
@ -0,0 +1,593 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/fatih/color"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/go-cs-lib/coalesce"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
|
)
|
||||||
|
|
||||||
|
type cmdHelp struct {
|
||||||
|
// Example is required, the others have a default value
|
||||||
|
// generated from the item type
|
||||||
|
use string
|
||||||
|
short string
|
||||||
|
long string
|
||||||
|
example string
|
||||||
|
}
|
||||||
|
|
||||||
|
type hubItemType struct {
|
||||||
|
name string // plural, as used in the hub index
|
||||||
|
singular string
|
||||||
|
oneOrMore string // parenthetical pluralizaion: "parser(s)"
|
||||||
|
help cmdHelp
|
||||||
|
installHelp cmdHelp
|
||||||
|
removeHelp cmdHelp
|
||||||
|
upgradeHelp cmdHelp
|
||||||
|
inspectHelp cmdHelp
|
||||||
|
listHelp cmdHelp
|
||||||
|
}
|
||||||
|
|
||||||
|
var hubItemTypes = map[string]hubItemType{
|
||||||
|
"parsers": {
|
||||||
|
name: "parsers",
|
||||||
|
singular: "parser",
|
||||||
|
oneOrMore: "parser(s)",
|
||||||
|
help: cmdHelp{
|
||||||
|
example: `cscli parsers list -a
|
||||||
|
cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
|
||||||
|
cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
|
||||||
|
cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
|
||||||
|
cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
installHelp: cmdHelp{
|
||||||
|
example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
|
||||||
|
},
|
||||||
|
removeHelp: cmdHelp{
|
||||||
|
example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
|
||||||
|
},
|
||||||
|
upgradeHelp: cmdHelp{
|
||||||
|
example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
|
||||||
|
},
|
||||||
|
inspectHelp: cmdHelp{
|
||||||
|
example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
|
||||||
|
},
|
||||||
|
listHelp: cmdHelp{
|
||||||
|
example: `cscli parsers list
|
||||||
|
cscli parsers list -a
|
||||||
|
cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
|
||||||
|
|
||||||
|
List only enabled parsers unless "-a" or names are specified.`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"postoverflows": {
|
||||||
|
name: "postoverflows",
|
||||||
|
singular: "postoverflow",
|
||||||
|
oneOrMore: "postoverflow(s)",
|
||||||
|
help: cmdHelp{
|
||||||
|
example: `cscli postoverflows list -a
|
||||||
|
cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
|
||||||
|
cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
|
||||||
|
cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
|
||||||
|
cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
installHelp: cmdHelp{
|
||||||
|
example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
|
||||||
|
},
|
||||||
|
removeHelp: cmdHelp{
|
||||||
|
example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
|
||||||
|
},
|
||||||
|
upgradeHelp: cmdHelp{
|
||||||
|
example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
|
||||||
|
},
|
||||||
|
inspectHelp: cmdHelp{
|
||||||
|
example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
|
||||||
|
},
|
||||||
|
listHelp: cmdHelp{
|
||||||
|
example: `cscli postoverflows list
|
||||||
|
cscli postoverflows list -a
|
||||||
|
cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
|
||||||
|
|
||||||
|
List only enabled postoverflows unless "-a" or names are specified.`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"scenarios": {
|
||||||
|
name: "scenarios",
|
||||||
|
singular: "scenario",
|
||||||
|
oneOrMore: "scenario(s)",
|
||||||
|
help: cmdHelp{
|
||||||
|
example: `cscli scenarios list -a
|
||||||
|
cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
|
||||||
|
cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
|
||||||
|
cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
|
||||||
|
cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
installHelp: cmdHelp{
|
||||||
|
example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
|
||||||
|
},
|
||||||
|
removeHelp: cmdHelp{
|
||||||
|
example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
|
||||||
|
},
|
||||||
|
upgradeHelp: cmdHelp{
|
||||||
|
example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
|
||||||
|
},
|
||||||
|
inspectHelp: cmdHelp{
|
||||||
|
example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
|
||||||
|
},
|
||||||
|
listHelp: cmdHelp{
|
||||||
|
example: `cscli scenarios list
|
||||||
|
cscli scenarios list -a
|
||||||
|
cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
|
||||||
|
|
||||||
|
List only enabled scenarios unless "-a" or names are specified.`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"collections": {
|
||||||
|
name: "collections",
|
||||||
|
singular: "collection",
|
||||||
|
oneOrMore: "collection(s)",
|
||||||
|
help: cmdHelp{
|
||||||
|
example: `cscli collections list -a
|
||||||
|
cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
|
||||||
|
cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
|
||||||
|
cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
|
||||||
|
cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
installHelp: cmdHelp{
|
||||||
|
example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
|
||||||
|
},
|
||||||
|
removeHelp: cmdHelp{
|
||||||
|
example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
|
||||||
|
},
|
||||||
|
upgradeHelp: cmdHelp{
|
||||||
|
example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
|
||||||
|
},
|
||||||
|
inspectHelp: cmdHelp{
|
||||||
|
example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
|
||||||
|
},
|
||||||
|
listHelp: cmdHelp{
|
||||||
|
example: `cscli collections list
|
||||||
|
cscli collections list -a
|
||||||
|
cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
|
||||||
|
|
||||||
|
List only enabled collections unless "-a" or names are specified.`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewItemsCmd(typeName string) *cobra.Command {
|
||||||
|
it := hubItemTypes[typeName]
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: coalesce.String(it.help.use, fmt.Sprintf("%s <action> [item]...", it.name)),
|
||||||
|
Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)),
|
||||||
|
Long: it.help.long,
|
||||||
|
Example: it.help.example,
|
||||||
|
Args: cobra.MinimumNArgs(1),
|
||||||
|
Aliases: []string{it.singular},
|
||||||
|
DisableAutoGenTag: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.AddCommand(NewItemsInstallCmd(typeName))
|
||||||
|
cmd.AddCommand(NewItemsRemoveCmd(typeName))
|
||||||
|
cmd.AddCommand(NewItemsUpgradeCmd(typeName))
|
||||||
|
cmd.AddCommand(NewItemsInspectCmd(typeName))
|
||||||
|
cmd.AddCommand(NewItemsListCmd(typeName))
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
|
||||||
|
run := func(cmd *cobra.Command, args []string) error {
|
||||||
|
flags := cmd.Flags()
|
||||||
|
|
||||||
|
downloadOnly, err := flags.GetBool("download-only")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
force, err := flags.GetBool("force")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
ignoreError, err := flags.GetBool("ignore")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range args {
|
||||||
|
item := hub.GetItem(it.name, name)
|
||||||
|
if item == nil {
|
||||||
|
msg := SuggestNearestMessage(hub, it.name, name)
|
||||||
|
if !ignoreError {
|
||||||
|
return fmt.Errorf(msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Errorf(msg)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := item.Install(force, downloadOnly); err != nil {
|
||||||
|
if !ignoreError {
|
||||||
|
return fmt.Errorf("error while installing '%s': %w", item.Name, err)
|
||||||
|
}
|
||||||
|
log.Errorf("Error while installing '%s': %s", item.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// XXX: only reload if we installed something
|
||||||
|
log.Infof(ReloadMessage())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return run
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewItemsInstallCmd(typeName string) *cobra.Command {
|
||||||
|
it := hubItemTypes[typeName]
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: coalesce.String(it.installHelp.use, "install [item]..."),
|
||||||
|
Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)),
|
||||||
|
Long: coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)),
|
||||||
|
Example: it.installHelp.example,
|
||||||
|
Args: cobra.MinimumNArgs(1),
|
||||||
|
DisableAutoGenTag: true,
|
||||||
|
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||||
|
return compAllItems(typeName, args, toComplete)
|
||||||
|
},
|
||||||
|
RunE: itemsInstallRunner(it),
|
||||||
|
}
|
||||||
|
|
||||||
|
flags := cmd.Flags()
|
||||||
|
flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
|
||||||
|
flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
|
||||||
|
flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name))
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
|
||||||
|
run := func(cmd *cobra.Command, args []string) error {
|
||||||
|
flags := cmd.Flags()
|
||||||
|
|
||||||
|
purge, err := flags.GetBool("purge")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
force, err := flags.GetBool("force")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
all, err := flags.GetBool("all")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if all {
|
||||||
|
getter := hub.GetInstalledItems
|
||||||
|
if purge {
|
||||||
|
getter = hub.GetAllItems
|
||||||
|
}
|
||||||
|
|
||||||
|
items, err := getter(it.name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
removed := 0
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
didRemove, err := item.Remove(purge, force)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if didRemove {
|
||||||
|
removed++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Infof("Removed %d %s", removed, it.name)
|
||||||
|
if removed > 0 {
|
||||||
|
log.Infof(ReloadMessage())
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(args) == 0 {
|
||||||
|
return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
|
||||||
|
}
|
||||||
|
|
||||||
|
removed := 0
|
||||||
|
|
||||||
|
for _, itemName := range args {
|
||||||
|
item := hub.GetItem(it.name, itemName)
|
||||||
|
if item == nil {
|
||||||
|
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !force && len(item.BelongsToCollections) > 0 {
|
||||||
|
log.Warningf("%s belongs to collections: %s", item.Name, item.BelongsToCollections)
|
||||||
|
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
didRemove, err := item.Remove(purge, force)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if didRemove {
|
||||||
|
log.Infof("Removed %s", item.Name)
|
||||||
|
removed++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if removed > 0 {
|
||||||
|
log.Infof(ReloadMessage())
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return run
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewItemsRemoveCmd(typeName string) *cobra.Command {
|
||||||
|
it := hubItemTypes[typeName]
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: coalesce.String(it.removeHelp.use, "remove [item]..."),
|
||||||
|
Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)),
|
||||||
|
Long: coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)),
|
||||||
|
Example: it.removeHelp.example,
|
||||||
|
Aliases: []string{"delete"},
|
||||||
|
DisableAutoGenTag: true,
|
||||||
|
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||||
|
return compInstalledItems(it.name, args, toComplete)
|
||||||
|
},
|
||||||
|
RunE: itemsRemoveRunner(it),
|
||||||
|
}
|
||||||
|
|
||||||
|
flags := cmd.Flags()
|
||||||
|
flags.Bool("purge", false, "Delete source file too")
|
||||||
|
flags.Bool("force", false, "Force remove: remove tainted and outdated files")
|
||||||
|
flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name))
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
|
||||||
|
run := func(cmd *cobra.Command, args []string) error {
|
||||||
|
flags := cmd.Flags()
|
||||||
|
|
||||||
|
force, err := flags.GetBool("force")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
all, err := flags.GetBool("all")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if all {
|
||||||
|
items, err := hub.GetInstalledItems(it.name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
updated := 0
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
didUpdate, err := item.Upgrade(force)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if didUpdate {
|
||||||
|
updated++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Infof("Updated %d %s", updated, it.name)
|
||||||
|
|
||||||
|
if updated > 0 {
|
||||||
|
log.Infof(ReloadMessage())
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(args) == 0 {
|
||||||
|
return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
|
||||||
|
}
|
||||||
|
|
||||||
|
updated := 0
|
||||||
|
|
||||||
|
for _, itemName := range args {
|
||||||
|
item := hub.GetItem(it.name, itemName)
|
||||||
|
if item == nil {
|
||||||
|
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
didUpdate, err := item.Upgrade(force)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if didUpdate {
|
||||||
|
log.Infof("Updated %s", item.Name)
|
||||||
|
updated++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if updated > 0 {
|
||||||
|
log.Infof(ReloadMessage())
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return run
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewItemsUpgradeCmd(typeName string) *cobra.Command {
|
||||||
|
it := hubItemTypes[typeName]
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."),
|
||||||
|
Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)),
|
||||||
|
Long: coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)),
|
||||||
|
Example: it.upgradeHelp.example,
|
||||||
|
DisableAutoGenTag: true,
|
||||||
|
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||||
|
return compInstalledItems(it.name, args, toComplete)
|
||||||
|
},
|
||||||
|
RunE: itemsUpgradeRunner(it),
|
||||||
|
}
|
||||||
|
|
||||||
|
flags := cmd.Flags()
|
||||||
|
flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name))
|
||||||
|
flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
|
||||||
|
run := func(cmd *cobra.Command, args []string) error {
|
||||||
|
flags := cmd.Flags()
|
||||||
|
|
||||||
|
url, err := flags.GetString("url")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if url != "" {
|
||||||
|
csConfig.Cscli.PrometheusUrl = url
|
||||||
|
}
|
||||||
|
|
||||||
|
noMetrics, err := flags.GetBool("no-metrics")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range args {
|
||||||
|
item := hub.GetItem(it.name, name)
|
||||||
|
if item == nil {
|
||||||
|
return fmt.Errorf("can't find '%s' in %s", name, it.name)
|
||||||
|
}
|
||||||
|
if err = InspectItem(item, !noMetrics); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return run
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewItemsInspectCmd(typeName string) *cobra.Command {
|
||||||
|
it := hubItemTypes[typeName]
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."),
|
||||||
|
Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)),
|
||||||
|
Long: coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)),
|
||||||
|
Example: it.inspectHelp.example,
|
||||||
|
Args: cobra.MinimumNArgs(1),
|
||||||
|
DisableAutoGenTag: true,
|
||||||
|
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||||
|
return compInstalledItems(it.name, args, toComplete)
|
||||||
|
},
|
||||||
|
RunE: itemsInspectRunner(it),
|
||||||
|
}
|
||||||
|
|
||||||
|
flags := cmd.Flags()
|
||||||
|
flags.StringP("url", "u", "", "Prometheus url")
|
||||||
|
flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
|
||||||
|
run := func(cmd *cobra.Command, args []string) error {
|
||||||
|
flags := cmd.Flags()
|
||||||
|
|
||||||
|
all, err := flags.GetBool("all")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
items := make(map[string][]*cwhub.Item)
|
||||||
|
|
||||||
|
items[it.name], err = selectItems(hub, it.name, args, !all)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = listItems(color.Output, []string{it.name}, items); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return run
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewItemsListCmd(typeName string) *cobra.Command {
|
||||||
|
it := hubItemTypes[typeName]
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: coalesce.String(it.listHelp.use, "list [item... | -a]"),
|
||||||
|
Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)),
|
||||||
|
Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)),
|
||||||
|
Example: it.listHelp.example,
|
||||||
|
DisableAutoGenTag: true,
|
||||||
|
RunE: itemsListRunner(it),
|
||||||
|
}
|
||||||
|
|
||||||
|
flags := cmd.Flags()
|
||||||
|
flags.BoolP("all", "a", false, "List disabled items as well")
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
157
cmd/crowdsec-cli/items.go
Normal file
157
cmd/crowdsec-cli/items.go
Normal file
|
@ -0,0 +1,157 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
"slices"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
|
)
|
||||||
|
|
||||||
|
// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name
|
||||||
|
func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) {
|
||||||
|
itemNames := hub.GetItemNames(itemType)
|
||||||
|
|
||||||
|
notExist := []string{}
|
||||||
|
|
||||||
|
if len(args) > 0 {
|
||||||
|
for _, arg := range args {
|
||||||
|
if !slices.Contains(itemNames, arg) {
|
||||||
|
notExist = append(notExist, arg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(notExist) > 0 {
|
||||||
|
return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(args) > 0 {
|
||||||
|
itemNames = args
|
||||||
|
installedOnly = false
|
||||||
|
}
|
||||||
|
|
||||||
|
items := make([]*cwhub.Item, 0, len(itemNames))
|
||||||
|
|
||||||
|
for _, itemName := range itemNames {
|
||||||
|
item := hub.GetItem(itemType, itemName)
|
||||||
|
if installedOnly && !item.Installed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
items = append(items, item)
|
||||||
|
}
|
||||||
|
|
||||||
|
cwhub.SortItemSlice(items)
|
||||||
|
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item) error {
|
||||||
|
switch csConfig.Cscli.Output {
|
||||||
|
case "human":
|
||||||
|
for _, itemType := range itemTypes {
|
||||||
|
listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
|
||||||
|
}
|
||||||
|
case "json":
|
||||||
|
type itemHubStatus struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
LocalVersion string `json:"local_version"`
|
||||||
|
LocalPath string `json:"local_path"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
UTF8Status string `json:"utf8_status"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
}
|
||||||
|
|
||||||
|
hubStatus := make(map[string][]itemHubStatus)
|
||||||
|
for _, itemType := range itemTypes {
|
||||||
|
// empty slice in case there are no items of this type
|
||||||
|
hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
|
||||||
|
|
||||||
|
for i, item := range items[itemType] {
|
||||||
|
status, emo := item.Status()
|
||||||
|
hubStatus[itemType][i] = itemHubStatus{
|
||||||
|
Name: item.Name,
|
||||||
|
LocalVersion: item.LocalVersion,
|
||||||
|
LocalPath: item.LocalPath,
|
||||||
|
Description: item.Description,
|
||||||
|
Status: status,
|
||||||
|
UTF8Status: fmt.Sprintf("%v %s", emo, status),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
x, err := json.MarshalIndent(hubStatus, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to unmarshal: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
out.Write(x)
|
||||||
|
case "raw":
|
||||||
|
csvwriter := csv.NewWriter(out)
|
||||||
|
|
||||||
|
header := []string{"name", "status", "version", "description"}
|
||||||
|
if len(itemTypes) > 1 {
|
||||||
|
header = append(header, "type")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := csvwriter.Write(header); err != nil {
|
||||||
|
return fmt.Errorf("failed to write header: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, itemType := range itemTypes {
|
||||||
|
for _, item := range items[itemType] {
|
||||||
|
status, _ := item.Status()
|
||||||
|
row := []string{
|
||||||
|
item.Name,
|
||||||
|
status,
|
||||||
|
item.LocalVersion,
|
||||||
|
item.Description,
|
||||||
|
}
|
||||||
|
if len(itemTypes) > 1 {
|
||||||
|
row = append(row, itemType)
|
||||||
|
}
|
||||||
|
if err := csvwriter.Write(row); err != nil {
|
||||||
|
return fmt.Errorf("failed to write raw output: %s", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
csvwriter.Flush()
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("unknown output format '%s'", csConfig.Cscli.Output)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func InspectItem(item *cwhub.Item, showMetrics bool) error {
|
||||||
|
switch csConfig.Cscli.Output {
|
||||||
|
case "human", "raw":
|
||||||
|
enc := yaml.NewEncoder(os.Stdout)
|
||||||
|
enc.SetIndent(2)
|
||||||
|
if err := enc.Encode(item); err != nil {
|
||||||
|
return fmt.Errorf("unable to encode item: %s", err)
|
||||||
|
}
|
||||||
|
case "json":
|
||||||
|
b, err := json.MarshalIndent(*item, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("unable to marshal item: %s", err)
|
||||||
|
}
|
||||||
|
fmt.Print(string(b))
|
||||||
|
}
|
||||||
|
|
||||||
|
if csConfig.Cscli.Output == "human" && showMetrics {
|
||||||
|
fmt.Printf("\nCurrent metrics: \n")
|
||||||
|
if err := ShowMetrics(item); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -38,11 +38,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
|
||||||
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
|
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
|
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("failed to get scenarios : %s", err)
|
log.Fatalf("failed to get scenarios : %s", err)
|
||||||
}
|
}
|
||||||
|
@ -338,12 +339,12 @@ cscli lapi context detect crowdsecurity/sshd-logs
|
||||||
log.Fatalf("Failed to init expr helpers : %s", err)
|
log.Fatalf("Failed to init expr helpers : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Populate cwhub package tools
|
hub, err := require.Hub(csConfig, nil)
|
||||||
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Failed to load hub index : %s", err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
csParsers := parser.NewParsers()
|
csParsers := parser.NewParsers(hub)
|
||||||
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
|
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
|
||||||
log.Fatalf("unable to load parsers: %s", err)
|
log.Fatalf("unable to load parsers: %s", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,6 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/fatih/color"
|
"github.com/fatih/color"
|
||||||
|
@ -12,9 +11,9 @@ import (
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"github.com/spf13/cobra/doc"
|
"github.com/spf13/cobra/doc"
|
||||||
|
"slices"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
|
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/database"
|
"github.com/crowdsecurity/crowdsec/pkg/database"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/fflag"
|
"github.com/crowdsecurity/crowdsec/pkg/fflag"
|
||||||
|
@ -29,15 +28,11 @@ var dbClient *database.Client
|
||||||
var OutputFormat string
|
var OutputFormat string
|
||||||
var OutputColor string
|
var OutputColor string
|
||||||
|
|
||||||
var downloadOnly bool
|
|
||||||
var forceAction bool
|
|
||||||
var purge bool
|
|
||||||
var all bool
|
|
||||||
|
|
||||||
var prometheusURL string
|
|
||||||
|
|
||||||
var mergedConfig string
|
var mergedConfig string
|
||||||
|
|
||||||
|
// flagBranch overrides the value in csConfig.Cscli.HubBranch
|
||||||
|
var flagBranch = ""
|
||||||
|
|
||||||
func initConfig() {
|
func initConfig() {
|
||||||
var err error
|
var err error
|
||||||
if trace_lvl {
|
if trace_lvl {
|
||||||
|
@ -58,10 +53,8 @@ func initConfig() {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := csConfig.LoadCSCLI(); err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
|
// XXX: check all the defaults
|
||||||
csConfig = csconfig.NewDefaultConfig()
|
csConfig = csconfig.NewDefaultConfig()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -71,13 +64,10 @@ func initConfig() {
|
||||||
log.Debugf("Enabled feature flags: %s", fflist)
|
log.Debugf("Enabled feature flags: %s", fflist)
|
||||||
}
|
}
|
||||||
|
|
||||||
if csConfig.Cscli == nil {
|
if flagBranch != "" {
|
||||||
log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
|
csConfig.Cscli.HubBranch = flagBranch
|
||||||
}
|
}
|
||||||
|
|
||||||
if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
|
|
||||||
cwhub.HubBranch = csConfig.Cscli.HubBranch
|
|
||||||
}
|
|
||||||
if OutputFormat != "" {
|
if OutputFormat != "" {
|
||||||
csConfig.Cscli.Output = OutputFormat
|
csConfig.Cscli.Output = OutputFormat
|
||||||
if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
|
if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
|
||||||
|
@ -206,7 +196,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
|
||||||
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
|
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
|
||||||
rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
|
rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
|
||||||
|
|
||||||
rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github")
|
rootCmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github")
|
||||||
if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
|
if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
|
||||||
log.Fatalf("failed to hide flag: %s", err)
|
log.Fatalf("failed to hide flag: %s", err)
|
||||||
}
|
}
|
||||||
|
@ -243,10 +233,6 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
|
||||||
rootCmd.AddCommand(NewSimulationCmds())
|
rootCmd.AddCommand(NewSimulationCmds())
|
||||||
rootCmd.AddCommand(NewBouncersCmd())
|
rootCmd.AddCommand(NewBouncersCmd())
|
||||||
rootCmd.AddCommand(NewMachinesCmd())
|
rootCmd.AddCommand(NewMachinesCmd())
|
||||||
rootCmd.AddCommand(NewParsersCmd())
|
|
||||||
rootCmd.AddCommand(NewScenariosCmd())
|
|
||||||
rootCmd.AddCommand(NewCollectionsCmd())
|
|
||||||
rootCmd.AddCommand(NewPostOverflowsCmd())
|
|
||||||
rootCmd.AddCommand(NewCapiCmd())
|
rootCmd.AddCommand(NewCapiCmd())
|
||||||
rootCmd.AddCommand(NewLapiCmd())
|
rootCmd.AddCommand(NewLapiCmd())
|
||||||
rootCmd.AddCommand(NewCompletionCmd())
|
rootCmd.AddCommand(NewCompletionCmd())
|
||||||
|
@ -255,6 +241,10 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
|
||||||
rootCmd.AddCommand(NewHubTestCmd())
|
rootCmd.AddCommand(NewHubTestCmd())
|
||||||
rootCmd.AddCommand(NewNotificationsCmd())
|
rootCmd.AddCommand(NewNotificationsCmd())
|
||||||
rootCmd.AddCommand(NewSupportCmd())
|
rootCmd.AddCommand(NewSupportCmd())
|
||||||
|
rootCmd.AddCommand(NewItemsCmd("collections"))
|
||||||
|
rootCmd.AddCommand(NewItemsCmd("parsers"))
|
||||||
|
rootCmd.AddCommand(NewItemsCmd("scenarios"))
|
||||||
|
rootCmd.AddCommand(NewItemsCmd("postoverflows"))
|
||||||
|
|
||||||
if fflag.CscliSetup.IsEnabled() {
|
if fflag.CscliSetup.IsEnabled() {
|
||||||
rootCmd.AddCommand(NewSetupCmd())
|
rootCmd.AddCommand(NewSetupCmd())
|
||||||
|
|
|
@ -284,8 +284,20 @@ var noUnit bool
|
||||||
|
|
||||||
|
|
||||||
func runMetrics(cmd *cobra.Command, args []string) error {
|
func runMetrics(cmd *cobra.Command, args []string) error {
|
||||||
if err := csConfig.LoadPrometheus(); err != nil {
|
flags := cmd.Flags()
|
||||||
return fmt.Errorf("failed to load prometheus config: %w", err)
|
|
||||||
|
url, err := flags.GetString("url")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if url != "" {
|
||||||
|
csConfig.Cscli.PrometheusUrl = url
|
||||||
|
}
|
||||||
|
|
||||||
|
noUnit, err = flags.GetBool("no-unit")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if csConfig.Prometheus == nil {
|
if csConfig.Prometheus == nil {
|
||||||
|
@ -296,17 +308,8 @@ func runMetrics(cmd *cobra.Command, args []string) error {
|
||||||
return fmt.Errorf("prometheus is not enabled, can't show metrics")
|
return fmt.Errorf("prometheus is not enabled, can't show metrics")
|
||||||
}
|
}
|
||||||
|
|
||||||
if prometheusURL == "" {
|
if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil {
|
||||||
prometheusURL = csConfig.Cscli.PrometheusUrl
|
return err
|
||||||
}
|
|
||||||
|
|
||||||
if prometheusURL == "" {
|
|
||||||
return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath)
|
|
||||||
}
|
|
||||||
|
|
||||||
err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("could not fetch prometheus metrics: %w", err)
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -321,8 +324,10 @@ func NewMetricsCmd() *cobra.Command {
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: runMetrics,
|
RunE: runMetrics,
|
||||||
}
|
}
|
||||||
cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
|
|
||||||
cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
|
flags := cmdMetrics.PersistentFlags()
|
||||||
|
flags.StringP("url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
|
||||||
|
flags.Bool("no-unit", false, "Show the real number instead of formatted with units")
|
||||||
|
|
||||||
return cmdMetrics
|
return cmdMetrics
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,194 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/fatih/color"
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
|
||||||
)
|
|
||||||
|
|
||||||
func NewParsersCmd() *cobra.Command {
|
|
||||||
var cmdParsers = &cobra.Command{
|
|
||||||
Use: "parsers [action] [config]",
|
|
||||||
Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
|
|
||||||
Example: `cscli parsers install crowdsecurity/sshd-logs
|
|
||||||
cscli parsers inspect crowdsecurity/sshd-logs
|
|
||||||
cscli parsers upgrade crowdsecurity/sshd-logs
|
|
||||||
cscli parsers list
|
|
||||||
cscli parsers remove crowdsecurity/sshd-logs
|
|
||||||
`,
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
Aliases: []string{"parser"},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
PersistentPostRun: func(cmd *cobra.Command, args []string) {
|
|
||||||
if cmd.Name() == "inspect" || cmd.Name() == "list" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
log.Infof(ReloadMessage())
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdParsers.AddCommand(NewParsersInstallCmd())
|
|
||||||
cmdParsers.AddCommand(NewParsersRemoveCmd())
|
|
||||||
cmdParsers.AddCommand(NewParsersUpgradeCmd())
|
|
||||||
cmdParsers.AddCommand(NewParsersInspectCmd())
|
|
||||||
cmdParsers.AddCommand(NewParsersListCmd())
|
|
||||||
|
|
||||||
return cmdParsers
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewParsersInstallCmd() *cobra.Command {
|
|
||||||
var ignoreError bool
|
|
||||||
|
|
||||||
var cmdParsersInstall = &cobra.Command{
|
|
||||||
Use: "install [config]",
|
|
||||||
Short: "Install given parser(s)",
|
|
||||||
Long: `Fetch and install given parser(s) from hub`,
|
|
||||||
Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`,
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compAllItems(cwhub.PARSERS, args, toComplete)
|
|
||||||
},
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
for _, name := range args {
|
|
||||||
t := cwhub.GetItem(cwhub.PARSERS, name)
|
|
||||||
if t == nil {
|
|
||||||
nearestItem, score := GetDistance(cwhub.PARSERS, name)
|
|
||||||
Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
|
|
||||||
if !ignoreError {
|
|
||||||
return fmt.Errorf("error while installing '%s': %w", name, err)
|
|
||||||
}
|
|
||||||
log.Errorf("Error while installing '%s': %s", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
|
|
||||||
cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
|
|
||||||
cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
|
|
||||||
|
|
||||||
return cmdParsersInstall
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewParsersRemoveCmd() *cobra.Command {
|
|
||||||
cmdParsersRemove := &cobra.Command{
|
|
||||||
Use: "remove [config]",
|
|
||||||
Short: "Remove given parser(s)",
|
|
||||||
Long: `Remove given parse(s) from hub`,
|
|
||||||
Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
|
|
||||||
Aliases: []string{"delete"},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.PARSERS, args, toComplete)
|
|
||||||
},
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if all {
|
|
||||||
cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(args) == 0 {
|
|
||||||
return fmt.Errorf("specify at least one parser to remove or '--all'")
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range args {
|
|
||||||
cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
|
|
||||||
cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
|
|
||||||
cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
|
|
||||||
|
|
||||||
return cmdParsersRemove
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewParsersUpgradeCmd() *cobra.Command {
|
|
||||||
cmdParsersUpgrade := &cobra.Command{
|
|
||||||
Use: "upgrade [config]",
|
|
||||||
Short: "Upgrade given parser(s)",
|
|
||||||
Long: `Fetch and upgrade given parser(s) from hub`,
|
|
||||||
Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.PARSERS, args, toComplete)
|
|
||||||
},
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if all {
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
|
|
||||||
} else {
|
|
||||||
if len(args) == 0 {
|
|
||||||
return fmt.Errorf("specify at least one parser to upgrade or '--all'")
|
|
||||||
}
|
|
||||||
for _, name := range args {
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
|
|
||||||
cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
|
|
||||||
|
|
||||||
return cmdParsersUpgrade
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewParsersInspectCmd() *cobra.Command {
|
|
||||||
var cmdParsersInspect = &cobra.Command{
|
|
||||||
Use: "inspect [name]",
|
|
||||||
Short: "Inspect given parser",
|
|
||||||
Long: `Inspect given parser`,
|
|
||||||
Example: `cscli parsers inspect crowdsec/xxx`,
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.PARSERS, args, toComplete)
|
|
||||||
},
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
InspectItem(args[0], cwhub.PARSERS)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
|
|
||||||
|
|
||||||
return cmdParsersInspect
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewParsersListCmd() *cobra.Command {
|
|
||||||
var cmdParsersList = &cobra.Command{
|
|
||||||
Use: "list [name]",
|
|
||||||
Short: "List all parsers or given one",
|
|
||||||
Long: `List all parsers or given one`,
|
|
||||||
Example: `cscli parsers list
|
|
||||||
cscli parser list crowdsecurity/xxx`,
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
|
|
||||||
|
|
||||||
return cmdParsersList
|
|
||||||
}
|
|
|
@ -1,191 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/fatih/color"
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
|
||||||
)
|
|
||||||
|
|
||||||
func NewPostOverflowsCmd() *cobra.Command {
|
|
||||||
cmdPostOverflows := &cobra.Command{
|
|
||||||
Use: "postoverflows [action] [config]",
|
|
||||||
Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
|
|
||||||
Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
|
|
||||||
cscli postoverflows inspect crowdsecurity/cdn-whitelist
|
|
||||||
cscli postoverflows upgrade crowdsecurity/cdn-whitelist
|
|
||||||
cscli postoverflows list
|
|
||||||
cscli postoverflows remove crowdsecurity/cdn-whitelist`,
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
Aliases: []string{"postoverflow"},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
PersistentPostRun: func(cmd *cobra.Command, args []string) {
|
|
||||||
if cmd.Name() == "inspect" || cmd.Name() == "list" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
log.Infof(ReloadMessage())
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
|
|
||||||
cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
|
|
||||||
cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
|
|
||||||
cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
|
|
||||||
cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
|
|
||||||
|
|
||||||
return cmdPostOverflows
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewPostOverflowsInstallCmd() *cobra.Command {
|
|
||||||
var ignoreError bool
|
|
||||||
|
|
||||||
cmdPostOverflowsInstall := &cobra.Command{
|
|
||||||
Use: "install [config]",
|
|
||||||
Short: "Install given postoverflow(s)",
|
|
||||||
Long: `Fetch and install given postoverflow(s) from hub`,
|
|
||||||
Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
|
|
||||||
},
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
for _, name := range args {
|
|
||||||
t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
|
|
||||||
if t == nil {
|
|
||||||
nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
|
|
||||||
Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
|
|
||||||
if !ignoreError {
|
|
||||||
return fmt.Errorf("error while installing '%s': %w", name, err)
|
|
||||||
}
|
|
||||||
log.Errorf("Error while installing '%s': %s", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
|
|
||||||
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
|
|
||||||
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows")
|
|
||||||
|
|
||||||
return cmdPostOverflowsInstall
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewPostOverflowsRemoveCmd() *cobra.Command {
|
|
||||||
cmdPostOverflowsRemove := &cobra.Command{
|
|
||||||
Use: "remove [config]",
|
|
||||||
Short: "Remove given postoverflow(s)",
|
|
||||||
Long: `remove given postoverflow(s)`,
|
|
||||||
Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
|
|
||||||
Aliases: []string{"delete"},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
|
|
||||||
},
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if all {
|
|
||||||
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(args) == 0 {
|
|
||||||
return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range args {
|
|
||||||
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
|
|
||||||
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
|
|
||||||
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows")
|
|
||||||
|
|
||||||
return cmdPostOverflowsRemove
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewPostOverflowsUpgradeCmd() *cobra.Command {
|
|
||||||
cmdPostOverflowsUpgrade := &cobra.Command{
|
|
||||||
Use: "upgrade [config]",
|
|
||||||
Short: "Upgrade given postoverflow(s)",
|
|
||||||
Long: `Fetch and Upgrade given postoverflow(s) from hub`,
|
|
||||||
Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
|
|
||||||
},
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if all {
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
|
|
||||||
} else {
|
|
||||||
if len(args) == 0 {
|
|
||||||
return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
|
|
||||||
}
|
|
||||||
for _, name := range args {
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows")
|
|
||||||
cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
|
|
||||||
|
|
||||||
return cmdPostOverflowsUpgrade
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewPostOverflowsInspectCmd() *cobra.Command {
|
|
||||||
cmdPostOverflowsInspect := &cobra.Command{
|
|
||||||
Use: "inspect [config]",
|
|
||||||
Short: "Inspect given postoverflow",
|
|
||||||
Long: `Inspect given postoverflow`,
|
|
||||||
Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
|
|
||||||
},
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
InspectItem(args[0], cwhub.PARSERS_OVFLW)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
return cmdPostOverflowsInspect
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewPostOverflowsListCmd() *cobra.Command {
|
|
||||||
cmdPostOverflowsList := &cobra.Command{
|
|
||||||
Use: "list [config]",
|
|
||||||
Short: "List all postoverflows or given one",
|
|
||||||
Long: `List all postoverflows or given one`,
|
|
||||||
Example: `cscli postoverflows list
|
|
||||||
cscli postoverflows list crowdsecurity/xxx`,
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
|
|
||||||
|
|
||||||
return cmdPostOverflowsList
|
|
||||||
}
|
|
58
cmd/crowdsec-cli/require/branch.go
Normal file
58
cmd/crowdsec-cli/require/branch.go
Normal file
|
@ -0,0 +1,58 @@
|
||||||
|
package require
|
||||||
|
|
||||||
|
// Set the appropriate hub branch according to config settings and crowdsec version
|
||||||
|
|
||||||
|
import (
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/mod/semver"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
|
)
|
||||||
|
|
||||||
|
func chooseBranch(cfg *csconfig.Config) string {
|
||||||
|
// this was set from config.yaml or flag
|
||||||
|
if cfg.Cscli.HubBranch != "" {
|
||||||
|
log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
|
||||||
|
return cfg.Cscli.HubBranch
|
||||||
|
}
|
||||||
|
|
||||||
|
latest, err := cwversion.Latest()
|
||||||
|
if err != nil {
|
||||||
|
log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
|
||||||
|
return "master"
|
||||||
|
}
|
||||||
|
|
||||||
|
csVersion := cwversion.VersionStrip()
|
||||||
|
if csVersion == latest {
|
||||||
|
log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
|
||||||
|
return "master"
|
||||||
|
}
|
||||||
|
|
||||||
|
// if current version is greater than the latest we are in pre-release
|
||||||
|
if semver.Compare(csVersion, latest) == 1 {
|
||||||
|
log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
|
||||||
|
return "master"
|
||||||
|
}
|
||||||
|
|
||||||
|
if csVersion == "" {
|
||||||
|
log.Warning("Crowdsec version is not set, using hub branch 'master'")
|
||||||
|
return "master"
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Warnf("A new CrowdSec release is available (%s). "+
|
||||||
|
"Your version is '%s'. Please update it to use new parsers/scenarios/collections.",
|
||||||
|
latest, csVersion)
|
||||||
|
return csVersion
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// HubBranch sets the branch (in cscli config) and returns its value
|
||||||
|
// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
|
||||||
|
func HubBranch(cfg *csconfig.Config) string {
|
||||||
|
branch := chooseBranch(cfg)
|
||||||
|
|
||||||
|
cfg.Cscli.HubBranch = branch
|
||||||
|
|
||||||
|
return branch
|
||||||
|
}
|
|
@ -23,6 +23,7 @@ func CAPI(c *csconfig.Config) error {
|
||||||
if c.API.Server.OnlineClient == nil {
|
if c.API.Server.OnlineClient == nil {
|
||||||
return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
|
return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -30,6 +31,7 @@ func PAPI(c *csconfig.Config) error {
|
||||||
if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
|
if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
|
||||||
return fmt.Errorf("no PAPI URL in configuration")
|
return fmt.Errorf("no PAPI URL in configuration")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -45,6 +47,7 @@ func DB(c *csconfig.Config) error {
|
||||||
if err := c.LoadDBConfig(); err != nil {
|
if err := c.LoadDBConfig(); err != nil {
|
||||||
return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
|
return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -64,20 +67,33 @@ func Notifications(c *csconfig.Config) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func Hub (c *csconfig.Config) error {
|
// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
|
||||||
if err := c.LoadHub(); err != nil {
|
func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
|
||||||
return err
|
// set branch in config, and log if necessary
|
||||||
|
branch := HubBranch(c)
|
||||||
|
remote := &cwhub.RemoteHubCfg {
|
||||||
|
Branch: branch,
|
||||||
|
URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
|
||||||
|
// URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s",
|
||||||
|
IndexPath: ".index.json",
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.Hub == nil {
|
return remote
|
||||||
return fmt.Errorf("you must configure cli before interacting with hub")
|
}
|
||||||
}
|
|
||||||
|
// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
|
||||||
cwhub.SetHubBranch()
|
// If no remote parameter is provided, the hub can only be used for local operations.
|
||||||
|
func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) {
|
||||||
if err := cwhub.GetHubIdx(c.Hub); err != nil {
|
local := c.Hub
|
||||||
return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
|
|
||||||
}
|
if local == nil {
|
||||||
|
return nil, fmt.Errorf("you must configure cli before interacting with hub")
|
||||||
return nil
|
}
|
||||||
|
|
||||||
|
hub, err := cwhub.NewHub(local, remote, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return hub, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,188 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/fatih/color"
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
|
||||||
)
|
|
||||||
|
|
||||||
func NewScenariosCmd() *cobra.Command {
|
|
||||||
var cmdScenarios = &cobra.Command{
|
|
||||||
Use: "scenarios [action] [config]",
|
|
||||||
Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
|
|
||||||
Example: `cscli scenarios list [-a]
|
|
||||||
cscli scenarios install crowdsecurity/ssh-bf
|
|
||||||
cscli scenarios inspect crowdsecurity/ssh-bf
|
|
||||||
cscli scenarios upgrade crowdsecurity/ssh-bf
|
|
||||||
cscli scenarios remove crowdsecurity/ssh-bf
|
|
||||||
`,
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
Aliases: []string{"scenario"},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
PersistentPostRun: func(cmd *cobra.Command, args []string) {
|
|
||||||
if cmd.Name() == "inspect" || cmd.Name() == "list" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
log.Infof(ReloadMessage())
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdScenarios.AddCommand(NewCmdScenariosInstall())
|
|
||||||
cmdScenarios.AddCommand(NewCmdScenariosRemove())
|
|
||||||
cmdScenarios.AddCommand(NewCmdScenariosUpgrade())
|
|
||||||
cmdScenarios.AddCommand(NewCmdScenariosInspect())
|
|
||||||
cmdScenarios.AddCommand(NewCmdScenariosList())
|
|
||||||
|
|
||||||
return cmdScenarios
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCmdScenariosInstall() *cobra.Command {
|
|
||||||
var ignoreError bool
|
|
||||||
|
|
||||||
var cmdScenariosInstall = &cobra.Command{
|
|
||||||
Use: "install [config]",
|
|
||||||
Short: "Install given scenario(s)",
|
|
||||||
Long: `Fetch and install given scenario(s) from hub`,
|
|
||||||
Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compAllItems(cwhub.SCENARIOS, args, toComplete)
|
|
||||||
},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
for _, name := range args {
|
|
||||||
t := cwhub.GetItem(cwhub.SCENARIOS, name)
|
|
||||||
if t == nil {
|
|
||||||
nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
|
|
||||||
Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
|
|
||||||
if !ignoreError {
|
|
||||||
return fmt.Errorf("error while installing '%s': %w", name, err)
|
|
||||||
}
|
|
||||||
log.Errorf("Error while installing '%s': %s", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
|
|
||||||
cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
|
|
||||||
cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios")
|
|
||||||
|
|
||||||
return cmdScenariosInstall
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCmdScenariosRemove() *cobra.Command {
|
|
||||||
var cmdScenariosRemove = &cobra.Command{
|
|
||||||
Use: "remove [config]",
|
|
||||||
Short: "Remove given scenario(s)",
|
|
||||||
Long: `remove given scenario(s)`,
|
|
||||||
Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
|
|
||||||
Aliases: []string{"delete"},
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
|
|
||||||
},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if all {
|
|
||||||
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(args) == 0 {
|
|
||||||
return fmt.Errorf("specify at least one scenario to remove or '--all'")
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range args {
|
|
||||||
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
|
|
||||||
cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
|
|
||||||
cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios")
|
|
||||||
|
|
||||||
return cmdScenariosRemove
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCmdScenariosUpgrade() *cobra.Command {
|
|
||||||
var cmdScenariosUpgrade = &cobra.Command{
|
|
||||||
Use: "upgrade [config]",
|
|
||||||
Short: "Upgrade given scenario(s)",
|
|
||||||
Long: `Fetch and Upgrade given scenario(s) from hub`,
|
|
||||||
Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
|
|
||||||
},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if all {
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
|
|
||||||
} else {
|
|
||||||
if len(args) == 0 {
|
|
||||||
return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
|
|
||||||
}
|
|
||||||
for _, name := range args {
|
|
||||||
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")
|
|
||||||
cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
|
|
||||||
|
|
||||||
return cmdScenariosUpgrade
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCmdScenariosInspect() *cobra.Command {
|
|
||||||
var cmdScenariosInspect = &cobra.Command{
|
|
||||||
Use: "inspect [config]",
|
|
||||||
Short: "Inspect given scenario",
|
|
||||||
Long: `Inspect given scenario`,
|
|
||||||
Example: `cscli scenarios inspect crowdsec/xxx`,
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
|
||||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
|
|
||||||
},
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
InspectItem(args[0], cwhub.SCENARIOS)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
|
|
||||||
|
|
||||||
return cmdScenariosInspect
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCmdScenariosList() *cobra.Command {
|
|
||||||
var cmdScenariosList = &cobra.Command{
|
|
||||||
Use: "list [config]",
|
|
||||||
Short: "List all scenario(s) or given one",
|
|
||||||
Long: `List all scenario(s) or given one`,
|
|
||||||
Example: `cscli scenarios list
|
|
||||||
cscli scenarios list crowdsecurity/xxx`,
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
|
|
||||||
|
|
||||||
return cmdScenariosList
|
|
||||||
}
|
|
|
@ -6,13 +6,15 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
|
||||||
|
goccyyaml "github.com/goccy/go-yaml"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
goccyyaml "github.com/goccy/go-yaml"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/setup"
|
"github.com/crowdsecurity/crowdsec/pkg/setup"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewSetupCmd defines the "cscli setup" command.
|
// NewSetupCmd defines the "cscli setup" command.
|
||||||
|
@ -303,7 +305,12 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
|
||||||
return fmt.Errorf("while reading file %s: %w", fromFile, err)
|
return fmt.Errorf("while reading file %s: %w", fromFile, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil {
|
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = setup.InstallHubItems(hub, input, dryRun); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,11 +3,11 @@ package main
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"slices"
|
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"gopkg.in/yaml.v2"
|
"gopkg.in/yaml.v2"
|
||||||
|
"slices"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
|
@ -112,9 +112,6 @@ cscli simulation disable crowdsecurity/ssh-bf`,
|
||||||
if err := csConfig.LoadSimulation(); err != nil {
|
if err := csConfig.LoadSimulation(); err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
if csConfig.Cscli == nil {
|
|
||||||
return fmt.Errorf("you must configure cli before using simulation")
|
|
||||||
}
|
|
||||||
if csConfig.Cscli.SimulationConfig == nil {
|
if csConfig.Cscli.SimulationConfig == nil {
|
||||||
return fmt.Errorf("no simulation configured")
|
return fmt.Errorf("no simulation configured")
|
||||||
}
|
}
|
||||||
|
@ -145,13 +142,14 @@ func NewSimulationEnableCmd() *cobra.Command {
|
||||||
Example: `cscli simulation enable`,
|
Example: `cscli simulation enable`,
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
if err := require.Hub(csConfig); err != nil {
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(args) > 0 {
|
if len(args) > 0 {
|
||||||
for _, scenario := range args {
|
for _, scenario := range args {
|
||||||
var item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
|
var item = hub.GetItem(cwhub.SCENARIOS, scenario)
|
||||||
if item == nil {
|
if item == nil {
|
||||||
log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
|
log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
|
||||||
continue
|
continue
|
||||||
|
|
|
@ -58,10 +58,6 @@ func stripAnsiString(str string) string {
|
||||||
|
|
||||||
func collectMetrics() ([]byte, []byte, error) {
|
func collectMetrics() ([]byte, []byte, error) {
|
||||||
log.Info("Collecting prometheus metrics")
|
log.Info("Collecting prometheus metrics")
|
||||||
err := csConfig.LoadPrometheus()
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if csConfig.Cscli.PrometheusUrl == "" {
|
if csConfig.Cscli.PrometheusUrl == "" {
|
||||||
log.Warn("No Prometheus URL configured, metrics will not be collected")
|
log.Warn("No Prometheus URL configured, metrics will not be collected")
|
||||||
|
@ -69,13 +65,13 @@ func collectMetrics() ([]byte, []byte, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
humanMetrics := bytes.NewBuffer(nil)
|
humanMetrics := bytes.NewBuffer(nil)
|
||||||
err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human")
|
err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human")
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
|
return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil)
|
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
|
return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
|
||||||
}
|
}
|
||||||
|
@ -132,10 +128,21 @@ func collectOSInfo() ([]byte, error) {
|
||||||
return w.Bytes(), nil
|
return w.Bytes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectHubItems(itemType string) []byte {
|
func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
|
||||||
|
var err error
|
||||||
|
|
||||||
out := bytes.NewBuffer(nil)
|
out := bytes.NewBuffer(nil)
|
||||||
log.Infof("Collecting %s list", itemType)
|
log.Infof("Collecting %s list", itemType)
|
||||||
ListItems(out, []string{itemType}, []string{}, false, true, all)
|
|
||||||
|
items := make(map[string][]*cwhub.Item)
|
||||||
|
|
||||||
|
if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil {
|
||||||
|
log.Warnf("could not collect %s list: %s", itemType, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := listItems(out, []string{itemType}, items); err != nil {
|
||||||
|
log.Warnf("could not collect %s list: %s", itemType, err)
|
||||||
|
}
|
||||||
return out.Bytes()
|
return out.Bytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -157,7 +164,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) {
|
||||||
return out.Bytes(), nil
|
return out.Bytes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte {
|
func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
|
||||||
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
|
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
|
||||||
return []byte("No agent credentials found, are we LAPI ?")
|
return []byte("No agent credentials found, are we LAPI ?")
|
||||||
}
|
}
|
||||||
|
@ -167,7 +174,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
|
return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
|
||||||
}
|
}
|
||||||
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
|
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
|
return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
|
||||||
}
|
}
|
||||||
|
@ -295,7 +302,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
|
||||||
skipAgent = true
|
skipAgent = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
hub, err := require.Hub(csConfig, nil)
|
||||||
|
if err != nil {
|
||||||
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
|
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
|
||||||
skipHub = true
|
skipHub = true
|
||||||
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
|
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
|
||||||
|
@ -333,10 +341,10 @@ cscli support dump -f /tmp/crowdsec-support.zip
|
||||||
infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
|
infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
|
||||||
|
|
||||||
if !skipHub {
|
if !skipHub {
|
||||||
infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
|
infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
|
||||||
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
|
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
|
||||||
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW)
|
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
|
||||||
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
|
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !skipDB {
|
if !skipDB {
|
||||||
|
@ -358,7 +366,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
|
||||||
infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
|
infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
|
||||||
csConfig.API.Server.OnlineClient.Credentials.Password,
|
csConfig.API.Server.OnlineClient.Credentials.Password,
|
||||||
csConfig.API.Server.OnlineClient.Credentials.URL,
|
csConfig.API.Server.OnlineClient.Credentials.URL,
|
||||||
CAPIURLPrefix)
|
CAPIURLPrefix,
|
||||||
|
hub)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !skipLAPI {
|
if !skipLAPI {
|
||||||
|
@ -366,7 +375,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
|
||||||
infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
|
infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
|
||||||
csConfig.API.Client.Credentials.Password,
|
csConfig.API.Client.Credentials.Password,
|
||||||
csConfig.API.Client.Credentials.URL,
|
csConfig.API.Client.Credentials.URL,
|
||||||
LAPIURLPrefix)
|
LAPIURLPrefix,
|
||||||
|
hub)
|
||||||
infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
|
infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,36 +1,17 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/csv"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"math"
|
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fatih/color"
|
|
||||||
dto "github.com/prometheus/client_model/go"
|
|
||||||
"github.com/prometheus/prom2json"
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"github.com/agext/levenshtein"
|
|
||||||
"gopkg.in/yaml.v2"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/go-cs-lib/trace"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/database"
|
"github.com/crowdsecurity/crowdsec/pkg/database"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
const MaxDistance = 7
|
|
||||||
|
|
||||||
func printHelp(cmd *cobra.Command) {
|
func printHelp(cmd *cobra.Command) {
|
||||||
err := cmd.Help()
|
err := cmd.Help()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -38,197 +19,6 @@ func printHelp(cmd *cobra.Command) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
|
|
||||||
errMsg := ""
|
|
||||||
if score < MaxDistance {
|
|
||||||
errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
|
|
||||||
} else {
|
|
||||||
errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
|
|
||||||
}
|
|
||||||
if ignoreErr {
|
|
||||||
log.Error(errMsg)
|
|
||||||
} else {
|
|
||||||
log.Fatalf(errMsg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
|
|
||||||
allItems := make([]string, 0)
|
|
||||||
nearestScore := 100
|
|
||||||
nearestItem := &cwhub.Item{}
|
|
||||||
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
|
|
||||||
for _, item := range hubItems {
|
|
||||||
allItems = append(allItems, item.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, s := range allItems {
|
|
||||||
d := levenshtein.Distance(itemName, s, nil)
|
|
||||||
if d < nearestScore {
|
|
||||||
nearestScore = d
|
|
||||||
nearestItem = cwhub.GetItem(itemType, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nearestItem, nearestScore
|
|
||||||
}
|
|
||||||
|
|
||||||
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
|
||||||
return nil, cobra.ShellCompDirectiveDefault
|
|
||||||
}
|
|
||||||
|
|
||||||
comp := make([]string, 0)
|
|
||||||
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
|
|
||||||
for _, item := range hubItems {
|
|
||||||
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
|
|
||||||
comp = append(comp, item.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
|
|
||||||
return comp, cobra.ShellCompDirectiveNoFileComp
|
|
||||||
}
|
|
||||||
|
|
||||||
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
|
||||||
if err := require.Hub(csConfig); err != nil {
|
|
||||||
return nil, cobra.ShellCompDirectiveDefault
|
|
||||||
}
|
|
||||||
|
|
||||||
items, err := cwhub.GetInstalledItemsAsString(itemType)
|
|
||||||
if err != nil {
|
|
||||||
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
|
|
||||||
return nil, cobra.ShellCompDirectiveDefault
|
|
||||||
}
|
|
||||||
|
|
||||||
comp := make([]string, 0)
|
|
||||||
|
|
||||||
if toComplete != "" {
|
|
||||||
for _, item := range items {
|
|
||||||
if strings.Contains(item, toComplete) {
|
|
||||||
comp = append(comp, item)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
comp = items
|
|
||||||
}
|
|
||||||
|
|
||||||
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
|
|
||||||
|
|
||||||
return comp, cobra.ShellCompDirectiveNoFileComp
|
|
||||||
}
|
|
||||||
|
|
||||||
func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
|
|
||||||
var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
|
|
||||||
|
|
||||||
for _, itemType := range itemTypes {
|
|
||||||
itemName := ""
|
|
||||||
if len(args) == 1 {
|
|
||||||
itemName = args[0]
|
|
||||||
}
|
|
||||||
hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
|
|
||||||
}
|
|
||||||
|
|
||||||
if csConfig.Cscli.Output == "human" {
|
|
||||||
for _, itemType := range itemTypes {
|
|
||||||
var statuses []cwhub.ItemHubStatus
|
|
||||||
var ok bool
|
|
||||||
if statuses, ok = hubStatusByItemType[itemType]; !ok {
|
|
||||||
log.Errorf("unknown item type: %s", itemType)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
|
|
||||||
}
|
|
||||||
} else if csConfig.Cscli.Output == "json" {
|
|
||||||
x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("failed to unmarshal")
|
|
||||||
}
|
|
||||||
out.Write(x)
|
|
||||||
} else if csConfig.Cscli.Output == "raw" {
|
|
||||||
csvwriter := csv.NewWriter(out)
|
|
||||||
if showHeader {
|
|
||||||
header := []string{"name", "status", "version", "description"}
|
|
||||||
if showType {
|
|
||||||
header = append(header, "type")
|
|
||||||
}
|
|
||||||
err := csvwriter.Write(header)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("failed to write header: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
for _, itemType := range itemTypes {
|
|
||||||
var statuses []cwhub.ItemHubStatus
|
|
||||||
var ok bool
|
|
||||||
if statuses, ok = hubStatusByItemType[itemType]; !ok {
|
|
||||||
log.Errorf("unknown item type: %s", itemType)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, status := range statuses {
|
|
||||||
if status.LocalVersion == "" {
|
|
||||||
status.LocalVersion = "n/a"
|
|
||||||
}
|
|
||||||
row := []string{
|
|
||||||
status.Name,
|
|
||||||
status.Status,
|
|
||||||
status.LocalVersion,
|
|
||||||
status.Description,
|
|
||||||
}
|
|
||||||
if showType {
|
|
||||||
row = append(row, itemType)
|
|
||||||
}
|
|
||||||
err := csvwriter.Write(row)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("failed to write raw output : %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
csvwriter.Flush()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func InspectItem(name string, objecitemType string) {
|
|
||||||
|
|
||||||
hubItem := cwhub.GetItem(objecitemType, name)
|
|
||||||
if hubItem == nil {
|
|
||||||
log.Fatalf("unable to retrieve item.")
|
|
||||||
}
|
|
||||||
var b []byte
|
|
||||||
var err error
|
|
||||||
switch csConfig.Cscli.Output {
|
|
||||||
case "human", "raw":
|
|
||||||
b, err = yaml.Marshal(*hubItem)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("unable to marshal item : %s", err)
|
|
||||||
}
|
|
||||||
case "json":
|
|
||||||
b, err = json.MarshalIndent(*hubItem, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("unable to marshal item : %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fmt.Printf("%s", string(b))
|
|
||||||
if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if prometheusURL == "" {
|
|
||||||
//This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus
|
|
||||||
//But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case
|
|
||||||
listenAddr := csConfig.Prometheus.ListenAddr
|
|
||||||
if listenAddr == "" {
|
|
||||||
listenAddr = "127.0.0.1"
|
|
||||||
}
|
|
||||||
listenPort := csConfig.Prometheus.ListenPort
|
|
||||||
if listenPort == 0 {
|
|
||||||
listenPort = 6060
|
|
||||||
}
|
|
||||||
prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort)
|
|
||||||
log.Debugf("No prometheus URL provided using: %s", prometheusURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("\nCurrent metrics : \n")
|
|
||||||
ShowMetrics(hubItem)
|
|
||||||
}
|
|
||||||
|
|
||||||
func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
|
func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
|
||||||
|
|
||||||
/*if a range is provided, change the scope*/
|
/*if a range is provided, change the scope*/
|
||||||
|
@ -259,232 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func ShowMetrics(hubItem *cwhub.Item) {
|
|
||||||
switch hubItem.Type {
|
|
||||||
case cwhub.PARSERS:
|
|
||||||
metrics := GetParserMetric(prometheusURL, hubItem.Name)
|
|
||||||
parserMetricsTable(color.Output, hubItem.Name, metrics)
|
|
||||||
case cwhub.SCENARIOS:
|
|
||||||
metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
|
|
||||||
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
|
|
||||||
case cwhub.COLLECTIONS:
|
|
||||||
for _, item := range hubItem.Parsers {
|
|
||||||
metrics := GetParserMetric(prometheusURL, item)
|
|
||||||
parserMetricsTable(color.Output, item, metrics)
|
|
||||||
}
|
|
||||||
for _, item := range hubItem.Scenarios {
|
|
||||||
metrics := GetScenarioMetric(prometheusURL, item)
|
|
||||||
scenarioMetricsTable(color.Output, item, metrics)
|
|
||||||
}
|
|
||||||
for _, item := range hubItem.Collections {
|
|
||||||
hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
|
|
||||||
if hubItem == nil {
|
|
||||||
log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
|
|
||||||
}
|
|
||||||
ShowMetrics(hubItem)
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
log.Errorf("item of type '%s' is unknown", hubItem.Type)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetParserMetric is a complete rip from prom2json
|
|
||||||
func GetParserMetric(url string, itemName string) map[string]map[string]int {
|
|
||||||
stats := make(map[string]map[string]int)
|
|
||||||
|
|
||||||
result := GetPrometheusMetric(url)
|
|
||||||
for idx, fam := range result {
|
|
||||||
if !strings.HasPrefix(fam.Name, "cs_") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
log.Tracef("round %d", idx)
|
|
||||||
for _, m := range fam.Metrics {
|
|
||||||
metric, ok := m.(prom2json.Metric)
|
|
||||||
if !ok {
|
|
||||||
log.Debugf("failed to convert metric to prom2json.Metric")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
name, ok := metric.Labels["name"]
|
|
||||||
if !ok {
|
|
||||||
log.Debugf("no name in Metric %v", metric.Labels)
|
|
||||||
}
|
|
||||||
if name != itemName {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
source, ok := metric.Labels["source"]
|
|
||||||
if !ok {
|
|
||||||
log.Debugf("no source in Metric %v", metric.Labels)
|
|
||||||
} else {
|
|
||||||
if srctype, ok := metric.Labels["type"]; ok {
|
|
||||||
source = srctype + ":" + source
|
|
||||||
}
|
|
||||||
}
|
|
||||||
value := m.(prom2json.Metric).Value
|
|
||||||
fval, err := strconv.ParseFloat(value, 32)
|
|
||||||
if err != nil {
|
|
||||||
log.Errorf("Unexpected int value %s : %s", value, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ival := int(fval)
|
|
||||||
|
|
||||||
switch fam.Name {
|
|
||||||
case "cs_reader_hits_total":
|
|
||||||
if _, ok := stats[source]; !ok {
|
|
||||||
stats[source] = make(map[string]int)
|
|
||||||
stats[source]["parsed"] = 0
|
|
||||||
stats[source]["reads"] = 0
|
|
||||||
stats[source]["unparsed"] = 0
|
|
||||||
stats[source]["hits"] = 0
|
|
||||||
}
|
|
||||||
stats[source]["reads"] += ival
|
|
||||||
case "cs_parser_hits_ok_total":
|
|
||||||
if _, ok := stats[source]; !ok {
|
|
||||||
stats[source] = make(map[string]int)
|
|
||||||
}
|
|
||||||
stats[source]["parsed"] += ival
|
|
||||||
case "cs_parser_hits_ko_total":
|
|
||||||
if _, ok := stats[source]; !ok {
|
|
||||||
stats[source] = make(map[string]int)
|
|
||||||
}
|
|
||||||
stats[source]["unparsed"] += ival
|
|
||||||
case "cs_node_hits_total":
|
|
||||||
if _, ok := stats[source]; !ok {
|
|
||||||
stats[source] = make(map[string]int)
|
|
||||||
}
|
|
||||||
stats[source]["hits"] += ival
|
|
||||||
case "cs_node_hits_ok_total":
|
|
||||||
if _, ok := stats[source]; !ok {
|
|
||||||
stats[source] = make(map[string]int)
|
|
||||||
}
|
|
||||||
stats[source]["parsed"] += ival
|
|
||||||
case "cs_node_hits_ko_total":
|
|
||||||
if _, ok := stats[source]; !ok {
|
|
||||||
stats[source] = make(map[string]int)
|
|
||||||
}
|
|
||||||
stats[source]["unparsed"] += ival
|
|
||||||
default:
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return stats
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetScenarioMetric(url string, itemName string) map[string]int {
|
|
||||||
stats := make(map[string]int)
|
|
||||||
|
|
||||||
stats["instantiation"] = 0
|
|
||||||
stats["curr_count"] = 0
|
|
||||||
stats["overflow"] = 0
|
|
||||||
stats["pour"] = 0
|
|
||||||
stats["underflow"] = 0
|
|
||||||
|
|
||||||
result := GetPrometheusMetric(url)
|
|
||||||
for idx, fam := range result {
|
|
||||||
if !strings.HasPrefix(fam.Name, "cs_") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
log.Tracef("round %d", idx)
|
|
||||||
for _, m := range fam.Metrics {
|
|
||||||
metric, ok := m.(prom2json.Metric)
|
|
||||||
if !ok {
|
|
||||||
log.Debugf("failed to convert metric to prom2json.Metric")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
name, ok := metric.Labels["name"]
|
|
||||||
if !ok {
|
|
||||||
log.Debugf("no name in Metric %v", metric.Labels)
|
|
||||||
}
|
|
||||||
if name != itemName {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
value := m.(prom2json.Metric).Value
|
|
||||||
fval, err := strconv.ParseFloat(value, 32)
|
|
||||||
if err != nil {
|
|
||||||
log.Errorf("Unexpected int value %s : %s", value, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ival := int(fval)
|
|
||||||
|
|
||||||
switch fam.Name {
|
|
||||||
case "cs_bucket_created_total":
|
|
||||||
stats["instantiation"] += ival
|
|
||||||
case "cs_buckets":
|
|
||||||
stats["curr_count"] += ival
|
|
||||||
case "cs_bucket_overflowed_total":
|
|
||||||
stats["overflow"] += ival
|
|
||||||
case "cs_bucket_poured_total":
|
|
||||||
stats["pour"] += ival
|
|
||||||
case "cs_bucket_underflowed_total":
|
|
||||||
stats["underflow"] += ival
|
|
||||||
default:
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return stats
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetPrometheusMetric(url string) []*prom2json.Family {
|
|
||||||
mfChan := make(chan *dto.MetricFamily, 1024)
|
|
||||||
|
|
||||||
// Start with the DefaultTransport for sane defaults.
|
|
||||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
|
||||||
// Conservatively disable HTTP keep-alives as this program will only
|
|
||||||
// ever need a single HTTP request.
|
|
||||||
transport.DisableKeepAlives = true
|
|
||||||
// Timeout early if the server doesn't even return the headers.
|
|
||||||
transport.ResponseHeaderTimeout = time.Minute
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
|
|
||||||
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("failed to fetch prometheus metrics : %v", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
result := []*prom2json.Family{}
|
|
||||||
for mf := range mfChan {
|
|
||||||
result = append(result, prom2json.NewFamily(mf))
|
|
||||||
}
|
|
||||||
log.Debugf("Finished reading prometheus output, %d entries", len(result))
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
type unit struct {
|
|
||||||
value int64
|
|
||||||
symbol string
|
|
||||||
}
|
|
||||||
|
|
||||||
var ranges = []unit{
|
|
||||||
{value: 1e18, symbol: "E"},
|
|
||||||
{value: 1e15, symbol: "P"},
|
|
||||||
{value: 1e12, symbol: "T"},
|
|
||||||
{value: 1e9, symbol: "G"},
|
|
||||||
{value: 1e6, symbol: "M"},
|
|
||||||
{value: 1e3, symbol: "k"},
|
|
||||||
{value: 1, symbol: ""},
|
|
||||||
}
|
|
||||||
|
|
||||||
func formatNumber(num int) string {
|
|
||||||
goodUnit := unit{}
|
|
||||||
for _, u := range ranges {
|
|
||||||
if int64(num) >= u.value {
|
|
||||||
goodUnit = u
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if goodUnit.value == 1 {
|
|
||||||
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
|
|
||||||
}
|
|
||||||
|
|
||||||
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
|
|
||||||
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
|
|
||||||
}
|
|
||||||
|
|
||||||
func getDBClient() (*database.Client, error) {
|
func getDBClient() (*database.Client, error) {
|
||||||
var err error
|
var err error
|
||||||
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
|
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
|
||||||
|
@ -518,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string {
|
||||||
}
|
}
|
||||||
|
|
||||||
return slice
|
return slice
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ package main
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
"github.com/aquasecurity/table"
|
"github.com/aquasecurity/table"
|
||||||
"github.com/enescakir/emoji"
|
"github.com/enescakir/emoji"
|
||||||
|
@ -10,14 +11,15 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
)
|
)
|
||||||
|
|
||||||
func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
|
func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
|
||||||
t := newLightTable(out)
|
t := newLightTable(out)
|
||||||
t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
|
t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
|
||||||
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
||||||
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
||||||
|
|
||||||
for _, status := range statuses {
|
for _, item := range items {
|
||||||
t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
|
status, emo := item.Status()
|
||||||
|
t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.LocalVersion, item.LocalPath)
|
||||||
}
|
}
|
||||||
renderTableTitle(out, title)
|
renderTableTitle(out, title)
|
||||||
t.Render()
|
t.Render()
|
||||||
|
@ -31,11 +33,11 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
|
||||||
t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
|
t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
|
||||||
|
|
||||||
t.AddRow(
|
t.AddRow(
|
||||||
fmt.Sprintf("%d", metrics["curr_count"]),
|
strconv.Itoa(metrics["curr_count"]),
|
||||||
fmt.Sprintf("%d", metrics["overflow"]),
|
strconv.Itoa(metrics["overflow"]),
|
||||||
fmt.Sprintf("%d", metrics["instantiation"]),
|
strconv.Itoa(metrics["instantiation"]),
|
||||||
fmt.Sprintf("%d", metrics["pour"]),
|
strconv.Itoa(metrics["pour"]),
|
||||||
fmt.Sprintf("%d", metrics["underflow"]),
|
strconv.Itoa(metrics["underflow"]),
|
||||||
)
|
)
|
||||||
|
|
||||||
renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
|
renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
|
||||||
|
@ -43,23 +45,25 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
|
||||||
}
|
}
|
||||||
|
|
||||||
func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
|
func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
|
||||||
skip := true
|
|
||||||
t := newTable(out)
|
t := newTable(out)
|
||||||
t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
|
t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
|
||||||
|
|
||||||
|
// don't show table if no hits
|
||||||
|
showTable := false
|
||||||
|
|
||||||
for source, stats := range metrics {
|
for source, stats := range metrics {
|
||||||
if stats["hits"] > 0 {
|
if stats["hits"] > 0 {
|
||||||
t.AddRow(
|
t.AddRow(
|
||||||
source,
|
source,
|
||||||
fmt.Sprintf("%d", stats["hits"]),
|
strconv.Itoa(stats["hits"]),
|
||||||
fmt.Sprintf("%d", stats["parsed"]),
|
strconv.Itoa(stats["parsed"]),
|
||||||
fmt.Sprintf("%d", stats["unparsed"]),
|
strconv.Itoa(stats["unparsed"]),
|
||||||
)
|
)
|
||||||
skip = false
|
showTable = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !skip {
|
if showTable {
|
||||||
renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
|
renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,21 +20,16 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
|
func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
// Populate cwhub package tools
|
|
||||||
if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
|
|
||||||
return nil, fmt.Errorf("while loading hub index: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start loading configs
|
// Start loading configs
|
||||||
csParsers := parser.NewParsers()
|
csParsers := parser.NewParsers(hub)
|
||||||
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
|
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
|
||||||
return nil, fmt.Errorf("while loading parsers: %w", err)
|
return nil, fmt.Errorf("while loading parsers: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := LoadBuckets(cConfig); err != nil {
|
if err := LoadBuckets(cConfig, hub); err != nil {
|
||||||
return nil, fmt.Errorf("while loading scenarios: %w", err)
|
return nil, fmt.Errorf("while loading scenarios: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -44,7 +39,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
|
||||||
return csParsers, nil
|
return csParsers, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
|
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error {
|
||||||
inputEventChan = make(chan types.Event)
|
inputEventChan = make(chan types.Event)
|
||||||
inputLineChan = make(chan types.Event)
|
inputLineChan = make(chan types.Event)
|
||||||
|
|
||||||
|
@ -99,7 +94,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
|
||||||
for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
|
for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
|
||||||
outputsTomb.Go(func() error {
|
outputsTomb.Go(func() error {
|
||||||
defer trace.CatchPanic("crowdsec/runOutput")
|
defer trace.CatchPanic("crowdsec/runOutput")
|
||||||
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil {
|
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil {
|
||||||
log.Fatalf("starting outputs error : %s", err)
|
log.Fatalf("starting outputs error : %s", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -131,7 +126,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) {
|
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) {
|
||||||
crowdsecTomb.Go(func() error {
|
crowdsecTomb.Go(func() error {
|
||||||
defer trace.CatchPanic("crowdsec/serveCrowdsec")
|
defer trace.CatchPanic("crowdsec/serveCrowdsec")
|
||||||
go func() {
|
go func() {
|
||||||
|
@ -139,7 +134,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady
|
||||||
// this logs every time, even at config reload
|
// this logs every time, even at config reload
|
||||||
log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
|
log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
|
||||||
agentReady <- true
|
agentReady <- true
|
||||||
if err := runCrowdsec(cConfig, parsers); err != nil {
|
if err := runCrowdsec(cConfig, parsers, hub); err != nil {
|
||||||
log.Fatalf("unable to start crowdsec routines: %s", err)
|
log.Fatalf("unable to start crowdsec routines: %s", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
|
@ -75,12 +75,12 @@ type Flags struct {
|
||||||
|
|
||||||
type labelsMap map[string]string
|
type labelsMap map[string]string
|
||||||
|
|
||||||
func LoadBuckets(cConfig *csconfig.Config) error {
|
func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
files []string
|
files []string
|
||||||
)
|
)
|
||||||
for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
|
for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
|
||||||
if hubScenarioItem.Installed {
|
if hubScenarioItem.Installed {
|
||||||
files = append(files, hubScenarioItem.LocalPath)
|
files = append(files, hubScenarioItem.LocalPath)
|
||||||
}
|
}
|
||||||
|
@ -88,7 +88,7 @@ func LoadBuckets(cConfig *csconfig.Config) error {
|
||||||
buckets = leakybucket.NewBuckets()
|
buckets = leakybucket.NewBuckets()
|
||||||
|
|
||||||
log.Infof("Loading %d scenario files", len(files))
|
log.Infof("Loading %d scenario files", len(files))
|
||||||
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent)
|
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("scenario loading failed: %v", err)
|
return fmt.Errorf("scenario loading failed: %v", err)
|
||||||
|
@ -212,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
|
||||||
func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
|
func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
|
||||||
cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
|
cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("while loading configuration file: %w", err)
|
||||||
}
|
|
||||||
|
|
||||||
if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
|
|
||||||
return nil, fmt.Errorf("unable to load configuration: common section is empty")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
|
cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
|
||||||
|
@ -228,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
|
||||||
dumpStates = true
|
dumpStates = true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Configuration paths are dependency to load crowdsec configuration
|
|
||||||
if err := cConfig.LoadConfigurationPaths(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if flags.SingleFileType != "" && flags.OneShotDSN != "" {
|
if flags.SingleFileType != "" && flags.OneShotDSN != "" {
|
||||||
// if we're in time-machine mode, we don't want to log to file
|
// if we're in time-machine mode, we don't want to log to file
|
||||||
cConfig.Common.LogMedia = "stdout"
|
cConfig.Common.LogMedia = "stdout"
|
||||||
|
|
|
@ -151,14 +151,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
|
||||||
if !config.Enabled {
|
if !config.Enabled {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if config.ListenAddr == "" {
|
|
||||||
log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
|
|
||||||
config.ListenAddr = "127.0.0.1"
|
|
||||||
}
|
|
||||||
if config.ListenPort == 0 {
|
|
||||||
log.Warning("prometheus is enabled, but the listen port is empty, using '6060'")
|
|
||||||
config.ListenPort = 6060
|
|
||||||
}
|
|
||||||
|
|
||||||
// Registering prometheus
|
// Registering prometheus
|
||||||
// If in aggregated mode, do not register events associated with a source, to keep the cardinality low
|
// If in aggregated mode, do not register events associated with a source, to keep the cardinality low
|
||||||
|
|
|
@ -62,7 +62,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
|
||||||
var bucketOverflows []types.Event
|
var bucketOverflows []types.Event
|
||||||
|
|
||||||
func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
|
func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
|
||||||
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
|
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node,
|
||||||
|
apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error {
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
ticker := time.NewTicker(1 * time.Second)
|
ticker := time.NewTicker(1 * time.Second)
|
||||||
|
@ -70,7 +71,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
|
||||||
var cache []types.RuntimeAlert
|
var cache []types.RuntimeAlert
|
||||||
var cacheMutex sync.Mutex
|
var cacheMutex sync.Mutex
|
||||||
|
|
||||||
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
|
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("loading list of installed hub scenarios: %w", err)
|
return fmt.Errorf("loading list of installed hub scenarios: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -93,7 +94,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
|
||||||
URL: apiURL,
|
URL: apiURL,
|
||||||
PapiURL: papiURL,
|
PapiURL: papiURL,
|
||||||
VersionPrefix: "v1",
|
VersionPrefix: "v1",
|
||||||
UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
|
UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("new client api: %w", err)
|
return fmt.Errorf("new client api: %w", err)
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
"github.com/crowdsecurity/go-cs-lib/trace"
|
"github.com/crowdsecurity/go-cs-lib/trace"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/database"
|
"github.com/crowdsecurity/crowdsec/pkg/database"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||||
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
|
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
|
||||||
|
@ -76,7 +77,12 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if !cConfig.DisableAgent {
|
if !cConfig.DisableAgent {
|
||||||
csParsers, err := initCrowdsec(cConfig)
|
hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("while loading hub index: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
csParsers, err := initCrowdsec(cConfig, hub)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("unable to init crowdsec: %w", err)
|
return nil, fmt.Errorf("unable to init crowdsec: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -93,7 +99,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
agentReady := make(chan bool, 1)
|
agentReady := make(chan bool, 1)
|
||||||
serveCrowdsec(csParsers, cConfig, agentReady)
|
serveCrowdsec(csParsers, cConfig, hub, agentReady)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("Reload is finished")
|
log.Printf("Reload is finished")
|
||||||
|
@ -342,14 +348,19 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
|
||||||
}
|
}
|
||||||
|
|
||||||
if !cConfig.DisableAgent {
|
if !cConfig.DisableAgent {
|
||||||
csParsers, err := initCrowdsec(cConfig)
|
hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("while loading hub index: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
csParsers, err := initCrowdsec(cConfig, hub)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("crowdsec init: %w", err)
|
return fmt.Errorf("crowdsec init: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// if it's just linting, we're done
|
// if it's just linting, we're done
|
||||||
if !flags.TestMode {
|
if !flags.TestMode {
|
||||||
serveCrowdsec(csParsers, cConfig, agentReady)
|
serveCrowdsec(csParsers, cConfig, hub, agentReady)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
agentReady <- true
|
agentReady <- true
|
||||||
|
|
|
@ -6,7 +6,6 @@ common:
|
||||||
log_max_size: 20
|
log_max_size: 20
|
||||||
compress_logs: true
|
compress_logs: true
|
||||||
log_max_files: 10
|
log_max_files: 10
|
||||||
working_dir: .
|
|
||||||
config_paths:
|
config_paths:
|
||||||
config_dir: /etc/crowdsec/
|
config_dir: /etc/crowdsec/
|
||||||
data_dir: /var/lib/crowdsec/data/
|
data_dir: /var/lib/crowdsec/data/
|
||||||
|
|
|
@ -3,7 +3,6 @@ common:
|
||||||
log_media: file
|
log_media: file
|
||||||
log_level: info
|
log_level: info
|
||||||
log_dir: C:\ProgramData\CrowdSec\log\
|
log_dir: C:\ProgramData\CrowdSec\log\
|
||||||
working_dir: .
|
|
||||||
config_paths:
|
config_paths:
|
||||||
config_dir: C:\ProgramData\CrowdSec\config\
|
config_dir: C:\ProgramData\CrowdSec\config\
|
||||||
data_dir: C:\ProgramData\CrowdSec\data\
|
data_dir: C:\ProgramData\CrowdSec\data\
|
||||||
|
|
|
@ -3,7 +3,6 @@ common:
|
||||||
log_media: file
|
log_media: file
|
||||||
log_level: info
|
log_level: info
|
||||||
log_dir: C:\ProgramData\CrowdSec\log\
|
log_dir: C:\ProgramData\CrowdSec\log\
|
||||||
working_dir: .
|
|
||||||
config_paths:
|
config_paths:
|
||||||
config_dir: C:\ProgramData\CrowdSec\config\
|
config_dir: C:\ProgramData\CrowdSec\config\
|
||||||
data_dir: C:\ProgramData\CrowdSec\data\
|
data_dir: C:\ProgramData\CrowdSec\data\
|
||||||
|
|
|
@ -2,7 +2,6 @@ common:
|
||||||
daemonize: true
|
daemonize: true
|
||||||
log_media: stdout
|
log_media: stdout
|
||||||
log_level: info
|
log_level: info
|
||||||
working_dir: .
|
|
||||||
config_paths:
|
config_paths:
|
||||||
config_dir: ./config
|
config_dir: ./config
|
||||||
data_dir: ./data/
|
data_dir: ./data/
|
||||||
|
|
|
@ -3,7 +3,6 @@ common:
|
||||||
log_media: stdout
|
log_media: stdout
|
||||||
log_level: info
|
log_level: info
|
||||||
log_dir: /var/log/
|
log_dir: /var/log/
|
||||||
working_dir: .
|
|
||||||
config_paths:
|
config_paths:
|
||||||
config_dir: /etc/crowdsec/
|
config_dir: /etc/crowdsec/
|
||||||
data_dir: /var/lib/crowdsec/data
|
data_dir: /var/lib/crowdsec/data
|
||||||
|
|
|
@ -3,7 +3,6 @@ common:
|
||||||
log_media: stdout
|
log_media: stdout
|
||||||
log_level: info
|
log_level: info
|
||||||
log_dir: /var/log/
|
log_dir: /var/log/
|
||||||
working_dir: .
|
|
||||||
config_paths:
|
config_paths:
|
||||||
config_dir: /etc/crowdsec/
|
config_dir: /etc/crowdsec/
|
||||||
data_dir: /var/lib/crowdsec/data/
|
data_dir: /var/lib/crowdsec/data/
|
||||||
|
|
|
@ -101,19 +101,23 @@ register_bouncer() {
|
||||||
# $2 can be install, remove, upgrade
|
# $2 can be install, remove, upgrade
|
||||||
# $3 is a list of object names separated by space
|
# $3 is a list of object names separated by space
|
||||||
cscli_if_clean() {
|
cscli_if_clean() {
|
||||||
|
local itemtype="$1"
|
||||||
|
local action="$2"
|
||||||
|
local objs=$3
|
||||||
|
shift 3
|
||||||
# loop over all objects
|
# loop over all objects
|
||||||
for obj in $3; do
|
for obj in $objs; do
|
||||||
if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
|
if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
|
||||||
echo "Object $1/$obj is tainted, skipping"
|
echo "Object $itemtype/$obj is tainted, skipping"
|
||||||
else
|
else
|
||||||
# # Too verbose? Only show errors if not in debug mode
|
# # Too verbose? Only show errors if not in debug mode
|
||||||
# if [ "$DEBUG" != "true" ]; then
|
# if [ "$DEBUG" != "true" ]; then
|
||||||
# error_only=--error
|
# error_only=--error
|
||||||
# fi
|
# fi
|
||||||
error_only=""
|
error_only=""
|
||||||
echo "Running: cscli $error_only $1 $2 \"$obj\""
|
echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli $error_only "$1" "$2" "$obj"
|
cscli $error_only "$itemtype" "$action" "$obj" "$@"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
@ -327,22 +331,22 @@ fi
|
||||||
## Remove collections, parsers, scenarios & postoverflows
|
## Remove collections, parsers, scenarios & postoverflows
|
||||||
if [ "$DISABLE_COLLECTIONS" != "" ]; then
|
if [ "$DISABLE_COLLECTIONS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli_if_clean collections remove "$DISABLE_COLLECTIONS"
|
cscli_if_clean collections remove "$DISABLE_COLLECTIONS" --force
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$DISABLE_PARSERS" != "" ]; then
|
if [ "$DISABLE_PARSERS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli_if_clean parsers remove "$DISABLE_PARSERS"
|
cscli_if_clean parsers remove "$DISABLE_PARSERS" --force
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$DISABLE_SCENARIOS" != "" ]; then
|
if [ "$DISABLE_SCENARIOS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli_if_clean scenarios remove "$DISABLE_SCENARIOS"
|
cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" --force
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
|
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS"
|
cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
|
||||||
fi
|
fi
|
||||||
|
|
||||||
## Register bouncers via env
|
## Register bouncers via env
|
||||||
|
|
|
@ -30,8 +30,8 @@ def test_install_two_collections(crowdsec, flavor):
|
||||||
cs.wait_for_log([
|
cs.wait_for_log([
|
||||||
# f'*collections install "{it1}"*'
|
# f'*collections install "{it1}"*'
|
||||||
# f'*collections install "{it2}"*'
|
# f'*collections install "{it2}"*'
|
||||||
f'*Enabled collections : {it1}*',
|
f'*Enabled collections: {it1}*',
|
||||||
f'*Enabled collections : {it2}*',
|
f'*Enabled collections: {it2}*',
|
||||||
])
|
])
|
||||||
|
|
||||||
|
|
||||||
|
@ -72,7 +72,7 @@ def test_install_and_disable_collection(crowdsec, flavor):
|
||||||
assert it not in items
|
assert it not in items
|
||||||
logs = cs.log_lines()
|
logs = cs.log_lines()
|
||||||
# check that there was no attempt to install
|
# check that there was no attempt to install
|
||||||
assert not any(f'Enabled collections : {it}' in line for line in logs)
|
assert not any(f'Enabled collections: {it}' in line for line in logs)
|
||||||
|
|
||||||
|
|
||||||
# already done in bats, prividing here as example of a somewhat complex test
|
# already done in bats, prividing here as example of a somewhat complex test
|
||||||
|
@ -91,7 +91,7 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor):
|
||||||
# implicit check for tainted=False
|
# implicit check for tainted=False
|
||||||
assert items[coll]['status'] == 'enabled'
|
assert items[coll]['status'] == 'enabled'
|
||||||
cs.wait_for_log([
|
cs.wait_for_log([
|
||||||
f'*Enabled collections : {coll}*',
|
f'*Enabled collections: {coll}*',
|
||||||
])
|
])
|
||||||
|
|
||||||
scenario = 'crowdsecurity/http-crawl-non_statics'
|
scenario = 'crowdsecurity/http-crawl-non_statics'
|
||||||
|
|
|
@ -21,8 +21,8 @@ def test_install_two_scenarios(crowdsec, flavor):
|
||||||
}
|
}
|
||||||
with crowdsec(flavor=flavor, environment=env) as cs:
|
with crowdsec(flavor=flavor, environment=env) as cs:
|
||||||
cs.wait_for_log([
|
cs.wait_for_log([
|
||||||
f'*scenarios install "{it1}*"',
|
f'*scenarios install "{it1}"*',
|
||||||
f'*scenarios install "{it2}*"',
|
f'*scenarios install "{it2}"*',
|
||||||
"*Starting processing data*"
|
"*Starting processing data*"
|
||||||
])
|
])
|
||||||
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
|
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
|
||||||
|
|
2
go.mod
2
go.mod
|
@ -25,7 +25,7 @@ require (
|
||||||
github.com/c-robinson/iplib v1.0.3
|
github.com/c-robinson/iplib v1.0.3
|
||||||
github.com/cespare/xxhash/v2 v2.2.0
|
github.com/cespare/xxhash/v2 v2.2.0
|
||||||
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
|
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
|
||||||
github.com/crowdsecurity/go-cs-lib v0.0.4
|
github.com/crowdsecurity/go-cs-lib v0.0.5
|
||||||
github.com/crowdsecurity/grokky v0.2.1
|
github.com/crowdsecurity/grokky v0.2.1
|
||||||
github.com/crowdsecurity/machineid v1.0.2
|
github.com/crowdsecurity/machineid v1.0.2
|
||||||
github.com/davecgh/go-spew v1.1.1
|
github.com/davecgh/go-spew v1.1.1
|
||||||
|
|
4
go.sum
4
go.sum
|
@ -137,8 +137,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
|
||||||
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
|
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
|
||||||
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
|
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
|
||||||
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
|
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
|
||||||
github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims=
|
github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=
|
||||||
github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
|
github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
|
||||||
github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
|
github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
|
||||||
github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
|
github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
|
||||||
github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
|
github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
|
||||||
|
|
|
@ -286,10 +286,6 @@ func (c *Config) LoadAPIServer() error {
|
||||||
log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs))
|
log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.LoadCommon(); err != nil {
|
|
||||||
return fmt.Errorf("loading common configuration: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
c.API.Server.LogDir = c.Common.LogDir
|
c.API.Server.LogDir = c.Common.LogDir
|
||||||
c.API.Server.LogMedia = c.Common.LogMedia
|
c.API.Server.LogMedia = c.Common.LogMedia
|
||||||
c.API.Server.CompressLogs = c.Common.CompressLogs
|
c.API.Server.CompressLogs = c.Common.CompressLogs
|
||||||
|
|
|
@ -3,7 +3,6 @@ package csconfig
|
||||||
import (
|
import (
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
@ -142,9 +141,6 @@ func TestLoadAPIServer(t *testing.T) {
|
||||||
err := tmpLAPI.LoadProfiles()
|
err := tmpLAPI.LoadProfiles()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
LogDirFullPath, err := filepath.Abs("./testdata")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
logLevel := log.InfoLevel
|
logLevel := log.InfoLevel
|
||||||
config := &Config{}
|
config := &Config{}
|
||||||
fcontent, err := os.ReadFile("./testdata/config.yaml")
|
fcontent, err := os.ReadFile("./testdata/config.yaml")
|
||||||
|
@ -179,7 +175,7 @@ func TestLoadAPIServer(t *testing.T) {
|
||||||
DbPath: "./testdata/test.db",
|
DbPath: "./testdata/test.db",
|
||||||
},
|
},
|
||||||
Common: &CommonCfg{
|
Common: &CommonCfg{
|
||||||
LogDir: "./testdata/",
|
LogDir: "./testdata",
|
||||||
LogMedia: "stdout",
|
LogMedia: "stdout",
|
||||||
},
|
},
|
||||||
DisableAPI: false,
|
DisableAPI: false,
|
||||||
|
@ -202,7 +198,7 @@ func TestLoadAPIServer(t *testing.T) {
|
||||||
ShareContext: ptr.Of(false),
|
ShareContext: ptr.Of(false),
|
||||||
ConsoleManagement: ptr.Of(false),
|
ConsoleManagement: ptr.Of(false),
|
||||||
},
|
},
|
||||||
LogDir: LogDirFullPath,
|
LogDir: "./testdata",
|
||||||
LogMedia: "stdout",
|
LogMedia: "stdout",
|
||||||
OnlineClient: &OnlineApiClientCfg{
|
OnlineClient: &OnlineApiClientCfg{
|
||||||
CredentialsFilePath: "./testdata/online-api-secrets.yaml",
|
CredentialsFilePath: "./testdata/online-api-secrets.yaml",
|
||||||
|
|
|
@ -14,7 +14,7 @@ type CommonCfg struct {
|
||||||
LogMedia string `yaml:"log_media"`
|
LogMedia string `yaml:"log_media"`
|
||||||
LogDir string `yaml:"log_dir,omitempty"` //if LogMedia = file
|
LogDir string `yaml:"log_dir,omitempty"` //if LogMedia = file
|
||||||
LogLevel *log.Level `yaml:"log_level"`
|
LogLevel *log.Level `yaml:"log_level"`
|
||||||
WorkingDir string `yaml:"working_dir,omitempty"` ///var/run
|
WorkingDir string `yaml:"working_dir,omitempty"` // TODO: This is just for backward compat. Remove this later
|
||||||
CompressLogs *bool `yaml:"compress_logs,omitempty"`
|
CompressLogs *bool `yaml:"compress_logs,omitempty"`
|
||||||
LogMaxSize int `yaml:"log_max_size,omitempty"`
|
LogMaxSize int `yaml:"log_max_size,omitempty"`
|
||||||
LogMaxAge int `yaml:"log_max_age,omitempty"`
|
LogMaxAge int `yaml:"log_max_age,omitempty"`
|
||||||
|
@ -22,15 +22,18 @@ type CommonCfg struct {
|
||||||
ForceColorLogs bool `yaml:"force_color_logs,omitempty"`
|
ForceColorLogs bool `yaml:"force_color_logs,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) LoadCommon() error {
|
func (c *Config) loadCommon() error {
|
||||||
var err error
|
var err error
|
||||||
if c.Common == nil {
|
if c.Common == nil {
|
||||||
return fmt.Errorf("no common block provided in configuration file")
|
c.Common = &CommonCfg{}
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.Common.LogMedia == "" {
|
||||||
|
c.Common.LogMedia = "stdout"
|
||||||
}
|
}
|
||||||
|
|
||||||
var CommonCleanup = []*string{
|
var CommonCleanup = []*string{
|
||||||
&c.Common.LogDir,
|
&c.Common.LogDir,
|
||||||
&c.Common.WorkingDir,
|
|
||||||
}
|
}
|
||||||
for _, k := range CommonCleanup {
|
for _, k := range CommonCleanup {
|
||||||
if *k == "" {
|
if *k == "" {
|
||||||
|
|
|
@ -1,83 +0,0 @@
|
||||||
package csconfig
|
|
||||||
|
|
||||||
import (
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/go-cs-lib/cstest"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestLoadCommon(t *testing.T) {
|
|
||||||
pidDirPath := "./testdata"
|
|
||||||
LogDirFullPath, err := filepath.Abs("./testdata/log/")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
WorkingDirFullPath, err := filepath.Abs("./testdata")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
input *Config
|
|
||||||
expected *CommonCfg
|
|
||||||
expectedErr string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "basic valid configuration",
|
|
||||||
input: &Config{
|
|
||||||
Common: &CommonCfg{
|
|
||||||
Daemonize: true,
|
|
||||||
PidDir: "./testdata",
|
|
||||||
LogMedia: "file",
|
|
||||||
LogDir: "./testdata/log/",
|
|
||||||
WorkingDir: "./testdata/",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expected: &CommonCfg{
|
|
||||||
Daemonize: true,
|
|
||||||
PidDir: pidDirPath,
|
|
||||||
LogMedia: "file",
|
|
||||||
LogDir: LogDirFullPath,
|
|
||||||
WorkingDir: WorkingDirFullPath,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty working dir",
|
|
||||||
input: &Config{
|
|
||||||
Common: &CommonCfg{
|
|
||||||
Daemonize: true,
|
|
||||||
PidDir: "./testdata",
|
|
||||||
LogMedia: "file",
|
|
||||||
LogDir: "./testdata/log/",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expected: &CommonCfg{
|
|
||||||
Daemonize: true,
|
|
||||||
PidDir: pidDirPath,
|
|
||||||
LogMedia: "file",
|
|
||||||
LogDir: LogDirFullPath,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no common",
|
|
||||||
input: &Config{},
|
|
||||||
expected: nil,
|
|
||||||
expectedErr: "no common block provided in configuration file",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
tc := tc
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
err := tc.input.LoadCommon()
|
|
||||||
cstest.RequireErrorContains(t, err, tc.expectedErr)
|
|
||||||
if tc.expectedErr != "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, tc.expected, tc.input.Common)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -36,7 +36,7 @@ type Config struct {
|
||||||
PluginConfig *PluginCfg `yaml:"plugin_config,omitempty"`
|
PluginConfig *PluginCfg `yaml:"plugin_config,omitempty"`
|
||||||
DisableAPI bool `yaml:"-"`
|
DisableAPI bool `yaml:"-"`
|
||||||
DisableAgent bool `yaml:"-"`
|
DisableAgent bool `yaml:"-"`
|
||||||
Hub *Hub `yaml:"-"`
|
Hub *LocalHubCfg `yaml:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {
|
func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {
|
||||||
|
@ -58,18 +58,49 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool
|
||||||
// this is actually the "merged" yaml
|
// this is actually the "merged" yaml
|
||||||
return nil, "", fmt.Errorf("%s: %w", configFile, err)
|
return nil, "", fmt.Errorf("%s: %w", configFile, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if cfg.Prometheus == nil {
|
||||||
|
cfg.Prometheus = &PrometheusCfg{}
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Prometheus.ListenAddr == "" {
|
||||||
|
cfg.Prometheus.ListenAddr = "127.0.0.1"
|
||||||
|
log.Debugf("prometheus.listen_addr is empty, defaulting to %s", cfg.Prometheus.ListenAddr)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Prometheus.ListenPort == 0 {
|
||||||
|
cfg.Prometheus.ListenPort = 6060
|
||||||
|
log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = cfg.loadCommon(); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = cfg.loadConfigurationPaths(); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = cfg.loadHub(); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = cfg.loadCSCLI(); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
return &cfg, configData, nil
|
return &cfg, configData, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// XXX: We must not have a different behavior with an empty vs a missing configuration file.
|
||||||
|
// XXX: For this reason, all defaults have to come from NewConfig(). The following function should
|
||||||
|
// XXX: be replaced
|
||||||
func NewDefaultConfig() *Config {
|
func NewDefaultConfig() *Config {
|
||||||
logLevel := log.InfoLevel
|
logLevel := log.InfoLevel
|
||||||
commonCfg := CommonCfg{
|
commonCfg := CommonCfg{
|
||||||
Daemonize: false,
|
Daemonize: false,
|
||||||
PidDir: "/tmp/",
|
|
||||||
LogMedia: "stdout",
|
LogMedia: "stdout",
|
||||||
//LogDir unneeded
|
LogLevel: &logLevel,
|
||||||
LogLevel: &logLevel,
|
|
||||||
WorkingDir: ".",
|
|
||||||
}
|
}
|
||||||
prometheus := PrometheusCfg{
|
prometheus := PrometheusCfg{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
|
|
|
@ -15,21 +15,25 @@ type ConfigurationPaths struct {
|
||||||
NotificationDir string `yaml:"notification_dir,omitempty"`
|
NotificationDir string `yaml:"notification_dir,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) LoadConfigurationPaths() error {
|
func (c *Config) loadConfigurationPaths() error {
|
||||||
var err error
|
var err error
|
||||||
if c.ConfigPaths == nil {
|
if c.ConfigPaths == nil {
|
||||||
|
// XXX: test me
|
||||||
return fmt.Errorf("no configuration paths provided")
|
return fmt.Errorf("no configuration paths provided")
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.ConfigPaths.DataDir == "" {
|
if c.ConfigPaths.DataDir == "" {
|
||||||
|
// XXX: test me
|
||||||
return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section")
|
return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section")
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.ConfigPaths.HubDir == "" {
|
if c.ConfigPaths.HubDir == "" {
|
||||||
|
// XXX: test me
|
||||||
c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub")
|
c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub")
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.ConfigPaths.HubIndexFile == "" {
|
if c.ConfigPaths.HubIndexFile == "" {
|
||||||
|
// XXX: test me
|
||||||
c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json")
|
c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -15,10 +15,10 @@ func TestNormalLoad(t *testing.T) {
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
_, _, err = NewConfig("./testdata/xxx.yaml", false, false, false)
|
_, _, err = NewConfig("./testdata/xxx.yaml", false, false, false)
|
||||||
assert.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
|
require.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
|
||||||
|
|
||||||
_, _, err = NewConfig("./testdata/simulation.yaml", false, false, false)
|
_, _, err = NewConfig("./testdata/simulation.yaml", false, false, false)
|
||||||
assert.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config")
|
require.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewCrowdSecConfig(t *testing.T) {
|
func TestNewCrowdSecConfig(t *testing.T) {
|
||||||
|
|
|
@ -28,10 +28,6 @@ type CrowdsecServiceCfg struct {
|
||||||
BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
|
BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
|
||||||
BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode
|
BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode
|
||||||
|
|
||||||
HubDir string `yaml:"-"`
|
|
||||||
DataDir string `yaml:"-"`
|
|
||||||
ConfigDir string `yaml:"-"`
|
|
||||||
HubIndexFile string `yaml:"-"`
|
|
||||||
SimulationFilePath string `yaml:"-"`
|
SimulationFilePath string `yaml:"-"`
|
||||||
ContextToSend map[string][]string `yaml:"-"`
|
ContextToSend map[string][]string `yaml:"-"`
|
||||||
}
|
}
|
||||||
|
@ -101,11 +97,6 @@ func (c *Config) LoadCrowdsec() error {
|
||||||
return fmt.Errorf("load error (simulation): %w", err)
|
return fmt.Errorf("load error (simulation): %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
|
|
||||||
c.Crowdsec.DataDir = c.ConfigPaths.DataDir
|
|
||||||
c.Crowdsec.HubDir = c.ConfigPaths.HubDir
|
|
||||||
c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile
|
|
||||||
|
|
||||||
if c.Crowdsec.ParserRoutinesCount <= 0 {
|
if c.Crowdsec.ParserRoutinesCount <= 0 {
|
||||||
c.Crowdsec.ParserRoutinesCount = 1
|
c.Crowdsec.ParserRoutinesCount = 1
|
||||||
}
|
}
|
||||||
|
@ -145,15 +136,11 @@ func (c *Config) LoadCrowdsec() error {
|
||||||
return fmt.Errorf("loading api client: %s", err)
|
return fmt.Errorf("loading api client: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.LoadHub(); err != nil {
|
|
||||||
return fmt.Errorf("while loading hub: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
c.Crowdsec.ContextToSend = make(map[string][]string, 0)
|
c.Crowdsec.ContextToSend = make(map[string][]string, 0)
|
||||||
fallback := false
|
fallback := false
|
||||||
if c.Crowdsec.ConsoleContextPath == "" {
|
if c.Crowdsec.ConsoleContextPath == "" {
|
||||||
// fallback to default config file
|
// fallback to default config file
|
||||||
c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml")
|
c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
|
||||||
fallback = true
|
fallback = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -20,18 +20,6 @@ func TestLoadCrowdsec(t *testing.T) {
|
||||||
acquisDirFullPath, err := filepath.Abs("./testdata/acquis")
|
acquisDirFullPath, err := filepath.Abs("./testdata/acquis")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
hubFullPath, err := filepath.Abs("./hub")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
dataFullPath, err := filepath.Abs("./data")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
configDirFullPath, err := filepath.Abs("./testdata")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
contextFileFullPath, err := filepath.Abs("./testdata/context.yaml")
|
contextFileFullPath, err := filepath.Abs("./testdata/context.yaml")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -66,10 +54,6 @@ func TestLoadCrowdsec(t *testing.T) {
|
||||||
AcquisitionDirPath: "",
|
AcquisitionDirPath: "",
|
||||||
ConsoleContextPath: contextFileFullPath,
|
ConsoleContextPath: contextFileFullPath,
|
||||||
AcquisitionFilePath: acquisFullPath,
|
AcquisitionFilePath: acquisFullPath,
|
||||||
ConfigDir: configDirFullPath,
|
|
||||||
DataDir: dataFullPath,
|
|
||||||
HubDir: hubFullPath,
|
|
||||||
HubIndexFile: hubIndexFileFullPath,
|
|
||||||
BucketsRoutinesCount: 1,
|
BucketsRoutinesCount: 1,
|
||||||
ParserRoutinesCount: 1,
|
ParserRoutinesCount: 1,
|
||||||
OutputRoutinesCount: 1,
|
OutputRoutinesCount: 1,
|
||||||
|
@ -109,10 +93,6 @@ func TestLoadCrowdsec(t *testing.T) {
|
||||||
AcquisitionDirPath: acquisDirFullPath,
|
AcquisitionDirPath: acquisDirFullPath,
|
||||||
AcquisitionFilePath: acquisFullPath,
|
AcquisitionFilePath: acquisFullPath,
|
||||||
ConsoleContextPath: contextFileFullPath,
|
ConsoleContextPath: contextFileFullPath,
|
||||||
ConfigDir: configDirFullPath,
|
|
||||||
HubIndexFile: hubIndexFileFullPath,
|
|
||||||
DataDir: dataFullPath,
|
|
||||||
HubDir: hubFullPath,
|
|
||||||
BucketsRoutinesCount: 1,
|
BucketsRoutinesCount: 1,
|
||||||
ParserRoutinesCount: 1,
|
ParserRoutinesCount: 1,
|
||||||
OutputRoutinesCount: 1,
|
OutputRoutinesCount: 1,
|
||||||
|
@ -141,7 +121,7 @@ func TestLoadCrowdsec(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Crowdsec: &CrowdsecServiceCfg{
|
Crowdsec: &CrowdsecServiceCfg{
|
||||||
ConsoleContextPath: contextFileFullPath,
|
ConsoleContextPath: "./testdata/context.yaml",
|
||||||
ConsoleContextValueLength: 10,
|
ConsoleContextValueLength: 10,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -149,10 +129,6 @@ func TestLoadCrowdsec(t *testing.T) {
|
||||||
Enable: ptr.Of(true),
|
Enable: ptr.Of(true),
|
||||||
AcquisitionDirPath: "",
|
AcquisitionDirPath: "",
|
||||||
AcquisitionFilePath: "",
|
AcquisitionFilePath: "",
|
||||||
ConfigDir: configDirFullPath,
|
|
||||||
HubIndexFile: hubIndexFileFullPath,
|
|
||||||
DataDir: dataFullPath,
|
|
||||||
HubDir: hubFullPath,
|
|
||||||
ConsoleContextPath: contextFileFullPath,
|
ConsoleContextPath: contextFileFullPath,
|
||||||
BucketsRoutinesCount: 1,
|
BucketsRoutinesCount: 1,
|
||||||
ParserRoutinesCount: 1,
|
ParserRoutinesCount: 1,
|
||||||
|
|
|
@ -1,5 +1,9 @@
|
||||||
package csconfig
|
package csconfig
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
/*cscli specific config, such as hub directory*/
|
/*cscli specific config, such as hub directory*/
|
||||||
type CscliCfg struct {
|
type CscliCfg struct {
|
||||||
Output string `yaml:"output,omitempty"`
|
Output string `yaml:"output,omitempty"`
|
||||||
|
@ -7,25 +11,19 @@ type CscliCfg struct {
|
||||||
HubBranch string `yaml:"hub_branch"`
|
HubBranch string `yaml:"hub_branch"`
|
||||||
SimulationConfig *SimulationConfig `yaml:"-"`
|
SimulationConfig *SimulationConfig `yaml:"-"`
|
||||||
DbConfig *DatabaseCfg `yaml:"-"`
|
DbConfig *DatabaseCfg `yaml:"-"`
|
||||||
HubDir string `yaml:"-"`
|
|
||||||
DataDir string `yaml:"-"`
|
|
||||||
ConfigDir string `yaml:"-"`
|
|
||||||
HubIndexFile string `yaml:"-"`
|
|
||||||
SimulationFilePath string `yaml:"-"`
|
SimulationFilePath string `yaml:"-"`
|
||||||
PrometheusUrl string `yaml:"prometheus_uri"`
|
PrometheusUrl string `yaml:"prometheus_uri"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) LoadCSCLI() error {
|
func (c *Config) loadCSCLI() error {
|
||||||
if c.Cscli == nil {
|
if c.Cscli == nil {
|
||||||
c.Cscli = &CscliCfg{}
|
c.Cscli = &CscliCfg{}
|
||||||
}
|
}
|
||||||
if err := c.LoadConfigurationPaths(); err != nil {
|
|
||||||
return err
|
if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
|
||||||
|
c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
|
||||||
}
|
}
|
||||||
c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
|
|
||||||
c.Cscli.DataDir = c.ConfigPaths.DataDir
|
|
||||||
c.Cscli.HubDir = c.ConfigPaths.HubDir
|
|
||||||
c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,28 +1,14 @@
|
||||||
package csconfig
|
package csconfig
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/go-cs-lib/cstest"
|
"github.com/crowdsecurity/go-cs-lib/cstest"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLoadCSCLI(t *testing.T) {
|
func TestLoadCSCLI(t *testing.T) {
|
||||||
hubFullPath, err := filepath.Abs("./hub")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
dataFullPath, err := filepath.Abs("./data")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
configDirFullPath, err := filepath.Abs("./testdata")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input *Config
|
input *Config
|
||||||
|
@ -38,26 +24,23 @@ func TestLoadCSCLI(t *testing.T) {
|
||||||
HubDir: "./hub",
|
HubDir: "./hub",
|
||||||
HubIndexFile: "./hub/.index.json",
|
HubIndexFile: "./hub/.index.json",
|
||||||
},
|
},
|
||||||
|
Prometheus: &PrometheusCfg{
|
||||||
|
Enabled: true,
|
||||||
|
Level: "full",
|
||||||
|
ListenAddr: "127.0.0.1",
|
||||||
|
ListenPort: 6060,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
expected: &CscliCfg{
|
expected: &CscliCfg{
|
||||||
ConfigDir: configDirFullPath,
|
PrometheusUrl: "http://127.0.0.1:6060/metrics",
|
||||||
DataDir: dataFullPath,
|
|
||||||
HubDir: hubFullPath,
|
|
||||||
HubIndexFile: hubIndexFileFullPath,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "no configuration path",
|
|
||||||
input: &Config{},
|
|
||||||
expected: &CscliCfg{},
|
|
||||||
expectedErr: "no configuration paths provided",
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
tc := tc
|
tc := tc
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
err := tc.input.LoadCSCLI()
|
err := tc.input.loadCSCLI()
|
||||||
cstest.RequireErrorContains(t, err, tc.expectedErr)
|
cstest.RequireErrorContains(t, err, tc.expectedErr)
|
||||||
if tc.expectedErr != "" {
|
if tc.expectedErr != "" {
|
||||||
return
|
return
|
||||||
|
|
|
@ -1,19 +1,15 @@
|
||||||
package csconfig
|
package csconfig
|
||||||
|
|
||||||
/*cscli specific config, such as hub directory*/
|
// LocalHubCfg holds the configuration for a local hub: where to download etc.
|
||||||
type Hub struct {
|
type LocalHubCfg struct {
|
||||||
HubIndexFile string
|
HubIndexFile string // Path to the local index file
|
||||||
HubDir string
|
HubDir string // Where the hub items are downloaded
|
||||||
InstallDir string
|
InstallDir string // Where to install items
|
||||||
InstallDataDir string
|
InstallDataDir string // Where to install data
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) LoadHub() error {
|
func (c *Config) loadHub() error {
|
||||||
if err := c.LoadConfigurationPaths(); err != nil {
|
c.Hub = &LocalHubCfg{
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
c.Hub = &Hub{
|
|
||||||
HubIndexFile: c.ConfigPaths.HubIndexFile,
|
HubIndexFile: c.ConfigPaths.HubIndexFile,
|
||||||
HubDir: c.ConfigPaths.HubDir,
|
HubDir: c.ConfigPaths.HubDir,
|
||||||
InstallDir: c.ConfigPaths.ConfigDir,
|
InstallDir: c.ConfigPaths.ConfigDir,
|
||||||
|
|
|
@ -1,32 +1,18 @@
|
||||||
package csconfig
|
package csconfig
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/go-cs-lib/cstest"
|
"github.com/crowdsecurity/go-cs-lib/cstest"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLoadHub(t *testing.T) {
|
func TestLoadHub(t *testing.T) {
|
||||||
hubFullPath, err := filepath.Abs("./hub")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
dataFullPath, err := filepath.Abs("./data")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
configDirFullPath, err := filepath.Abs("./testdata")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input *Config
|
input *Config
|
||||||
expected *Hub
|
expected *LocalHubCfg
|
||||||
expectedErr string
|
expectedErr string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
|
@ -39,35 +25,19 @@ func TestLoadHub(t *testing.T) {
|
||||||
HubIndexFile: "./hub/.index.json",
|
HubIndexFile: "./hub/.index.json",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
expected: &Hub{
|
expected: &LocalHubCfg{
|
||||||
HubDir: hubFullPath,
|
HubDir: "./hub",
|
||||||
HubIndexFile: hubIndexFileFullPath,
|
HubIndexFile: "./hub/.index.json",
|
||||||
InstallDir: configDirFullPath,
|
InstallDir: "./testdata",
|
||||||
InstallDataDir: dataFullPath,
|
InstallDataDir: "./data",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "no data dir",
|
|
||||||
input: &Config{
|
|
||||||
ConfigPaths: &ConfigurationPaths{
|
|
||||||
ConfigDir: "./testdata",
|
|
||||||
HubDir: "./hub",
|
|
||||||
HubIndexFile: "./hub/.index.json",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no configuration path",
|
|
||||||
input: &Config{},
|
|
||||||
expectedErr: "no configuration paths provided",
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
tc := tc
|
tc := tc
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
err := tc.input.LoadHub()
|
err := tc.input.loadHub()
|
||||||
cstest.RequireErrorContains(t, err, tc.expectedErr)
|
cstest.RequireErrorContains(t, err, tc.expectedErr)
|
||||||
if tc.expectedErr != "" {
|
if tc.expectedErr != "" {
|
||||||
return
|
return
|
||||||
|
|
|
@ -1,19 +1,8 @@
|
||||||
package csconfig
|
package csconfig
|
||||||
|
|
||||||
import "fmt"
|
|
||||||
|
|
||||||
type PrometheusCfg struct {
|
type PrometheusCfg struct {
|
||||||
Enabled bool `yaml:"enabled"`
|
Enabled bool `yaml:"enabled"`
|
||||||
Level string `yaml:"level"` //aggregated|full
|
Level string `yaml:"level"` //aggregated|full
|
||||||
ListenAddr string `yaml:"listen_addr"`
|
ListenAddr string `yaml:"listen_addr"`
|
||||||
ListenPort int `yaml:"listen_port"`
|
ListenPort int `yaml:"listen_port"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) LoadPrometheus() error {
|
|
||||||
if c.Cscli != nil && c.Cscli.PrometheusUrl == "" && c.Prometheus != nil {
|
|
||||||
if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
|
|
||||||
c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,42 +0,0 @@
|
||||||
package csconfig
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/go-cs-lib/cstest"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestLoadPrometheus(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
input *Config
|
|
||||||
expectedURL string
|
|
||||||
expectedErr string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "basic valid configuration",
|
|
||||||
input: &Config{
|
|
||||||
Prometheus: &PrometheusCfg{
|
|
||||||
Enabled: true,
|
|
||||||
Level: "full",
|
|
||||||
ListenAddr: "127.0.0.1",
|
|
||||||
ListenPort: 6060,
|
|
||||||
},
|
|
||||||
Cscli: &CscliCfg{},
|
|
||||||
},
|
|
||||||
expectedURL: "http://127.0.0.1:6060",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
tc := tc
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
err := tc.input.LoadPrometheus()
|
|
||||||
cstest.RequireErrorContains(t, err, tc.expectedErr)
|
|
||||||
|
|
||||||
require.Equal(t, tc.expectedURL, tc.input.Cscli.PrometheusUrl)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -30,11 +30,6 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) LoadSimulation() error {
|
func (c *Config) LoadSimulation() error {
|
||||||
|
|
||||||
if err := c.LoadConfigurationPaths(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
simCfg := SimulationConfig{}
|
simCfg := SimulationConfig{}
|
||||||
if c.ConfigPaths.SimulationFilePath == "" {
|
if c.ConfigPaths.SimulationFilePath == "" {
|
||||||
c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")
|
c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")
|
||||||
|
|
|
@ -2,7 +2,6 @@ package csconfig
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
@ -12,12 +11,6 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSimulationLoading(t *testing.T) {
|
func TestSimulationLoading(t *testing.T) {
|
||||||
testXXFullPath, err := filepath.Abs("./testdata/xxx.yaml")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
badYamlFullPath, err := filepath.Abs("./testdata/config.yaml")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input *Config
|
input *Config
|
||||||
|
@ -56,7 +49,7 @@ func TestSimulationLoading(t *testing.T) {
|
||||||
},
|
},
|
||||||
Crowdsec: &CrowdsecServiceCfg{},
|
Crowdsec: &CrowdsecServiceCfg{},
|
||||||
},
|
},
|
||||||
expectedErr: fmt.Sprintf("while reading yaml file: open %s: %s", testXXFullPath, cstest.FileNotFoundMessage),
|
expectedErr: fmt.Sprintf("while reading yaml file: open ./testdata/xxx.yaml: %s", cstest.FileNotFoundMessage),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "basic bad file content",
|
name: "basic bad file content",
|
||||||
|
@ -67,7 +60,7 @@ func TestSimulationLoading(t *testing.T) {
|
||||||
},
|
},
|
||||||
Crowdsec: &CrowdsecServiceCfg{},
|
Crowdsec: &CrowdsecServiceCfg{},
|
||||||
},
|
},
|
||||||
expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
|
expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "basic bad file content",
|
name: "basic bad file content",
|
||||||
|
@ -78,7 +71,7 @@ func TestSimulationLoading(t *testing.T) {
|
||||||
},
|
},
|
||||||
Crowdsec: &CrowdsecServiceCfg{},
|
Crowdsec: &CrowdsecServiceCfg{},
|
||||||
},
|
},
|
||||||
expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
|
expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
1
pkg/csconfig/testdata/config.yaml
vendored
1
pkg/csconfig/testdata/config.yaml
vendored
|
@ -2,7 +2,6 @@ common:
|
||||||
daemonize: false
|
daemonize: false
|
||||||
log_media: stdout
|
log_media: stdout
|
||||||
log_level: info
|
log_level: info
|
||||||
working_dir: .
|
|
||||||
prometheus:
|
prometheus:
|
||||||
enabled: true
|
enabled: true
|
||||||
level: full
|
level: full
|
||||||
|
|
|
@ -1,288 +1,14 @@
|
||||||
|
// Package cwhub is responsible for installing and upgrading the local hub files.
|
||||||
|
//
|
||||||
|
// This includes retrieving the index, the items to install (parsers, scenarios, data files...)
|
||||||
|
// and managing the dependencies and taints.
|
||||||
package cwhub
|
package cwhub
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"net/http"
|
||||||
"os"
|
"time"
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/enescakir/emoji"
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
"golang.org/x/mod/semver"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
var hubClient = &http.Client{
|
||||||
HubIndexFile = ".index.json"
|
Timeout: 120 * time.Second,
|
||||||
|
|
||||||
// managed item types
|
|
||||||
PARSERS = "parsers"
|
|
||||||
PARSERS_OVFLW = "postoverflows"
|
|
||||||
SCENARIOS = "scenarios"
|
|
||||||
COLLECTIONS = "collections"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS}
|
|
||||||
|
|
||||||
ErrMissingReference = errors.New("Reference(s) missing in collection")
|
|
||||||
|
|
||||||
// XXX: can we remove these globals?
|
|
||||||
skippedLocal = 0
|
|
||||||
skippedTainted = 0
|
|
||||||
RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
|
|
||||||
HubBranch = "master"
|
|
||||||
hubIdx map[string]map[string]Item
|
|
||||||
)
|
|
||||||
|
|
||||||
type ItemVersion struct {
|
|
||||||
Digest string `json:"digest,omitempty"` // meow
|
|
||||||
Deprecated bool `json:"deprecated,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ItemHubStatus struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
LocalVersion string `json:"local_version"`
|
|
||||||
LocalPath string `json:"local_path"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
UTF8Status string `json:"utf8_status"`
|
|
||||||
Status string `json:"status"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Item can be: parser, scenario, collection..
|
|
||||||
type Item struct {
|
|
||||||
// descriptive info
|
|
||||||
Type string `json:"type,omitempty" yaml:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich)
|
|
||||||
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
|
|
||||||
Name string `json:"name,omitempty"` // as seen in .config.json, usually "author/name"
|
|
||||||
FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml
|
|
||||||
Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .config.json
|
|
||||||
Author string `json:"author,omitempty"` // as seen in .config.json
|
|
||||||
References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .config.json
|
|
||||||
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
|
|
||||||
|
|
||||||
// remote (hub) info
|
|
||||||
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml
|
|
||||||
Version string `json:"version,omitempty"` // the last version
|
|
||||||
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions
|
|
||||||
|
|
||||||
// local (deployed) info
|
|
||||||
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
|
|
||||||
LocalVersion string `json:"local_version,omitempty"`
|
|
||||||
LocalHash string `json:"local_hash,omitempty"` // the local meow
|
|
||||||
Installed bool `json:"installed,omitempty"`
|
|
||||||
Downloaded bool `json:"downloaded,omitempty"`
|
|
||||||
UpToDate bool `json:"up_to_date,omitempty"`
|
|
||||||
Tainted bool `json:"tainted,omitempty"` // has it been locally modified
|
|
||||||
Local bool `json:"local,omitempty"` // if it's a non versioned control one
|
|
||||||
|
|
||||||
// if it's a collection, it's not a single file
|
|
||||||
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
|
|
||||||
PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
|
|
||||||
Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"`
|
|
||||||
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (i *Item) status() (string, emoji.Emoji) {
|
|
||||||
status := "disabled"
|
|
||||||
ok := false
|
|
||||||
|
|
||||||
if i.Installed {
|
|
||||||
ok = true
|
|
||||||
status = "enabled"
|
|
||||||
}
|
|
||||||
|
|
||||||
managed := true
|
|
||||||
if i.Local {
|
|
||||||
managed = false
|
|
||||||
status += ",local"
|
|
||||||
}
|
|
||||||
|
|
||||||
warning := false
|
|
||||||
if i.Tainted {
|
|
||||||
warning = true
|
|
||||||
status += ",tainted"
|
|
||||||
} else if !i.UpToDate && !i.Local {
|
|
||||||
warning = true
|
|
||||||
status += ",update-available"
|
|
||||||
}
|
|
||||||
|
|
||||||
emo := emoji.QuestionMark
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case !managed:
|
|
||||||
emo = emoji.House
|
|
||||||
case !i.Installed:
|
|
||||||
emo = emoji.Prohibited
|
|
||||||
case warning:
|
|
||||||
emo = emoji.Warning
|
|
||||||
case ok:
|
|
||||||
emo = emoji.CheckMark
|
|
||||||
}
|
|
||||||
|
|
||||||
return status, emo
|
|
||||||
}
|
|
||||||
|
|
||||||
func (i *Item) hubStatus() ItemHubStatus {
|
|
||||||
status, emo := i.status()
|
|
||||||
|
|
||||||
return ItemHubStatus{
|
|
||||||
Name: i.Name,
|
|
||||||
LocalVersion: i.LocalVersion,
|
|
||||||
LocalPath: i.LocalPath,
|
|
||||||
Description: i.Description,
|
|
||||||
Status: status,
|
|
||||||
UTF8Status: fmt.Sprintf("%v %s", emo, status),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// versionStatus: semver requires 'v' prefix
|
|
||||||
func (i *Item) versionStatus() int {
|
|
||||||
return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetItemMap(itemType string) map[string]Item {
|
|
||||||
m, ok := hubIdx[itemType]
|
|
||||||
if !ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// Given a FileInfo, extract the map key. Follow a symlink if necessary
|
|
||||||
func itemKey(itemPath string) (string, error) {
|
|
||||||
f, err := os.Lstat(itemPath)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if f.Mode()&os.ModeSymlink == 0 {
|
|
||||||
// it's not a symlink, so the filename itsef should be the key
|
|
||||||
return filepath.Base(itemPath), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolve the symlink to hub file
|
|
||||||
pathInHub, err := os.Readlink(itemPath)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
author := filepath.Base(filepath.Dir(pathInHub))
|
|
||||||
|
|
||||||
fname := filepath.Base(pathInHub)
|
|
||||||
fname = strings.TrimSuffix(fname, ".yaml")
|
|
||||||
fname = strings.TrimSuffix(fname, ".yml")
|
|
||||||
|
|
||||||
return fmt.Sprintf("%s/%s", author, fname), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
|
|
||||||
func GetItemByPath(itemType string, itemPath string) (*Item, error) {
|
|
||||||
itemKey, err := itemKey(itemPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
m := GetItemMap(itemType)
|
|
||||||
if m == nil {
|
|
||||||
return nil, fmt.Errorf("item type %s doesn't exist", itemType)
|
|
||||||
}
|
|
||||||
|
|
||||||
v, ok := m[itemKey]
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%s not found in %s", itemKey, itemType)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &v, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetItem(itemType string, itemName string) *Item {
|
|
||||||
if m, ok := GetItemMap(itemType)[itemName]; ok {
|
|
||||||
return &m
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func AddItem(itemType string, item Item) error {
|
|
||||||
for _, itype := range ItemTypes {
|
|
||||||
if itype == itemType {
|
|
||||||
hubIdx[itemType][item.Name] = item
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Errorf("ItemType %s is unknown", itemType)
|
|
||||||
}
|
|
||||||
|
|
||||||
func DisplaySummary() {
|
|
||||||
log.Infof("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
|
|
||||||
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
|
|
||||||
|
|
||||||
if skippedLocal > 0 || skippedTainted > 0 {
|
|
||||||
log.Infof("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetInstalledItems(itemType string) ([]Item, error) {
|
|
||||||
items, ok := hubIdx[itemType]
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("no %s in hubIdx", itemType)
|
|
||||||
}
|
|
||||||
|
|
||||||
retItems := make([]Item, 0)
|
|
||||||
|
|
||||||
for _, item := range items {
|
|
||||||
if item.Installed {
|
|
||||||
retItems = append(retItems, item)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return retItems, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetInstalledItemsAsString(itemType string) ([]string, error) {
|
|
||||||
items, err := GetInstalledItems(itemType)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
retStr := make([]string, len(items))
|
|
||||||
|
|
||||||
for i, it := range items {
|
|
||||||
retStr[i] = it.Name
|
|
||||||
}
|
|
||||||
|
|
||||||
return retStr, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Returns a slice of entries for packages: name, status, local_path, local_version, utf8_status (fancy)
|
|
||||||
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
|
|
||||||
if _, ok := hubIdx[itemType]; !ok {
|
|
||||||
log.Errorf("type %s doesn't exist", itemType)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
ret := make([]ItemHubStatus, 0)
|
|
||||||
|
|
||||||
// remember, you do it for the user :)
|
|
||||||
for _, item := range hubIdx[itemType] {
|
|
||||||
if name != "" && name != item.Name {
|
|
||||||
// user has requested a specific name
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Only enabled items ?
|
|
||||||
if !all && !item.Installed {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Check the item status
|
|
||||||
ret = append(ret, item.hubStatus())
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
|
|
||||||
|
|
||||||
return ret
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,14 +9,13 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
"github.com/crowdsecurity/go-cs-lib/cstest"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
|
||||||
|
|
||||||
/*
|
/*
|
||||||
To test :
|
To test :
|
||||||
- Download 'first' hub index
|
- Download 'first' hub index
|
||||||
|
@ -28,294 +27,63 @@ import (
|
||||||
|
|
||||||
var responseByPath map[string]string
|
var responseByPath map[string]string
|
||||||
|
|
||||||
func TestItemStatus(t *testing.T) {
|
// testHub initializes a temporary hub with an empty json file, optionally updating it
|
||||||
cfg := envSetup(t)
|
func testHub(t *testing.T, update bool) *Hub {
|
||||||
defer envTearDown(cfg)
|
tmpDir, err := os.MkdirTemp("", "testhub")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
// DownloadHubIdx()
|
local := &csconfig.LocalHubCfg{
|
||||||
err := UpdateHubIdx(cfg.Hub)
|
HubDir: filepath.Join(tmpDir, "crowdsec", "hub"),
|
||||||
require.NoError(t, err, "failed to download index")
|
HubIndexFile: filepath.Join(tmpDir, "crowdsec", "hub", ".index.json"),
|
||||||
|
InstallDir: filepath.Join(tmpDir, "crowdsec"),
|
||||||
err = GetHubIdx(cfg.Hub)
|
InstallDataDir: filepath.Join(tmpDir, "installed-data"),
|
||||||
require.NoError(t, err, "failed to load hub index")
|
|
||||||
|
|
||||||
// get existing map
|
|
||||||
x := GetItemMap(COLLECTIONS)
|
|
||||||
require.NotEmpty(t, x)
|
|
||||||
|
|
||||||
// Get item : good and bad
|
|
||||||
for k := range x {
|
|
||||||
item := GetItem(COLLECTIONS, k)
|
|
||||||
require.NotNil(t, item)
|
|
||||||
|
|
||||||
item.Installed = true
|
|
||||||
item.UpToDate = false
|
|
||||||
item.Local = false
|
|
||||||
item.Tainted = false
|
|
||||||
|
|
||||||
txt, _ := item.status()
|
|
||||||
require.Equal(t, "enabled,update-available", txt)
|
|
||||||
|
|
||||||
item.Installed = false
|
|
||||||
item.UpToDate = false
|
|
||||||
item.Local = true
|
|
||||||
item.Tainted = false
|
|
||||||
|
|
||||||
txt, _ = item.status()
|
|
||||||
require.Equal(t, "disabled,local", txt)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
DisplaySummary()
|
err = os.MkdirAll(local.HubDir, 0o700)
|
||||||
}
|
require.NoError(t, err)
|
||||||
|
|
||||||
func TestGetters(t *testing.T) {
|
err = os.MkdirAll(local.InstallDir, 0o700)
|
||||||
cfg := envSetup(t)
|
require.NoError(t, err)
|
||||||
defer envTearDown(cfg)
|
|
||||||
|
|
||||||
// DownloadHubIdx()
|
err = os.MkdirAll(local.InstallDataDir, 0o700)
|
||||||
err := UpdateHubIdx(cfg.Hub)
|
require.NoError(t, err)
|
||||||
require.NoError(t, err, "failed to download index")
|
|
||||||
|
|
||||||
err = GetHubIdx(cfg.Hub)
|
err = os.WriteFile(local.HubIndexFile, []byte("{}"), 0o644)
|
||||||
require.NoError(t, err, "failed to load hub index")
|
require.NoError(t, err)
|
||||||
|
|
||||||
// get non existing map
|
|
||||||
empty := GetItemMap("ratata")
|
|
||||||
require.Nil(t, empty)
|
|
||||||
|
|
||||||
// get existing map
|
|
||||||
x := GetItemMap(COLLECTIONS)
|
|
||||||
require.NotEmpty(t, x)
|
|
||||||
|
|
||||||
// Get item : good and bad
|
|
||||||
for k := range x {
|
|
||||||
empty := GetItem(COLLECTIONS, k+"nope")
|
|
||||||
require.Nil(t, empty)
|
|
||||||
|
|
||||||
item := GetItem(COLLECTIONS, k)
|
|
||||||
require.NotNil(t, item)
|
|
||||||
|
|
||||||
// Add item and get it
|
|
||||||
item.Name += "nope"
|
|
||||||
err := AddItem(COLLECTIONS, *item)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
newitem := GetItem(COLLECTIONS, item.Name)
|
|
||||||
require.NotNil(t, newitem)
|
|
||||||
|
|
||||||
err = AddItem("ratata", *item)
|
|
||||||
cstest.RequireErrorContains(t, err, "ItemType ratata is unknown")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIndexDownload(t *testing.T) {
|
|
||||||
cfg := envSetup(t)
|
|
||||||
defer envTearDown(cfg)
|
|
||||||
|
|
||||||
// DownloadHubIdx()
|
|
||||||
err := UpdateHubIdx(cfg.Hub)
|
|
||||||
require.NoError(t, err, "failed to download index")
|
|
||||||
|
|
||||||
err = GetHubIdx(cfg.Hub)
|
|
||||||
require.NoError(t, err, "failed to load hub index")
|
|
||||||
}
|
|
||||||
|
|
||||||
func getTestCfg() *csconfig.Config {
|
|
||||||
cfg := &csconfig.Config{Hub: &csconfig.Hub{}}
|
|
||||||
cfg.Hub.InstallDir, _ = filepath.Abs("./install")
|
|
||||||
cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
|
|
||||||
cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
|
|
||||||
|
|
||||||
return cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
func envSetup(t *testing.T) *csconfig.Config {
|
|
||||||
resetResponseByPath()
|
|
||||||
log.SetLevel(log.DebugLevel)
|
|
||||||
|
|
||||||
cfg := getTestCfg()
|
|
||||||
|
|
||||||
defaultTransport := http.DefaultClient.Transport
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
http.DefaultClient.Transport = defaultTransport
|
os.RemoveAll(tmpDir)
|
||||||
|
})
|
||||||
|
|
||||||
|
remote := &RemoteHubCfg{
|
||||||
|
Branch: "master",
|
||||||
|
URLTemplate: mockURLTemplate,
|
||||||
|
IndexPath: ".index.json",
|
||||||
|
}
|
||||||
|
|
||||||
|
hub, err := NewHub(local, remote, update)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return hub
|
||||||
|
}
|
||||||
|
|
||||||
|
// envSetup initializes the temporary hub and mocks the http client
|
||||||
|
func envSetup(t *testing.T) *Hub {
|
||||||
|
setResponseByPath()
|
||||||
|
log.SetLevel(log.DebugLevel)
|
||||||
|
|
||||||
|
defaultTransport := hubClient.Transport
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
hubClient.Transport = defaultTransport
|
||||||
})
|
})
|
||||||
|
|
||||||
// Mock the http client
|
// Mock the http client
|
||||||
http.DefaultClient.Transport = newMockTransport()
|
hubClient.Transport = newMockTransport()
|
||||||
|
|
||||||
err := os.MkdirAll(cfg.Hub.InstallDir, 0700)
|
hub := testHub(t, true)
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
err = os.MkdirAll(cfg.Hub.HubDir, 0700)
|
return hub
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
err = UpdateHubIdx(cfg.Hub)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
|
|
||||||
// log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
|
|
||||||
// }
|
|
||||||
// if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil {
|
|
||||||
// log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err)
|
|
||||||
// }
|
|
||||||
return cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
func envTearDown(cfg *csconfig.Config) {
|
|
||||||
if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
|
|
||||||
log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := os.RemoveAll(cfg.Hub.HubDir); err != nil {
|
|
||||||
log.Fatalf("failed to remove %s : %s", cfg.Hub.HubDir, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
|
||||||
// Install the parser
|
|
||||||
err := DownloadLatest(cfg, &item, false, false)
|
|
||||||
require.NoError(t, err, "failed to download %s", item.Name)
|
|
||||||
|
|
||||||
_, err = LocalSync(cfg)
|
|
||||||
require.NoError(t, err, "failed to run localSync")
|
|
||||||
|
|
||||||
assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
|
|
||||||
assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
|
|
||||||
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
|
|
||||||
|
|
||||||
err = EnableItem(cfg, &item)
|
|
||||||
require.NoError(t, err, "failed to enable %s", item.Name)
|
|
||||||
|
|
||||||
_, err = LocalSync(cfg)
|
|
||||||
require.NoError(t, err, "failed to run localSync")
|
|
||||||
|
|
||||||
assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
|
||||||
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
|
|
||||||
|
|
||||||
f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
|
|
||||||
require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name)
|
|
||||||
|
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
_, err = f.WriteString("tainted")
|
|
||||||
require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name)
|
|
||||||
|
|
||||||
// Local sync and check status
|
|
||||||
_, err = LocalSync(cfg)
|
|
||||||
require.NoError(t, err, "failed to run localSync")
|
|
||||||
|
|
||||||
assert.True(t, hubIdx[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
|
||||||
assert.False(t, hubIdx[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
|
|
||||||
|
|
||||||
// Update it + check status
|
|
||||||
err := DownloadLatest(cfg, &item, true, true)
|
|
||||||
require.NoError(t, err, "failed to update %s", item.Name)
|
|
||||||
|
|
||||||
// Local sync and check status
|
|
||||||
_, err = LocalSync(cfg)
|
|
||||||
require.NoError(t, err, "failed to run localSync")
|
|
||||||
|
|
||||||
assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
|
|
||||||
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
|
|
||||||
assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
|
|
||||||
|
|
||||||
// Remove
|
|
||||||
err := DisableItem(cfg, &item, false, false)
|
|
||||||
require.NoError(t, err, "failed to disable %s", item.Name)
|
|
||||||
|
|
||||||
// Local sync and check status
|
|
||||||
warns, err := LocalSync(cfg)
|
|
||||||
require.NoError(t, err, "failed to run localSync")
|
|
||||||
require.Empty(t, warns, "unexpected warnings : %+v", warns)
|
|
||||||
|
|
||||||
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
|
|
||||||
assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
|
|
||||||
assert.True(t, hubIdx[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
|
|
||||||
|
|
||||||
// Purge
|
|
||||||
err = DisableItem(cfg, &item, true, false)
|
|
||||||
require.NoError(t, err, "failed to purge %s", item.Name)
|
|
||||||
|
|
||||||
// Local sync and check status
|
|
||||||
warns, err = LocalSync(cfg)
|
|
||||||
require.NoError(t, err, "failed to run localSync")
|
|
||||||
require.Empty(t, warns, "unexpected warnings : %+v", warns)
|
|
||||||
|
|
||||||
assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
|
|
||||||
assert.False(t, hubIdx[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInstallParser(t *testing.T) {
|
|
||||||
/*
|
|
||||||
- install a random parser
|
|
||||||
- check its status
|
|
||||||
- taint it
|
|
||||||
- check its status
|
|
||||||
- force update it
|
|
||||||
- check its status
|
|
||||||
- remove it
|
|
||||||
*/
|
|
||||||
cfg := envSetup(t)
|
|
||||||
defer envTearDown(cfg)
|
|
||||||
|
|
||||||
getHubIdxOrFail(t)
|
|
||||||
// map iteration is random by itself
|
|
||||||
for _, it := range hubIdx[PARSERS] {
|
|
||||||
testInstallItem(cfg.Hub, t, it)
|
|
||||||
it = hubIdx[PARSERS][it.Name]
|
|
||||||
_ = GetHubStatusForItemType(PARSERS, it.Name, false)
|
|
||||||
testTaintItem(cfg.Hub, t, it)
|
|
||||||
it = hubIdx[PARSERS][it.Name]
|
|
||||||
_ = GetHubStatusForItemType(PARSERS, it.Name, false)
|
|
||||||
testUpdateItem(cfg.Hub, t, it)
|
|
||||||
it = hubIdx[PARSERS][it.Name]
|
|
||||||
testDisableItem(cfg.Hub, t, it)
|
|
||||||
it = hubIdx[PARSERS][it.Name]
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInstallCollection(t *testing.T) {
|
|
||||||
/*
|
|
||||||
- install a random parser
|
|
||||||
- check its status
|
|
||||||
- taint it
|
|
||||||
- check its status
|
|
||||||
- force update it
|
|
||||||
- check its status
|
|
||||||
- remove it
|
|
||||||
*/
|
|
||||||
cfg := envSetup(t)
|
|
||||||
defer envTearDown(cfg)
|
|
||||||
|
|
||||||
getHubIdxOrFail(t)
|
|
||||||
// map iteration is random by itself
|
|
||||||
for _, it := range hubIdx[COLLECTIONS] {
|
|
||||||
testInstallItem(cfg.Hub, t, it)
|
|
||||||
it = hubIdx[COLLECTIONS][it.Name]
|
|
||||||
testTaintItem(cfg.Hub, t, it)
|
|
||||||
it = hubIdx[COLLECTIONS][it.Name]
|
|
||||||
testUpdateItem(cfg.Hub, t, it)
|
|
||||||
it = hubIdx[COLLECTIONS][it.Name]
|
|
||||||
testDisableItem(cfg.Hub, t, it)
|
|
||||||
|
|
||||||
it = hubIdx[COLLECTIONS][it.Name]
|
|
||||||
x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
|
|
||||||
log.Infof("%+v", x)
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type mockTransport struct{}
|
type mockTransport struct{}
|
||||||
|
@ -362,7 +130,7 @@ func fileToStringX(path string) string {
|
||||||
return strings.ReplaceAll(string(data), "\r\n", "\n")
|
return strings.ReplaceAll(string(data), "\r\n", "\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
func resetResponseByPath() {
|
func setResponseByPath() {
|
||||||
responseByPath = map[string]string{
|
responseByPath = map[string]string{
|
||||||
"/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml": fileToStringX("./testdata/foobar_parser.yaml"),
|
"/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml": fileToStringX("./testdata/foobar_parser.yaml"),
|
||||||
"/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml": fileToStringX("./testdata/foobar_parser.yaml"),
|
"/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml": fileToStringX("./testdata/foobar_parser.yaml"),
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package cwhub
|
package cwhub
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -8,63 +9,71 @@ import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
"gopkg.in/yaml.v2"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
type DataSet struct {
|
type DataSet struct {
|
||||||
Data []*types.DataSource `yaml:"data,omitempty"`
|
Data []types.DataSource `yaml:"data,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func downloadFile(url string, destPath string) error {
|
func downloadFile(url string, destPath string) error {
|
||||||
log.Debugf("downloading %s in %s", url, destPath)
|
log.Debugf("downloading %s in %s", url, destPath)
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
resp, err := hubClient.Get(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return fmt.Errorf("while downloading %s: %w", url, err)
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body))
|
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
|
||||||
}
|
}
|
||||||
|
|
||||||
file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
|
file, err := os.Create(destPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
_, err = io.Copy(file, resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = file.Write(body)
|
if err = file.Sync(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = file.Sync()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetData(data []*types.DataSource, dataDir string) error {
|
// downloadData downloads the data files for an item
|
||||||
for _, dataS := range data {
|
func downloadData(dataFolder string, force bool, reader io.Reader) error {
|
||||||
destPath := filepath.Join(dataDir, dataS.DestPath)
|
dec := yaml.NewDecoder(reader)
|
||||||
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
|
|
||||||
|
|
||||||
err := downloadFile(dataS.SourceURL, destPath)
|
for {
|
||||||
if err != nil {
|
data := &DataSet{}
|
||||||
return err
|
|
||||||
|
if err := dec.Decode(data); err != nil {
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("while reading file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, dataS := range data.Data {
|
||||||
|
destPath := filepath.Join(dataFolder, dataS.DestPath)
|
||||||
|
|
||||||
|
if _, err := os.Stat(destPath); os.IsNotExist(err) || force {
|
||||||
|
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
|
||||||
|
|
||||||
|
if err := downloadFile(dataS.SourceURL, destPath); err != nil {
|
||||||
|
return fmt.Errorf("while getting data: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
|
|
||||||
"github.com/jarcoal/httpmock"
|
"github.com/jarcoal/httpmock"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDownloadFile(t *testing.T) {
|
func TestDownloadFile(t *testing.T) {
|
||||||
|
@ -14,12 +15,14 @@ func TestDownloadFile(t *testing.T) {
|
||||||
|
|
||||||
httpmock.Activate()
|
httpmock.Activate()
|
||||||
defer httpmock.DeactivateAndReset()
|
defer httpmock.DeactivateAndReset()
|
||||||
|
|
||||||
//OK
|
//OK
|
||||||
httpmock.RegisterResponder(
|
httpmock.RegisterResponder(
|
||||||
"GET",
|
"GET",
|
||||||
"https://example.com/xx",
|
"https://example.com/xx",
|
||||||
httpmock.NewStringResponder(200, "example content oneoneone"),
|
httpmock.NewStringResponder(200, "example content oneoneone"),
|
||||||
)
|
)
|
||||||
|
|
||||||
httpmock.RegisterResponder(
|
httpmock.RegisterResponder(
|
||||||
"GET",
|
"GET",
|
||||||
"https://example.com/x",
|
"https://example.com/x",
|
||||||
|
@ -27,17 +30,21 @@ func TestDownloadFile(t *testing.T) {
|
||||||
)
|
)
|
||||||
|
|
||||||
err := downloadFile("https://example.com/xx", examplePath)
|
err := downloadFile("https://example.com/xx", examplePath)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
content, err := os.ReadFile(examplePath)
|
content, err := os.ReadFile(examplePath)
|
||||||
assert.Equal(t, "example content oneoneone", string(content))
|
assert.Equal(t, "example content oneoneone", string(content))
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
//bad uri
|
//bad uri
|
||||||
err = downloadFile("https://zz.com", examplePath)
|
err = downloadFile("https://zz.com", examplePath)
|
||||||
assert.Error(t, err)
|
require.Error(t, err)
|
||||||
|
|
||||||
//404
|
//404
|
||||||
err = downloadFile("https://example.com/x", examplePath)
|
err = downloadFile("https://example.com/x", examplePath)
|
||||||
assert.Error(t, err)
|
require.Error(t, err)
|
||||||
|
|
||||||
//bad target
|
//bad target
|
||||||
err = downloadFile("https://example.com/xx", "")
|
err = downloadFile("https://example.com/xx", "")
|
||||||
assert.Error(t, err)
|
require.Error(t, err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,324 +0,0 @@
|
||||||
package cwhub
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"crypto/sha256"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
"gopkg.in/yaml.v2"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
|
||||||
)
|
|
||||||
|
|
||||||
var ErrIndexNotFound = fmt.Errorf("index not found")
|
|
||||||
|
|
||||||
func UpdateHubIdx(hub *csconfig.Hub) error {
|
|
||||||
bidx, err := DownloadHubIdx(hub)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to download index: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ret, err := LoadPkgIndex(bidx)
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, ErrMissingReference) {
|
|
||||||
return fmt.Errorf("failed to read index: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
hubIdx = ret
|
|
||||||
|
|
||||||
if _, err := LocalSync(hub); err != nil {
|
|
||||||
return fmt.Errorf("failed to sync: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
|
|
||||||
log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
|
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to build request for hub index: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed http request for hub index: %w", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
if resp.StatusCode == http.StatusNotFound {
|
|
||||||
return nil, ErrIndexNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
oldContent, err := os.ReadFile(hub.HubIndexFile)
|
|
||||||
if err != nil {
|
|
||||||
if !os.IsNotExist(err) {
|
|
||||||
log.Warningf("failed to read hub index: %s", err)
|
|
||||||
}
|
|
||||||
} else if bytes.Equal(body, oldContent) {
|
|
||||||
log.Info("hub index is up to date")
|
|
||||||
// write it anyway, can't hurt
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.OpenFile(hub.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("while opening hub index file: %w", err)
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
wsize, err := file.Write(body)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("while writing hub index file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
|
|
||||||
|
|
||||||
return body, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DownloadLatest will download the latest version of Item to the tdir directory
|
|
||||||
func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly bool) error {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
log.Debugf("Downloading %s %s", target.Type, target.Name)
|
|
||||||
|
|
||||||
if target.Type != COLLECTIONS {
|
|
||||||
if !target.Installed && updateOnly && target.Downloaded {
|
|
||||||
log.Debugf("skipping upgrade of %s : not installed", target.Name)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return DownloadItem(hub, target, overwrite)
|
|
||||||
}
|
|
||||||
|
|
||||||
// collection
|
|
||||||
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
|
|
||||||
for idx, ptr := range tmp {
|
|
||||||
ptrtype := ItemTypes[idx]
|
|
||||||
for _, p := range ptr {
|
|
||||||
val, ok := hubIdx[ptrtype][p]
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !val.Installed && updateOnly && val.Downloaded {
|
|
||||||
log.Debugf("skipping upgrade of %s : not installed", target.Name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly)
|
|
||||||
//recurse as it's a collection
|
|
||||||
if ptrtype == COLLECTIONS {
|
|
||||||
log.Tracef("collection, recurse")
|
|
||||||
|
|
||||||
err = DownloadLatest(hub, &val, overwrite, updateOnly)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while downloading %s: %w", val.Name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
downloaded := val.Downloaded
|
|
||||||
|
|
||||||
err = DownloadItem(hub, &val, overwrite)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while downloading %s: %w", val.Name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// We need to enable an item when it has been added to a collection since latest release of the collection.
|
|
||||||
// We check if val.Downloaded is false because maybe the item has been disabled by the user.
|
|
||||||
if !val.Installed && !downloaded {
|
|
||||||
if err = EnableItem(hub, &val); err != nil {
|
|
||||||
return fmt.Errorf("enabling '%s': %w", val.Name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
hubIdx[ptrtype][p] = val
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = DownloadItem(hub, target, overwrite)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to download item: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error {
|
|
||||||
tdir := hub.HubDir
|
|
||||||
|
|
||||||
// if user didn't --force, don't overwrite local, tainted, up-to-date files
|
|
||||||
if !overwrite {
|
|
||||||
if target.Tainted {
|
|
||||||
log.Debugf("%s : tainted, not updated", target.Name)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if target.UpToDate {
|
|
||||||
// We still have to check if data files are present
|
|
||||||
log.Debugf("%s : up-to-date, not updated", target.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while reading %s: %w", req.URL.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
h := sha256.New()
|
|
||||||
if _, err = h.Write(body); err != nil {
|
|
||||||
return fmt.Errorf("while hashing %s: %w", target.Name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
meow := fmt.Sprintf("%x", h.Sum(nil))
|
|
||||||
if meow != target.Versions[target.Version].Digest {
|
|
||||||
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
|
|
||||||
log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
|
|
||||||
|
|
||||||
return fmt.Errorf("invalid download hash for %s", target.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
//all good, install
|
|
||||||
//check if parent dir exists
|
|
||||||
tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/")
|
|
||||||
parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
|
|
||||||
|
|
||||||
// ensure that target file is within target dir
|
|
||||||
finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.HasPrefix(finalPath, tdir) {
|
|
||||||
return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir)
|
|
||||||
}
|
|
||||||
|
|
||||||
// check dir
|
|
||||||
if _, err = os.Stat(parentDir); os.IsNotExist(err) {
|
|
||||||
log.Debugf("%s doesn't exist, create", parentDir)
|
|
||||||
|
|
||||||
if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
|
|
||||||
return fmt.Errorf("while creating parent directories: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// check actual file
|
|
||||||
if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
|
|
||||||
log.Warningf("%s : overwrite", target.Name)
|
|
||||||
log.Debugf("target: %s/%s", tdir, target.RemotePath)
|
|
||||||
} else {
|
|
||||||
log.Infof("%s : OK", target.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while opening file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
_, err = f.Write(body)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while writing file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
target.Downloaded = true
|
|
||||||
target.Tainted = false
|
|
||||||
target.UpToDate = true
|
|
||||||
|
|
||||||
if err = downloadData(hub.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
|
|
||||||
return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
hubIdx[target.Type][target.Name] = *target
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
|
|
||||||
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName)
|
|
||||||
|
|
||||||
itemFile, err := os.Open(itemFilePath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while opening %s: %w", itemFilePath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer itemFile.Close()
|
|
||||||
|
|
||||||
if err = downloadData(hub.InstallDataDir, force, itemFile); err != nil {
|
|
||||||
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func downloadData(dataFolder string, force bool, reader io.Reader) error {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
dec := yaml.NewDecoder(reader)
|
|
||||||
|
|
||||||
for {
|
|
||||||
data := &DataSet{}
|
|
||||||
|
|
||||||
err = dec.Decode(data)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, io.EOF) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Errorf("while reading file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
download := false
|
|
||||||
|
|
||||||
for _, dataS := range data.Data {
|
|
||||||
if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
|
|
||||||
download = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if download || force {
|
|
||||||
err = GetData(data.Data, dataFolder)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while getting data: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
|
@ -1,52 +0,0 @@
|
||||||
package cwhub
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestDownloadHubIdx(t *testing.T) {
|
|
||||||
back := RawFileURLTemplate
|
|
||||||
// bad url template
|
|
||||||
fmt.Println("Test 'bad URL'")
|
|
||||||
|
|
||||||
RawFileURLTemplate = "x"
|
|
||||||
|
|
||||||
ret, err := DownloadHubIdx(&csconfig.Hub{})
|
|
||||||
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
|
|
||||||
log.Errorf("unexpected error %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("->%+v", ret)
|
|
||||||
|
|
||||||
// bad domain
|
|
||||||
fmt.Println("Test 'bad domain'")
|
|
||||||
|
|
||||||
RawFileURLTemplate = "https://baddomain/%s/%s"
|
|
||||||
|
|
||||||
ret, err = DownloadHubIdx(&csconfig.Hub{})
|
|
||||||
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
|
|
||||||
log.Errorf("unexpected error %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("->%+v", ret)
|
|
||||||
|
|
||||||
// bad target path
|
|
||||||
fmt.Println("Test 'bad target path'")
|
|
||||||
|
|
||||||
RawFileURLTemplate = back
|
|
||||||
|
|
||||||
ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
|
|
||||||
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
|
|
||||||
log.Errorf("unexpected error %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
RawFileURLTemplate = back
|
|
||||||
|
|
||||||
fmt.Printf("->%+v", ret)
|
|
||||||
}
|
|
167
pkg/cwhub/enable.go
Normal file
167
pkg/cwhub/enable.go
Normal file
|
@ -0,0 +1,167 @@
|
||||||
|
package cwhub
|
||||||
|
|
||||||
|
// Enable/disable items already downloaded
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// installLink returns the location of the symlink to the actual config file (eg. /etc/crowdsec/collections/xyz.yaml)
|
||||||
|
func (i *Item) installLink() string {
|
||||||
|
return filepath.Join(i.hub.local.InstallDir, i.Type, i.Stage, i.FileName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir
|
||||||
|
func (i *Item) createInstallLink() error {
|
||||||
|
dest, err := filepath.Abs(i.installLink())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
destDir := filepath.Dir(dest)
|
||||||
|
if err = os.MkdirAll(destDir, os.ModePerm); err != nil {
|
||||||
|
return fmt.Errorf("while creating %s: %w", destDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err = os.Lstat(dest); !os.IsNotExist(err) {
|
||||||
|
log.Infof("%s already exists.", dest)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
src, err := filepath.Abs(filepath.Join(i.hub.local.HubDir, i.RemotePath))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = os.Symlink(src, dest); err != nil {
|
||||||
|
return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items
|
||||||
|
func (i *Item) enable() error {
|
||||||
|
if i.Installed {
|
||||||
|
if i.Tainted {
|
||||||
|
return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if i.IsLocal() {
|
||||||
|
return fmt.Errorf("%s is local, won't enable", i.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// if it's a collection, check sub-items even if the collection file itself is up-to-date
|
||||||
|
if i.UpToDate && !i.HasSubItems() {
|
||||||
|
log.Tracef("%s is installed and up-to-date, skip.", i.Name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, sub := range i.SubItems() {
|
||||||
|
if err := sub.enable(); err != nil {
|
||||||
|
return fmt.Errorf("while installing %s: %w", sub.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := i.createInstallLink(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Infof("Enabled %s: %s", i.Type, i.Name)
|
||||||
|
i.Installed = true
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// purge removes the actual config file that was downloaded
|
||||||
|
func (i *Item) purge() error {
|
||||||
|
if !i.Downloaded {
|
||||||
|
log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
src := filepath.Join(i.hub.local.HubDir, i.RemotePath)
|
||||||
|
|
||||||
|
if err := os.Remove(src); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
log.Debugf("%s doesn't exist, no need to remove", src)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("while removing file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
i.Downloaded = false
|
||||||
|
log.Infof("Removed source file [%s]: %s", i.Name, src)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Item) removeInstallLink() error {
|
||||||
|
syml, err := filepath.Abs(i.installLink())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
stat, err := os.Lstat(syml)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
|
||||||
|
if stat.Mode()&os.ModeSymlink == 0 {
|
||||||
|
log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
|
||||||
|
return fmt.Errorf("%s isn't managed by hub", i.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
hubpath, err := os.Readlink(syml)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("while reading symlink: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
src, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if hubpath != src {
|
||||||
|
log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src)
|
||||||
|
return fmt.Errorf("%s isn't managed by hub", i.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Remove(syml); err != nil {
|
||||||
|
return fmt.Errorf("while removing symlink: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Infof("Removed symlink [%s]: %s", i.Name, syml)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// disable removes the symlink to the downloaded content, also removes the content if purge is true
|
||||||
|
func (i *Item) disable(purge bool, force bool) error {
|
||||||
|
// XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not
|
||||||
|
err := i.removeInstallLink()
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
if !purge && !force {
|
||||||
|
return fmt.Errorf("link %s does not exist (override with --force or --purge)", i.installLink())
|
||||||
|
}
|
||||||
|
} else if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
i.Installed = false
|
||||||
|
|
||||||
|
if purge {
|
||||||
|
if err := i.purge(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
141
pkg/cwhub/enable_test.go
Normal file
141
pkg/cwhub/enable_test.go
Normal file
|
@ -0,0 +1,141 @@
|
||||||
|
package cwhub
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func testInstall(hub *Hub, t *testing.T, item *Item) {
|
||||||
|
// Install the parser
|
||||||
|
err := item.downloadLatest(false, false)
|
||||||
|
require.NoError(t, err, "failed to download %s", item.Name)
|
||||||
|
|
||||||
|
err = hub.localSync()
|
||||||
|
require.NoError(t, err, "failed to run localSync")
|
||||||
|
|
||||||
|
assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
|
||||||
|
assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
|
||||||
|
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
|
||||||
|
|
||||||
|
err = item.enable()
|
||||||
|
require.NoError(t, err, "failed to enable %s", item.Name)
|
||||||
|
|
||||||
|
err = hub.localSync()
|
||||||
|
require.NoError(t, err, "failed to run localSync")
|
||||||
|
|
||||||
|
assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testTaint(hub *Hub, t *testing.T, item *Item) {
|
||||||
|
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
|
||||||
|
|
||||||
|
// truncate the file
|
||||||
|
f, err := os.Create(item.LocalPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
f.Close()
|
||||||
|
|
||||||
|
// Local sync and check status
|
||||||
|
err = hub.localSync()
|
||||||
|
require.NoError(t, err, "failed to run localSync")
|
||||||
|
|
||||||
|
assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testUpdate(hub *Hub, t *testing.T, item *Item) {
|
||||||
|
assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
|
||||||
|
|
||||||
|
// Update it + check status
|
||||||
|
err := item.downloadLatest(true, true)
|
||||||
|
require.NoError(t, err, "failed to update %s", item.Name)
|
||||||
|
|
||||||
|
// Local sync and check status
|
||||||
|
err = hub.localSync()
|
||||||
|
require.NoError(t, err, "failed to run localSync")
|
||||||
|
|
||||||
|
assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
|
||||||
|
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testDisable(hub *Hub, t *testing.T, item *Item) {
|
||||||
|
assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
|
||||||
|
|
||||||
|
// Remove
|
||||||
|
err := item.disable(false, false)
|
||||||
|
require.NoError(t, err, "failed to disable %s", item.Name)
|
||||||
|
|
||||||
|
// Local sync and check status
|
||||||
|
err = hub.localSync()
|
||||||
|
require.NoError(t, err, "failed to run localSync")
|
||||||
|
require.Empty(t, hub.Warnings)
|
||||||
|
|
||||||
|
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
|
||||||
|
assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
|
||||||
|
assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
|
||||||
|
|
||||||
|
// Purge
|
||||||
|
err = item.disable(true, false)
|
||||||
|
require.NoError(t, err, "failed to purge %s", item.Name)
|
||||||
|
|
||||||
|
// Local sync and check status
|
||||||
|
err = hub.localSync()
|
||||||
|
require.NoError(t, err, "failed to run localSync")
|
||||||
|
require.Empty(t, hub.Warnings)
|
||||||
|
|
||||||
|
assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
|
||||||
|
assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInstallParser(t *testing.T) {
|
||||||
|
/*
|
||||||
|
- install a random parser
|
||||||
|
- check its status
|
||||||
|
- taint it
|
||||||
|
- check its status
|
||||||
|
- force update it
|
||||||
|
- check its status
|
||||||
|
- remove it
|
||||||
|
*/
|
||||||
|
hub := envSetup(t)
|
||||||
|
|
||||||
|
// map iteration is random by itself
|
||||||
|
for _, it := range hub.Items[PARSERS] {
|
||||||
|
testInstall(hub, t, it)
|
||||||
|
it = hub.Items[PARSERS][it.Name]
|
||||||
|
testTaint(hub, t, it)
|
||||||
|
it = hub.Items[PARSERS][it.Name]
|
||||||
|
testUpdate(hub, t, it)
|
||||||
|
it = hub.Items[PARSERS][it.Name]
|
||||||
|
testDisable(hub, t, it)
|
||||||
|
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInstallCollection(t *testing.T) {
|
||||||
|
/*
|
||||||
|
- install a random parser
|
||||||
|
- check its status
|
||||||
|
- taint it
|
||||||
|
- check its status
|
||||||
|
- force update it
|
||||||
|
- check its status
|
||||||
|
- remove it
|
||||||
|
*/
|
||||||
|
hub := envSetup(t)
|
||||||
|
|
||||||
|
// map iteration is random by itself
|
||||||
|
for _, it := range hub.Items[COLLECTIONS] {
|
||||||
|
testInstall(hub, t, it)
|
||||||
|
it = hub.Items[COLLECTIONS][it.Name]
|
||||||
|
testTaint(hub, t, it)
|
||||||
|
it = hub.Items[COLLECTIONS][it.Name]
|
||||||
|
testUpdate(hub, t, it)
|
||||||
|
it = hub.Items[COLLECTIONS][it.Name]
|
||||||
|
testDisable(hub, t, it)
|
||||||
|
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
21
pkg/cwhub/errors.go
Normal file
21
pkg/cwhub/errors.go
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
package cwhub
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor.
|
||||||
|
// All attempts to download index or items will return this error.
|
||||||
|
ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
|
||||||
|
)
|
||||||
|
|
||||||
|
type IndexNotFoundError struct {
|
||||||
|
URL string
|
||||||
|
Branch string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e IndexNotFoundError) Error() string {
|
||||||
|
return fmt.Sprintf("index not found at %s, branch '%s'. Please check the .cscli.hub_branch value if you specified it in config.yaml, or use 'master' if not sure", e.URL, e.Branch)
|
||||||
|
}
|
|
@ -1,222 +1,340 @@
|
||||||
package cwhub
|
package cwhub
|
||||||
|
|
||||||
|
// Install, upgrade and remove items from the hub to the local configuration
|
||||||
|
|
||||||
|
// XXX: this file could use a better name
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/enescakir/emoji"
|
"github.com/enescakir/emoji"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"golang.org/x/mod/semver"
|
"slices"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// pick a hub branch corresponding to the current crowdsec version.
|
// Install installs the item from the hub, downloading it if needed
|
||||||
func chooseHubBranch() string {
|
func (i *Item) Install(force bool, downloadOnly bool) error {
|
||||||
latest, err := cwversion.Latest()
|
if downloadOnly && i.Downloaded && i.UpToDate {
|
||||||
if err != nil {
|
log.Infof("%s is already downloaded and up-to-date", i.Name)
|
||||||
log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
|
|
||||||
//lint:ignore nilerr
|
|
||||||
return "master"
|
|
||||||
}
|
|
||||||
|
|
||||||
csVersion := cwversion.VersionStrip()
|
|
||||||
if csVersion == latest {
|
|
||||||
log.Debugf("current version is equal to latest (%s)", csVersion)
|
|
||||||
return "master"
|
|
||||||
}
|
|
||||||
|
|
||||||
// if current version is greater than the latest we are in pre-release
|
|
||||||
if semver.Compare(csVersion, latest) == 1 {
|
|
||||||
log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
|
|
||||||
return "master"
|
|
||||||
}
|
|
||||||
|
|
||||||
if csVersion == "" {
|
|
||||||
log.Warning("Crowdsec version is not set, using master branch for the hub")
|
|
||||||
return "master"
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Warnf("Crowdsec is not the latest version. "+
|
|
||||||
"Current version is '%s' and the latest stable version is '%s'. Please update it!",
|
|
||||||
csVersion, latest)
|
|
||||||
|
|
||||||
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
|
|
||||||
"added to Crowdsec Hub after CrowdSec %s", latest)
|
|
||||||
|
|
||||||
return csVersion
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetHubBranch sets the package variable that points to the hub branch.
|
|
||||||
func SetHubBranch() {
|
|
||||||
// a branch is already set, or specified from the flags
|
|
||||||
if HubBranch != "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// use the branch corresponding to the crowdsec version
|
|
||||||
HubBranch = chooseHubBranch()
|
|
||||||
|
|
||||||
log.Debugf("Using branch '%s' for the hub", HubBranch)
|
|
||||||
}
|
|
||||||
|
|
||||||
func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error {
|
|
||||||
item := GetItem(obtype, name)
|
|
||||||
if item == nil {
|
|
||||||
return fmt.Errorf("unable to retrieve item: %s", name)
|
|
||||||
}
|
|
||||||
|
|
||||||
if downloadOnly && item.Downloaded && item.UpToDate {
|
|
||||||
log.Warningf("%s is already downloaded and up-to-date", item.Name)
|
|
||||||
|
|
||||||
if !force {
|
if !force {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err := DownloadLatest(csConfig.Hub, item, force, true)
|
// XXX: confusing semantic between force and updateOnly?
|
||||||
if err != nil {
|
if err := i.downloadLatest(force, true); err != nil {
|
||||||
return fmt.Errorf("while downloading %s: %w", item.Name, err)
|
return fmt.Errorf("while downloading %s: %w", i.Name, err)
|
||||||
}
|
|
||||||
|
|
||||||
if err = AddItem(obtype, *item); err != nil {
|
|
||||||
return fmt.Errorf("while adding %s: %w", item.Name, err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if downloadOnly {
|
if downloadOnly {
|
||||||
log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath))
|
// XXX: should get the path from downloadLatest
|
||||||
|
log.Infof("Downloaded %s to %s", i.Name, filepath.Join(i.hub.local.HubDir, i.RemotePath))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err = EnableItem(csConfig.Hub, item)
|
// XXX: should we stop here if the item is already installed?
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while enabling %s: %w", item.Name, err)
|
if err := i.enable(); err != nil {
|
||||||
|
return fmt.Errorf("while enabling %s: %w", i.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := AddItem(obtype, *item); err != nil {
|
log.Infof("Enabled %s", i.Name)
|
||||||
return fmt.Errorf("while adding %s: %w", item.Name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("Enabled %s", item.Name)
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// XXX this must return errors instead of log.Fatal
|
// allDependencies return a list of all dependencies and sub-dependencies of the item
|
||||||
func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) {
|
func (i *Item) allDependencies() []*Item {
|
||||||
if name != "" {
|
var deps []*Item
|
||||||
item := GetItem(itemType, name)
|
|
||||||
if item == nil {
|
|
||||||
log.Fatalf("unable to retrieve: %s", name)
|
|
||||||
}
|
|
||||||
|
|
||||||
err := DisableItem(csConfig.Hub, item, purge, forceAction)
|
for _, dep := range i.SubItems() {
|
||||||
|
if dep == i {
|
||||||
if err != nil {
|
log.Errorf("circular dependency detected: %s depends on %s", dep.Name, i.Name)
|
||||||
log.Fatalf("unable to disable %s : %v", item.Name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = AddItem(itemType, *item); err != nil {
|
|
||||||
log.Fatalf("unable to add %s: %v", item.Name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !all {
|
|
||||||
log.Fatal("removing item: no item specified")
|
|
||||||
}
|
|
||||||
|
|
||||||
disabled := 0
|
|
||||||
|
|
||||||
// remove all
|
|
||||||
for _, v := range GetItemMap(itemType) {
|
|
||||||
if !v.Installed {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
err := DisableItem(csConfig.Hub, &v, purge, forceAction)
|
deps = append(deps, dep.allDependencies()...)
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("unable to disable %s : %v", v.Name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := AddItem(itemType, v); err != nil {
|
|
||||||
log.Fatalf("unable to add %s: %v", v.Name, err)
|
|
||||||
}
|
|
||||||
disabled++
|
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("Disabled %d items", disabled)
|
return append(deps, i)
|
||||||
}
|
}
|
||||||
|
|
||||||
func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) {
|
// Remove disables the item, optionally removing the downloaded content
|
||||||
updated := 0
|
func (i *Item) Remove(purge bool, force bool) (bool, error) {
|
||||||
found := false
|
if i.IsLocal() {
|
||||||
|
return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
|
||||||
|
}
|
||||||
|
|
||||||
for _, v := range GetItemMap(itemType) {
|
if i.Tainted && !force {
|
||||||
if name != "" && name != v.Name {
|
return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !i.Installed && !purge {
|
||||||
|
log.Infof("removing %s: not installed -- no need to remove", i.Name)
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
removed := false
|
||||||
|
|
||||||
|
allDeps := i.allDependencies()
|
||||||
|
|
||||||
|
for _, sub := range i.SubItems() {
|
||||||
|
if !sub.Installed {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if !v.Installed {
|
// if the other collection(s) are direct or indirect dependencies of the current one, it's good to go
|
||||||
log.Tracef("skip %s, not installed", v.Name)
|
// log parent collections
|
||||||
continue
|
for _, subParent := range sub.parentCollections() {
|
||||||
}
|
if subParent == i {
|
||||||
|
continue
|
||||||
if !v.Downloaded {
|
|
||||||
log.Warningf("%s : not downloaded, please install.", v.Name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
found = true
|
|
||||||
|
|
||||||
if v.UpToDate {
|
|
||||||
log.Infof("%s : up-to-date", v.Name)
|
|
||||||
|
|
||||||
if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil {
|
|
||||||
log.Fatalf("%s : download failed : %v", v.Name, err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !force {
|
if !slices.Contains(allDeps, subParent) {
|
||||||
|
log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil {
|
subRemoved, err := sub.Remove(purge, force)
|
||||||
log.Fatalf("%s : download failed : %v", v.Name, err)
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !v.UpToDate {
|
removed = removed || subRemoved
|
||||||
if v.Tainted {
|
|
||||||
log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name)
|
|
||||||
} else if v.Local {
|
|
||||||
log.Infof("%v %s is local", emoji.Prohibited, v.Name)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// this is used while scripting to know if the hub has been upgraded
|
|
||||||
// and a configuration reload is required
|
|
||||||
fmt.Printf("updated %s\n", v.Name)
|
|
||||||
log.Infof("%v %s : updated", emoji.Package, v.Name)
|
|
||||||
updated++
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := AddItem(itemType, v); err != nil {
|
|
||||||
log.Fatalf("unable to add %s: %v", v.Name, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !found && name == "" {
|
err := i.disable(purge, force)
|
||||||
log.Infof("No %s installed, nothing to upgrade", itemType)
|
if err != nil {
|
||||||
} else if !found {
|
return false, fmt.Errorf("while removing %s: %w", i.Name, err)
|
||||||
log.Errorf("Item '%s' not found in hub", name)
|
|
||||||
} else if updated == 0 && found {
|
|
||||||
if name == "" {
|
|
||||||
log.Infof("All %s are already up-to-date", itemType)
|
|
||||||
} else {
|
|
||||||
log.Infof("Item '%s' is up-to-date", name)
|
|
||||||
}
|
|
||||||
} else if updated != 0 {
|
|
||||||
log.Infof("Upgraded %d items", updated)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// XXX: should take the value from disable()
|
||||||
|
removed = true
|
||||||
|
|
||||||
|
return removed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upgrade downloads and applies the last version from the hub
|
||||||
|
func (i *Item) Upgrade(force bool) (bool, error) {
|
||||||
|
updated := false
|
||||||
|
|
||||||
|
if !i.Downloaded {
|
||||||
|
return false, fmt.Errorf("can't upgrade %s: not installed", i.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !i.Installed {
|
||||||
|
return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", i.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if i.UpToDate {
|
||||||
|
log.Infof("%s: up-to-date", i.Name)
|
||||||
|
|
||||||
|
if err := i.DownloadDataIfNeeded(force); err != nil {
|
||||||
|
return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !force {
|
||||||
|
// no upgrade needed
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := i.downloadLatest(force, true); err != nil {
|
||||||
|
return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !i.UpToDate {
|
||||||
|
if i.Tainted {
|
||||||
|
log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name)
|
||||||
|
} else if i.IsLocal() {
|
||||||
|
log.Infof("%v %s is local", emoji.Prohibited, i.Name)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// a check on stdout is used while scripting to know if the hub has been upgraded
|
||||||
|
// and a configuration reload is required
|
||||||
|
// TODO: use a better way to communicate this
|
||||||
|
fmt.Printf("updated %s\n", i.Name)
|
||||||
|
log.Infof("%v %s: updated", emoji.Package, i.Name)
|
||||||
|
updated = true
|
||||||
|
}
|
||||||
|
|
||||||
|
return updated, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// downloadLatest will download the latest version of Item to the tdir directory
|
||||||
|
func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error {
|
||||||
|
// XXX: should return the path of the downloaded file (taken from download())
|
||||||
|
log.Debugf("Downloading %s %s", i.Type, i.Name)
|
||||||
|
|
||||||
|
for _, sub := range i.SubItems() {
|
||||||
|
if !sub.Installed && updateOnly && sub.Downloaded {
|
||||||
|
log.Debugf("skipping upgrade of %s: not installed", i.Name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.Installed, updateOnly)
|
||||||
|
|
||||||
|
// recurse as it's a collection
|
||||||
|
if sub.HasSubItems() {
|
||||||
|
log.Tracef("collection, recurse")
|
||||||
|
|
||||||
|
if err := sub.downloadLatest(overwrite, updateOnly); err != nil {
|
||||||
|
return fmt.Errorf("while downloading %s: %w", sub.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
downloaded := sub.Downloaded
|
||||||
|
|
||||||
|
if err := sub.download(overwrite); err != nil {
|
||||||
|
return fmt.Errorf("while downloading %s: %w", sub.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// We need to enable an item when it has been added to a collection since latest release of the collection.
|
||||||
|
// We check if sub.Downloaded is false because maybe the item has been disabled by the user.
|
||||||
|
if !sub.Installed && !downloaded {
|
||||||
|
if err := sub.enable(); err != nil {
|
||||||
|
return fmt.Errorf("enabling '%s': %w", sub.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !i.Installed && updateOnly && i.Downloaded {
|
||||||
|
log.Debugf("skipping upgrade of %s: not installed", i.Name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := i.download(overwrite); err != nil {
|
||||||
|
return fmt.Errorf("failed to download item: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetch downloads the item from the hub, verifies the hash and returns the body
|
||||||
|
func (i *Item) fetch() ([]byte, error) {
|
||||||
|
url, err := i.hub.remote.urlTo(i.RemotePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to build hub item request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := hubClient.Get(url)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("while downloading %s: %w", url, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("while downloading %s: %w", url, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
hash := sha256.New()
|
||||||
|
if _, err = hash.Write(body); err != nil {
|
||||||
|
return nil, fmt.Errorf("while hashing %s: %w", i.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
meow := hex.EncodeToString(hash.Sum(nil))
|
||||||
|
if meow != i.Versions[i.Version].Digest {
|
||||||
|
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
|
||||||
|
log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("invalid download hash for %s", i.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// download downloads the item from the hub and writes it to the hub directory
|
||||||
|
func (i *Item) download(overwrite bool) error {
|
||||||
|
// if user didn't --force, don't overwrite local, tainted, up-to-date files
|
||||||
|
if !overwrite {
|
||||||
|
if i.Tainted {
|
||||||
|
log.Debugf("%s: tainted, not updated", i.Name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if i.UpToDate {
|
||||||
|
// We still have to check if data files are present
|
||||||
|
log.Debugf("%s: up-to-date, not updated", i.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := i.fetch()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
tdir := i.hub.local.HubDir
|
||||||
|
|
||||||
|
//all good, install
|
||||||
|
|
||||||
|
finalPath, err := filepath.Abs(filepath.Join(tdir, i.RemotePath))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensure that target file is within target dir
|
||||||
|
if !strings.HasPrefix(finalPath, tdir) {
|
||||||
|
return fmt.Errorf("path %s escapes %s, abort", i.RemotePath, tdir)
|
||||||
|
}
|
||||||
|
|
||||||
|
parentDir := filepath.Dir(finalPath)
|
||||||
|
|
||||||
|
if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
|
||||||
|
return fmt.Errorf("while creating %s: %w", parentDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// check actual file
|
||||||
|
if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
|
||||||
|
log.Warningf("%s: overwrite", i.Name)
|
||||||
|
log.Debugf("target: %s/%s", tdir, i.RemotePath)
|
||||||
|
} else {
|
||||||
|
log.Infof("%s: OK", i.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = os.WriteFile(finalPath, body, 0o644); err != nil {
|
||||||
|
return fmt.Errorf("while writing %s: %w", finalPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
i.Downloaded = true
|
||||||
|
i.Tainted = false
|
||||||
|
i.UpToDate = true
|
||||||
|
|
||||||
|
if err = downloadData(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
|
||||||
|
return fmt.Errorf("while downloading data for %s: %w", i.FileName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadDataIfNeeded downloads the data files for the item
|
||||||
|
func (i *Item) DownloadDataIfNeeded(force bool) error {
|
||||||
|
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", i.hub.local.InstallDir, i.Type, i.Stage, i.FileName)
|
||||||
|
|
||||||
|
itemFile, err := os.Open(itemFilePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("while opening %s: %w", itemFilePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer itemFile.Close()
|
||||||
|
|
||||||
|
if err = downloadData(i.hub.local.InstallDataDir, force, itemFile); err != nil {
|
||||||
|
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,157 +4,187 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
|
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
|
||||||
// We expect the new scenario to be installed
|
// We expect the new scenario to be installed
|
||||||
func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
|
func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
|
||||||
cfg := envSetup(t)
|
hub := envSetup(t)
|
||||||
defer envTearDown(cfg)
|
|
||||||
|
|
||||||
// fresh install of collection
|
// fresh install of collection
|
||||||
getHubIdxOrFail(t)
|
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
||||||
|
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
||||||
|
|
||||||
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
|
||||||
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
require.NoError(t, item.Install(false, false))
|
||||||
|
|
||||||
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
||||||
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
|
||||||
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
|
||||||
|
|
||||||
// This is the scenario that gets added in next version of collection
|
// This is the scenario that gets added in next version of collection
|
||||||
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
|
require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"])
|
||||||
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
|
|
||||||
|
|
||||||
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
|
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
|
||||||
|
|
||||||
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
|
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
|
||||||
pushUpdateToCollectionInHub()
|
pushUpdateToCollectionInHub()
|
||||||
|
|
||||||
if err := UpdateHubIdx(cfg.Hub); err != nil {
|
remote := &RemoteHubCfg{
|
||||||
t.Fatalf("failed to download index : %s", err)
|
URLTemplate: mockURLTemplate,
|
||||||
|
Branch: "master",
|
||||||
|
IndexPath: ".index.json",
|
||||||
}
|
}
|
||||||
|
|
||||||
getHubIdxOrFail(t)
|
hub, err := NewHub(hub.local, remote, true)
|
||||||
|
require.NoError(t, err, "failed to download index: %s", err)
|
||||||
|
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
hub = getHubOrFail(t, hub.local, remote)
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
|
||||||
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
|
||||||
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
|
||||||
|
|
||||||
UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
||||||
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
||||||
|
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
||||||
|
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
||||||
|
|
||||||
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
|
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
|
||||||
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
|
didUpdate, err := item.Upgrade(false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.True(t, didUpdate)
|
||||||
|
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
|
||||||
|
|
||||||
|
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
|
||||||
|
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Install a collection, disable a scenario.
|
// Install a collection, disable a scenario.
|
||||||
// Upgrade should install should not enable/download the disabled scenario.
|
// Upgrade should install should not enable/download the disabled scenario.
|
||||||
func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
|
func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
|
||||||
cfg := envSetup(t)
|
hub := envSetup(t)
|
||||||
defer envTearDown(cfg)
|
|
||||||
|
|
||||||
// fresh install of collection
|
// fresh install of collection
|
||||||
getHubIdxOrFail(t)
|
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
||||||
|
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
||||||
|
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
||||||
|
|
||||||
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
|
||||||
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
require.NoError(t, item.Install(false, false))
|
||||||
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
|
||||||
|
|
||||||
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
||||||
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
||||||
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
||||||
|
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
||||||
|
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
||||||
|
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
|
||||||
|
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
didRemove, err := item.Remove(false, false)
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
require.NoError(t, err)
|
||||||
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
require.True(t, didRemove)
|
||||||
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
|
||||||
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
|
|
||||||
|
|
||||||
RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
|
remote := &RemoteHubCfg{
|
||||||
getHubIdxOrFail(t)
|
URLTemplate: mockURLTemplate,
|
||||||
// scenario referenced by collection was deleted hence, collection should be tainted
|
Branch: "master",
|
||||||
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
IndexPath: ".index.json",
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
|
||||||
|
|
||||||
if err := UpdateHubIdx(cfg.Hub); err != nil {
|
|
||||||
t.Fatalf("failed to download index : %s", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
|
hub = getHubOrFail(t, hub.local, remote)
|
||||||
|
// scenario referenced by collection was deleted hence, collection should be tainted
|
||||||
|
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
||||||
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
||||||
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
||||||
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
||||||
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
||||||
|
|
||||||
getHubIdxOrFail(t)
|
hub, err = NewHub(hub.local, remote, true)
|
||||||
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
require.NoError(t, err, "failed to download index: %s", err)
|
||||||
|
|
||||||
|
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
|
||||||
|
didUpdate, err := item.Upgrade(false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.False(t, didUpdate)
|
||||||
|
|
||||||
|
hub = getHubOrFail(t, hub.local, remote)
|
||||||
|
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getHubIdxOrFail(t *testing.T) {
|
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test
|
||||||
if err := GetHubIdx(getTestCfg().Hub); err != nil {
|
func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
|
||||||
t.Fatalf("failed to load hub index")
|
hub, err := NewHub(local, remote, false)
|
||||||
}
|
require.NoError(t, err, "failed to load hub index")
|
||||||
|
|
||||||
|
return hub
|
||||||
}
|
}
|
||||||
|
|
||||||
// Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario
|
// Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario
|
||||||
// Upgrade should not enable/download the disabled scenario.
|
// Upgrade should not enable/download the disabled scenario.
|
||||||
// Upgrade should install and enable the newly added scenario.
|
// Upgrade should install and enable the newly added scenario.
|
||||||
func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
|
func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
|
||||||
cfg := envSetup(t)
|
hub := envSetup(t)
|
||||||
defer envTearDown(cfg)
|
|
||||||
|
|
||||||
// fresh install of collection
|
// fresh install of collection
|
||||||
getHubIdxOrFail(t)
|
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
||||||
|
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
||||||
|
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
||||||
|
|
||||||
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
|
||||||
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
require.NoError(t, item.Install(false, false))
|
||||||
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
|
||||||
|
|
||||||
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
||||||
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
||||||
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
||||||
|
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
||||||
|
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
||||||
|
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
|
||||||
|
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
didRemove, err := item.Remove(false, false)
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
require.NoError(t, err)
|
||||||
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
require.True(t, didRemove)
|
||||||
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
|
||||||
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
|
|
||||||
|
|
||||||
RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
|
remote := &RemoteHubCfg{
|
||||||
getHubIdxOrFail(t)
|
URLTemplate: mockURLTemplate,
|
||||||
|
Branch: "master",
|
||||||
|
IndexPath: ".index.json",
|
||||||
|
}
|
||||||
|
|
||||||
|
hub = getHubOrFail(t, hub.local, remote)
|
||||||
// scenario referenced by collection was deleted hence, collection should be tainted
|
// scenario referenced by collection was deleted hence, collection should be tainted
|
||||||
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
||||||
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
|
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
|
||||||
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
|
||||||
|
|
||||||
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
|
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
|
||||||
// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
|
// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
|
||||||
// we just removed. Nor should it install the newly added scenario
|
// we just removed. Nor should it install the newly added scenario
|
||||||
pushUpdateToCollectionInHub()
|
pushUpdateToCollectionInHub()
|
||||||
|
|
||||||
if err := UpdateHubIdx(cfg.Hub); err != nil {
|
hub, err = NewHub(hub.local, remote, true)
|
||||||
t.Fatalf("failed to download index : %s", err)
|
require.NoError(t, err, "failed to download index: %s", err)
|
||||||
}
|
|
||||||
|
|
||||||
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
||||||
getHubIdxOrFail(t)
|
hub = getHubOrFail(t, hub.local, remote)
|
||||||
|
|
||||||
UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
|
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
|
||||||
getHubIdxOrFail(t)
|
didUpdate, err := item.Upgrade(false)
|
||||||
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
require.NoError(t, err)
|
||||||
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
|
require.True(t, didUpdate)
|
||||||
|
|
||||||
|
hub = getHubOrFail(t, hub.local, remote)
|
||||||
|
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
|
||||||
|
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertCollectionDepsInstalled(t *testing.T, collection string) {
|
func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
c := hubIdx[COLLECTIONS][collection]
|
c := hub.Items[COLLECTIONS][collection]
|
||||||
require.NoError(t, CollecDepsCheck(&c))
|
require.NoError(t, hub.checkSubItems(c))
|
||||||
}
|
}
|
||||||
|
|
||||||
func pushUpdateToCollectionInHub() {
|
func pushUpdateToCollectionInHub() {
|
||||||
|
|
122
pkg/cwhub/hub.go
Normal file
122
pkg/cwhub/hub.go
Normal file
|
@ -0,0 +1,122 @@
|
||||||
|
package cwhub
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Hub struct {
|
||||||
|
Items HubItems
|
||||||
|
local *csconfig.LocalHubCfg
|
||||||
|
remote *RemoteHubCfg
|
||||||
|
skippedLocal int
|
||||||
|
skippedTainted int
|
||||||
|
Warnings []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Hub) GetDataDir() string {
|
||||||
|
return h.local.InstallDataDir
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state
|
||||||
|
// It also downloads the index if downloadIndex is true
|
||||||
|
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, downloadIndex bool) (*Hub, error) {
|
||||||
|
if local == nil {
|
||||||
|
return nil, fmt.Errorf("no hub configuration found")
|
||||||
|
}
|
||||||
|
|
||||||
|
if downloadIndex {
|
||||||
|
if err := remote.downloadIndex(local.HubIndexFile); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debugf("loading hub idx %s", local.HubIndexFile)
|
||||||
|
|
||||||
|
hub := &Hub{
|
||||||
|
local: local,
|
||||||
|
remote: remote,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := hub.parseIndex(); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to load index: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := hub.localSync(); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to sync items: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return hub, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections
|
||||||
|
func (h *Hub) parseIndex() error {
|
||||||
|
bidx, err := os.ReadFile(h.local.HubIndexFile)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("unable to read index file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := json.Unmarshal(bidx, &h.Items); err != nil {
|
||||||
|
return fmt.Errorf("failed to unmarshal index: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debugf("%d item types in hub index", len(ItemTypes))
|
||||||
|
|
||||||
|
// Iterate over the different types to complete the struct
|
||||||
|
for _, itemType := range ItemTypes {
|
||||||
|
log.Tracef("%s: %d items", itemType, len(h.Items[itemType]))
|
||||||
|
|
||||||
|
for name, item := range h.Items[itemType] {
|
||||||
|
item.hub = h
|
||||||
|
item.Name = name
|
||||||
|
|
||||||
|
// if the item has no (redundant) author, take it from the json key
|
||||||
|
if item.Author == "" && strings.Contains(name, "/") {
|
||||||
|
item.Author = strings.Split(name, "/")[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
item.Type = itemType
|
||||||
|
x := strings.Split(item.RemotePath, "/")
|
||||||
|
item.FileName = x[len(x)-1]
|
||||||
|
|
||||||
|
item.logMissingSubItems()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ItemStats returns total counts of the hub items
|
||||||
|
func (h *Hub) ItemStats() []string {
|
||||||
|
loaded := ""
|
||||||
|
|
||||||
|
for _, itemType := range ItemTypes {
|
||||||
|
if len(h.Items[itemType]) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded = strings.Trim(loaded, ", ")
|
||||||
|
if loaded == "" {
|
||||||
|
// empty hub
|
||||||
|
loaded = "0 items"
|
||||||
|
}
|
||||||
|
|
||||||
|
ret := []string{
|
||||||
|
fmt.Sprintf("Loaded: %s", loaded),
|
||||||
|
}
|
||||||
|
|
||||||
|
if h.skippedLocal > 0 || h.skippedTainted > 0 {
|
||||||
|
ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted))
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret
|
||||||
|
}
|
73
pkg/cwhub/hub_test.go
Normal file
73
pkg/cwhub/hub_test.go
Normal file
|
@ -0,0 +1,73 @@
|
||||||
|
package cwhub
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/go-cs-lib/cstest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestInitHubUpdate(t *testing.T) {
|
||||||
|
hub := envSetup(t)
|
||||||
|
|
||||||
|
remote := &RemoteHubCfg{
|
||||||
|
URLTemplate: mockURLTemplate,
|
||||||
|
Branch: "master",
|
||||||
|
IndexPath: ".index.json",
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := NewHub(hub.local, remote, true)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDownloadIndex(t *testing.T) {
|
||||||
|
// bad url template
|
||||||
|
fmt.Println("Test 'bad URL'")
|
||||||
|
|
||||||
|
tmpIndex, err := os.CreateTemp("", "index.json")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
os.Remove(tmpIndex.Name())
|
||||||
|
})
|
||||||
|
|
||||||
|
hub := envSetup(t)
|
||||||
|
|
||||||
|
hub.remote = &RemoteHubCfg{
|
||||||
|
URLTemplate: "x",
|
||||||
|
Branch: "",
|
||||||
|
IndexPath: "",
|
||||||
|
}
|
||||||
|
|
||||||
|
err = hub.remote.downloadIndex(tmpIndex.Name())
|
||||||
|
cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'")
|
||||||
|
|
||||||
|
// bad domain
|
||||||
|
fmt.Println("Test 'bad domain'")
|
||||||
|
|
||||||
|
hub.remote = &RemoteHubCfg{
|
||||||
|
URLTemplate: "https://baddomain/%s/%s",
|
||||||
|
Branch: "master",
|
||||||
|
IndexPath: ".index.json",
|
||||||
|
}
|
||||||
|
|
||||||
|
err = hub.remote.downloadIndex(tmpIndex.Name())
|
||||||
|
require.NoError(t, err)
|
||||||
|
// XXX: this is not failing
|
||||||
|
// cstest.RequireErrorContains(t, err, "failed http request for hub index: Get")
|
||||||
|
|
||||||
|
// bad target path
|
||||||
|
fmt.Println("Test 'bad target path'")
|
||||||
|
|
||||||
|
hub.remote = &RemoteHubCfg{
|
||||||
|
URLTemplate: mockURLTemplate,
|
||||||
|
Branch: "master",
|
||||||
|
IndexPath: ".index.json",
|
||||||
|
}
|
||||||
|
|
||||||
|
err = hub.remote.downloadIndex("/does/not/exist/index.json")
|
||||||
|
cstest.RequireErrorContains(t, err, "failed to write hub index: open /does/not/exist/index.json:")
|
||||||
|
}
|
|
@ -1,214 +0,0 @@
|
||||||
package cwhub
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
|
||||||
)
|
|
||||||
|
|
||||||
func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
|
|
||||||
itempath := hub.HubDir + "/" + target.RemotePath
|
|
||||||
|
|
||||||
// disable hub file
|
|
||||||
if err := os.Remove(itempath); err != nil {
|
|
||||||
return target, fmt.Errorf("while removing file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
target.Downloaded = false
|
|
||||||
log.Infof("Removed source file [%s]: %s", target.Name, itempath)
|
|
||||||
hubIdx[target.Type][target.Name] = target
|
|
||||||
|
|
||||||
return target, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DisableItem to disable an item managed by the hub, removes the symlink if purge is true
|
|
||||||
func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
// already disabled, noop unless purge
|
|
||||||
if !target.Installed {
|
|
||||||
if purge {
|
|
||||||
*target, err = purgeItem(hub, *target)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if target.Local {
|
|
||||||
return fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
if target.Tainted && !force {
|
|
||||||
return fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// for a COLLECTIONS, disable sub-items
|
|
||||||
if target.Type == COLLECTIONS {
|
|
||||||
for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
|
|
||||||
ptrtype := ItemTypes[idx]
|
|
||||||
for _, p := range ptr {
|
|
||||||
if val, ok := hubIdx[ptrtype][p]; ok {
|
|
||||||
// check if the item doesn't belong to another collection before removing it
|
|
||||||
toRemove := true
|
|
||||||
|
|
||||||
for _, collection := range val.BelongsToCollections {
|
|
||||||
if collection != target.Name {
|
|
||||||
toRemove = false
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if toRemove {
|
|
||||||
err = DisableItem(hub, &val, purge, force)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while disabling %s: %w", p, err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
log.Infof("%s was not removed because it belongs to another collection", val.Name)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
syml, err := filepath.Abs(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
stat, err := os.Lstat(syml)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
// we only accept to "delete" non existing items if it's a forced purge
|
|
||||||
if !purge && !force {
|
|
||||||
return fmt.Errorf("can't delete %s : %s doesn't exist", target.Name, syml)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
|
|
||||||
if stat.Mode()&os.ModeSymlink == 0 {
|
|
||||||
log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml)
|
|
||||||
return fmt.Errorf("%s isn't managed by hub", target.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
hubpath, err := os.Readlink(syml)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while reading symlink: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
absPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while abs path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if hubpath != absPath {
|
|
||||||
log.Warningf("%s (%s) isn't a symlink to %s", target.Name, syml, absPath)
|
|
||||||
return fmt.Errorf("%s isn't managed by hub", target.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// remove the symlink
|
|
||||||
if err = os.Remove(syml); err != nil {
|
|
||||||
return fmt.Errorf("while removing symlink: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("Removed symlink [%s] : %s", target.Name, syml)
|
|
||||||
}
|
|
||||||
|
|
||||||
target.Installed = false
|
|
||||||
|
|
||||||
if purge {
|
|
||||||
*target, err = purgeItem(hub, *target)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
hubIdx[target.Type][target.Name] = *target
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// creates symlink between actual config file at hub.HubDir and hub.ConfigDir
|
|
||||||
// Handles collections recursively
|
|
||||||
func EnableItem(hub *csconfig.Hub, target *Item) error {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
|
|
||||||
|
|
||||||
// create directories if needed
|
|
||||||
if target.Installed {
|
|
||||||
if target.Tainted {
|
|
||||||
return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
if target.Local {
|
|
||||||
return fmt.Errorf("%s is local, won't enable", target.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// if it's a collection, check sub-items even if the collection file itself is up-to-date
|
|
||||||
if target.UpToDate && target.Type != COLLECTIONS {
|
|
||||||
log.Tracef("%s is installed and up-to-date, skip.", target.Name)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err = os.Stat(parentDir); os.IsNotExist(err) {
|
|
||||||
log.Infof("%s doesn't exist, create", parentDir)
|
|
||||||
|
|
||||||
if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
|
|
||||||
return fmt.Errorf("while creating directory: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// install sub-items if it's a collection
|
|
||||||
if target.Type == COLLECTIONS {
|
|
||||||
for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
|
|
||||||
ptrtype := ItemTypes[idx]
|
|
||||||
for _, p := range ptr {
|
|
||||||
val, ok := hubIdx[ptrtype][p]
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = EnableItem(hub, &val)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while installing %s: %w", p, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// check if file already exists where it should in configdir (eg /etc/crowdsec/collections/)
|
|
||||||
if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) {
|
|
||||||
log.Infof("%s already exists.", parentDir+"/"+target.FileName)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// hub.ConfigDir + target.RemotePath
|
|
||||||
srcPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while getting source path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
dstPath, err := filepath.Abs(parentDir + "/" + target.FileName)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("while getting destination path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = os.Symlink(srcPath, dstPath); err != nil {
|
|
||||||
return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("Enabled %s : %s", target.Type, target.Name)
|
|
||||||
target.Installed = true
|
|
||||||
hubIdx[target.Type][target.Name] = *target
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
359
pkg/cwhub/items.go
Normal file
359
pkg/cwhub/items.go
Normal file
|
@ -0,0 +1,359 @@
|
||||||
|
package cwhub
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/Masterminds/semver/v3"
|
||||||
|
"github.com/enescakir/emoji"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// managed item types
|
||||||
|
COLLECTIONS = "collections"
|
||||||
|
PARSERS = "parsers"
|
||||||
|
POSTOVERFLOWS = "postoverflows"
|
||||||
|
SCENARIOS = "scenarios"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
VersionUpToDate = iota
|
||||||
|
VersionUpdateAvailable
|
||||||
|
VersionUnknown
|
||||||
|
VersionFuture
|
||||||
|
)
|
||||||
|
|
||||||
|
// The order is important, as it is used to range over sub-items in collections
|
||||||
|
var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
|
||||||
|
|
||||||
|
type HubItems map[string]map[string]*Item
|
||||||
|
|
||||||
|
// ItemVersion is used to detect the version of a given item
|
||||||
|
// by comparing the hash of each version to the local file.
|
||||||
|
// If the item does not match any known version, it is considered tainted.
|
||||||
|
type ItemVersion struct {
|
||||||
|
Digest string `json:"digest,omitempty"` // meow
|
||||||
|
Deprecated bool `json:"deprecated,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Item represents an object managed in the hub. It can be a parser, scenario, collection..
|
||||||
|
type Item struct {
|
||||||
|
// back pointer to the hub, to retrieve subitems and call install/remove methods
|
||||||
|
hub *Hub
|
||||||
|
|
||||||
|
// descriptive info
|
||||||
|
Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes
|
||||||
|
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
|
||||||
|
Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name"
|
||||||
|
FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml
|
||||||
|
Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json
|
||||||
|
Author string `json:"author,omitempty"` // as seen in .index.json
|
||||||
|
References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json
|
||||||
|
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
|
||||||
|
|
||||||
|
// remote (hub) info
|
||||||
|
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml
|
||||||
|
Version string `json:"version,omitempty"` // the last version
|
||||||
|
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions
|
||||||
|
|
||||||
|
// local (deployed) info
|
||||||
|
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
|
||||||
|
LocalVersion string `json:"local_version,omitempty"`
|
||||||
|
LocalHash string `json:"local_hash,omitempty"` // the local meow
|
||||||
|
Installed bool `json:"installed"`
|
||||||
|
Downloaded bool `json:"downloaded"`
|
||||||
|
UpToDate bool `json:"up_to_date"`
|
||||||
|
Tainted bool `json:"tainted"` // has it been locally modified?
|
||||||
|
|
||||||
|
// if it's a collection, it can have sub items
|
||||||
|
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
|
||||||
|
PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
|
||||||
|
Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"`
|
||||||
|
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Item) HasSubItems() bool {
|
||||||
|
return i.Type == COLLECTIONS
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Item) IsLocal() bool {
|
||||||
|
return i.Installed && !i.Downloaded
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalJSON is used to add the "local" field to the json output
|
||||||
|
// (i.e. with cscli ... inspect -o json)
|
||||||
|
// It must not use a pointer receiver
|
||||||
|
func (i Item) MarshalJSON() ([]byte, error) {
|
||||||
|
type Alias Item
|
||||||
|
|
||||||
|
return json.Marshal(&struct {
|
||||||
|
Alias
|
||||||
|
Local bool `json:"local"`
|
||||||
|
}{
|
||||||
|
Alias: Alias(i),
|
||||||
|
Local: i.IsLocal(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalYAML is used to add the "local" field to the yaml output
|
||||||
|
// (i.e. with cscli ... inspect -o raw)
|
||||||
|
// It must not use a pointer receiver
|
||||||
|
func (i Item) MarshalYAML() (interface{}, error) {
|
||||||
|
type Alias Item
|
||||||
|
|
||||||
|
return &struct {
|
||||||
|
Alias `yaml:",inline"`
|
||||||
|
Local bool `yaml:"local"`
|
||||||
|
}{
|
||||||
|
Alias: Alias(i),
|
||||||
|
Local: i.IsLocal(),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SubItems returns a slice of sub-item pointers, excluding the ones that were not found
|
||||||
|
func (i *Item) SubItems() []*Item {
|
||||||
|
sub := make([]*Item, 0)
|
||||||
|
|
||||||
|
for _, name := range i.Parsers {
|
||||||
|
s := i.hub.GetItem(PARSERS, name)
|
||||||
|
if s == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
sub = append(sub, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range i.PostOverflows {
|
||||||
|
s := i.hub.GetItem(POSTOVERFLOWS, name)
|
||||||
|
if s == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
sub = append(sub, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range i.Scenarios {
|
||||||
|
s := i.hub.GetItem(SCENARIOS, name)
|
||||||
|
if s == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
sub = append(sub, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range i.Collections {
|
||||||
|
s := i.hub.GetItem(COLLECTIONS, name)
|
||||||
|
if s == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
sub = append(sub, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
return sub
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Item) logMissingSubItems() {
|
||||||
|
if !i.HasSubItems() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, subName := range i.Parsers {
|
||||||
|
if i.hub.GetItem(PARSERS, subName) == nil {
|
||||||
|
log.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, subName := range i.Scenarios {
|
||||||
|
if i.hub.GetItem(SCENARIOS, subName) == nil {
|
||||||
|
log.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, subName := range i.PostOverflows {
|
||||||
|
if i.hub.GetItem(POSTOVERFLOWS, subName) == nil {
|
||||||
|
log.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, subName := range i.Collections {
|
||||||
|
if i.hub.GetItem(COLLECTIONS, subName) == nil {
|
||||||
|
log.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Item) parentCollections() []*Item {
|
||||||
|
ret := make([]*Item, 0)
|
||||||
|
|
||||||
|
for _, parentName := range i.BelongsToCollections {
|
||||||
|
parent := i.hub.GetItem(COLLECTIONS, parentName)
|
||||||
|
if parent == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
ret = append(ret, parent)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status returns the status of the item as a string and an emoji
|
||||||
|
// ie. "enabled,update-available" and emoji.Warning
|
||||||
|
func (i *Item) Status() (string, emoji.Emoji) {
|
||||||
|
status := "disabled"
|
||||||
|
ok := false
|
||||||
|
|
||||||
|
if i.Installed {
|
||||||
|
ok = true
|
||||||
|
status = "enabled"
|
||||||
|
}
|
||||||
|
|
||||||
|
managed := true
|
||||||
|
if i.IsLocal() {
|
||||||
|
managed = false
|
||||||
|
status += ",local"
|
||||||
|
}
|
||||||
|
|
||||||
|
warning := false
|
||||||
|
if i.Tainted {
|
||||||
|
warning = true
|
||||||
|
status += ",tainted"
|
||||||
|
} else if !i.UpToDate && !i.IsLocal() {
|
||||||
|
warning = true
|
||||||
|
status += ",update-available"
|
||||||
|
}
|
||||||
|
|
||||||
|
emo := emoji.QuestionMark
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case !managed:
|
||||||
|
emo = emoji.House
|
||||||
|
case !i.Installed:
|
||||||
|
emo = emoji.Prohibited
|
||||||
|
case warning:
|
||||||
|
emo = emoji.Warning
|
||||||
|
case ok:
|
||||||
|
emo = emoji.CheckMark
|
||||||
|
}
|
||||||
|
|
||||||
|
return status, emo
|
||||||
|
}
|
||||||
|
|
||||||
|
// versionStatus: semver requires 'v' prefix
|
||||||
|
func (i *Item) versionStatus() int {
|
||||||
|
local, err := semver.NewVersion(i.LocalVersion)
|
||||||
|
if err != nil {
|
||||||
|
return VersionUnknown
|
||||||
|
}
|
||||||
|
|
||||||
|
// hub versions are already validated while syncing, ignore errors
|
||||||
|
latest, _ := semver.NewVersion(i.Version)
|
||||||
|
|
||||||
|
if local.LessThan(latest) {
|
||||||
|
return VersionUpdateAvailable
|
||||||
|
}
|
||||||
|
|
||||||
|
if local.Equal(latest) {
|
||||||
|
return VersionUpToDate
|
||||||
|
}
|
||||||
|
|
||||||
|
return VersionFuture
|
||||||
|
}
|
||||||
|
|
||||||
|
// validPath returns true if the (relative) path is allowed for the item
|
||||||
|
// dirNname: the directory name (ie. crowdsecurity)
|
||||||
|
// fileName: the filename (ie. apache2-logs.yaml)
|
||||||
|
func (i *Item) validPath(dirName, fileName string) bool {
|
||||||
|
return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetItemMap returns the map of items for a given type
|
||||||
|
func (h *Hub) GetItemMap(itemType string) map[string]*Item {
|
||||||
|
return h.Items[itemType]
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetItem returns the item from hub based on its type and full name (author/name)
|
||||||
|
func (h *Hub) GetItem(itemType string, itemName string) *Item {
|
||||||
|
return h.GetItemMap(itemType)[itemName]
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetItemNames returns the list of item (full) names for a given type
|
||||||
|
// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx
|
||||||
|
// The names can be used to retrieve the item with GetItem()
|
||||||
|
func (h *Hub) GetItemNames(itemType string) []string {
|
||||||
|
m := h.GetItemMap(itemType)
|
||||||
|
if m == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
names := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
names = append(names, k)
|
||||||
|
}
|
||||||
|
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAllItems returns a slice of all the items, installed or not
|
||||||
|
func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
|
||||||
|
items, ok := h.Items[itemType]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("no %s in the hub index", itemType)
|
||||||
|
}
|
||||||
|
|
||||||
|
ret := make([]*Item, len(items))
|
||||||
|
|
||||||
|
idx := 0
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
ret[idx] = item
|
||||||
|
idx++
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetInstalledItems returns the list of installed items
|
||||||
|
func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
|
||||||
|
items, ok := h.Items[itemType]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("no %s in the hub index", itemType)
|
||||||
|
}
|
||||||
|
|
||||||
|
retItems := make([]*Item, 0)
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
if item.Installed {
|
||||||
|
retItems = append(retItems, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return retItems, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetInstalledItemsAsString returns the names of the installed items
|
||||||
|
func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) {
|
||||||
|
items, err := h.GetInstalledItems(itemType)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
retStr := make([]string, len(items))
|
||||||
|
|
||||||
|
for idx, it := range items {
|
||||||
|
retStr[idx] = it.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
return retStr, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SortItemSlice sorts a slice of items by name, case insensitive
|
||||||
|
func SortItemSlice(items []*Item) {
|
||||||
|
sort.Slice(items, func(i, j int) bool {
|
||||||
|
return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
|
||||||
|
})
|
||||||
|
}
|
68
pkg/cwhub/items_test.go
Normal file
68
pkg/cwhub/items_test.go
Normal file
|
@ -0,0 +1,68 @@
|
||||||
|
package cwhub
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestItemStatus(t *testing.T) {
|
||||||
|
hub := envSetup(t)
|
||||||
|
|
||||||
|
// get existing map
|
||||||
|
x := hub.GetItemMap(COLLECTIONS)
|
||||||
|
require.NotEmpty(t, x)
|
||||||
|
|
||||||
|
// Get item: good and bad
|
||||||
|
for k := range x {
|
||||||
|
item := hub.GetItem(COLLECTIONS, k)
|
||||||
|
require.NotNil(t, item)
|
||||||
|
|
||||||
|
item.Installed = true
|
||||||
|
item.UpToDate = false
|
||||||
|
item.Tainted = false
|
||||||
|
item.Downloaded = true
|
||||||
|
|
||||||
|
txt, _ := item.Status()
|
||||||
|
require.Equal(t, "enabled,update-available", txt)
|
||||||
|
|
||||||
|
item.Installed = true
|
||||||
|
item.UpToDate = false
|
||||||
|
item.Tainted = false
|
||||||
|
item.Downloaded = false
|
||||||
|
|
||||||
|
txt, _ = item.Status()
|
||||||
|
require.Equal(t, "enabled,local", txt)
|
||||||
|
}
|
||||||
|
|
||||||
|
stats := hub.ItemStats()
|
||||||
|
require.Equal(t, []string{"Loaded: 2 parsers, 1 scenarios, 3 collections"}, stats)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetters(t *testing.T) {
|
||||||
|
hub := envSetup(t)
|
||||||
|
|
||||||
|
// get non existing map
|
||||||
|
empty := hub.GetItemMap("ratata")
|
||||||
|
require.Nil(t, empty)
|
||||||
|
|
||||||
|
// get existing map
|
||||||
|
x := hub.GetItemMap(COLLECTIONS)
|
||||||
|
require.NotEmpty(t, x)
|
||||||
|
|
||||||
|
// Get item: good and bad
|
||||||
|
for k := range x {
|
||||||
|
empty := hub.GetItem(COLLECTIONS, k+"nope")
|
||||||
|
require.Nil(t, empty)
|
||||||
|
|
||||||
|
item := hub.GetItem(COLLECTIONS, k)
|
||||||
|
require.NotNil(t, item)
|
||||||
|
|
||||||
|
// Add item and get it
|
||||||
|
item.Name += "nope"
|
||||||
|
hub.Items[item.Type][item.Name] = item
|
||||||
|
|
||||||
|
newitem := hub.GetItem(COLLECTIONS, item.Name)
|
||||||
|
require.NotNil(t, newitem)
|
||||||
|
}
|
||||||
|
}
|
53
pkg/cwhub/leakybucket.go
Normal file
53
pkg/cwhub/leakybucket.go
Normal file
|
@ -0,0 +1,53 @@
|
||||||
|
package cwhub
|
||||||
|
|
||||||
|
// Resolve a symlink to find the hub item it points to.
|
||||||
|
// This file is used only by pkg/leakybucket
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary
|
||||||
|
func itemKey(itemPath string) (string, error) {
|
||||||
|
f, err := os.Lstat(itemPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if f.Mode()&os.ModeSymlink == 0 {
|
||||||
|
// it's not a symlink, so the filename itsef should be the key
|
||||||
|
return filepath.Base(itemPath), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolve the symlink to hub file
|
||||||
|
pathInHub, err := os.Readlink(itemPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
author := filepath.Base(filepath.Dir(pathInHub))
|
||||||
|
|
||||||
|
fname := filepath.Base(pathInHub)
|
||||||
|
fname = strings.TrimSuffix(fname, ".yaml")
|
||||||
|
fname = strings.TrimSuffix(fname, ".yml")
|
||||||
|
|
||||||
|
return fmt.Sprintf("%s/%s", author, fname), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetItemByPath retrieves the item from the hub index based on its path.
|
||||||
|
func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
|
||||||
|
itemKey, err := itemKey(itemPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
item := h.GetItem(itemType, itemKey)
|
||||||
|
if item == nil {
|
||||||
|
return nil, fmt.Errorf("%s not found in %s", itemKey, itemType)
|
||||||
|
}
|
||||||
|
|
||||||
|
return item, nil
|
||||||
|
}
|
|
@ -1,552 +0,0 @@
|
||||||
package cwhub
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
|
||||||
)
|
|
||||||
|
|
||||||
func isYAMLFileName(path string) bool {
|
|
||||||
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
|
|
||||||
}
|
|
||||||
|
|
||||||
func validItemFileName(vname string, fauthor string, fname string) bool {
|
|
||||||
return (fauthor+"/"+fname == vname+".yaml") || (fauthor+"/"+fname == vname+".yml")
|
|
||||||
}
|
|
||||||
|
|
||||||
func handleSymlink(path string) (string, error) {
|
|
||||||
hubpath, err := os.Readlink(path)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("unable to read symlink of %s", path)
|
|
||||||
}
|
|
||||||
// the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
|
|
||||||
_, err = os.Lstat(hubpath)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
|
|
||||||
// remove the symlink
|
|
||||||
if err = os.Remove(path); err != nil {
|
|
||||||
return "", fmt.Errorf("failed to unlink %s: %w", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// XXX: is this correct?
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return hubpath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func getSHA256(filepath string) (string, error) {
|
|
||||||
f, err := os.Open(filepath)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
h := sha256.New()
|
|
||||||
if _, err := io.Copy(h, f); err != nil {
|
|
||||||
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("%x", h.Sum(nil)), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type Walker struct {
|
|
||||||
// the walk/parserVisit function can't receive extra args
|
|
||||||
hubdir string
|
|
||||||
installdir string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewWalker(hub *csconfig.Hub) Walker {
|
|
||||||
return Walker{
|
|
||||||
hubdir: hub.HubDir,
|
|
||||||
installdir: hub.InstallDir,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type itemFileInfo struct {
|
|
||||||
fname string
|
|
||||||
stage string
|
|
||||||
ftype string
|
|
||||||
fauthor string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
|
|
||||||
ret := itemFileInfo{}
|
|
||||||
inhub := false
|
|
||||||
|
|
||||||
subs := strings.Split(path, string(os.PathSeparator))
|
|
||||||
|
|
||||||
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, w.hubdir, w.installdir)
|
|
||||||
log.Tracef("subs:%v", subs)
|
|
||||||
// we're in hub (~/.hub/hub/)
|
|
||||||
if strings.HasPrefix(path, w.hubdir) {
|
|
||||||
log.Tracef("in hub dir")
|
|
||||||
|
|
||||||
inhub = true
|
|
||||||
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
|
|
||||||
//.../hub/scenarios/crowdsec/ssh_bf.yaml
|
|
||||||
//.../hub/profiles/crowdsec/linux.yaml
|
|
||||||
if len(subs) < 4 {
|
|
||||||
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
|
||||||
}
|
|
||||||
|
|
||||||
ret.fname = subs[len(subs)-1]
|
|
||||||
ret.fauthor = subs[len(subs)-2]
|
|
||||||
ret.stage = subs[len(subs)-3]
|
|
||||||
ret.ftype = subs[len(subs)-4]
|
|
||||||
} else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec/<type>/...
|
|
||||||
log.Tracef("in install dir")
|
|
||||||
if len(subs) < 3 {
|
|
||||||
log.Fatalf("path is too short : %s (%d)", path, len(subs))
|
|
||||||
}
|
|
||||||
///.../config/parser/stage/file.yaml
|
|
||||||
///.../config/postoverflow/stage/file.yaml
|
|
||||||
///.../config/scenarios/scenar.yaml
|
|
||||||
///.../config/collections/linux.yaml //file is empty
|
|
||||||
ret.fname = subs[len(subs)-1]
|
|
||||||
ret.stage = subs[len(subs)-2]
|
|
||||||
ret.ftype = subs[len(subs)-3]
|
|
||||||
ret.fauthor = ""
|
|
||||||
} else {
|
|
||||||
return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, w.hubdir, w.installdir)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
|
|
||||||
// log.Infof("%s -> name:%s stage:%s", path, fname, stage)
|
|
||||||
|
|
||||||
if ret.stage == SCENARIOS {
|
|
||||||
ret.ftype = SCENARIOS
|
|
||||||
ret.stage = ""
|
|
||||||
} else if ret.stage == COLLECTIONS {
|
|
||||||
ret.ftype = COLLECTIONS
|
|
||||||
ret.stage = ""
|
|
||||||
} else if ret.ftype != PARSERS && ret.ftype != PARSERS_OVFLW {
|
|
||||||
// its a PARSER / PARSER_OVFLW with a stage
|
|
||||||
return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
|
|
||||||
|
|
||||||
return ret, inhub, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
|
|
||||||
var (
|
|
||||||
local bool
|
|
||||||
hubpath string
|
|
||||||
)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Debugf("while syncing hub dir: %s", err)
|
|
||||||
// there is a path error, we ignore the file
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
path, err = filepath.Abs(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// we only care about files
|
|
||||||
if f == nil || f.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if !isYAMLFileName(f.Name()) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
info, inhub, err := w.getItemInfo(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
we can encounter 'collections' in the form of a symlink :
|
|
||||||
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
|
|
||||||
when the collection is installed, both files are created
|
|
||||||
*/
|
|
||||||
// non symlinks are local user files or hub files
|
|
||||||
if f.Type()&os.ModeSymlink == 0 {
|
|
||||||
local = true
|
|
||||||
|
|
||||||
log.Tracef("%s isn't a symlink", path)
|
|
||||||
} else {
|
|
||||||
hubpath, err = handleSymlink(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
log.Tracef("%s points to %s", path, hubpath)
|
|
||||||
|
|
||||||
if hubpath == "" {
|
|
||||||
// XXX: is this correct?
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// if it's not a symlink and not in hub, it's a local file, don't bother
|
|
||||||
if local && !inhub {
|
|
||||||
log.Tracef("%s is a local file, skip", path)
|
|
||||||
skippedLocal++
|
|
||||||
// log.Infof("local scenario, skip.")
|
|
||||||
|
|
||||||
_, fileName := filepath.Split(path)
|
|
||||||
|
|
||||||
hubIdx[info.ftype][info.fname] = Item{
|
|
||||||
Name: info.fname,
|
|
||||||
Stage: info.stage,
|
|
||||||
Installed: true,
|
|
||||||
Type: info.ftype,
|
|
||||||
Local: true,
|
|
||||||
LocalPath: path,
|
|
||||||
UpToDate: true,
|
|
||||||
FileName: fileName,
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// try to find which configuration item it is
|
|
||||||
log.Tracef("check [%s] of %s", info.fname, info.ftype)
|
|
||||||
|
|
||||||
match := false
|
|
||||||
|
|
||||||
for name, item := range hubIdx[info.ftype] {
|
|
||||||
log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml")
|
|
||||||
|
|
||||||
if info.fname != item.FileName {
|
|
||||||
log.Tracef("%s != %s (filename)", info.fname, item.FileName)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// wrong stage
|
|
||||||
if item.Stage != info.stage {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// if we are walking hub dir, just mark present files as downloaded
|
|
||||||
if inhub {
|
|
||||||
// wrong author
|
|
||||||
if info.fauthor != item.Author {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// wrong file
|
|
||||||
if !validItemFileName(item.Name, info.fauthor, info.fname) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if path == w.hubdir+"/"+item.RemotePath {
|
|
||||||
log.Tracef("marking %s as downloaded", item.Name)
|
|
||||||
item.Downloaded = true
|
|
||||||
}
|
|
||||||
} else if !hasPathSuffix(hubpath, item.RemotePath) {
|
|
||||||
// wrong file
|
|
||||||
// <type>/<stage>/<author>/<name>.yaml
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
sha, err := getSHA256(path)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to get sha of %s : %v", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// let's reverse sort the versions to deal with hash collisions (#154)
|
|
||||||
versions := make([]string, 0, len(item.Versions))
|
|
||||||
for k := range item.Versions {
|
|
||||||
versions = append(versions, k)
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
|
|
||||||
|
|
||||||
for _, version := range versions {
|
|
||||||
val := item.Versions[version]
|
|
||||||
if sha != val.Digest {
|
|
||||||
// log.Infof("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// we got an exact match, update struct
|
|
||||||
|
|
||||||
item.Downloaded = true
|
|
||||||
item.LocalHash = sha
|
|
||||||
|
|
||||||
if !inhub {
|
|
||||||
log.Tracef("found exact match for %s, version is %s, latest is %s", item.Name, version, item.Version)
|
|
||||||
item.LocalPath = path
|
|
||||||
item.LocalVersion = version
|
|
||||||
item.Tainted = false
|
|
||||||
// if we're walking the hub, present file doesn't means installed file
|
|
||||||
item.Installed = true
|
|
||||||
}
|
|
||||||
|
|
||||||
if version == item.Version {
|
|
||||||
log.Tracef("%s is up-to-date", item.Name)
|
|
||||||
item.UpToDate = true
|
|
||||||
}
|
|
||||||
|
|
||||||
match = true
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
if !match {
|
|
||||||
log.Tracef("got tainted match for %s: %s", item.Name, path)
|
|
||||||
|
|
||||||
skippedTainted++
|
|
||||||
// the file and the stage is right, but the hash is wrong, it has been tainted by user
|
|
||||||
if !inhub {
|
|
||||||
item.LocalPath = path
|
|
||||||
item.Installed = true
|
|
||||||
}
|
|
||||||
|
|
||||||
item.UpToDate = false
|
|
||||||
item.LocalVersion = "?"
|
|
||||||
item.Tainted = true
|
|
||||||
item.LocalHash = sha
|
|
||||||
}
|
|
||||||
|
|
||||||
// update the entry if appropriate
|
|
||||||
// if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D {
|
|
||||||
// fmt.Printf("Updating %s", k)
|
|
||||||
// hubIdx[ftype][k] = v
|
|
||||||
// } else if !inhub {
|
|
||||||
|
|
||||||
// } else if
|
|
||||||
hubIdx[info.ftype][name] = item
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("Ignoring file %s of type %s", path, info.ftype)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func CollecDepsCheck(v *Item) error {
|
|
||||||
if v.versionStatus() != 0 { // not up-to-date
|
|
||||||
log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if v.Type != COLLECTIONS {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// if it's a collection, ensure all the items are installed, or tag it as tainted
|
|
||||||
log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
|
|
||||||
|
|
||||||
for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} {
|
|
||||||
sliceType := ItemTypes[idx]
|
|
||||||
for _, subName := range itemSlice {
|
|
||||||
subItem, ok := hubIdx[sliceType][subName]
|
|
||||||
if !ok {
|
|
||||||
log.Fatalf("Referred %s %s in collection %s doesn't exist.", sliceType, subName, v.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed)
|
|
||||||
|
|
||||||
if !v.Installed {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if subItem.Type == COLLECTIONS {
|
|
||||||
log.Tracef("collec, recurse.")
|
|
||||||
|
|
||||||
if err := CollecDepsCheck(&subItem); err != nil {
|
|
||||||
if subItem.Tainted {
|
|
||||||
v.Tainted = true
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
hubIdx[sliceType][subName] = subItem
|
|
||||||
}
|
|
||||||
|
|
||||||
// propagate the state of sub-items to set
|
|
||||||
if subItem.Tainted {
|
|
||||||
v.Tainted = true
|
|
||||||
return fmt.Errorf("tainted %s %s, tainted", sliceType, subName)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !subItem.Installed && v.Installed {
|
|
||||||
v.Tainted = true
|
|
||||||
return fmt.Errorf("missing %s %s, tainted", sliceType, subName)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !subItem.UpToDate {
|
|
||||||
v.UpToDate = false
|
|
||||||
return fmt.Errorf("outdated %s %s", sliceType, subName)
|
|
||||||
}
|
|
||||||
|
|
||||||
skip := false
|
|
||||||
|
|
||||||
for idx := range subItem.BelongsToCollections {
|
|
||||||
if subItem.BelongsToCollections[idx] == v.Name {
|
|
||||||
skip = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !skip {
|
|
||||||
subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
hubIdx[sliceType][subName] = subItem
|
|
||||||
|
|
||||||
log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
|
|
||||||
warnings := []string{}
|
|
||||||
|
|
||||||
// For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last
|
|
||||||
for _, scan := range ItemTypes {
|
|
||||||
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
|
|
||||||
if err != nil {
|
|
||||||
log.Errorf("failed %s : %s", cpath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = filepath.WalkDir(cpath, NewWalker(hub).parserVisit)
|
|
||||||
if err != nil {
|
|
||||||
return warnings, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, item := range hubIdx[COLLECTIONS] {
|
|
||||||
if !item.Installed {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
vs := item.versionStatus()
|
|
||||||
switch vs {
|
|
||||||
case 0: // latest
|
|
||||||
if err := CollecDepsCheck(&item); err != nil {
|
|
||||||
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
|
|
||||||
hubIdx[COLLECTIONS][name] = item
|
|
||||||
}
|
|
||||||
case 1: // not up-to-date
|
|
||||||
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
|
|
||||||
default: // version is higher than the highest available from hub?
|
|
||||||
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions)
|
|
||||||
}
|
|
||||||
|
|
||||||
return warnings, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Updates the info from HubInit() with the local state
|
|
||||||
func LocalSync(hub *csconfig.Hub) ([]string, error) {
|
|
||||||
skippedLocal = 0
|
|
||||||
skippedTainted = 0
|
|
||||||
|
|
||||||
warnings, err := SyncDir(hub, hub.InstallDir)
|
|
||||||
if err != nil {
|
|
||||||
return warnings, fmt.Errorf("failed to scan %s: %w", hub.InstallDir, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = SyncDir(hub, hub.HubDir)
|
|
||||||
if err != nil {
|
|
||||||
return warnings, fmt.Errorf("failed to scan %s: %w", hub.HubDir, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return warnings, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetHubIdx(hub *csconfig.Hub) error {
|
|
||||||
if hub == nil {
|
|
||||||
return fmt.Errorf("no configuration found for hub")
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Debugf("loading hub idx %s", hub.HubIndexFile)
|
|
||||||
|
|
||||||
bidx, err := os.ReadFile(hub.HubIndexFile)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("unable to read index file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ret, err := LoadPkgIndex(bidx)
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, ErrMissingReference) {
|
|
||||||
return fmt.Errorf("unable to load existing index: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// XXX: why the error check if we bail out anyway?
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
hubIdx = ret
|
|
||||||
|
|
||||||
_, err = LocalSync(hub)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to sync Hub index with local deployment : %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// LoadPkgIndex loads a local .index.json file and returns the map of associated parsers/scenarios/collections
|
|
||||||
func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
|
|
||||||
var (
|
|
||||||
RawIndex map[string]map[string]Item
|
|
||||||
missingItems []string
|
|
||||||
)
|
|
||||||
|
|
||||||
if err := json.Unmarshal(buff, &RawIndex); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to unmarshal index: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Debugf("%d item types in hub index", len(ItemTypes))
|
|
||||||
|
|
||||||
// Iterate over the different types to complete the struct
|
|
||||||
for _, itemType := range ItemTypes {
|
|
||||||
log.Tracef("%d item", len(RawIndex[itemType]))
|
|
||||||
|
|
||||||
for name, item := range RawIndex[itemType] {
|
|
||||||
item.Name = name
|
|
||||||
item.Type = itemType
|
|
||||||
x := strings.Split(item.RemotePath, "/")
|
|
||||||
item.FileName = x[len(x)-1]
|
|
||||||
RawIndex[itemType][name] = item
|
|
||||||
|
|
||||||
if itemType != COLLECTIONS {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// if it's a collection, check its sub-items are present
|
|
||||||
// XXX should be done later
|
|
||||||
for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} {
|
|
||||||
ptrtype := ItemTypes[idx]
|
|
||||||
for _, p := range ptr {
|
|
||||||
if _, ok := RawIndex[ptrtype][p]; !ok {
|
|
||||||
log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name)
|
|
||||||
missingItems = append(missingItems, p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(missingItems) > 0 {
|
|
||||||
return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference)
|
|
||||||
}
|
|
||||||
|
|
||||||
return RawIndex, nil
|
|
||||||
}
|
|
80
pkg/cwhub/remote.go
Normal file
80
pkg/cwhub/remote.go
Normal file
|
@ -0,0 +1,80 @@
|
||||||
|
package cwhub
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RemoteHubCfg contains where to find the remote hub, which branch etc.
|
||||||
|
type RemoteHubCfg struct {
|
||||||
|
Branch string
|
||||||
|
URLTemplate string
|
||||||
|
IndexPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
|
||||||
|
if r == nil {
|
||||||
|
return "", ErrNilRemoteHub
|
||||||
|
}
|
||||||
|
|
||||||
|
// the template must contain two string placeholders
|
||||||
|
if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate {
|
||||||
|
return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate)
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// downloadIndex downloads the latest version of the index
|
||||||
|
func (r *RemoteHubCfg) downloadIndex(localPath string) error {
|
||||||
|
if r == nil {
|
||||||
|
return ErrNilRemoteHub
|
||||||
|
}
|
||||||
|
|
||||||
|
url, err := r.urlTo(r.IndexPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to build hub index request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := hubClient.Get(url)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed http request for hub index: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
if resp.StatusCode == http.StatusNotFound {
|
||||||
|
return IndexNotFoundError{url, r.Branch}
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to read request answer for hub index: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
oldContent, err := os.ReadFile(localPath)
|
||||||
|
if err != nil {
|
||||||
|
if !os.IsNotExist(err) {
|
||||||
|
log.Warningf("failed to read hub index: %s", err)
|
||||||
|
}
|
||||||
|
} else if bytes.Equal(body, oldContent) {
|
||||||
|
log.Info("hub index is up to date")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = os.WriteFile(localPath, body, 0o644); err != nil {
|
||||||
|
return fmt.Errorf("failed to write hub index: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Infof("Wrote index to %s, %d bytes", localPath, len(body))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
479
pkg/cwhub/sync.go
Normal file
479
pkg/cwhub/sync.go
Normal file
|
@ -0,0 +1,479 @@
|
||||||
|
package cwhub
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"slices"
|
||||||
|
|
||||||
|
"github.com/Masterminds/semver/v3"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
type localItem struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func isYAMLFileName(path string) bool {
|
||||||
|
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleSymlink(path string) (string, error) {
|
||||||
|
hubpath, err := os.Readlink(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("unable to read symlink of %s", path)
|
||||||
|
}
|
||||||
|
// the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
|
||||||
|
_, err = os.Lstat(hubpath)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
|
||||||
|
// remove the symlink
|
||||||
|
if err = os.Remove(path); err != nil {
|
||||||
|
return "", fmt.Errorf("failed to unlink %s: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ignore this file
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return hubpath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSHA256(filepath string) (string, error) {
|
||||||
|
f, err := os.Open(filepath)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
h := sha256.New()
|
||||||
|
if _, err := io.Copy(h, f); err != nil {
|
||||||
|
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return hex.EncodeToString(h.Sum(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type itemFileInfo struct {
|
||||||
|
fname string
|
||||||
|
stage string
|
||||||
|
ftype string
|
||||||
|
fauthor string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
|
||||||
|
ret := itemFileInfo{}
|
||||||
|
inhub := false
|
||||||
|
|
||||||
|
hubDir := h.local.HubDir
|
||||||
|
installDir := h.local.InstallDir
|
||||||
|
|
||||||
|
subs := strings.Split(path, string(os.PathSeparator))
|
||||||
|
|
||||||
|
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir)
|
||||||
|
log.Tracef("subs:%v", subs)
|
||||||
|
// we're in hub (~/.hub/hub/)
|
||||||
|
if strings.HasPrefix(path, hubDir) {
|
||||||
|
log.Tracef("in hub dir")
|
||||||
|
|
||||||
|
inhub = true
|
||||||
|
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
|
||||||
|
//.../hub/scenarios/crowdsec/ssh_bf.yaml
|
||||||
|
//.../hub/profiles/crowdsec/linux.yaml
|
||||||
|
if len(subs) < 4 {
|
||||||
|
return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
|
||||||
|
}
|
||||||
|
|
||||||
|
ret.fname = subs[len(subs)-1]
|
||||||
|
ret.fauthor = subs[len(subs)-2]
|
||||||
|
ret.stage = subs[len(subs)-3]
|
||||||
|
ret.ftype = subs[len(subs)-4]
|
||||||
|
} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
|
||||||
|
log.Tracef("in install dir")
|
||||||
|
if len(subs) < 3 {
|
||||||
|
return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
|
||||||
|
}
|
||||||
|
///.../config/parser/stage/file.yaml
|
||||||
|
///.../config/postoverflow/stage/file.yaml
|
||||||
|
///.../config/scenarios/scenar.yaml
|
||||||
|
///.../config/collections/linux.yaml //file is empty
|
||||||
|
ret.fname = subs[len(subs)-1]
|
||||||
|
ret.stage = subs[len(subs)-2]
|
||||||
|
ret.ftype = subs[len(subs)-3]
|
||||||
|
ret.fauthor = ""
|
||||||
|
} else {
|
||||||
|
return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
|
||||||
|
// log.Infof("%s -> name:%s stage:%s", path, fname, stage)
|
||||||
|
|
||||||
|
if ret.stage == SCENARIOS {
|
||||||
|
ret.ftype = SCENARIOS
|
||||||
|
ret.stage = ""
|
||||||
|
} else if ret.stage == COLLECTIONS {
|
||||||
|
ret.ftype = COLLECTIONS
|
||||||
|
ret.stage = ""
|
||||||
|
} else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
|
||||||
|
// it's a PARSER / POSTOVERFLOW with a stage
|
||||||
|
return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
|
||||||
|
|
||||||
|
return ret, inhub, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sortedVersions returns the input data, sorted in reverse order by semver
|
||||||
|
func sortedVersions(raw []string) ([]string, error) {
|
||||||
|
vs := make([]*semver.Version, len(raw))
|
||||||
|
|
||||||
|
for idx, r := range raw {
|
||||||
|
v, err := semver.NewVersion(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", r, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
vs[idx] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Sort(sort.Reverse(semver.Collection(vs)))
|
||||||
|
|
||||||
|
ret := make([]string, len(vs))
|
||||||
|
for idx, v := range vs {
|
||||||
|
ret[idx] = v.Original()
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
|
||||||
|
local := false
|
||||||
|
hubpath := ""
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Debugf("while syncing hub dir: %s", err)
|
||||||
|
// there is a path error, we ignore the file
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
path, err = filepath.Abs(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// we only care about files
|
||||||
|
if f == nil || f.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if !isYAMLFileName(f.Name()) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
info, inhub, err := h.getItemInfo(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
we can encounter 'collections' in the form of a symlink:
|
||||||
|
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
|
||||||
|
when the collection is installed, both files are created
|
||||||
|
*/
|
||||||
|
// non symlinks are local user files or hub files
|
||||||
|
if f.Type()&os.ModeSymlink == 0 {
|
||||||
|
local = true
|
||||||
|
|
||||||
|
log.Tracef("%s isn't a symlink", path)
|
||||||
|
} else {
|
||||||
|
hubpath, err = handleSymlink(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Tracef("%s points to %s", path, hubpath)
|
||||||
|
|
||||||
|
if hubpath == "" {
|
||||||
|
// ignore this file
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// if it's not a symlink and not in hub, it's a local file, don't bother
|
||||||
|
if local && !inhub {
|
||||||
|
log.Tracef("%s is a local file, skip", path)
|
||||||
|
h.skippedLocal++
|
||||||
|
|
||||||
|
_, fileName := filepath.Split(path)
|
||||||
|
|
||||||
|
item := localItem{}
|
||||||
|
itemContent, err := os.ReadFile(path)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to read %s: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = yaml.Unmarshal(itemContent, &item)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to unmarshal %s: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
h.Items[info.ftype][info.fname] = &Item{
|
||||||
|
hub: h,
|
||||||
|
Name: item.Name,
|
||||||
|
Stage: info.stage,
|
||||||
|
Installed: true,
|
||||||
|
Type: info.ftype,
|
||||||
|
LocalPath: path,
|
||||||
|
UpToDate: true,
|
||||||
|
FileName: fileName,
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// try to find which configuration item it is
|
||||||
|
log.Tracef("check [%s] of %s", info.fname, info.ftype)
|
||||||
|
|
||||||
|
match := false
|
||||||
|
|
||||||
|
for name, item := range h.Items[info.ftype] {
|
||||||
|
log.Tracef("check [%s] vs [%s]: %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml")
|
||||||
|
|
||||||
|
if info.fname != item.FileName {
|
||||||
|
log.Tracef("%s != %s (filename)", info.fname, item.FileName)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// wrong stage
|
||||||
|
if item.Stage != info.stage {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// if we are walking hub dir, just mark present files as downloaded
|
||||||
|
if inhub {
|
||||||
|
// wrong author
|
||||||
|
if info.fauthor != item.Author {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// not the item we're looking for
|
||||||
|
if !item.validPath(info.fauthor, info.fname) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if path == h.local.HubDir+"/"+item.RemotePath {
|
||||||
|
log.Tracef("marking %s as downloaded", item.Name)
|
||||||
|
item.Downloaded = true
|
||||||
|
}
|
||||||
|
} else if !hasPathSuffix(hubpath, item.RemotePath) {
|
||||||
|
// wrong file
|
||||||
|
// <type>/<stage>/<author>/<name>.yaml
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
sha, err := getSHA256(path)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to get sha of %s: %v", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// let's reverse sort the versions to deal with hash collisions (#154)
|
||||||
|
versions := make([]string, 0, len(item.Versions))
|
||||||
|
for k := range item.Versions {
|
||||||
|
versions = append(versions, k)
|
||||||
|
}
|
||||||
|
|
||||||
|
versions, err = sortedVersions(versions)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("while syncing %s %s: %w", info.ftype, info.fname, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, version := range versions {
|
||||||
|
if item.Versions[version].Digest != sha {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// we got an exact match, update struct
|
||||||
|
|
||||||
|
item.Downloaded = true
|
||||||
|
item.LocalHash = sha
|
||||||
|
|
||||||
|
if !inhub {
|
||||||
|
log.Tracef("found exact match for %s, version is %s, latest is %s", item.Name, version, item.Version)
|
||||||
|
item.LocalPath = path
|
||||||
|
item.LocalVersion = version
|
||||||
|
item.Tainted = false
|
||||||
|
// if we're walking the hub, present file doesn't means installed file
|
||||||
|
item.Installed = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if version == item.Version {
|
||||||
|
log.Tracef("%s is up-to-date", item.Name)
|
||||||
|
item.UpToDate = true
|
||||||
|
}
|
||||||
|
|
||||||
|
match = true
|
||||||
|
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
if !match {
|
||||||
|
log.Tracef("got tainted match for %s: %s", item.Name, path)
|
||||||
|
|
||||||
|
h.skippedTainted++
|
||||||
|
|
||||||
|
// the file and the stage is right, but the hash is wrong, it has been tainted by user
|
||||||
|
if !inhub {
|
||||||
|
item.LocalPath = path
|
||||||
|
item.Installed = true
|
||||||
|
}
|
||||||
|
|
||||||
|
item.UpToDate = false
|
||||||
|
item.LocalVersion = "?"
|
||||||
|
item.Tainted = true
|
||||||
|
item.LocalHash = sha
|
||||||
|
}
|
||||||
|
|
||||||
|
h.Items[info.ftype][name] = item
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Infof("Ignoring file %s of type %s", path, info.ftype)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkSubItems checks for the presence, taint and version state of sub-items
|
||||||
|
func (h *Hub) checkSubItems(v *Item) error {
|
||||||
|
if !v.HasSubItems() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.versionStatus() != VersionUpToDate {
|
||||||
|
log.Debugf("%s dependencies not checked: not up-to-date", v.Name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensure all the sub-items are installed, or tag the parent as tainted
|
||||||
|
log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
|
||||||
|
|
||||||
|
for _, sub := range v.SubItems() {
|
||||||
|
log.Tracef("check %s installed:%t", sub.Name, sub.Installed)
|
||||||
|
|
||||||
|
if !v.Installed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := h.checkSubItems(sub); err != nil {
|
||||||
|
if sub.Tainted {
|
||||||
|
v.Tainted = true
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("sub collection %s is broken: %w", sub.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if sub.Tainted {
|
||||||
|
v.Tainted = true
|
||||||
|
return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !sub.Installed && v.Installed {
|
||||||
|
v.Tainted = true
|
||||||
|
return fmt.Errorf("missing %s %s, tainted", sub.Type, sub.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !sub.UpToDate {
|
||||||
|
v.UpToDate = false
|
||||||
|
return fmt.Errorf("outdated %s %s", sub.Type, sub.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Contains(sub.BelongsToCollections, v.Name) {
|
||||||
|
sub.BelongsToCollections = append(sub.BelongsToCollections, v.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Hub) syncDir(dir string) ([]string, error) {
|
||||||
|
warnings := []string{}
|
||||||
|
|
||||||
|
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
|
||||||
|
for _, scan := range ItemTypes {
|
||||||
|
// cpath: top-level item directory, either downloaded or installed items.
|
||||||
|
// i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ...
|
||||||
|
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed %s: %s", cpath, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// explicit check for non existing directory, avoid spamming log.Debug
|
||||||
|
if _, err = os.Stat(cpath); os.IsNotExist(err) {
|
||||||
|
log.Tracef("directory %s doesn't exist, skipping", cpath)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = filepath.WalkDir(cpath, h.itemVisit); err != nil {
|
||||||
|
return warnings, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range h.Items[COLLECTIONS] {
|
||||||
|
if !item.Installed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
vs := item.versionStatus()
|
||||||
|
switch vs {
|
||||||
|
case VersionUpToDate: // latest
|
||||||
|
if err := h.checkSubItems(item); err != nil {
|
||||||
|
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
|
||||||
|
}
|
||||||
|
case VersionUpdateAvailable: // not up-to-date
|
||||||
|
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
|
||||||
|
case VersionFuture:
|
||||||
|
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
|
||||||
|
case VersionUnknown:
|
||||||
|
warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions)
|
||||||
|
}
|
||||||
|
|
||||||
|
return warnings, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Updates the info from HubInit() with the local state
|
||||||
|
func (h *Hub) localSync() error {
|
||||||
|
h.skippedLocal = 0
|
||||||
|
h.skippedTainted = 0
|
||||||
|
h.Warnings = []string{}
|
||||||
|
|
||||||
|
warnings, err := h.syncDir(h.local.InstallDir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
h.Warnings = append(h.Warnings, warnings...)
|
||||||
|
|
||||||
|
if warnings, err = h.syncDir(h.local.HubDir); err != nil {
|
||||||
|
return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
h.Warnings = append(h.Warnings, warnings...)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -5,173 +5,194 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ParserCoverage struct {
|
type Coverage struct {
|
||||||
Parser string
|
Name string
|
||||||
TestsCount int
|
TestsCount int
|
||||||
PresentIn map[string]bool //poorman's set
|
PresentIn map[string]bool //poorman's set
|
||||||
}
|
}
|
||||||
|
|
||||||
type ScenarioCoverage struct {
|
func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
|
||||||
Scenario string
|
if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok {
|
||||||
TestsCount int
|
return nil, fmt.Errorf("no parsers in hub index")
|
||||||
PresentIn map[string]bool
|
}
|
||||||
}
|
|
||||||
|
|
||||||
func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) {
|
// populate from hub, iterate in alphabetical order
|
||||||
var coverage []ParserCoverage
|
pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.PARSERS])
|
||||||
if _, ok := h.HubIndex.Data[cwhub.PARSERS]; !ok {
|
coverage := make([]Coverage, len(pkeys))
|
||||||
return coverage, fmt.Errorf("no parsers in hub index")
|
|
||||||
}
|
for i, name := range pkeys {
|
||||||
//populate from hub, iterate in alphabetical order
|
coverage[i] = Coverage{
|
||||||
var pkeys []string
|
Name: name,
|
||||||
for pname := range h.HubIndex.Data[cwhub.PARSERS] {
|
|
||||||
pkeys = append(pkeys, pname)
|
|
||||||
}
|
|
||||||
sort.Strings(pkeys)
|
|
||||||
for _, pname := range pkeys {
|
|
||||||
coverage = append(coverage, ParserCoverage{
|
|
||||||
Parser: pname,
|
|
||||||
TestsCount: 0,
|
TestsCount: 0,
|
||||||
PresentIn: make(map[string]bool),
|
PresentIn: make(map[string]bool),
|
||||||
})
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//parser the expressions a-la-oneagain
|
// parser the expressions a-la-oneagain
|
||||||
passerts, err := filepath.Glob(".tests/*/parser.assert")
|
passerts, err := filepath.Glob(".tests/*/parser.assert")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return coverage, fmt.Errorf("while find parser asserts : %s", err)
|
return nil, fmt.Errorf("while find parser asserts : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, assert := range passerts {
|
for _, assert := range passerts {
|
||||||
file, err := os.Open(assert)
|
file, err := os.Open(assert)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return coverage, fmt.Errorf("while reading %s : %s", assert, err)
|
return nil, fmt.Errorf("while reading %s : %s", assert, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
scanner := bufio.NewScanner(file)
|
scanner := bufio.NewScanner(file)
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
assertLine := regexp.MustCompile(`^results\["[^"]+"\]\["(?P<parser>[^"]+)"\]\[[0-9]+\]\.Evt\..*`)
|
|
||||||
line := scanner.Text()
|
line := scanner.Text()
|
||||||
log.Debugf("assert line : %s", line)
|
log.Debugf("assert line : %s", line)
|
||||||
match := assertLine.FindStringSubmatch(line)
|
|
||||||
|
match := parserResultRE.FindStringSubmatch(line)
|
||||||
if len(match) == 0 {
|
if len(match) == 0 {
|
||||||
log.Debugf("%s doesn't match", line)
|
log.Debugf("%s doesn't match", line)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
sidx := assertLine.SubexpIndex("parser")
|
|
||||||
|
sidx := parserResultRE.SubexpIndex("parser")
|
||||||
capturedParser := match[sidx]
|
capturedParser := match[sidx]
|
||||||
|
|
||||||
for idx, pcover := range coverage {
|
for idx, pcover := range coverage {
|
||||||
if pcover.Parser == capturedParser {
|
if pcover.Name == capturedParser {
|
||||||
coverage[idx].TestsCount++
|
coverage[idx].TestsCount++
|
||||||
coverage[idx].PresentIn[assert] = true
|
coverage[idx].PresentIn[assert] = true
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
parserNameSplit := strings.Split(pcover.Parser, "/")
|
|
||||||
|
parserNameSplit := strings.Split(pcover.Name, "/")
|
||||||
parserNameOnly := parserNameSplit[len(parserNameSplit)-1]
|
parserNameOnly := parserNameSplit[len(parserNameSplit)-1]
|
||||||
|
|
||||||
if parserNameOnly == capturedParser {
|
if parserNameOnly == capturedParser {
|
||||||
coverage[idx].TestsCount++
|
coverage[idx].TestsCount++
|
||||||
coverage[idx].PresentIn[assert] = true
|
coverage[idx].PresentIn[assert] = true
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
capturedParserSplit := strings.Split(capturedParser, "/")
|
capturedParserSplit := strings.Split(capturedParser, "/")
|
||||||
capturedParserName := capturedParserSplit[len(capturedParserSplit)-1]
|
capturedParserName := capturedParserSplit[len(capturedParserSplit)-1]
|
||||||
|
|
||||||
if capturedParserName == parserNameOnly {
|
if capturedParserName == parserNameOnly {
|
||||||
coverage[idx].TestsCount++
|
coverage[idx].TestsCount++
|
||||||
coverage[idx].PresentIn[assert] = true
|
coverage[idx].PresentIn[assert] = true
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if capturedParserName == parserNameOnly+"-logs" {
|
if capturedParserName == parserNameOnly+"-logs" {
|
||||||
coverage[idx].TestsCount++
|
coverage[idx].TestsCount++
|
||||||
coverage[idx].PresentIn[assert] = true
|
coverage[idx].PresentIn[assert] = true
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
file.Close()
|
file.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
return coverage, nil
|
return coverage, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) {
|
func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
|
||||||
var coverage []ScenarioCoverage
|
if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok {
|
||||||
if _, ok := h.HubIndex.Data[cwhub.SCENARIOS]; !ok {
|
return nil, fmt.Errorf("no scenarios in hub index")
|
||||||
return coverage, fmt.Errorf("no scenarios in hub index")
|
|
||||||
}
|
|
||||||
//populate from hub, iterate in alphabetical order
|
|
||||||
var pkeys []string
|
|
||||||
for scenarioName := range h.HubIndex.Data[cwhub.SCENARIOS] {
|
|
||||||
pkeys = append(pkeys, scenarioName)
|
|
||||||
}
|
|
||||||
sort.Strings(pkeys)
|
|
||||||
for _, scenarioName := range pkeys {
|
|
||||||
coverage = append(coverage, ScenarioCoverage{
|
|
||||||
Scenario: scenarioName,
|
|
||||||
TestsCount: 0,
|
|
||||||
PresentIn: make(map[string]bool),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//parser the expressions a-la-oneagain
|
// populate from hub, iterate in alphabetical order
|
||||||
|
pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.SCENARIOS])
|
||||||
|
coverage := make([]Coverage, len(pkeys))
|
||||||
|
|
||||||
|
for i, name := range pkeys {
|
||||||
|
coverage[i] = Coverage{
|
||||||
|
Name: name,
|
||||||
|
TestsCount: 0,
|
||||||
|
PresentIn: make(map[string]bool),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parser the expressions a-la-oneagain
|
||||||
passerts, err := filepath.Glob(".tests/*/scenario.assert")
|
passerts, err := filepath.Glob(".tests/*/scenario.assert")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return coverage, fmt.Errorf("while find scenario asserts : %s", err)
|
return nil, fmt.Errorf("while find scenario asserts : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
for _, assert := range passerts {
|
for _, assert := range passerts {
|
||||||
file, err := os.Open(assert)
|
file, err := os.Open(assert)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return coverage, fmt.Errorf("while reading %s : %s", assert, err)
|
return nil, fmt.Errorf("while reading %s : %s", assert, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
scanner := bufio.NewScanner(file)
|
scanner := bufio.NewScanner(file)
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
assertLine := regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P<scenario>[^"]+)"`)
|
|
||||||
line := scanner.Text()
|
line := scanner.Text()
|
||||||
log.Debugf("assert line : %s", line)
|
log.Debugf("assert line : %s", line)
|
||||||
match := assertLine.FindStringSubmatch(line)
|
match := scenarioResultRE.FindStringSubmatch(line)
|
||||||
|
|
||||||
if len(match) == 0 {
|
if len(match) == 0 {
|
||||||
log.Debugf("%s doesn't match", line)
|
log.Debugf("%s doesn't match", line)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
sidx := assertLine.SubexpIndex("scenario")
|
|
||||||
scanner_name := match[sidx]
|
sidx := scenarioResultRE.SubexpIndex("scenario")
|
||||||
|
scannerName := match[sidx]
|
||||||
|
|
||||||
for idx, pcover := range coverage {
|
for idx, pcover := range coverage {
|
||||||
if pcover.Scenario == scanner_name {
|
if pcover.Name == scannerName {
|
||||||
coverage[idx].TestsCount++
|
coverage[idx].TestsCount++
|
||||||
coverage[idx].PresentIn[assert] = true
|
coverage[idx].PresentIn[assert] = true
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
scenarioNameSplit := strings.Split(pcover.Scenario, "/")
|
|
||||||
|
scenarioNameSplit := strings.Split(pcover.Name, "/")
|
||||||
scenarioNameOnly := scenarioNameSplit[len(scenarioNameSplit)-1]
|
scenarioNameOnly := scenarioNameSplit[len(scenarioNameSplit)-1]
|
||||||
if scenarioNameOnly == scanner_name {
|
|
||||||
|
if scenarioNameOnly == scannerName {
|
||||||
coverage[idx].TestsCount++
|
coverage[idx].TestsCount++
|
||||||
coverage[idx].PresentIn[assert] = true
|
coverage[idx].PresentIn[assert] = true
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
fixedProbingWord := strings.ReplaceAll(pcover.Scenario, "probbing", "probing")
|
|
||||||
fixedProbingAssert := strings.ReplaceAll(scanner_name, "probbing", "probing")
|
fixedProbingWord := strings.ReplaceAll(pcover.Name, "probbing", "probing")
|
||||||
|
fixedProbingAssert := strings.ReplaceAll(scannerName, "probbing", "probing")
|
||||||
|
|
||||||
if fixedProbingWord == fixedProbingAssert {
|
if fixedProbingWord == fixedProbingAssert {
|
||||||
coverage[idx].TestsCount++
|
coverage[idx].TestsCount++
|
||||||
coverage[idx].PresentIn[assert] = true
|
coverage[idx].PresentIn[assert] = true
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if fmt.Sprintf("%s-detection", pcover.Scenario) == scanner_name {
|
|
||||||
|
if fmt.Sprintf("%s-detection", pcover.Name) == scannerName {
|
||||||
coverage[idx].TestsCount++
|
coverage[idx].TestsCount++
|
||||||
coverage[idx].PresentIn[assert] = true
|
coverage[idx].PresentIn[assert] = true
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if fmt.Sprintf("%s-detection", fixedProbingWord) == fixedProbingAssert {
|
if fmt.Sprintf("%s-detection", fixedProbingWord) == fixedProbingAssert {
|
||||||
coverage[idx].TestsCount++
|
coverage[idx].TestsCount++
|
||||||
coverage[idx].PresentIn[assert] = true
|
coverage[idx].PresentIn[assert] = true
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
file.Close()
|
file.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
return coverage, nil
|
return coverage, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -18,7 +19,7 @@ type HubTest struct {
|
||||||
TemplateConfigPath string
|
TemplateConfigPath string
|
||||||
TemplateProfilePath string
|
TemplateProfilePath string
|
||||||
TemplateSimulationPath string
|
TemplateSimulationPath string
|
||||||
HubIndex *HubIndex
|
HubIndex *cwhub.Hub
|
||||||
Tests []*HubTestItem
|
Tests []*HubTestItem
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -29,42 +30,44 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) {
|
func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) {
|
||||||
var err error
|
hubPath, err := filepath.Abs(hubPath)
|
||||||
|
|
||||||
hubPath, err = filepath.Abs(hubPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err)
|
return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// we can't use hubtest without the hub
|
// we can't use hubtest without the hub
|
||||||
if _, err := os.Stat(hubPath); os.IsNotExist(err) {
|
if _, err = os.Stat(hubPath); os.IsNotExist(err) {
|
||||||
return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath)
|
return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
HubTestPath := filepath.Join(hubPath, "./.tests/")
|
HubTestPath := filepath.Join(hubPath, "./.tests/")
|
||||||
|
|
||||||
// we can't use hubtest without crowdsec binary
|
// we can't use hubtest without crowdsec binary
|
||||||
if _, err := exec.LookPath(crowdsecPath); err != nil {
|
if _, err = exec.LookPath(crowdsecPath); err != nil {
|
||||||
if _, err := os.Stat(crowdsecPath); os.IsNotExist(err) {
|
if _, err = os.Stat(crowdsecPath); os.IsNotExist(err) {
|
||||||
return HubTest{}, fmt.Errorf("path to crowdsec binary '%s' doesn't exist or is not in $PATH, can't run", crowdsecPath)
|
return HubTest{}, fmt.Errorf("path to crowdsec binary '%s' doesn't exist or is not in $PATH, can't run", crowdsecPath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// we can't use hubtest without cscli binary
|
// we can't use hubtest without cscli binary
|
||||||
if _, err := exec.LookPath(cscliPath); err != nil {
|
if _, err = exec.LookPath(cscliPath); err != nil {
|
||||||
if _, err := os.Stat(cscliPath); os.IsNotExist(err) {
|
if _, err = os.Stat(cscliPath); os.IsNotExist(err) {
|
||||||
return HubTest{}, fmt.Errorf("path to cscli binary '%s' doesn't exist or is not in $PATH, can't run", cscliPath)
|
return HubTest{}, fmt.Errorf("path to cscli binary '%s' doesn't exist or is not in $PATH, can't run", cscliPath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
hubIndexFile := filepath.Join(hubPath, ".index.json")
|
hubIndexFile := filepath.Join(hubPath, ".index.json")
|
||||||
bidx, err := os.ReadFile(hubIndexFile)
|
|
||||||
if err != nil {
|
local := &csconfig.LocalHubCfg{
|
||||||
return HubTest{}, fmt.Errorf("unable to read index file: %s", err)
|
HubDir: hubPath,
|
||||||
|
HubIndexFile: hubIndexFile,
|
||||||
|
InstallDir: HubTestPath,
|
||||||
|
InstallDataDir: HubTestPath,
|
||||||
}
|
}
|
||||||
|
|
||||||
// load hub index
|
hub, err := cwhub.NewHub(local, nil, false)
|
||||||
hubIndex, err := cwhub.LoadPkgIndex(bidx)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return HubTest{}, fmt.Errorf("unable to load hub index file: %s", err)
|
return HubTest{}, fmt.Errorf("unable to load hub: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile)
|
templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile)
|
||||||
|
@ -80,16 +83,18 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
|
||||||
TemplateConfigPath: templateConfigFilePath,
|
TemplateConfigPath: templateConfigFilePath,
|
||||||
TemplateProfilePath: templateProfilePath,
|
TemplateProfilePath: templateProfilePath,
|
||||||
TemplateSimulationPath: templateSimulationPath,
|
TemplateSimulationPath: templateSimulationPath,
|
||||||
HubIndex: &HubIndex{Data: hubIndex},
|
HubIndex: hub,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) {
|
func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) {
|
||||||
HubTestItem := &HubTestItem{}
|
HubTestItem := &HubTestItem{}
|
||||||
|
|
||||||
testItem, err := NewTest(name, h)
|
testItem, err := NewTest(name, h)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return HubTestItem, err
|
return HubTestItem, err
|
||||||
}
|
}
|
||||||
|
|
||||||
h.Tests = append(h.Tests, testItem)
|
h.Tests = append(h.Tests, testItem)
|
||||||
|
|
||||||
return testItem, nil
|
return testItem, nil
|
||||||
|
@ -108,5 +113,6 @@ func (h *HubTest) LoadAllTests() error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,11 +7,12 @@ import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
"gopkg.in/yaml.v2"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/parser"
|
"github.com/crowdsecurity/crowdsec/pkg/parser"
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
"gopkg.in/yaml.v2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type HubTestItemConfig struct {
|
type HubTestItemConfig struct {
|
||||||
|
@ -25,10 +26,6 @@ type HubTestItemConfig struct {
|
||||||
OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00
|
OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00
|
||||||
}
|
}
|
||||||
|
|
||||||
type HubIndex struct {
|
|
||||||
Data map[string]map[string]cwhub.Item
|
|
||||||
}
|
|
||||||
|
|
||||||
type HubTestItem struct {
|
type HubTestItem struct {
|
||||||
Name string
|
Name string
|
||||||
Path string
|
Path string
|
||||||
|
@ -43,7 +40,7 @@ type HubTestItem struct {
|
||||||
RuntimeConfigFilePath string
|
RuntimeConfigFilePath string
|
||||||
RuntimeProfileFilePath string
|
RuntimeProfileFilePath string
|
||||||
RuntimeSimulationFilePath string
|
RuntimeSimulationFilePath string
|
||||||
RuntimeHubConfig *csconfig.Hub
|
RuntimeHubConfig *csconfig.LocalHubCfg
|
||||||
|
|
||||||
ResultsPath string
|
ResultsPath string
|
||||||
ParserResultFile string
|
ParserResultFile string
|
||||||
|
@ -56,7 +53,7 @@ type HubTestItem struct {
|
||||||
TemplateConfigPath string
|
TemplateConfigPath string
|
||||||
TemplateProfilePath string
|
TemplateProfilePath string
|
||||||
TemplateSimulationPath string
|
TemplateSimulationPath string
|
||||||
HubIndex *HubIndex
|
HubIndex *cwhub.Hub
|
||||||
|
|
||||||
Config *HubTestItemConfig
|
Config *HubTestItemConfig
|
||||||
|
|
||||||
|
@ -80,8 +77,6 @@ const (
|
||||||
BucketPourResultFileName = "bucketpour-dump.yaml"
|
BucketPourResultFileName = "bucketpour-dump.yaml"
|
||||||
)
|
)
|
||||||
|
|
||||||
var crowdsecPatternsFolder = csconfig.DefaultConfigPath("patterns")
|
|
||||||
|
|
||||||
func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
|
func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
|
||||||
testPath := filepath.Join(hubTest.HubTestPath, name)
|
testPath := filepath.Join(hubTest.HubTestPath, name)
|
||||||
runtimeFolder := filepath.Join(testPath, "runtime")
|
runtimeFolder := filepath.Join(testPath, "runtime")
|
||||||
|
@ -91,10 +86,12 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
|
||||||
|
|
||||||
// read test configuration file
|
// read test configuration file
|
||||||
configFileData := &HubTestItemConfig{}
|
configFileData := &HubTestItemConfig{}
|
||||||
|
|
||||||
yamlFile, err := os.ReadFile(configFilePath)
|
yamlFile, err := os.ReadFile(configFilePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("no config file found in '%s': %v", testPath, err)
|
log.Printf("no config file found in '%s': %v", testPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = yaml.Unmarshal(yamlFile, configFileData)
|
err = yaml.Unmarshal(yamlFile, configFileData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("unmarshal: %v", err)
|
return nil, fmt.Errorf("unmarshal: %v", err)
|
||||||
|
@ -105,6 +102,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
|
||||||
|
|
||||||
scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName)
|
scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName)
|
||||||
ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath)
|
ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath)
|
||||||
|
|
||||||
return &HubTestItem{
|
return &HubTestItem{
|
||||||
Name: name,
|
Name: name,
|
||||||
Path: testPath,
|
Path: testPath,
|
||||||
|
@ -121,7 +119,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
|
||||||
ParserResultFile: filepath.Join(resultPath, ParserResultFileName),
|
ParserResultFile: filepath.Join(resultPath, ParserResultFileName),
|
||||||
ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName),
|
ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName),
|
||||||
BucketPourResultFile: filepath.Join(resultPath, BucketPourResultFileName),
|
BucketPourResultFile: filepath.Join(resultPath, BucketPourResultFileName),
|
||||||
RuntimeHubConfig: &csconfig.Hub{
|
RuntimeHubConfig: &csconfig.LocalHubCfg{
|
||||||
HubDir: runtimeHubFolder,
|
HubDir: runtimeHubFolder,
|
||||||
HubIndexFile: hubTest.HubIndexFile,
|
HubIndexFile: hubTest.HubIndexFile,
|
||||||
InstallDir: runtimeFolder,
|
InstallDir: runtimeFolder,
|
||||||
|
@ -147,23 +145,25 @@ func (t *HubTestItem) InstallHub() error {
|
||||||
if parser == "" {
|
if parser == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var parserDirDest string
|
|
||||||
if hubParser, ok := t.HubIndex.Data[cwhub.PARSERS][parser]; ok {
|
if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok {
|
||||||
parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
|
parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
|
return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
parserFileName := filepath.Base(parserSource)
|
parserFileName := filepath.Base(parserSource)
|
||||||
|
|
||||||
// runtime/hub/parsers/s00-raw/crowdsecurity/
|
// runtime/hub/parsers/s00-raw/crowdsecurity/
|
||||||
hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubParser.RemotePath))
|
hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubParser.RemotePath))
|
||||||
|
|
||||||
// runtime/parsers/s00-raw/
|
// runtime/parsers/s00-raw/
|
||||||
parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage)
|
parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage)
|
||||||
|
|
||||||
if err := os.MkdirAll(hubDirParserDest, os.ModePerm); err != nil {
|
if err := os.MkdirAll(hubDirParserDest, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to create folder '%s': %s", hubDirParserDest, err)
|
return fmt.Errorf("unable to create folder '%s': %s", hubDirParserDest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
|
if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
|
return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
|
||||||
}
|
}
|
||||||
|
@ -204,7 +204,7 @@ func (t *HubTestItem) InstallHub() error {
|
||||||
//return fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath)
|
//return fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage)
|
parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage)
|
||||||
if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
|
if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
|
||||||
continue
|
continue
|
||||||
//return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
|
//return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
|
||||||
|
@ -231,23 +231,25 @@ func (t *HubTestItem) InstallHub() error {
|
||||||
if scenario == "" {
|
if scenario == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var scenarioDirDest string
|
|
||||||
if hubScenario, ok := t.HubIndex.Data[cwhub.SCENARIOS][scenario]; ok {
|
if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok {
|
||||||
scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath))
|
scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("can't get absolute path to: %s", scenarioSource)
|
return fmt.Errorf("can't get absolute path to: %s", scenarioSource)
|
||||||
}
|
}
|
||||||
|
|
||||||
scenarioFileName := filepath.Base(scenarioSource)
|
scenarioFileName := filepath.Base(scenarioSource)
|
||||||
|
|
||||||
// runtime/hub/scenarios/crowdsecurity/
|
// runtime/hub/scenarios/crowdsecurity/
|
||||||
hubDirScenarioDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubScenario.RemotePath))
|
hubDirScenarioDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubScenario.RemotePath))
|
||||||
|
|
||||||
// runtime/parsers/scenarios/
|
// runtime/parsers/scenarios/
|
||||||
scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath)
|
scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath)
|
||||||
|
|
||||||
if err := os.MkdirAll(hubDirScenarioDest, os.ModePerm); err != nil {
|
if err := os.MkdirAll(hubDirScenarioDest, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to create folder '%s': %s", hubDirScenarioDest, err)
|
return fmt.Errorf("unable to create folder '%s': %s", hubDirScenarioDest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
|
if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
|
return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
|
||||||
}
|
}
|
||||||
|
@ -275,7 +277,7 @@ func (t *HubTestItem) InstallHub() error {
|
||||||
//return fmt.Errorf("scenarios '%s' doesn't exist in the hub and doesn't appear to be a custom one.", scenario)
|
//return fmt.Errorf("scenarios '%s' doesn't exist in the hub and doesn't appear to be a custom one.", scenario)
|
||||||
}
|
}
|
||||||
|
|
||||||
scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath)
|
scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath)
|
||||||
if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
|
if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
|
return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
|
||||||
}
|
}
|
||||||
|
@ -300,23 +302,25 @@ func (t *HubTestItem) InstallHub() error {
|
||||||
if postoverflow == "" {
|
if postoverflow == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var postoverflowDirDest string
|
|
||||||
if hubPostOverflow, ok := t.HubIndex.Data[cwhub.PARSERS_OVFLW][postoverflow]; ok {
|
if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok {
|
||||||
postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
|
postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)
|
return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
postoverflowFileName := filepath.Base(postoverflowSource)
|
postoverflowFileName := filepath.Base(postoverflowSource)
|
||||||
|
|
||||||
// runtime/hub/postoverflows/s00-enrich/crowdsecurity/
|
// runtime/hub/postoverflows/s00-enrich/crowdsecurity/
|
||||||
hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubPostOverflow.RemotePath))
|
hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubPostOverflow.RemotePath))
|
||||||
|
|
||||||
// runtime/postoverflows/s00-enrich
|
// runtime/postoverflows/s00-enrich
|
||||||
postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage)
|
postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage)
|
||||||
|
|
||||||
if err := os.MkdirAll(hubDirPostoverflowDest, os.ModePerm); err != nil {
|
if err := os.MkdirAll(hubDirPostoverflowDest, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to create folder '%s': %s", hubDirPostoverflowDest, err)
|
return fmt.Errorf("unable to create folder '%s': %s", hubDirPostoverflowDest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
|
if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
|
return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
|
||||||
}
|
}
|
||||||
|
@ -357,7 +361,7 @@ func (t *HubTestItem) InstallHub() error {
|
||||||
//return fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath)
|
//return fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage)
|
postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage)
|
||||||
if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
|
if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
|
||||||
continue
|
continue
|
||||||
//return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
|
//return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
|
||||||
|
@ -384,10 +388,12 @@ func (t *HubTestItem) InstallHub() error {
|
||||||
Filter: "1==1",
|
Filter: "1==1",
|
||||||
Statics: t.Config.OverrideStatics,
|
Statics: t.Config.OverrideStatics,
|
||||||
}
|
}
|
||||||
|
|
||||||
b, err := yaml.Marshal(n)
|
b, err := yaml.Marshal(n)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to marshal overrides: %s", err)
|
return fmt.Errorf("unable to marshal overrides: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
tgtFilename := fmt.Sprintf("%s/parsers/s00-raw/00_overrides.yaml", t.RuntimePath)
|
tgtFilename := fmt.Sprintf("%s/parsers/s00-raw/00_overrides.yaml", t.RuntimePath)
|
||||||
if err := os.WriteFile(tgtFilename, b, os.ModePerm); err != nil {
|
if err := os.WriteFile(tgtFilename, b, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to write overrides to '%s': %s", tgtFilename, err)
|
return fmt.Errorf("unable to write overrides to '%s': %s", tgtFilename, err)
|
||||||
|
@ -395,40 +401,43 @@ func (t *HubTestItem) InstallHub() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// load installed hub
|
// load installed hub
|
||||||
err := cwhub.GetHubIdx(t.RuntimeHubConfig)
|
hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("can't local sync the hub: %+v", err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// install data for parsers if needed
|
// install data for parsers if needed
|
||||||
ret := cwhub.GetItemMap(cwhub.PARSERS)
|
ret := hub.GetItemMap(cwhub.PARSERS)
|
||||||
for parserName, item := range ret {
|
for parserName, item := range ret {
|
||||||
if item.Installed {
|
if item.Installed {
|
||||||
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
|
if err := item.DownloadDataIfNeeded(true); err != nil {
|
||||||
return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err)
|
return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debugf("parser '%s' installed successfully in runtime environment", parserName)
|
log.Debugf("parser '%s' installed successfully in runtime environment", parserName)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// install data for scenarios if needed
|
// install data for scenarios if needed
|
||||||
ret = cwhub.GetItemMap(cwhub.SCENARIOS)
|
ret = hub.GetItemMap(cwhub.SCENARIOS)
|
||||||
for scenarioName, item := range ret {
|
for scenarioName, item := range ret {
|
||||||
if item.Installed {
|
if item.Installed {
|
||||||
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
|
if err := item.DownloadDataIfNeeded(true); err != nil {
|
||||||
return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err)
|
return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName)
|
log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// install data for postoverflows if needed
|
// install data for postoverflows if needed
|
||||||
ret = cwhub.GetItemMap(cwhub.PARSERS_OVFLW)
|
ret = hub.GetItemMap(cwhub.POSTOVERFLOWS)
|
||||||
for postoverflowName, item := range ret {
|
for postoverflowName, item := range ret {
|
||||||
if item.Installed {
|
if item.Installed {
|
||||||
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
|
if err := item.DownloadDataIfNeeded(true); err != nil {
|
||||||
return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err)
|
return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName)
|
log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -455,51 +464,53 @@ func (t *HubTestItem) Run() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// create runtime folder
|
// create runtime folder
|
||||||
if err := os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil {
|
if err = os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err)
|
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// create runtime data folder
|
// create runtime data folder
|
||||||
if err := os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil {
|
if err = os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err)
|
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// create runtime hub folder
|
// create runtime hub folder
|
||||||
if err := os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil {
|
if err = os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err)
|
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
|
if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
|
||||||
return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err)
|
return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// create results folder
|
// create results folder
|
||||||
if err := os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil {
|
if err = os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil {
|
||||||
return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err)
|
return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// copy template config file to runtime folder
|
// copy template config file to runtime folder
|
||||||
if err := Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
|
if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
|
||||||
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err)
|
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// copy template profile file to runtime folder
|
// copy template profile file to runtime folder
|
||||||
if err := Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
|
if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
|
||||||
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err)
|
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// copy template simulation file to runtime folder
|
// copy template simulation file to runtime folder
|
||||||
if err := Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
|
if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
|
||||||
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err)
|
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns")
|
||||||
|
|
||||||
// copy template patterns folder to runtime folder
|
// copy template patterns folder to runtime folder
|
||||||
if err := CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil {
|
if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil {
|
||||||
return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err)
|
return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// install the hub in the runtime folder
|
// install the hub in the runtime folder
|
||||||
if err := t.InstallHub(); err != nil {
|
if err = t.InstallHub(); err != nil {
|
||||||
return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err)
|
return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -507,7 +518,7 @@ func (t *HubTestItem) Run() error {
|
||||||
logType := t.Config.LogType
|
logType := t.Config.LogType
|
||||||
dsn := fmt.Sprintf("file://%s", logFile)
|
dsn := fmt.Sprintf("file://%s", logFile)
|
||||||
|
|
||||||
if err := os.Chdir(testPath); err != nil {
|
if err = os.Chdir(testPath); err != nil {
|
||||||
return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err)
|
return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -515,6 +526,7 @@ func (t *HubTestItem) Run() error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to stat log file '%s': %s", logFile, err)
|
return fmt.Errorf("unable to stat log file '%s': %s", logFile, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if logFileStat.Size() == 0 {
|
if logFileStat.Size() == 0 {
|
||||||
return fmt.Errorf("log file '%s' is empty, please fill it with log", logFile)
|
return fmt.Errorf("log file '%s' is empty, please fill it with log", logFile)
|
||||||
}
|
}
|
||||||
|
@ -522,6 +534,7 @@ func (t *HubTestItem) Run() error {
|
||||||
cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"}
|
cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"}
|
||||||
cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...)
|
cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...)
|
||||||
log.Debugf("%s", cscliRegisterCmd.String())
|
log.Debugf("%s", cscliRegisterCmd.String())
|
||||||
|
|
||||||
output, err := cscliRegisterCmd.CombinedOutput()
|
output, err := cscliRegisterCmd.CombinedOutput()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") {
|
if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") {
|
||||||
|
@ -531,16 +544,20 @@ func (t *HubTestItem) Run() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", t.ResultsPath, "-order-event"}
|
cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", t.ResultsPath, "-order-event"}
|
||||||
|
|
||||||
for labelKey, labelValue := range t.Config.Labels {
|
for labelKey, labelValue := range t.Config.Labels {
|
||||||
arg := fmt.Sprintf("%s:%s", labelKey, labelValue)
|
arg := fmt.Sprintf("%s:%s", labelKey, labelValue)
|
||||||
cmdArgs = append(cmdArgs, "-label", arg)
|
cmdArgs = append(cmdArgs, "-label", arg)
|
||||||
}
|
}
|
||||||
|
|
||||||
crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...)
|
crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...)
|
||||||
log.Debugf("%s", crowdsecCmd.String())
|
log.Debugf("%s", crowdsecCmd.String())
|
||||||
output, err = crowdsecCmd.CombinedOutput()
|
output, err = crowdsecCmd.CombinedOutput()
|
||||||
|
|
||||||
if log.GetLevel() >= log.DebugLevel || err != nil {
|
if log.GetLevel() >= log.DebugLevel || err != nil {
|
||||||
fmt.Println(string(output))
|
fmt.Println(string(output))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err)
|
return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err)
|
||||||
}
|
}
|
||||||
|
@ -557,8 +574,10 @@ func (t *HubTestItem) Run() error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
parserAssertFile.Close()
|
parserAssertFile.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
assertFileStat, err := os.Stat(t.ParserAssert.File)
|
assertFileStat, err := os.Stat(t.ParserAssert.File)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error while stats '%s': %s", t.ParserAssert.File, err)
|
return fmt.Errorf("error while stats '%s': %s", t.ParserAssert.File, err)
|
||||||
|
@ -569,6 +588,7 @@ func (t *HubTestItem) Run() error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("couldn't generate assertion: %s", err)
|
return fmt.Errorf("couldn't generate assertion: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.ParserAssert.AutoGenAssertData = assertData
|
t.ParserAssert.AutoGenAssertData = assertData
|
||||||
t.ParserAssert.AutoGenAssert = true
|
t.ParserAssert.AutoGenAssert = true
|
||||||
} else {
|
} else {
|
||||||
|
@ -580,12 +600,15 @@ func (t *HubTestItem) Run() error {
|
||||||
|
|
||||||
// assert scenarios
|
// assert scenarios
|
||||||
nbScenario := 0
|
nbScenario := 0
|
||||||
|
|
||||||
for _, scenario := range t.Config.Scenarios {
|
for _, scenario := range t.Config.Scenarios {
|
||||||
if scenario == "" {
|
if scenario == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
nbScenario += 1
|
|
||||||
|
nbScenario++
|
||||||
}
|
}
|
||||||
|
|
||||||
if nbScenario > 0 {
|
if nbScenario > 0 {
|
||||||
_, err := os.Stat(t.ScenarioAssert.File)
|
_, err := os.Stat(t.ScenarioAssert.File)
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
|
@ -593,8 +616,10 @@ func (t *HubTestItem) Run() error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
scenarioAssertFile.Close()
|
scenarioAssertFile.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
assertFileStat, err := os.Stat(t.ScenarioAssert.File)
|
assertFileStat, err := os.Stat(t.ScenarioAssert.File)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error while stats '%s': %s", t.ScenarioAssert.File, err)
|
return fmt.Errorf("error while stats '%s': %s", t.ScenarioAssert.File, err)
|
||||||
|
@ -605,6 +630,7 @@ func (t *HubTestItem) Run() error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("couldn't generate assertion: %s", err)
|
return fmt.Errorf("couldn't generate assertion: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.ScenarioAssert.AutoGenAssertData = assertData
|
t.ScenarioAssert.AutoGenAssertData = assertData
|
||||||
t.ScenarioAssert.AutoGenAssert = true
|
t.ScenarioAssert.AutoGenAssert = true
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -5,13 +5,11 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"regexp"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/antonmedv/expr"
|
"github.com/antonmedv/expr"
|
||||||
"github.com/antonmedv/expr/vm"
|
|
||||||
"github.com/enescakir/emoji"
|
"github.com/enescakir/emoji"
|
||||||
"github.com/fatih/color"
|
"github.com/fatih/color"
|
||||||
diff "github.com/r3labs/diff/v2"
|
diff "github.com/r3labs/diff/v2"
|
||||||
|
@ -43,10 +41,10 @@ type ParserResult struct {
|
||||||
Evt types.Event
|
Evt types.Event
|
||||||
Success bool
|
Success bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type ParserResults map[string]map[string][]ParserResult
|
type ParserResults map[string]map[string][]ParserResult
|
||||||
|
|
||||||
func NewParserAssert(file string) *ParserAssert {
|
func NewParserAssert(file string) *ParserAssert {
|
||||||
|
|
||||||
ParserAssert := &ParserAssert{
|
ParserAssert := &ParserAssert{
|
||||||
File: file,
|
File: file,
|
||||||
NbAssert: 0,
|
NbAssert: 0,
|
||||||
|
@ -55,6 +53,7 @@ func NewParserAssert(file string) *ParserAssert {
|
||||||
AutoGenAssert: false,
|
AutoGenAssert: false,
|
||||||
TestData: &ParserResults{},
|
TestData: &ParserResults{},
|
||||||
}
|
}
|
||||||
|
|
||||||
return ParserAssert
|
return ParserAssert
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -63,22 +62,24 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
ret := p.AutoGenParserAssert()
|
ret := p.AutoGenParserAssert()
|
||||||
|
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *ParserAssert) LoadTest(filename string) error {
|
func (p *ParserAssert) LoadTest(filename string) error {
|
||||||
var err error
|
|
||||||
parserDump, err := LoadParserDump(filename)
|
parserDump, err := LoadParserDump(filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("loading parser dump file: %+v", err)
|
return fmt.Errorf("loading parser dump file: %+v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
p.TestData = parserDump
|
p.TestData = parserDump
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *ParserAssert) AssertFile(testFile string) error {
|
func (p *ParserAssert) AssertFile(testFile string) error {
|
||||||
|
|
||||||
file, err := os.Open(p.File)
|
file, err := os.Open(p.File)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -88,19 +89,26 @@ func (p *ParserAssert) AssertFile(testFile string) error {
|
||||||
if err := p.LoadTest(testFile); err != nil {
|
if err := p.LoadTest(testFile); err != nil {
|
||||||
return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
|
return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
scanner := bufio.NewScanner(file)
|
scanner := bufio.NewScanner(file)
|
||||||
scanner.Split(bufio.ScanLines)
|
scanner.Split(bufio.ScanLines)
|
||||||
|
|
||||||
nbLine := 0
|
nbLine := 0
|
||||||
|
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
nbLine += 1
|
nbLine++
|
||||||
|
|
||||||
if scanner.Text() == "" {
|
if scanner.Text() == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
ok, err := p.Run(scanner.Text())
|
ok, err := p.Run(scanner.Text())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
|
return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
|
||||||
}
|
}
|
||||||
p.NbAssert += 1
|
|
||||||
|
p.NbAssert++
|
||||||
|
|
||||||
if !ok {
|
if !ok {
|
||||||
log.Debugf("%s is FALSE", scanner.Text())
|
log.Debugf("%s is FALSE", scanner.Text())
|
||||||
failedAssert := &AssertFail{
|
failedAssert := &AssertFail{
|
||||||
|
@ -109,37 +117,43 @@ func (p *ParserAssert) AssertFile(testFile string) error {
|
||||||
Expression: scanner.Text(),
|
Expression: scanner.Text(),
|
||||||
Debug: make(map[string]string),
|
Debug: make(map[string]string),
|
||||||
}
|
}
|
||||||
variableRE := regexp.MustCompile(`(?P<variable>[^ =]+) == .*`)
|
|
||||||
match := variableRE.FindStringSubmatch(scanner.Text())
|
match := variableRE.FindStringSubmatch(scanner.Text())
|
||||||
variable := ""
|
variable := ""
|
||||||
|
|
||||||
if len(match) == 0 {
|
if len(match) == 0 {
|
||||||
log.Infof("Couldn't get variable of line '%s'", scanner.Text())
|
log.Infof("Couldn't get variable of line '%s'", scanner.Text())
|
||||||
variable = scanner.Text()
|
variable = scanner.Text()
|
||||||
} else {
|
} else {
|
||||||
variable = match[1]
|
variable = match[1]
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := p.EvalExpression(variable)
|
result, err := p.EvalExpression(variable)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("unable to evaluate variable '%s': %s", variable, err)
|
log.Errorf("unable to evaluate variable '%s': %s", variable, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
failedAssert.Debug[variable] = result
|
failedAssert.Debug[variable] = result
|
||||||
p.Fails = append(p.Fails, *failedAssert)
|
p.Fails = append(p.Fails, *failedAssert)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
|
//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
file.Close()
|
file.Close()
|
||||||
|
|
||||||
if p.NbAssert == 0 {
|
if p.NbAssert == 0 {
|
||||||
assertData, err := p.AutoGenFromFile(testFile)
|
assertData, err := p.AutoGenFromFile(testFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("couldn't generate assertion: %s", err)
|
return fmt.Errorf("couldn't generate assertion: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
p.AutoGenAssertData = assertData
|
p.AutoGenAssertData = assertData
|
||||||
p.AutoGenAssert = true
|
p.AutoGenAssert = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(p.Fails) == 0 {
|
if len(p.Fails) == 0 {
|
||||||
p.Success = true
|
p.Success = true
|
||||||
}
|
}
|
||||||
|
@ -148,15 +162,14 @@ func (p *ParserAssert) AssertFile(testFile string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
|
func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
|
||||||
var err error
|
|
||||||
//debug doesn't make much sense with the ability to evaluate "on the fly"
|
//debug doesn't make much sense with the ability to evaluate "on the fly"
|
||||||
//var debugFilter *exprhelpers.ExprDebugger
|
//var debugFilter *exprhelpers.ExprDebugger
|
||||||
var runtimeFilter *vm.Program
|
|
||||||
var output interface{}
|
var output interface{}
|
||||||
|
|
||||||
env := map[string]interface{}{"results": *p.TestData}
|
env := map[string]interface{}{"results": *p.TestData}
|
||||||
|
|
||||||
if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil {
|
runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...)
|
||||||
|
if err != nil {
|
||||||
log.Errorf("failed to compile '%s' : %s", expression, err)
|
log.Errorf("failed to compile '%s' : %s", expression, err)
|
||||||
return output, err
|
return output, err
|
||||||
}
|
}
|
||||||
|
@ -168,8 +181,10 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warningf("running : %s", expression)
|
log.Warningf("running : %s", expression)
|
||||||
log.Warningf("runtime error : %s", err)
|
log.Warningf("runtime error : %s", err)
|
||||||
|
|
||||||
return output, fmt.Errorf("while running expression %s: %w", expression, err)
|
return output, fmt.Errorf("while running expression %s: %w", expression, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return output, nil
|
return output, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -178,10 +193,13 @@ func (p *ParserAssert) EvalExpression(expression string) (string, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
ret, err := yaml.Marshal(output)
|
ret, err := yaml.Marshal(output)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
return string(ret), nil
|
return string(ret), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -190,6 +208,7 @@ func (p *ParserAssert) Run(assert string) (bool, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
switch out := output.(type) {
|
switch out := output.(type) {
|
||||||
case bool:
|
case bool:
|
||||||
return out, nil
|
return out, nil
|
||||||
|
@ -201,80 +220,89 @@ func (p *ParserAssert) Run(assert string) (bool, error) {
|
||||||
func Escape(val string) string {
|
func Escape(val string) string {
|
||||||
val = strings.ReplaceAll(val, `\`, `\\`)
|
val = strings.ReplaceAll(val, `\`, `\\`)
|
||||||
val = strings.ReplaceAll(val, `"`, `\"`)
|
val = strings.ReplaceAll(val, `"`, `\"`)
|
||||||
|
|
||||||
return val
|
return val
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *ParserAssert) AutoGenParserAssert() string {
|
func (p *ParserAssert) AutoGenParserAssert() string {
|
||||||
//attempt to autogen parser asserts
|
//attempt to autogen parser asserts
|
||||||
var ret string
|
ret := fmt.Sprintf("len(results) == %d\n", len(*p.TestData))
|
||||||
|
|
||||||
|
//sort map keys for consistent order
|
||||||
|
stages := sortedMapKeys(*p.TestData)
|
||||||
|
|
||||||
//sort map keys for consistent ordre
|
|
||||||
var stages []string
|
|
||||||
for stage := range *p.TestData {
|
|
||||||
stages = append(stages, stage)
|
|
||||||
}
|
|
||||||
sort.Strings(stages)
|
|
||||||
ret += fmt.Sprintf("len(results) == %d\n", len(*p.TestData))
|
|
||||||
for _, stage := range stages {
|
for _, stage := range stages {
|
||||||
parsers := (*p.TestData)[stage]
|
parsers := (*p.TestData)[stage]
|
||||||
//sort map keys for consistent ordre
|
|
||||||
var pnames []string
|
//sort map keys for consistent order
|
||||||
for pname := range parsers {
|
pnames := sortedMapKeys(parsers)
|
||||||
pnames = append(pnames, pname)
|
|
||||||
}
|
|
||||||
sort.Strings(pnames)
|
|
||||||
for _, parser := range pnames {
|
for _, parser := range pnames {
|
||||||
presults := parsers[parser]
|
presults := parsers[parser]
|
||||||
ret += fmt.Sprintf(`len(results["%s"]["%s"]) == %d`+"\n", stage, parser, len(presults))
|
ret += fmt.Sprintf(`len(results["%s"]["%s"]) == %d`+"\n", stage, parser, len(presults))
|
||||||
|
|
||||||
for pidx, result := range presults {
|
for pidx, result := range presults {
|
||||||
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Success == %t`+"\n", stage, parser, pidx, result.Success)
|
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Success == %t`+"\n", stage, parser, pidx, result.Success)
|
||||||
|
|
||||||
if !result.Success {
|
if !result.Success {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, pkey := range sortedMapKeys(result.Evt.Parsed) {
|
for _, pkey := range sortedMapKeys(result.Evt.Parsed) {
|
||||||
pval := result.Evt.Parsed[pkey]
|
pval := result.Evt.Parsed[pkey]
|
||||||
if pval == "" {
|
if pval == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval))
|
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval))
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, mkey := range sortedMapKeys(result.Evt.Meta) {
|
for _, mkey := range sortedMapKeys(result.Evt.Meta) {
|
||||||
mval := result.Evt.Meta[mkey]
|
mval := result.Evt.Meta[mkey]
|
||||||
if mval == "" {
|
if mval == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval))
|
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval))
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, ekey := range sortedMapKeys(result.Evt.Enriched) {
|
for _, ekey := range sortedMapKeys(result.Evt.Enriched) {
|
||||||
eval := result.Evt.Enriched[ekey]
|
eval := result.Evt.Enriched[ekey]
|
||||||
if eval == "" {
|
if eval == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval))
|
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval))
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) {
|
for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) {
|
||||||
uval := result.Evt.Unmarshaled[ukey]
|
uval := result.Evt.Unmarshaled[ukey]
|
||||||
if uval == "" {
|
if uval == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
base := fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Unmarshaled["%s"]`, stage, parser, pidx, ukey)
|
base := fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Unmarshaled["%s"]`, stage, parser, pidx, ukey)
|
||||||
|
|
||||||
for _, line := range p.buildUnmarshaledAssert(base, uval) {
|
for _, line := range p.buildUnmarshaledAssert(base, uval) {
|
||||||
ret += line
|
ret += line
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Whitelisted == %t`+"\n", stage, parser, pidx, result.Evt.Whitelisted)
|
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Whitelisted == %t`+"\n", stage, parser, pidx, result.Evt.Whitelisted)
|
||||||
|
|
||||||
if result.Evt.WhitelistReason != "" {
|
if result.Evt.WhitelistReason != "" {
|
||||||
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.WhitelistReason == "%s"`+"\n", stage, parser, pidx, Escape(result.Evt.WhitelistReason))
|
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.WhitelistReason == "%s"`+"\n", stage, parser, pidx, Escape(result.Evt.WhitelistReason))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []string {
|
func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []string {
|
||||||
ret := make([]string, 0)
|
ret := make([]string, 0)
|
||||||
|
|
||||||
switch val := eval.(type) {
|
switch val := eval.(type) {
|
||||||
case map[string]interface{}:
|
case map[string]interface{}:
|
||||||
for k, v := range val {
|
for k, v := range val {
|
||||||
|
@ -297,12 +325,11 @@ func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []s
|
||||||
default:
|
default:
|
||||||
log.Warningf("unknown type '%T' for key '%s'", val, ekey)
|
log.Warningf("unknown type '%T' for key '%s'", val, ekey)
|
||||||
}
|
}
|
||||||
|
|
||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
|
|
||||||
func LoadParserDump(filepath string) (*ParserResults, error) {
|
func LoadParserDump(filepath string) (*ParserResults, error) {
|
||||||
var pdump ParserResults
|
|
||||||
|
|
||||||
dumpData, err := os.Open(filepath)
|
dumpData, err := os.Open(filepath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -314,18 +341,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pdump := ParserResults{}
|
||||||
|
|
||||||
if err := yaml.Unmarshal(results, &pdump); err != nil {
|
if err := yaml.Unmarshal(results, &pdump); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
/* we know that some variables should always be set,
|
/* we know that some variables should always be set,
|
||||||
let's check if they're present in last parser output of last stage */
|
let's check if they're present in last parser output of last stage */
|
||||||
stages := make([]string, 0, len(pdump))
|
|
||||||
for k := range pdump {
|
stages := sortedMapKeys(pdump)
|
||||||
stages = append(stages, k)
|
|
||||||
}
|
|
||||||
sort.Strings(stages)
|
|
||||||
var lastStage string
|
var lastStage string
|
||||||
|
|
||||||
//Loop over stages to find last successful one with at least one parser
|
//Loop over stages to find last successful one with at least one parser
|
||||||
for i := len(stages) - 2; i >= 0; i-- {
|
for i := len(stages) - 2; i >= 0; i-- {
|
||||||
if len(pdump[stages[i]]) != 0 {
|
if len(pdump[stages[i]]) != 0 {
|
||||||
|
@ -333,11 +361,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
parsers := make([]string, 0, len(pdump[lastStage]))
|
parsers := make([]string, 0, len(pdump[lastStage]))
|
||||||
|
|
||||||
for k := range pdump[lastStage] {
|
for k := range pdump[lastStage] {
|
||||||
parsers = append(parsers, k)
|
parsers = append(parsers, k)
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Strings(parsers)
|
sort.Strings(parsers)
|
||||||
|
|
||||||
|
if len(parsers) == 0 {
|
||||||
|
return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry")
|
||||||
|
}
|
||||||
|
|
||||||
lastParser := parsers[len(parsers)-1]
|
lastParser := parsers[len(parsers)-1]
|
||||||
|
|
||||||
for idx, result := range pdump[lastStage][lastParser] {
|
for idx, result := range pdump[lastStage][lastParser] {
|
||||||
|
@ -357,47 +393,51 @@ type DumpOpts struct {
|
||||||
ShowNotOkParsers bool
|
ShowNotOkParsers bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts DumpOpts) {
|
func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) {
|
||||||
//note : we can use line -> time as the unique identifier (of acquisition)
|
//note : we can use line -> time as the unique identifier (of acquisition)
|
||||||
|
|
||||||
state := make(map[time.Time]map[string]map[string]ParserResult)
|
state := make(map[time.Time]map[string]map[string]ParserResult)
|
||||||
assoc := make(map[time.Time]string, 0)
|
assoc := make(map[time.Time]string, 0)
|
||||||
|
|
||||||
for stage, parsers := range parser_results {
|
for stage, parsers := range parserResults {
|
||||||
for parser, results := range parsers {
|
for parser, results := range parsers {
|
||||||
for _, parser_res := range results {
|
for _, parserRes := range results {
|
||||||
evt := parser_res.Evt
|
evt := parserRes.Evt
|
||||||
if _, ok := state[evt.Line.Time]; !ok {
|
if _, ok := state[evt.Line.Time]; !ok {
|
||||||
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
|
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
|
||||||
assoc[evt.Line.Time] = evt.Line.Raw
|
assoc[evt.Line.Time] = evt.Line.Raw
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := state[evt.Line.Time][stage]; !ok {
|
if _, ok := state[evt.Line.Time][stage]; !ok {
|
||||||
state[evt.Line.Time][stage] = make(map[string]ParserResult)
|
state[evt.Line.Time][stage] = make(map[string]ParserResult)
|
||||||
}
|
}
|
||||||
state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parser_res.Success}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for bname, evtlist := range bucket_pour {
|
for bname, evtlist := range bucketPour {
|
||||||
for _, evt := range evtlist {
|
for _, evt := range evtlist {
|
||||||
if evt.Line.Raw == "" {
|
if evt.Line.Raw == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
//it might be bucket overflow being reprocessed, skip this
|
//it might be bucket overflow being reprocessed, skip this
|
||||||
if _, ok := state[evt.Line.Time]; !ok {
|
if _, ok := state[evt.Line.Time]; !ok {
|
||||||
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
|
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
|
||||||
assoc[evt.Line.Time] = evt.Line.Raw
|
assoc[evt.Line.Time] = evt.Line.Raw
|
||||||
}
|
}
|
||||||
|
|
||||||
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
|
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
|
||||||
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
|
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
|
||||||
if _, ok := state[evt.Line.Time]["buckets"]; !ok {
|
if _, ok := state[evt.Line.Time]["buckets"]; !ok {
|
||||||
state[evt.Line.Time]["buckets"] = make(map[string]ParserResult)
|
state[evt.Line.Time]["buckets"] = make(map[string]ParserResult)
|
||||||
}
|
}
|
||||||
|
|
||||||
state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true}
|
state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
yellow := color.New(color.FgYellow).SprintFunc()
|
yellow := color.New(color.FgYellow).SprintFunc()
|
||||||
red := color.New(color.FgRed).SprintFunc()
|
red := color.New(color.FgRed).SprintFunc()
|
||||||
green := color.New(color.FgGreen).SprintFunc()
|
green := color.New(color.FgGreen).SprintFunc()
|
||||||
|
@ -409,19 +449,25 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("line: %s\n", rawstr)
|
fmt.Printf("line: %s\n", rawstr)
|
||||||
|
|
||||||
skeys := make([]string, 0, len(state[tstamp]))
|
skeys := make([]string, 0, len(state[tstamp]))
|
||||||
|
|
||||||
for k := range state[tstamp] {
|
for k := range state[tstamp] {
|
||||||
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
|
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
|
||||||
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
|
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
|
||||||
if k == "buckets" {
|
if k == "buckets" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
skeys = append(skeys, k)
|
skeys = append(skeys, k)
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Strings(skeys)
|
sort.Strings(skeys)
|
||||||
//iterate stage
|
|
||||||
var prev_item types.Event
|
// iterate stage
|
||||||
|
var prevItem types.Event
|
||||||
|
|
||||||
for _, stage := range skeys {
|
for _, stage := range skeys {
|
||||||
parsers := state[tstamp][stage]
|
parsers := state[tstamp][stage]
|
||||||
|
@ -431,18 +477,16 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
|
||||||
|
|
||||||
fmt.Printf("\t%s %s\n", sep, stage)
|
fmt.Printf("\t%s %s\n", sep, stage)
|
||||||
|
|
||||||
pkeys := make([]string, 0, len(parsers))
|
pkeys := sortedMapKeys(parsers)
|
||||||
for k := range parsers {
|
|
||||||
pkeys = append(pkeys, k)
|
|
||||||
}
|
|
||||||
sort.Strings(pkeys)
|
|
||||||
|
|
||||||
for idx, parser := range pkeys {
|
for idx, parser := range pkeys {
|
||||||
res := parsers[parser].Success
|
res := parsers[parser].Success
|
||||||
sep := "├"
|
sep := "├"
|
||||||
|
|
||||||
if idx == len(pkeys)-1 {
|
if idx == len(pkeys)-1 {
|
||||||
sep = "└"
|
sep = "└"
|
||||||
}
|
}
|
||||||
|
|
||||||
created := 0
|
created := 0
|
||||||
updated := 0
|
updated := 0
|
||||||
deleted := 0
|
deleted := 0
|
||||||
|
@ -451,16 +495,19 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
|
||||||
detailsDisplay := ""
|
detailsDisplay := ""
|
||||||
|
|
||||||
if res {
|
if res {
|
||||||
changelog, _ := diff.Diff(prev_item, parsers[parser].Evt)
|
changelog, _ := diff.Diff(prevItem, parsers[parser].Evt)
|
||||||
for _, change := range changelog {
|
for _, change := range changelog {
|
||||||
switch change.Type {
|
switch change.Type {
|
||||||
case "create":
|
case "create":
|
||||||
created++
|
created++
|
||||||
|
|
||||||
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To))
|
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To))
|
||||||
case "update":
|
case "update":
|
||||||
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To))
|
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To))
|
||||||
|
|
||||||
if change.Path[0] == "Whitelisted" && change.To == true {
|
if change.Path[0] == "Whitelisted" && change.To == true {
|
||||||
whitelisted = true
|
whitelisted = true
|
||||||
|
|
||||||
if whitelistReason == "" {
|
if whitelistReason == "" {
|
||||||
whitelistReason = parsers[parser].Evt.WhitelistReason
|
whitelistReason = parsers[parser].Evt.WhitelistReason
|
||||||
}
|
}
|
||||||
|
@ -468,51 +515,64 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
|
||||||
updated++
|
updated++
|
||||||
case "delete":
|
case "delete":
|
||||||
deleted++
|
deleted++
|
||||||
|
|
||||||
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, ".")))
|
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, ".")))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
prev_item = parsers[parser].Evt
|
|
||||||
|
prevItem = parsers[parser].Evt
|
||||||
}
|
}
|
||||||
|
|
||||||
if created > 0 {
|
if created > 0 {
|
||||||
changeStr += green(fmt.Sprintf("+%d", created))
|
changeStr += green(fmt.Sprintf("+%d", created))
|
||||||
}
|
}
|
||||||
|
|
||||||
if updated > 0 {
|
if updated > 0 {
|
||||||
if len(changeStr) > 0 {
|
if len(changeStr) > 0 {
|
||||||
changeStr += " "
|
changeStr += " "
|
||||||
}
|
}
|
||||||
|
|
||||||
changeStr += yellow(fmt.Sprintf("~%d", updated))
|
changeStr += yellow(fmt.Sprintf("~%d", updated))
|
||||||
}
|
}
|
||||||
|
|
||||||
if deleted > 0 {
|
if deleted > 0 {
|
||||||
if len(changeStr) > 0 {
|
if len(changeStr) > 0 {
|
||||||
changeStr += " "
|
changeStr += " "
|
||||||
}
|
}
|
||||||
|
|
||||||
changeStr += red(fmt.Sprintf("-%d", deleted))
|
changeStr += red(fmt.Sprintf("-%d", deleted))
|
||||||
}
|
}
|
||||||
|
|
||||||
if whitelisted {
|
if whitelisted {
|
||||||
if len(changeStr) > 0 {
|
if len(changeStr) > 0 {
|
||||||
changeStr += " "
|
changeStr += " "
|
||||||
}
|
}
|
||||||
|
|
||||||
changeStr += red("[whitelisted]")
|
changeStr += red("[whitelisted]")
|
||||||
}
|
}
|
||||||
|
|
||||||
if changeStr == "" {
|
if changeStr == "" {
|
||||||
changeStr = yellow("unchanged")
|
changeStr = yellow("unchanged")
|
||||||
}
|
}
|
||||||
|
|
||||||
if res {
|
if res {
|
||||||
fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr)
|
fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr)
|
||||||
|
|
||||||
if opts.Details {
|
if opts.Details {
|
||||||
fmt.Print(detailsDisplay)
|
fmt.Print(detailsDisplay)
|
||||||
}
|
}
|
||||||
} else if opts.ShowNotOkParsers {
|
} else if opts.ShowNotOkParsers {
|
||||||
fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)
|
fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sep := "└"
|
sep := "└"
|
||||||
|
|
||||||
if len(state[tstamp]["buckets"]) > 0 {
|
if len(state[tstamp]["buckets"]) > 0 {
|
||||||
sep = "├"
|
sep = "├"
|
||||||
}
|
}
|
||||||
|
|
||||||
//did the event enter the bucket pour phase ?
|
//did the event enter the bucket pour phase ?
|
||||||
if _, ok := state[tstamp]["buckets"]["OK"]; ok {
|
if _, ok := state[tstamp]["buckets"]["OK"]; ok {
|
||||||
fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle)
|
fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle)
|
||||||
|
@ -521,27 +581,35 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle)
|
fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle)
|
||||||
}
|
}
|
||||||
|
|
||||||
//now print bucket info
|
//now print bucket info
|
||||||
if len(state[tstamp]["buckets"]) > 0 {
|
if len(state[tstamp]["buckets"]) > 0 {
|
||||||
fmt.Printf("\t├ Scenarios\n")
|
fmt.Printf("\t├ Scenarios\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
bnames := make([]string, 0, len(state[tstamp]["buckets"]))
|
bnames := make([]string, 0, len(state[tstamp]["buckets"]))
|
||||||
|
|
||||||
for k := range state[tstamp]["buckets"] {
|
for k := range state[tstamp]["buckets"] {
|
||||||
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
|
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
|
||||||
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
|
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
|
||||||
if k == "OK" {
|
if k == "OK" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
bnames = append(bnames, k)
|
bnames = append(bnames, k)
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Strings(bnames)
|
sort.Strings(bnames)
|
||||||
|
|
||||||
for idx, bname := range bnames {
|
for idx, bname := range bnames {
|
||||||
sep := "├"
|
sep := "├"
|
||||||
if idx == len(bnames)-1 {
|
if idx == len(bnames)-1 {
|
||||||
sep = "└"
|
sep = "└"
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname)
|
fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
11
pkg/hubtest/regexp.go
Normal file
11
pkg/hubtest/regexp.go
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
package hubtest
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
variableRE = regexp.MustCompile(`(?P<variable>[^ =]+) == .*`)
|
||||||
|
parserResultRE = regexp.MustCompile(`^results\["[^"]+"\]\["(?P<parser>[^"]+)"\]\[[0-9]+\]\.Evt\..*`)
|
||||||
|
scenarioResultRE = regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P<scenario>[^"]+)"`)
|
||||||
|
)
|
|
@ -5,12 +5,10 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"regexp"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/antonmedv/expr"
|
"github.com/antonmedv/expr"
|
||||||
"github.com/antonmedv/expr/vm"
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"gopkg.in/yaml.v2"
|
"gopkg.in/yaml.v2"
|
||||||
|
|
||||||
|
@ -42,6 +40,7 @@ func NewScenarioAssert(file string) *ScenarioAssert {
|
||||||
TestData: &BucketResults{},
|
TestData: &BucketResults{},
|
||||||
PourData: &BucketPourInfo{},
|
PourData: &BucketPourInfo{},
|
||||||
}
|
}
|
||||||
|
|
||||||
return ScenarioAssert
|
return ScenarioAssert
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -50,7 +49,9 @@ func (s *ScenarioAssert) AutoGenFromFile(filename string) (string, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
ret := s.AutoGenScenarioAssert()
|
ret := s.AutoGenScenarioAssert()
|
||||||
|
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -59,6 +60,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err)
|
return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.TestData = bucketDump
|
s.TestData = bucketDump
|
||||||
|
|
||||||
if bucketpour != "" {
|
if bucketpour != "" {
|
||||||
|
@ -66,8 +68,10 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err)
|
return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.PourData = pourDump
|
s.PourData = pourDump
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -81,19 +85,26 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
|
||||||
if err := s.LoadTest(testFile, ""); err != nil {
|
if err := s.LoadTest(testFile, ""); err != nil {
|
||||||
return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
|
return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
scanner := bufio.NewScanner(file)
|
scanner := bufio.NewScanner(file)
|
||||||
scanner.Split(bufio.ScanLines)
|
scanner.Split(bufio.ScanLines)
|
||||||
|
|
||||||
nbLine := 0
|
nbLine := 0
|
||||||
|
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
nbLine += 1
|
nbLine++
|
||||||
|
|
||||||
if scanner.Text() == "" {
|
if scanner.Text() == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
ok, err := s.Run(scanner.Text())
|
ok, err := s.Run(scanner.Text())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
|
return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
|
||||||
}
|
}
|
||||||
s.NbAssert += 1
|
|
||||||
|
s.NbAssert++
|
||||||
|
|
||||||
if !ok {
|
if !ok {
|
||||||
log.Debugf("%s is FALSE", scanner.Text())
|
log.Debugf("%s is FALSE", scanner.Text())
|
||||||
failedAssert := &AssertFail{
|
failedAssert := &AssertFail{
|
||||||
|
@ -102,31 +113,38 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
|
||||||
Expression: scanner.Text(),
|
Expression: scanner.Text(),
|
||||||
Debug: make(map[string]string),
|
Debug: make(map[string]string),
|
||||||
}
|
}
|
||||||
variableRE := regexp.MustCompile(`(?P<variable>[^ ]+) == .*`)
|
|
||||||
match := variableRE.FindStringSubmatch(scanner.Text())
|
match := variableRE.FindStringSubmatch(scanner.Text())
|
||||||
|
|
||||||
if len(match) == 0 {
|
if len(match) == 0 {
|
||||||
log.Infof("Couldn't get variable of line '%s'", scanner.Text())
|
log.Infof("Couldn't get variable of line '%s'", scanner.Text())
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
variable := match[1]
|
variable := match[1]
|
||||||
|
|
||||||
result, err := s.EvalExpression(variable)
|
result, err := s.EvalExpression(variable)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("unable to evaluate variable '%s': %s", variable, err)
|
log.Errorf("unable to evaluate variable '%s': %s", variable, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
failedAssert.Debug[variable] = result
|
failedAssert.Debug[variable] = result
|
||||||
s.Fails = append(s.Fails, *failedAssert)
|
s.Fails = append(s.Fails, *failedAssert)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
|
//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
file.Close()
|
file.Close()
|
||||||
|
|
||||||
if s.NbAssert == 0 {
|
if s.NbAssert == 0 {
|
||||||
assertData, err := s.AutoGenFromFile(testFile)
|
assertData, err := s.AutoGenFromFile(testFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("couldn't generate assertion: %s", err)
|
return fmt.Errorf("couldn't generate assertion: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.AutoGenAssertData = assertData
|
s.AutoGenAssertData = assertData
|
||||||
s.AutoGenAssert = true
|
s.AutoGenAssert = true
|
||||||
}
|
}
|
||||||
|
@ -139,15 +157,14 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) {
|
func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) {
|
||||||
var err error
|
|
||||||
//debug doesn't make much sense with the ability to evaluate "on the fly"
|
//debug doesn't make much sense with the ability to evaluate "on the fly"
|
||||||
//var debugFilter *exprhelpers.ExprDebugger
|
//var debugFilter *exprhelpers.ExprDebugger
|
||||||
var runtimeFilter *vm.Program
|
|
||||||
var output interface{}
|
var output interface{}
|
||||||
|
|
||||||
env := map[string]interface{}{"results": *s.TestData}
|
env := map[string]interface{}{"results": *s.TestData}
|
||||||
|
|
||||||
if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil {
|
runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...)
|
||||||
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil {
|
// if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil {
|
||||||
|
@ -161,8 +178,10 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warningf("running : %s", expression)
|
log.Warningf("running : %s", expression)
|
||||||
log.Warningf("runtime error : %s", err)
|
log.Warningf("runtime error : %s", err)
|
||||||
|
|
||||||
return nil, fmt.Errorf("while running expression %s: %w", expression, err)
|
return nil, fmt.Errorf("while running expression %s: %w", expression, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return output, nil
|
return output, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -171,10 +190,12 @@ func (s *ScenarioAssert) EvalExpression(expression string) (string, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
ret, err := yaml.Marshal(output)
|
ret, err := yaml.Marshal(output)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
return string(ret), nil
|
return string(ret), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -183,6 +204,7 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
switch out := output.(type) {
|
switch out := output.(type) {
|
||||||
case bool:
|
case bool:
|
||||||
return out, nil
|
return out, nil
|
||||||
|
@ -192,9 +214,9 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ScenarioAssert) AutoGenScenarioAssert() string {
|
func (s *ScenarioAssert) AutoGenScenarioAssert() string {
|
||||||
//attempt to autogen parser asserts
|
// attempt to autogen scenario asserts
|
||||||
var ret string
|
ret := fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData))
|
||||||
ret += fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData))
|
|
||||||
for eventIndex, event := range *s.TestData {
|
for eventIndex, event := range *s.TestData {
|
||||||
for ipSrc, source := range event.Overflow.Sources {
|
for ipSrc, source := range event.Overflow.Sources {
|
||||||
ret += fmt.Sprintf(`"%s" in results[%d].Overflow.GetSources()`+"\n", ipSrc, eventIndex)
|
ret += fmt.Sprintf(`"%s" in results[%d].Overflow.GetSources()`+"\n", ipSrc, eventIndex)
|
||||||
|
@ -203,15 +225,18 @@ func (s *ScenarioAssert) AutoGenScenarioAssert() string {
|
||||||
ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetScope() == "%s"`+"\n", eventIndex, ipSrc, *source.Scope)
|
ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetScope() == "%s"`+"\n", eventIndex, ipSrc, *source.Scope)
|
||||||
ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetValue() == "%s"`+"\n", eventIndex, ipSrc, *source.Value)
|
ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetValue() == "%s"`+"\n", eventIndex, ipSrc, *source.Value)
|
||||||
}
|
}
|
||||||
|
|
||||||
for evtIndex, evt := range event.Overflow.Alert.Events {
|
for evtIndex, evt := range event.Overflow.Alert.Events {
|
||||||
for _, meta := range evt.Meta {
|
for _, meta := range evt.Meta {
|
||||||
ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value))
|
ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetScenario() == "%s"`+"\n", eventIndex, *event.Overflow.Alert.Scenario)
|
ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetScenario() == "%s"`+"\n", eventIndex, *event.Overflow.Alert.Scenario)
|
||||||
ret += fmt.Sprintf(`results[%d].Overflow.Alert.Remediation == %t`+"\n", eventIndex, event.Overflow.Alert.Remediation)
|
ret += fmt.Sprintf(`results[%d].Overflow.Alert.Remediation == %t`+"\n", eventIndex, event.Overflow.Alert.Remediation)
|
||||||
ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetEventsCount() == %d`+"\n", eventIndex, *event.Overflow.Alert.EventsCount)
|
ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetEventsCount() == %d`+"\n", eventIndex, *event.Overflow.Alert.EventsCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -228,8 +253,6 @@ func (b BucketResults) Swap(i, j int) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
|
func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
|
||||||
var bucketDump BucketPourInfo
|
|
||||||
|
|
||||||
dumpData, err := os.Open(filepath)
|
dumpData, err := os.Open(filepath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -241,6 +264,8 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var bucketDump BucketPourInfo
|
||||||
|
|
||||||
if err := yaml.Unmarshal(results, &bucketDump); err != nil {
|
if err := yaml.Unmarshal(results, &bucketDump); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -249,8 +274,6 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func LoadScenarioDump(filepath string) (*BucketResults, error) {
|
func LoadScenarioDump(filepath string) (*BucketResults, error) {
|
||||||
var bucketDump BucketResults
|
|
||||||
|
|
||||||
dumpData, err := os.Open(filepath)
|
dumpData, err := os.Open(filepath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -262,6 +285,8 @@ func LoadScenarioDump(filepath string) (*BucketResults, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var bucketDump BucketResults
|
||||||
|
|
||||||
if err := yaml.Unmarshal(results, &bucketDump); err != nil {
|
if err := yaml.Unmarshal(results, &bucketDump); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,7 +12,9 @@ func sortedMapKeys[V any](m map[string]V) []string {
|
||||||
for k := range m {
|
for k := range m {
|
||||||
keys = append(keys, k)
|
keys = append(keys, k)
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
|
|
||||||
return keys
|
return keys
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -22,7 +24,7 @@ func Copy(src string, dst string) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = os.WriteFile(dst, content, 0644)
|
err = os.WriteFile(dst, content, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -43,16 +45,20 @@ func checkPathNotContained(path string, subpath string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
current := absSubPath
|
current := absSubPath
|
||||||
|
|
||||||
for {
|
for {
|
||||||
if current == absPath {
|
if current == absPath {
|
||||||
return fmt.Errorf("cannot copy a folder onto itself")
|
return fmt.Errorf("cannot copy a folder onto itself")
|
||||||
}
|
}
|
||||||
|
|
||||||
up := filepath.Dir(current)
|
up := filepath.Dir(current)
|
||||||
if current == up {
|
if current == up {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
current = up
|
current = up
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,16 +3,16 @@ package hubtest
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCheckPathNotContained(t *testing.T) {
|
func TestCheckPathNotContained(t *testing.T) {
|
||||||
assert.Nil(t, checkPathNotContained("/foo", "/bar"))
|
require.NoError(t, checkPathNotContained("/foo", "/bar"))
|
||||||
assert.Nil(t, checkPathNotContained("/foo/bar", "/foo"))
|
require.NoError(t, checkPathNotContained("/foo/bar", "/foo"))
|
||||||
assert.Nil(t, checkPathNotContained("/foo/bar", "/"))
|
require.NoError(t, checkPathNotContained("/foo/bar", "/"))
|
||||||
assert.Nil(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else"))
|
require.NoError(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else"))
|
||||||
assert.Nil(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else"))
|
require.NoError(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else"))
|
||||||
assert.NotNil(t, checkPathNotContained("/foo", "/foo/bar"))
|
require.Error(t, checkPathNotContained("/foo", "/foo/bar"))
|
||||||
assert.NotNil(t, checkPathNotContained("/", "/foo"))
|
require.Error(t, checkPathNotContained("/", "/foo"))
|
||||||
assert.NotNil(t, checkPathNotContained("/", "/foo/bar/baz"))
|
require.Error(t, checkPathNotContained("/", "/foo/bar/baz"))
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,12 +8,14 @@ import (
|
||||||
"html/template"
|
"html/template"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/parser"
|
"github.com/crowdsecurity/crowdsec/pkg/parser"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
@ -33,28 +35,45 @@ func TestBucket(t *testing.T) {
|
||||||
envSetting = os.Getenv("TEST_ONLY")
|
envSetting = os.Getenv("TEST_ONLY")
|
||||||
tomb = &tomb.Tomb{}
|
tomb = &tomb.Tomb{}
|
||||||
)
|
)
|
||||||
err := exprhelpers.Init(nil)
|
|
||||||
|
testdata := "./tests"
|
||||||
|
|
||||||
|
hubCfg := &csconfig.LocalHubCfg{
|
||||||
|
HubDir: filepath.Join(testdata, "hub"),
|
||||||
|
HubIndexFile: filepath.Join(testdata, "hub", "index.json"),
|
||||||
|
InstallDataDir: testdata,
|
||||||
|
}
|
||||||
|
|
||||||
|
hub, err := cwhub.NewHub(hubCfg, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to init hub: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = exprhelpers.Init(nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("exprhelpers init failed: %s", err)
|
log.Fatalf("exprhelpers init failed: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if envSetting != "" {
|
if envSetting != "" {
|
||||||
if err := testOneBucket(t, envSetting, tomb); err != nil {
|
if err := testOneBucket(t, hub, envSetting, tomb); err != nil {
|
||||||
t.Fatalf("Test '%s' failed : %s", envSetting, err)
|
t.Fatalf("Test '%s' failed : %s", envSetting, err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
wg := new(sync.WaitGroup)
|
wg := new(sync.WaitGroup)
|
||||||
fds, err := os.ReadDir("./tests/")
|
fds, err := os.ReadDir(testdata)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Unable to read test directory : %s", err)
|
t.Fatalf("Unable to read test directory : %s", err)
|
||||||
}
|
}
|
||||||
for _, fd := range fds {
|
for _, fd := range fds {
|
||||||
fname := "./tests/" + fd.Name()
|
if fd.Name() == "hub" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fname := filepath.Join(testdata, fd.Name())
|
||||||
log.Infof("Running test on %s", fname)
|
log.Infof("Running test on %s", fname)
|
||||||
tomb.Go(func() error {
|
tomb.Go(func() error {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if err := testOneBucket(t, fname, tomb); err != nil {
|
if err := testOneBucket(t, hub, fname, tomb); err != nil {
|
||||||
t.Fatalf("Test '%s' failed : %s", fname, err)
|
t.Fatalf("Test '%s' failed : %s", fname, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
@ -76,7 +95,7 @@ func watchTomb(tomb *tomb.Tomb) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
|
func testOneBucket(t *testing.T, hub *cwhub.Hub, dir string, tomb *tomb.Tomb) error {
|
||||||
|
|
||||||
var (
|
var (
|
||||||
holders []BucketFactory
|
holders []BucketFactory
|
||||||
|
@ -112,10 +131,8 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
|
||||||
files = append(files, x.Filename)
|
files = append(files, x.Filename)
|
||||||
}
|
}
|
||||||
|
|
||||||
cscfg := &csconfig.CrowdsecServiceCfg{
|
cscfg := &csconfig.CrowdsecServiceCfg{}
|
||||||
DataDir: "tests",
|
holders, response, err := LoadBuckets(cscfg, hub, files, tomb, buckets, false)
|
||||||
}
|
|
||||||
holders, response, err := LoadBuckets(cscfg, files, tomb, buckets, false)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed loading bucket : %s", err)
|
t.Fatalf("failed loading bucket : %s", err)
|
||||||
}
|
}
|
||||||
|
@ -123,7 +140,7 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
|
||||||
watchTomb(tomb)
|
watchTomb(tomb)
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if !testFile(t, dir+"/test.json", dir+"/in-buckets_state.json", holders, response, buckets) {
|
if !testFile(t, filepath.Join(dir, "test.json"), filepath.Join(dir, "in-buckets_state.json"), holders, response, buckets) {
|
||||||
return fmt.Errorf("tests from %s failed", dir)
|
return fmt.Errorf("tests from %s failed", dir)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|
|
@ -179,7 +179,7 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
|
func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, hub *cwhub.Hub, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
|
||||||
var (
|
var (
|
||||||
ret = []BucketFactory{}
|
ret = []BucketFactory{}
|
||||||
response chan types.Event
|
response chan types.Event
|
||||||
|
@ -212,7 +212,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
|
||||||
log.Tracef("End of yaml file")
|
log.Tracef("End of yaml file")
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
bucketFactory.DataDir = cscfg.DataDir
|
bucketFactory.DataDir = hub.GetDataDir()
|
||||||
//check empty
|
//check empty
|
||||||
if bucketFactory.Name == "" {
|
if bucketFactory.Name == "" {
|
||||||
log.Errorf("Won't load nameless bucket")
|
log.Errorf("Won't load nameless bucket")
|
||||||
|
@ -235,7 +235,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
|
||||||
bucketFactory.Filename = filepath.Clean(f)
|
bucketFactory.Filename = filepath.Clean(f)
|
||||||
bucketFactory.BucketName = seed.Generate()
|
bucketFactory.BucketName = seed.Generate()
|
||||||
bucketFactory.ret = response
|
bucketFactory.ret = response
|
||||||
hubItem, err := cwhub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename)
|
hubItem, err := hub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename)
|
log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename)
|
||||||
} else {
|
} else {
|
||||||
|
|
1
pkg/leakybucket/tests/hub/index.json
Normal file
1
pkg/leakybucket/tests/hub/index.json
Normal file
|
@ -0,0 +1 @@
|
||||||
|
{}
|
|
@ -57,15 +57,16 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) {
|
||||||
|
|
||||||
// Return new parsers
|
// Return new parsers
|
||||||
// nodes and povfwnodes are already initialized in parser.LoadStages
|
// nodes and povfwnodes are already initialized in parser.LoadStages
|
||||||
func NewParsers() *Parsers {
|
func NewParsers(hub *cwhub.Hub) *Parsers {
|
||||||
parsers := &Parsers{
|
parsers := &Parsers{
|
||||||
Ctx: &UnixParserCtx{},
|
Ctx: &UnixParserCtx{},
|
||||||
Povfwctx: &UnixParserCtx{},
|
Povfwctx: &UnixParserCtx{},
|
||||||
StageFiles: make([]Stagefile, 0),
|
StageFiles: make([]Stagefile, 0),
|
||||||
PovfwStageFiles: make([]Stagefile, 0),
|
PovfwStageFiles: make([]Stagefile, 0),
|
||||||
}
|
}
|
||||||
for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
|
|
||||||
for _, hubParserItem := range cwhub.GetItemMap(itemType) {
|
for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} {
|
||||||
|
for _, hubParserItem := range hub.GetItemMap(itemType) {
|
||||||
if hubParserItem.Installed {
|
if hubParserItem.Installed {
|
||||||
stagefile := Stagefile{
|
stagefile := Stagefile{
|
||||||
Filename: hubParserItem.LocalPath,
|
Filename: hubParserItem.LocalPath,
|
||||||
|
@ -74,7 +75,7 @@ func NewParsers() *Parsers {
|
||||||
if itemType == cwhub.PARSERS {
|
if itemType == cwhub.PARSERS {
|
||||||
parsers.StageFiles = append(parsers.StageFiles, stagefile)
|
parsers.StageFiles = append(parsers.StageFiles, stagefile)
|
||||||
}
|
}
|
||||||
if itemType == cwhub.PARSERS_OVFLW {
|
if itemType == cwhub.POSTOVERFLOWS {
|
||||||
parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
|
parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -97,16 +98,16 @@ func NewParsers() *Parsers {
|
||||||
func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
|
func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
patternsDir := filepath.Join(cConfig.Crowdsec.ConfigDir, "patterns/")
|
patternsDir := filepath.Join(cConfig.ConfigPaths.ConfigDir, "patterns/")
|
||||||
log.Infof("Loading grok library %s", patternsDir)
|
log.Infof("Loading grok library %s", patternsDir)
|
||||||
/* load base regexps for two grok parsers */
|
/* load base regexps for two grok parsers */
|
||||||
parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
|
parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
|
||||||
"data": cConfig.Crowdsec.DataDir})
|
"data": cConfig.ConfigPaths.DataDir})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
|
return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
|
||||||
}
|
}
|
||||||
parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
|
parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
|
||||||
"data": cConfig.Crowdsec.DataDir})
|
"data": cConfig.ConfigPaths.DataDir})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
|
return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
|
||||||
}
|
}
|
||||||
|
@ -116,7 +117,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
|
||||||
*/
|
*/
|
||||||
log.Infof("Loading enrich plugins")
|
log.Infof("Loading enrich plugins")
|
||||||
|
|
||||||
parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir)
|
parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
|
return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,7 +10,6 @@ import (
|
||||||
goccyyaml "github.com/goccy/go-yaml"
|
goccyyaml "github.com/goccy/go-yaml"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -46,22 +45,12 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// InstallHubItems installs the objects recommended in a setup file.
|
// InstallHubItems installs the objects recommended in a setup file.
|
||||||
func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error {
|
func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
|
||||||
setupEnvelope, err := decodeSetup(input, false)
|
setupEnvelope, err := decodeSetup(input, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := csConfig.LoadHub(); err != nil {
|
|
||||||
return fmt.Errorf("loading hub: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cwhub.SetHubBranch()
|
|
||||||
|
|
||||||
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
|
|
||||||
return fmt.Errorf("getting hub index: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, setupItem := range setupEnvelope.Setup {
|
for _, setupItem := range setupEnvelope.Setup {
|
||||||
forceAction := false
|
forceAction := false
|
||||||
downloadOnly := false
|
downloadOnly := false
|
||||||
|
@ -73,14 +62,19 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
|
||||||
|
|
||||||
if len(install.Collections) > 0 {
|
if len(install.Collections) > 0 {
|
||||||
for _, collection := range setupItem.Install.Collections {
|
for _, collection := range setupItem.Install.Collections {
|
||||||
|
item := hub.GetItem(cwhub.COLLECTIONS, collection)
|
||||||
|
if item == nil {
|
||||||
|
return fmt.Errorf("collection %s not found", collection)
|
||||||
|
}
|
||||||
|
|
||||||
if dryRun {
|
if dryRun {
|
||||||
fmt.Println("dry-run: would install collection", collection)
|
fmt.Println("dry-run: would install collection", collection)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cwhub.InstallItem(csConfig, collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
|
if err := item.Install(forceAction, downloadOnly); err != nil {
|
||||||
return fmt.Errorf("while installing collection %s: %w", collection, err)
|
return fmt.Errorf("while installing collection %s: %w", item.Name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -93,8 +87,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cwhub.InstallItem(csConfig, parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
|
item := hub.GetItem(cwhub.PARSERS, parser)
|
||||||
return fmt.Errorf("while installing parser %s: %w", parser, err)
|
if item == nil {
|
||||||
|
return fmt.Errorf("parser %s not found", parser)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := item.Install(forceAction, downloadOnly); err != nil {
|
||||||
|
return fmt.Errorf("while installing parser %s: %w", item.Name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -107,8 +106,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cwhub.InstallItem(csConfig, scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
|
item := hub.GetItem(cwhub.SCENARIOS, scenario)
|
||||||
return fmt.Errorf("while installing scenario %s: %w", scenario, err)
|
if item == nil {
|
||||||
|
return fmt.Errorf("scenario %s not found", scenario)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := item.Install(forceAction, downloadOnly); err != nil {
|
||||||
|
return fmt.Errorf("while installing scenario %s: %w", item.Name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -121,8 +125,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
|
item := hub.GetItem(cwhub.POSTOVERFLOWS, postoverflow)
|
||||||
return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err)
|
if item == nil {
|
||||||
|
return fmt.Errorf("postoverflow %s not found", postoverflow)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := item.Install(forceAction, downloadOnly); err != nil {
|
||||||
|
return fmt.Errorf("while installing postoverflow %s: %w", item.Name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
71
test/bats/00_wait_for.bats
Normal file
71
test/bats/00_wait_for.bats
Normal file
|
@ -0,0 +1,71 @@
|
||||||
|
#!/usr/bin/env bats
|
||||||
|
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
|
||||||
|
|
||||||
|
set -u
|
||||||
|
|
||||||
|
setup_file() {
|
||||||
|
load "../lib/setup_file.sh"
|
||||||
|
}
|
||||||
|
|
||||||
|
setup() {
|
||||||
|
load "../lib/setup.sh"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "run a command and capture its stdout" {
|
||||||
|
run -0 wait-for seq 1 3
|
||||||
|
assert_output - <<-EOT
|
||||||
|
1
|
||||||
|
2
|
||||||
|
3
|
||||||
|
EOT
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "run a command and capture its stderr" {
|
||||||
|
rune -0 wait-for sh -c 'seq 1 3 >&2'
|
||||||
|
assert_stderr - <<-EOT
|
||||||
|
1
|
||||||
|
2
|
||||||
|
3
|
||||||
|
EOT
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "run a command until a pattern is found in stdout" {
|
||||||
|
run -0 wait-for --out "1[12]0" seq 1 200
|
||||||
|
assert_line --index 0 "1"
|
||||||
|
assert_line --index -1 "110"
|
||||||
|
refute_line "111"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "run a command until a pattern is found in stderr" {
|
||||||
|
rune -0 wait-for --err "10" sh -c 'seq 1 20 >&2'
|
||||||
|
assert_stderr - <<-EOT
|
||||||
|
1
|
||||||
|
2
|
||||||
|
3
|
||||||
|
4
|
||||||
|
5
|
||||||
|
6
|
||||||
|
7
|
||||||
|
8
|
||||||
|
9
|
||||||
|
10
|
||||||
|
EOT
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "run a command with timeout (no match)" {
|
||||||
|
# when the process is terminated without a match, it returns
|
||||||
|
# 256 - 15 (SIGTERM) = 241
|
||||||
|
rune -241 wait-for --timeout 0.1 --out "10" sh -c 'echo 1; sleep 3; echo 2'
|
||||||
|
assert_line 1
|
||||||
|
# there may be more, but we don't care
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "run a command with timeout (match)" {
|
||||||
|
# when the process is terminated with a match, return code is 128
|
||||||
|
rune -128 wait-for --timeout .4 --out "2" sh -c 'echo 1; sleep .1; echo 2; echo 3; echo 4; sleep 10'
|
||||||
|
assert_output - <<-EOT
|
||||||
|
1
|
||||||
|
2
|
||||||
|
EOT
|
||||||
|
}
|
||||||
|
|
|
@ -24,28 +24,22 @@ teardown() {
|
||||||
#----------
|
#----------
|
||||||
|
|
||||||
@test "crowdsec (usage)" {
|
@test "crowdsec (usage)" {
|
||||||
rune -0 timeout 2s "${CROWDSEC}" -h
|
rune -0 wait-for --out "Usage of " "${CROWDSEC}" -h
|
||||||
assert_stderr_line --regexp "Usage of .*:"
|
rune -0 wait-for --out "Usage of " "${CROWDSEC}" --help
|
||||||
|
|
||||||
rune -0 timeout 2s "${CROWDSEC}" --help
|
|
||||||
assert_stderr_line --regexp "Usage of .*:"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "crowdsec (unknown flag)" {
|
@test "crowdsec (unknown flag)" {
|
||||||
rune -2 timeout 2s "${CROWDSEC}" --foobar
|
rune -0 wait-for --err "flag provided but not defined: -foobar" "$CROWDSEC" --foobar
|
||||||
assert_stderr_line "flag provided but not defined: -foobar"
|
|
||||||
assert_stderr_line --regexp "Usage of .*"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "crowdsec (unknown argument)" {
|
@test "crowdsec (unknown argument)" {
|
||||||
rune -2 timeout 2s "${CROWDSEC}" trololo
|
rune -0 wait-for --err "argument provided but not defined: trololo" "${CROWDSEC}" trololo
|
||||||
assert_stderr_line "argument provided but not defined: trololo"
|
|
||||||
assert_stderr_line --regexp "Usage of .*"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "crowdsec (no api and no agent)" {
|
@test "crowdsec (no api and no agent)" {
|
||||||
rune -1 timeout 2s "${CROWDSEC}" -no-api -no-cs
|
rune -0 wait-for \
|
||||||
assert_stderr_line --partial "You must run at least the API Server or crowdsec"
|
--err "You must run at least the API Server or crowdsec" \
|
||||||
|
"${CROWDSEC}" -no-api -no-cs
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "crowdsec - print error on exit" {
|
@test "crowdsec - print error on exit" {
|
||||||
|
@ -55,20 +49,22 @@ teardown() {
|
||||||
assert_stderr --partial "unable to create database client: unknown database type 'meh'"
|
assert_stderr --partial "unable to create database client: unknown database type 'meh'"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "crowdsec - bad configuration (empty/missing common section)" {
|
@test "crowdsec - default logging configuration (empty/missing common section)" {
|
||||||
config_set '.common={}'
|
config_set '.common={}'
|
||||||
rune -1 "${CROWDSEC}"
|
rune -0 wait-for \
|
||||||
|
--err "Starting processing data" \
|
||||||
|
"${CROWDSEC}"
|
||||||
refute_output
|
refute_output
|
||||||
assert_stderr --partial "unable to load configuration: common section is empty"
|
|
||||||
|
|
||||||
config_set 'del(.common)'
|
config_set 'del(.common)'
|
||||||
rune -1 "${CROWDSEC}"
|
rune -0 wait-for \
|
||||||
|
--err "Starting processing data" \
|
||||||
|
"${CROWDSEC}"
|
||||||
refute_output
|
refute_output
|
||||||
assert_stderr --partial "unable to load configuration: common section is empty"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "CS_LAPI_SECRET not strong enough" {
|
@test "CS_LAPI_SECRET not strong enough" {
|
||||||
CS_LAPI_SECRET=foo rune -1 timeout 2s "${CROWDSEC}"
|
CS_LAPI_SECRET=foo rune -1 wait-for "${CROWDSEC}"
|
||||||
assert_stderr --partial "api server init: unable to run local API: controller init: CS_LAPI_SECRET not strong enough"
|
assert_stderr --partial "api server init: unable to run local API: controller init: CS_LAPI_SECRET not strong enough"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -138,8 +134,8 @@ teardown() {
|
||||||
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
||||||
rm -f "$ACQUIS_YAML"
|
rm -f "$ACQUIS_YAML"
|
||||||
|
|
||||||
rune -1 timeout 2s "${CROWDSEC}"
|
rune -1 wait-for "${CROWDSEC}"
|
||||||
assert_stderr_line --partial "acquis.yaml: no such file or directory"
|
assert_stderr --partial "acquis.yaml: no such file or directory"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "crowdsec (error if acquisition_path is not defined and acquisition_dir is empty)" {
|
@test "crowdsec (error if acquisition_path is not defined and acquisition_dir is empty)" {
|
||||||
|
@ -151,7 +147,7 @@ teardown() {
|
||||||
rm -f "$ACQUIS_DIR"
|
rm -f "$ACQUIS_DIR"
|
||||||
|
|
||||||
config_set '.common.log_media="stdout"'
|
config_set '.common.log_media="stdout"'
|
||||||
rune -1 timeout 2s "${CROWDSEC}"
|
rune -1 wait-for "${CROWDSEC}"
|
||||||
# check warning
|
# check warning
|
||||||
assert_stderr --partial "no acquisition file found"
|
assert_stderr --partial "no acquisition file found"
|
||||||
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
|
@ -167,13 +163,15 @@ teardown() {
|
||||||
config_set '.crowdsec_service.acquisition_dir=""'
|
config_set '.crowdsec_service.acquisition_dir=""'
|
||||||
|
|
||||||
config_set '.common.log_media="stdout"'
|
config_set '.common.log_media="stdout"'
|
||||||
rune -1 timeout 2s "${CROWDSEC}"
|
rune -1 wait-for "${CROWDSEC}"
|
||||||
# check warning
|
# check warning
|
||||||
assert_stderr --partial "no acquisition_path or acquisition_dir specified"
|
assert_stderr --partial "no acquisition_path or acquisition_dir specified"
|
||||||
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
|
@test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
|
||||||
|
config_set '.common.log_media="stdout"'
|
||||||
|
|
||||||
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
||||||
config_set '.crowdsec_service.acquisition_path=""'
|
config_set '.crowdsec_service.acquisition_path=""'
|
||||||
|
|
||||||
|
@ -181,13 +179,15 @@ teardown() {
|
||||||
mkdir -p "$ACQUIS_DIR"
|
mkdir -p "$ACQUIS_DIR"
|
||||||
mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
|
mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
|
||||||
|
|
||||||
rune -124 timeout 2s "${CROWDSEC}"
|
rune -0 wait-for \
|
||||||
|
--err "Starting processing data" \
|
||||||
|
"${CROWDSEC}"
|
||||||
|
|
||||||
# now, if foo.yaml is empty instead, there won't be valid datasources.
|
# now, if foo.yaml is empty instead, there won't be valid datasources.
|
||||||
|
|
||||||
cat /dev/null >"$ACQUIS_DIR"/foo.yaml
|
cat /dev/null >"$ACQUIS_DIR"/foo.yaml
|
||||||
|
|
||||||
rune -1 timeout 2s "${CROWDSEC}"
|
rune -1 wait-for "${CROWDSEC}"
|
||||||
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -212,9 +212,10 @@ teardown() {
|
||||||
type: syslog
|
type: syslog
|
||||||
EOT
|
EOT
|
||||||
|
|
||||||
rune -124 timeout 2s env PATH='' "${CROWDSEC}"
|
|
||||||
#shellcheck disable=SC2016
|
#shellcheck disable=SC2016
|
||||||
assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
|
rune -0 wait-for \
|
||||||
|
--err 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in ' \
|
||||||
|
env PATH='' "${CROWDSEC}"
|
||||||
|
|
||||||
# if all datasources are disabled, crowdsec should exit
|
# if all datasources are disabled, crowdsec should exit
|
||||||
|
|
||||||
|
@ -222,7 +223,7 @@ teardown() {
|
||||||
rm -f "$ACQUIS_YAML"
|
rm -f "$ACQUIS_YAML"
|
||||||
config_set '.crowdsec_service.acquisition_path=""'
|
config_set '.crowdsec_service.acquisition_path=""'
|
||||||
|
|
||||||
rune -1 timeout 2s env PATH='' "${CROWDSEC}"
|
rune -1 wait-for env PATH='' "${CROWDSEC}"
|
||||||
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue