cwhub: context type (#2631)

* add hub type "context"
* cscli lapi: log.Fatal -> fmt.Errorf; lint
* tests for context.yaml
* load console context from hub
* original & compiled context
* deprecate "cscli lapi context delete"
$ cscli lapi context delete
Command "delete" is deprecated, please manually edit the context file.
* cscli completion: add appsec-rules, appsec-configs, explain, hubtest
This commit is contained in:
mmetc 2023-12-07 16:20:13 +01:00 committed by GitHub
parent 3e86f52250
commit 4acb4f8df3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 427 additions and 151 deletions

View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewContextCLI() *itemCLI {
return &itemCLI{
name: cwhub.CONTEXTS,
singular: "context",
oneOrMore: "context(s)",
help: cliHelp{
example: `cscli contexts list -a
cscli contexts install crowdsecurity/yyy crowdsecurity/zzz
cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz
cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz
cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz
`,
},
installHelp: cliHelp{
example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`,
},
removeHelp: cliHelp{
example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`,
},
upgradeHelp: cliHelp{
example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`,
},
inspectHelp: cliHelp{
example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`,
},
listHelp: cliHelp{
example: `cscli contexts list
cscli contexts list -a
cscli contexts list crowdsecurity/yyy crowdsecurity/zzz
List only enabled contexts unless "-a" or names are specified.`,
},
}
}

View file

@ -2,10 +2,10 @@ package main
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"net/url" "net/url"
"os" "os"
"slices"
"sort" "sort"
"strings" "strings"
@ -13,6 +13,7 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"slices"
"github.com/crowdsecurity/go-cs-lib/version" "github.com/crowdsecurity/go-cs-lib/version"
@ -26,26 +27,24 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/parser"
) )
var LAPIURLPrefix string = "v1" const LAPIURLPrefix = "v1"
func runLapiStatus(cmd *cobra.Command, args []string) error { func runLapiStatus(cmd *cobra.Command, args []string) error {
var err error
password := strfmt.Password(csConfig.API.Client.Credentials.Password) password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL) apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
login := csConfig.API.Client.Credentials.Login login := csConfig.API.Client.Credentials.Login
if err != nil { if err != nil {
log.Fatalf("parsing api url ('%s'): %s", apiurl, err) return fmt.Errorf("parsing api url: %w", err)
} }
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil)
if err != nil { if err != nil {
log.Fatal(err) return err
} }
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil { if err != nil {
log.Fatalf("failed to get scenarios : %s", err) return fmt.Errorf("failed to get scenarios: %w", err)
} }
Client, err = apiclient.NewDefaultClient(apiurl, Client, err = apiclient.NewDefaultClient(apiurl,
@ -53,28 +52,27 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
fmt.Sprintf("crowdsec/%s", version.String()), fmt.Sprintf("crowdsec/%s", version.String()),
nil) nil)
if err != nil { if err != nil {
log.Fatalf("init default client: %s", err) return fmt.Errorf("init default client: %w", err)
} }
t := models.WatcherAuthRequest{ t := models.WatcherAuthRequest{
MachineID: &login, MachineID: &login,
Password: &password, Password: &password,
Scenarios: scenarios, Scenarios: scenarios,
} }
log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath) log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", login, apiurl) log.Infof("Trying to authenticate with username %s on %s", login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t) _, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil { if err != nil {
log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err) return fmt.Errorf("failed to authenticate to Local API (LAPI): %w", err)
} else {
log.Infof("You can successfully interact with Local API (LAPI)")
} }
log.Infof("You can successfully interact with Local API (LAPI)")
return nil return nil
} }
func runLapiRegister(cmd *cobra.Command, args []string) error { func runLapiRegister(cmd *cobra.Command, args []string) error {
var err error
flags := cmd.Flags() flags := cmd.Flags()
apiURL, err := flags.GetString("url") apiURL, err := flags.GetString("url")
@ -95,16 +93,15 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
if lapiUser == "" { if lapiUser == "" {
lapiUser, err = generateID("") lapiUser, err = generateID("")
if err != nil { if err != nil {
log.Fatalf("unable to generate machine id: %s", err) return fmt.Errorf("unable to generate machine id: %w", err)
} }
} }
password := strfmt.Password(generatePassword(passwordLength)) password := strfmt.Password(generatePassword(passwordLength))
if apiURL == "" { if apiURL == "" {
if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" { if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil || csConfig.API.Client.Credentials.URL == "" {
apiURL = csConfig.API.Client.Credentials.URL return fmt.Errorf("no Local API URL. Please provide it in your configuration or with the -u parameter")
} else {
log.Fatalf("No Local API URL. Please provide it in your configuration or with the -u parameter")
} }
apiURL = csConfig.API.Client.Credentials.URL
} }
/*URL needs to end with /, but user doesn't care*/ /*URL needs to end with /, but user doesn't care*/
if !strings.HasSuffix(apiURL, "/") { if !strings.HasSuffix(apiURL, "/") {
@ -116,7 +113,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
} }
apiurl, err := url.Parse(apiURL) apiurl, err := url.Parse(apiURL)
if err != nil { if err != nil {
log.Fatalf("parsing api url: %s", err) return fmt.Errorf("parsing api url: %w", err)
} }
_, err = apiclient.RegisterClient(&apiclient.Config{ _, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: lapiUser, MachineID: lapiUser,
@ -127,7 +124,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
}, nil) }, nil)
if err != nil { if err != nil {
log.Fatalf("api client register: %s", err) return fmt.Errorf("api client register: %w", err)
} }
log.Printf("Successfully registered to Local API (LAPI)") log.Printf("Successfully registered to Local API (LAPI)")
@ -147,12 +144,12 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
} }
apiConfigDump, err := yaml.Marshal(apiCfg) apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil { if err != nil {
log.Fatalf("unable to marshal api credentials: %s", err) return fmt.Errorf("unable to marshal api credentials: %w", err)
} }
if dumpFile != "" { if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0o600) err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
if err != nil { if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err) return fmt.Errorf("write api credentials to '%s' failed: %w", dumpFile, err)
} }
log.Printf("Local API credentials written to '%s'", dumpFile) log.Printf("Local API credentials written to '%s'", dumpFile)
} else { } else {
@ -195,7 +192,7 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
} }
func NewLapiCmd() *cobra.Command { func NewLapiCmd() *cobra.Command {
var cmdLapi = &cobra.Command{ cmdLapi := &cobra.Command{
Use: "lapi [action]", Use: "lapi [action]",
Short: "Manage interaction with Local API (LAPI)", Short: "Manage interaction with Local API (LAPI)",
Args: cobra.MinimumNArgs(1), Args: cobra.MinimumNArgs(1),
@ -221,6 +218,7 @@ func AddContext(key string, values []string) error {
} }
if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok { if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok {
csConfig.Crowdsec.ContextToSend[key] = make([]string, 0) csConfig.Crowdsec.ContextToSend[key] = make([]string, 0)
log.Infof("key '%s' added", key) log.Infof("key '%s' added", key)
} }
data := csConfig.Crowdsec.ContextToSend[key] data := csConfig.Crowdsec.ContextToSend[key]
@ -247,11 +245,11 @@ func NewLapiContextCmd() *cobra.Command {
if err := csConfig.LoadCrowdsec(); err != nil { if err := csConfig.LoadCrowdsec(); err != nil {
fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath) fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
if err.Error() != fileNotFoundMessage { if err.Error() != fileNotFoundMessage {
log.Fatalf("Unable to load CrowdSec Agent: %s", err) return fmt.Errorf("unable to start CrowdSec agent: %w", err)
} }
} }
if csConfig.DisableAgent { if csConfig.DisableAgent {
log.Fatalf("Agent is disabled and lapi context can only be used on the agent") return errors.New("agent is disabled and lapi context can only be used on the agent")
} }
return nil return nil
@ -271,12 +269,21 @@ cscli lapi context add --key file_source --value evt.Line.Src
cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) { RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if keyToAdd != "" { if keyToAdd != "" {
if err := AddContext(keyToAdd, valuesToAdd); err != nil { if err := AddContext(keyToAdd, valuesToAdd); err != nil {
log.Fatalf(err.Error()) return err
} }
return return nil
} }
for _, v := range valuesToAdd { for _, v := range valuesToAdd {
@ -284,9 +291,11 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
key := keySlice[len(keySlice)-1] key := keySlice[len(keySlice)-1]
value := []string{v} value := []string{v}
if err := AddContext(key, value); err != nil { if err := AddContext(key, value); err != nil {
log.Fatalf(err.Error()) return err
} }
} }
return nil
}, },
} }
cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send") cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
@ -298,19 +307,29 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
Use: "status", Use: "status",
Short: "List context to send with alerts", Short: "List context to send with alerts",
DisableAutoGenTag: true, DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) { RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if len(csConfig.Crowdsec.ContextToSend) == 0 { if len(csConfig.Crowdsec.ContextToSend) == 0 {
fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.") fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
return return nil
} }
dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend) dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
if err != nil { if err != nil {
log.Fatalf("unable to show context status: %s", err) return fmt.Errorf("unable to show context status: %w", err)
} }
fmt.Println(string(dump)) fmt.Print(string(dump))
return nil
}, },
} }
cmdContext.AddCommand(cmdContextStatus) cmdContext.AddCommand(cmdContextStatus)
@ -323,9 +342,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
cscli lapi context detect crowdsecurity/sshd-logs cscli lapi context detect crowdsecurity/sshd-logs
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) { RunE: func(cmd *cobra.Command, args []string) error {
var err error
if !detectAll && len(args) == 0 { if !detectAll && len(args) == 0 {
log.Infof("Please provide parsers to detect or --all flag.") log.Infof("Please provide parsers to detect or --all flag.")
printHelp(cmd) printHelp(cmd)
@ -334,19 +351,18 @@ cscli lapi context detect crowdsecurity/sshd-logs
// to avoid all the log.Info from the loaders functions // to avoid all the log.Info from the loaders functions
log.SetLevel(log.WarnLevel) log.SetLevel(log.WarnLevel)
err = exprhelpers.Init(nil) if err := exprhelpers.Init(nil); err != nil {
if err != nil { return fmt.Errorf("failed to init expr helpers: %w", err)
log.Fatalf("Failed to init expr helpers : %s", err)
} }
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil)
if err != nil { if err != nil {
log.Fatal(err) return err
} }
csParsers := parser.NewParsers(hub) csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil { if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
log.Fatalf("unable to load parsers: %s", err) return fmt.Errorf("unable to load parsers: %w", err)
} }
fieldByParsers := make(map[string][]string) fieldByParsers := make(map[string][]string)
@ -366,7 +382,6 @@ cscli lapi context detect crowdsecurity/sshd-logs
fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field) fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
} }
} }
} }
fmt.Printf("Acquisition :\n\n") fmt.Printf("Acquisition :\n\n")
@ -399,59 +414,17 @@ cscli lapi context detect crowdsecurity/sshd-logs
log.Errorf("parser '%s' not found, can't detect fields", parserNotFound) log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
} }
} }
return nil
}, },
} }
cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser") cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
cmdContext.AddCommand(cmdContextDetect) cmdContext.AddCommand(cmdContextDetect)
var keysToDelete []string
var valuesToDelete []string
cmdContextDelete := &cobra.Command{ cmdContextDelete := &cobra.Command{
Use: "delete", Use: "delete",
Short: "Delete context to send with alerts", Deprecated: "please manually edit the context file.",
Example: `cscli lapi context delete --key source_ip
cscli lapi context delete --value evt.Line.Src
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
log.Fatalf("please provide at least a key or a value to delete")
} }
for _, key := range keysToDelete {
if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
delete(csConfig.Crowdsec.ContextToSend, key)
log.Infof("key '%s' has been removed", key)
} else {
log.Warningf("key '%s' doesn't exist", key)
}
}
for _, value := range valuesToDelete {
valueFound := false
for key, context := range csConfig.Crowdsec.ContextToSend {
if slices.Contains(context, value) {
valueFound = true
csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
log.Infof("value '%s' has been removed from key '%s'", value, key)
}
if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
delete(csConfig.Crowdsec.ContextToSend, key)
}
}
if !valueFound {
log.Warningf("value '%s' not found", value)
}
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
}
},
}
cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
cmdContext.AddCommand(cmdContextDelete) cmdContext.AddCommand(cmdContextDelete)
return cmdContext return cmdContext
@ -459,6 +432,7 @@ cscli lapi context delete --value evt.Line.Src
func detectStaticField(GrokStatics []parser.ExtraField) []string { func detectStaticField(GrokStatics []parser.ExtraField) []string {
ret := make([]string, 0) ret := make([]string, 0)
for _, static := range GrokStatics { for _, static := range GrokStatics {
if static.Parsed != "" { if static.Parsed != "" {
fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed) fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
@ -487,7 +461,8 @@ func detectStaticField(GrokStatics []parser.ExtraField) []string {
} }
func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string { func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
var ret = make([]string, 0) ret := make([]string, 0)
if node.Grok.RunTimeRegexp != nil { if node.Grok.RunTimeRegexp != nil {
for _, capturedField := range node.Grok.RunTimeRegexp.Names() { for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)

View file

@ -93,9 +93,10 @@ func initConfig() {
} }
var validArgs = []string{ var validArgs = []string{
"scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines", "alerts", "appsec-configs", "appsec-rules", "bouncers", "capi", "collections",
"metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard", "completion", "config", "console", "contexts", "dashboard", "decisions", "explain",
"config", "completion", "version", "console", "notifications", "support", "hub", "hubtest", "lapi", "machines", "metrics", "notifications", "parsers",
"postoverflows", "scenarios", "simulation", "support", "version",
} }
func prepender(filename string) string { func prepender(filename string) string {
@ -246,6 +247,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.AddCommand(NewParserCLI().NewCommand()) rootCmd.AddCommand(NewParserCLI().NewCommand())
rootCmd.AddCommand(NewScenarioCLI().NewCommand()) rootCmd.AddCommand(NewScenarioCLI().NewCommand())
rootCmd.AddCommand(NewPostOverflowCLI().NewCommand()) rootCmd.AddCommand(NewPostOverflowCLI().NewCommand())
rootCmd.AddCommand(NewContextCLI().NewCommand())
rootCmd.AddCommand(NewAppsecConfigCLI().NewCommand()) rootCmd.AddCommand(NewAppsecConfigCLI().NewCommand())
rootCmd.AddCommand(NewAppsecRuleCLI().NewCommand()) rootCmd.AddCommand(NewAppsecRuleCLI().NewCommand())

View file

@ -37,6 +37,7 @@ const (
SUPPORT_OS_INFO_PATH = "osinfo.txt" SUPPORT_OS_INFO_PATH = "osinfo.txt"
SUPPORT_PARSERS_PATH = "hub/parsers.txt" SUPPORT_PARSERS_PATH = "hub/parsers.txt"
SUPPORT_SCENARIOS_PATH = "hub/scenarios.txt" SUPPORT_SCENARIOS_PATH = "hub/scenarios.txt"
SUPPORT_CONTEXTS_PATH = "hub/scenarios.txt"
SUPPORT_COLLECTIONS_PATH = "hub/collections.txt" SUPPORT_COLLECTIONS_PATH = "hub/collections.txt"
SUPPORT_POSTOVERFLOWS_PATH = "hub/postoverflows.txt" SUPPORT_POSTOVERFLOWS_PATH = "hub/postoverflows.txt"
SUPPORT_BOUNCERS_PATH = "lapi/bouncers.txt" SUPPORT_BOUNCERS_PATH = "lapi/bouncers.txt"
@ -272,6 +273,7 @@ func (cli cliSupport) NewDumpCmd() *cobra.Command {
- Installed parsers list - Installed parsers list
- Installed scenarios list - Installed scenarios list
- Installed postoverflows list - Installed postoverflows list
- Installed context list
- Bouncers list - Bouncers list
- Machines list - Machines list
- CAPI status - CAPI status
@ -321,6 +323,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error()) infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error()) infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error())
infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error()) infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error())
infos[SUPPORT_CONTEXTS_PATH] = []byte(err.Error())
infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error()) infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error())
} }
@ -356,6 +359,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS) infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS) infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS) infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_CONTEXTS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS) infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
} }

View file

@ -14,6 +14,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/acquisition"
"github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@ -24,6 +25,10 @@ import (
func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) { func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
var err error var err error
if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
return nil, fmt.Errorf("while loading context: %w", err)
}
// Start loading configs // Start loading configs
csParsers := parser.NewParsers(hub) csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
@ -41,6 +46,7 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er
if err := LoadAcquisition(cConfig); err != nil { if err := LoadAcquisition(cConfig); err != nil {
return nil, fmt.Errorf("while loading acquisition config: %w", err) return nil, fmt.Errorf("while loading acquisition config: %w", err)
} }
return csParsers, nil return csParsers, nil
} }

View file

@ -320,10 +320,12 @@ config.yaml) each time the container is run.
| `PARSERS` | | Parsers to install, separated by space | | `PARSERS` | | Parsers to install, separated by space |
| `SCENARIOS` | | Scenarios to install, separated by space | | `SCENARIOS` | | Scenarios to install, separated by space |
| `POSTOVERFLOWS` | | Postoverflows to install, separated by space | | `POSTOVERFLOWS` | | Postoverflows to install, separated by space |
| `CONTEXTS` | | Context files to install, separated by space |
| `DISABLE_COLLECTIONS` | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` | | `DISABLE_COLLECTIONS` | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
| `DISABLE_PARSERS` | | Parsers to remove, separated by space | | `DISABLE_PARSERS` | | Parsers to remove, separated by space |
| `DISABLE_SCENARIOS` | | Scenarios to remove, separated by space | | `DISABLE_SCENARIOS` | | Scenarios to remove, separated by space |
| `DISABLE_POSTOVERFLOWS` | | Postoverflows to remove, separated by space | | `DISABLE_POSTOVERFLOWS` | | Postoverflows to remove, separated by space |
| `DISABLE_POSTOVERFLOWS` | | Context files to remove, separated by space |
| | | | | | | |
| __Log verbosity__ | | | | __Log verbosity__ | | |
| `LEVEL_INFO` | false | Force INFO level for the container log | | `LEVEL_INFO` | false | Force INFO level for the container log |

View file

@ -300,7 +300,7 @@ fi
conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)' conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
## Install collections, parsers, scenarios & postoverflows ## Install hub items
cscli hub update cscli hub update
cscli_if_clean collections upgrade crowdsecurity/linux cscli_if_clean collections upgrade crowdsecurity/linux
@ -328,6 +328,11 @@ if [ "$POSTOVERFLOWS" != "" ]; then
cscli_if_clean postoverflows install "$(difference "$POSTOVERFLOWS" "$DISABLE_POSTOVERFLOWS")" cscli_if_clean postoverflows install "$(difference "$POSTOVERFLOWS" "$DISABLE_POSTOVERFLOWS")"
fi fi
if [ "$CONTEXTS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean contexts install "$(difference "$CONTEXTS" "$DISABLE_CONTEXTS")"
fi
## Remove collections, parsers, scenarios & postoverflows ## Remove collections, parsers, scenarios & postoverflows
if [ "$DISABLE_COLLECTIONS" != "" ]; then if [ "$DISABLE_COLLECTIONS" != "" ]; then
# shellcheck disable=SC2086 # shellcheck disable=SC2086
@ -349,6 +354,11 @@ if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
fi fi
if [ "$DISABLE_CONTEXTS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean contexts remove "$DISABLE_CONTEXTS" --force
fi
## Register bouncers via env ## Register bouncers via env
for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
KEY=$(printf '%s' "${!BOUNCER}") KEY=$(printf '%s' "${!BOUNCER}")

View file

@ -63,13 +63,21 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error {
} }
for key, values := range contextToSend { for key, values := range contextToSend {
if _, ok := alertContext.ContextToSend[key]; !ok {
alertContext.ContextToSend[key] = make([]string, 0)
}
if _, ok := alertContext.ContextToSendCompiled[key]; !ok {
alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0) alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0)
}
for _, value := range values { for _, value := range values {
valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
return fmt.Errorf("compilation of '%s' context value failed: %v", value, err) return fmt.Errorf("compilation of '%s' context value failed: %v", value, err)
} }
alertContext.ContextToSendCompiled[key] = append(alertContext.ContextToSendCompiled[key], valueCompiled) alertContext.ContextToSendCompiled[key] = append(alertContext.ContextToSendCompiled[key], valueCompiled)
alertContext.ContextToSend[key] = append(alertContext.ContextToSend[key], value)
} }
} }

125
pkg/alertcontext/config.go Normal file
View file

@ -0,0 +1,125 @@
package alertcontext
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// this file is here to avoid circular dependencies between the configuration and the hub
// HubItemWrapper is a wrapper around a hub item to unmarshal only the context part
// because there are other fields like name etc.
type HubItemWrapper struct {
Context map[string][]string `yaml:"context"`
}
// mergeContext adds the context from src to dest.
func mergeContext(dest map[string][]string, src map[string][]string) {
for k, v := range src {
if _, ok := dest[k]; !ok {
dest[k] = make([]string, 0)
}
for _, s := range v {
if !slices.Contains(dest[k], s) {
dest[k] = append(dest[k], s)
}
}
}
}
// addContextFromItem merges the context from an item into the context to send to the console.
func addContextFromItem(toSend map[string][]string, item *cwhub.Item) error {
filePath := item.State.LocalPath
log.Tracef("loading console context from %s", filePath)
content, err := os.ReadFile(filePath)
if err != nil {
return err
}
wrapper := &HubItemWrapper{}
err = yaml.Unmarshal(content, wrapper)
if err != nil {
return fmt.Errorf("%s: %w", filePath, err)
}
mergeContext(toSend, wrapper.Context)
return nil
}
// addContextFromFile merges the context from a file into the context to send to the console.
func addContextFromFile(toSend map[string][]string, filePath string) error {
log.Tracef("loading console context from %s", filePath)
content, err := os.ReadFile(filePath)
if err != nil {
return err
}
newContext := make(map[string][]string, 0)
err = yaml.Unmarshal(content, newContext)
if err != nil {
return fmt.Errorf("%s: %w", filePath, err)
}
mergeContext(toSend, newContext)
return nil
}
// LoadConsoleContext loads the context from the hub (if provided) and the file console_context_path.
func LoadConsoleContext(c *csconfig.Config, hub *cwhub.Hub) error {
c.Crowdsec.ContextToSend = make(map[string][]string, 0)
if hub != nil {
items, err := hub.GetInstalledItems(cwhub.CONTEXTS)
if err != nil {
return err
}
for _, item := range items {
// context in item files goes under the key 'context'
if err = addContextFromItem(c.Crowdsec.ContextToSend, item); err != nil {
return err
}
}
}
ignoreMissing := false
if c.Crowdsec.ConsoleContextPath != "" {
// if it's provided, it must exist
if _, err := os.Stat(c.Crowdsec.ConsoleContextPath); err != nil {
return fmt.Errorf("while checking console_context_path: %w", err)
}
} else {
c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
ignoreMissing = true
}
if err := addContextFromFile(c.Crowdsec.ContextToSend, c.Crowdsec.ConsoleContextPath); err != nil {
if !ignoreMissing || !os.IsNotExist(err) {
return err
}
}
feedback, err := json.Marshal(c.Crowdsec.ContextToSend)
if err != nil {
return fmt.Errorf("marshaling console context: %s", err)
}
log.Debugf("console context to send: %s", feedback)
return nil
}

View file

@ -108,8 +108,9 @@ func (c *Config) LoadCrowdsec() error {
c.Crowdsec.OutputRoutinesCount = 1 c.Crowdsec.OutputRoutinesCount = 1
} }
var crowdsecCleanup = []*string{ crowdsecCleanup := []*string{
&c.Crowdsec.AcquisitionFilePath, &c.Crowdsec.AcquisitionFilePath,
&c.Crowdsec.ConsoleContextPath,
} }
for _, k := range crowdsecCleanup { for _, k := range crowdsecCleanup {
@ -131,38 +132,10 @@ func (c *Config) LoadCrowdsec() error {
c.Crowdsec.AcquisitionFiles[i] = f c.Crowdsec.AcquisitionFiles[i] = f
} }
if err := c.LoadAPIClient(); err != nil { if err = c.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %s", err) return fmt.Errorf("loading api client: %s", err)
} }
c.Crowdsec.ContextToSend = make(map[string][]string, 0)
fallback := false
if c.Crowdsec.ConsoleContextPath == "" {
// fallback to default config file
c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
fallback = true
}
f, err := filepath.Abs(c.Crowdsec.ConsoleContextPath)
if err != nil {
return fmt.Errorf("fail to get absolute path of %s: %s", c.Crowdsec.ConsoleContextPath, err)
}
c.Crowdsec.ConsoleContextPath = f
yamlFile, err := os.ReadFile(c.Crowdsec.ConsoleContextPath)
if err != nil {
if fallback {
log.Debugf("Default context config file doesn't exist, will not use it")
} else {
return fmt.Errorf("failed to open context file: %s", err)
}
} else {
err = yaml.Unmarshal(yamlFile, c.Crowdsec.ContextToSend)
if err != nil {
return fmt.Errorf("unmarshaling labels console config file '%s': %s", c.Crowdsec.ConsoleContextPath, err)
}
}
return nil return nil
} }
@ -170,10 +143,16 @@ func (c *CrowdsecServiceCfg) DumpContextConfigFile() error {
var out []byte var out []byte
var err error var err error
// XXX: MakeDirs
if out, err = yaml.Marshal(c.ContextToSend); err != nil { if out, err = yaml.Marshal(c.ContextToSend); err != nil {
return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleContextPath, err) return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleContextPath, err)
} }
if err = os.MkdirAll(filepath.Dir(c.ConsoleContextPath), 0700); err != nil {
return fmt.Errorf("while creating directories for %s: %w", c.ConsoleContextPath, err)
}
if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil { if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil {
return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleContextPath, err) return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleContextPath, err)
} }

View file

@ -60,9 +60,10 @@ func TestLoadCrowdsec(t *testing.T) {
ConsoleContextValueLength: 2500, ConsoleContextValueLength: 2500,
AcquisitionFiles: []string{acquisFullPath}, AcquisitionFiles: []string{acquisFullPath},
SimulationFilePath: "./testdata/simulation.yaml", SimulationFilePath: "./testdata/simulation.yaml",
ContextToSend: map[string][]string{ // context is loaded in pkg/alertcontext
"source_ip": {"evt.Parsed.source_ip"}, // ContextToSend: map[string][]string{
}, // "source_ip": {"evt.Parsed.source_ip"},
// },
SimulationConfig: &SimulationConfig{ SimulationConfig: &SimulationConfig{
Simulation: ptr.Of(false), Simulation: ptr.Of(false),
}, },
@ -98,9 +99,10 @@ func TestLoadCrowdsec(t *testing.T) {
OutputRoutinesCount: 1, OutputRoutinesCount: 1,
ConsoleContextValueLength: 0, ConsoleContextValueLength: 0,
AcquisitionFiles: []string{acquisFullPath, acquisInDirFullPath}, AcquisitionFiles: []string{acquisFullPath, acquisInDirFullPath},
ContextToSend: map[string][]string{ // context is loaded in pkg/alertcontext
"source_ip": {"evt.Parsed.source_ip"}, // ContextToSend: map[string][]string{
}, // "source_ip": {"evt.Parsed.source_ip"},
// },
SimulationFilePath: "./testdata/simulation.yaml", SimulationFilePath: "./testdata/simulation.yaml",
SimulationConfig: &SimulationConfig{ SimulationConfig: &SimulationConfig{
Simulation: ptr.Of(false), Simulation: ptr.Of(false),
@ -136,9 +138,10 @@ func TestLoadCrowdsec(t *testing.T) {
ConsoleContextValueLength: 10, ConsoleContextValueLength: 10,
AcquisitionFiles: []string{}, AcquisitionFiles: []string{},
SimulationFilePath: "", SimulationFilePath: "",
ContextToSend: map[string][]string{ // context is loaded in pkg/alertcontext
"source_ip": {"evt.Parsed.source_ip"}, // ContextToSend: map[string][]string{
}, // "source_ip": {"evt.Parsed.source_ip"},
// },
SimulationConfig: &SimulationConfig{ SimulationConfig: &SimulationConfig{
Simulation: ptr.Of(false), Simulation: ptr.Of(false),
}, },

View file

@ -16,6 +16,7 @@ const (
PARSERS = "parsers" PARSERS = "parsers"
POSTOVERFLOWS = "postoverflows" POSTOVERFLOWS = "postoverflows"
SCENARIOS = "scenarios" SCENARIOS = "scenarios"
CONTEXTS = "contexts"
APPSEC_CONFIGS = "appsec-configs" APPSEC_CONFIGS = "appsec-configs"
APPSEC_RULES = "appsec-rules" APPSEC_RULES = "appsec-rules"
) )
@ -29,7 +30,7 @@ const (
var ( var (
// The order is important, as it is used to range over sub-items in collections. // The order is important, as it is used to range over sub-items in collections.
ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS} ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, CONTEXTS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS}
) )
type HubItems map[string]map[string]*Item type HubItems map[string]map[string]*Item
@ -120,6 +121,7 @@ type Item struct {
PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"`
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
Contexts []string `json:"contexts,omitempty" yaml:"contexts,omitempty"`
AppsecConfigs []string `json:"appsec-configs,omitempty" yaml:"appsec-configs,omitempty"` AppsecConfigs []string `json:"appsec-configs,omitempty" yaml:"appsec-configs,omitempty"`
AppsecRules []string `json:"appsec-rules,omitempty" yaml:"appsec-rules,omitempty"` AppsecRules []string `json:"appsec-rules,omitempty" yaml:"appsec-rules,omitempty"`
} }
@ -231,6 +233,15 @@ func (i *Item) SubItems() []*Item {
sub = append(sub, s) sub = append(sub, s)
} }
for _, name := range i.Contexts {
s := i.hub.GetItem(CONTEXTS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.AppsecConfigs { for _, name := range i.AppsecConfigs {
s := i.hub.GetItem(APPSEC_CONFIGS, name) s := i.hub.GetItem(APPSEC_CONFIGS, name)
if s == nil { if s == nil {
@ -284,6 +295,12 @@ func (i *Item) logMissingSubItems() {
} }
} }
for _, subName := range i.Contexts {
if i.hub.GetItem(CONTEXTS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, CONTEXTS, i.Name)
}
}
for _, subName := range i.AppsecConfigs { for _, subName := range i.AppsecConfigs {
if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil { if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name) log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name)

View file

@ -346,7 +346,7 @@ func (i *Item) checkSubItemVersions() error {
// syncDir scans a directory for items, and updates the Hub state accordingly. // syncDir scans a directory for items, and updates the Hub state accordingly.
func (h *Hub) syncDir(dir string) error { func (h *Hub) syncDir(dir string) error {
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last // For each, scan PARSERS, POSTOVERFLOWS... and COLLECTIONS last
for _, scan := range ItemTypes { for _, scan := range ItemTypes {
// cpath: top-level item directory, either downloaded or installed items. // cpath: top-level item directory, either downloaded or installed items.
// i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ... // i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ...

View file

@ -83,7 +83,7 @@ bats-build: bats-environment
# Create a reusable package with initial configuration + data # Create a reusable package with initial configuration + data
bats-fixture: bats-check-requirements bats-update-tools bats-fixture: bats-check-requirements bats-update-tools
@echo "Creating functional test fixture..." @echo "Creating functional test fixture."
@$(TEST_DIR)/instance-data make @$(TEST_DIR)/instance-data make
# Remove the local crowdsec installation and the fixture config + data # Remove the local crowdsec installation and the fixture config + data

View file

@ -252,19 +252,20 @@ teardown() {
@test "cscli - malformed LAPI url" { @test "cscli - malformed LAPI url" {
LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path') LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path')
config_set "${LOCAL_API_CREDENTIALS}" '.url="https://127.0.0.1:-80"' config_set "${LOCAL_API_CREDENTIALS}" '.url="http://127.0.0.1:-80"'
rune -1 cscli lapi status rune -1 cscli lapi status -o json
assert_stderr --partial 'parsing api url' rune -0 jq -r '.msg' <(stderr)
assert_stderr --partial 'invalid port \":-80\" after host' assert_output 'parsing api url: parse "http://127.0.0.1:-80/": invalid port ":-80" after host'
}
rune -1 cscli alerts list @test "cscli - bad LAPI password" {
assert_stderr --partial 'parsing api url' LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path')
assert_stderr --partial 'invalid port \":-80\" after host' config_set "${LOCAL_API_CREDENTIALS}" '.password="meh"'
rune -1 cscli decisions list rune -1 cscli lapi status -o json
assert_stderr --partial 'parsing api url' rune -0 jq -r '.msg' <(stderr)
assert_stderr --partial 'invalid port \":-80\" after host' assert_output 'failed to authenticate to Local API (LAPI): API error: incorrect Username or Password'
} }
@test "cscli metrics" { @test "cscli metrics" {

95
test/bats/09_context.bats Normal file
View file

@ -0,0 +1,95 @@
#!/usr/bin/env bats
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
set -u
setup_file() {
load "../lib/setup_file.sh"
CONFIG_DIR=$(config_get '.config_paths.config_dir')
export CONFIG_DIR
CONTEXT_YAML="$CONFIG_DIR/console/context.yaml"
export CONTEXT_YAML
}
teardown_file() {
load "../lib/teardown_file.sh"
}
setup() {
load "../lib/setup.sh"
load "../lib/bats-file/load.bash"
./instance-data load
config_set '.common.log_media="stdout"'
mkdir -p "$CONFIG_DIR/console"
}
teardown() {
./instance-crowdsec stop
}
#----------
@test "detect available context" {
rune -0 cscli lapi context detect -a
rune -0 yq -o json <(output)
assert_json '{"Acquisition":["evt.Line.Module","evt.Line.Raw","evt.Line.Src"]}'
rune -0 cscli parsers install crowdsecurity/dateparse-enrich
rune -0 cscli lapi context detect crowdsecurity/dateparse-enrich
rune -0 yq -o json '.crowdsecurity/dateparse-enrich' <(output)
assert_json '["evt.MarshaledTime","evt.Meta.timestamp"]'
}
@test "attempt to load from default context file, ignore if missing" {
rune -0 rm -f "$CONTEXT_YAML"
rune -0 "$CROWDSEC" -t --trace
assert_stderr --partial "loading console context from $CONTEXT_YAML"
}
@test "error if context file is explicitly set but does not exist" {
config_set ".crowdsec_service.console_context_path=strenv(CONTEXT_YAML)"
rune -0 rm -f "$CONTEXT_YAML"
rune -1 "$CROWDSEC" -t
assert_stderr --partial "while checking console_context_path: stat $CONTEXT_YAML: no such file or directory"
}
@test "context file is bad" {
echo "bad yaml" > "$CONTEXT_YAML"
rune -1 "$CROWDSEC" -t
assert_stderr --partial "while loading context: $CONTEXT_YAML: yaml: unmarshal errors"
}
@test "context file is good" {
echo '{"source_ip":["evt.Parsed.source_ip"]}' > "$CONTEXT_YAML"
rune -0 "$CROWDSEC" -t --debug
assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
}
@test "context file is from hub (local item)" {
mkdir -p "$CONFIG_DIR/contexts"
config_set "del(.crowdsec_service.console_context_path)"
echo '{"context":{"source_ip":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/foobar.yaml"
rune -0 "$CROWDSEC" -t --trace
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/foobar.yaml"
assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
}
@test "merge multiple contexts" {
mkdir -p "$CONFIG_DIR/contexts"
echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
echo '{"context":{"two":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/two.yaml"
rune -0 "$CROWDSEC" -t --trace
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/two.yaml"
assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip"],"two":["evt.Parsed.source_ip"]}'
}
@test "merge contexts from hub and context.yaml file" {
mkdir -p "$CONFIG_DIR/contexts"
echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
echo '{"one":["evt.Parsed.source_ip_2"]}' > "$CONFIG_DIR/console/context.yaml"
rune -0 "$CROWDSEC" -t --trace
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
assert_stderr --partial "loading console context from $CONFIG_DIR/console/context.yaml"
assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip","evt.Parsed.source_ip_2"]}'
}

View file

@ -36,7 +36,7 @@ teardown() {
rune -0 cscli hub list rune -0 cscli hub list
assert_output "No items to display" assert_output "No items to display"
rune -0 cscli hub list -o json rune -0 cscli hub list -o json
assert_json '{"appsec-configs":[],"appsec-rules":[],parsers:[],scenarios:[],collections:[],postoverflows:[]}' assert_json '{"appsec-configs":[],"appsec-rules":[],parsers:[],scenarios:[],collections:[],contexts:[],postoverflows:[]}'
rune -0 cscli hub list -o raw rune -0 cscli hub list -o raw
assert_output 'name,status,version,description,type' assert_output 'name,status,version,description,type'
@ -47,6 +47,7 @@ teardown() {
assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*" assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*"
refute_output --partial 'POSTOVERFLOWS' refute_output --partial 'POSTOVERFLOWS'
refute_output --partial 'COLLECTIONS' refute_output --partial 'COLLECTIONS'
rune -0 cscli hub list -o json rune -0 cscli hub list -o json
rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output) rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output)
rune -0 cscli hub list -o raw rune -0 cscli hub list -o raw
@ -55,8 +56,11 @@ teardown() {
refute_output --partial 'crowdsecurity/iptables' refute_output --partial 'crowdsecurity/iptables'
# all items # all items
mkdir -p "$CONFIG_DIR/contexts"
# there are no contexts yet, so we create a local one
touch "$CONFIG_DIR/contexts/mycontext.yaml"
rune -0 cscli hub list -a rune -0 cscli hub list -a
assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*crowdsecurity/iptables.*" assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*CONTEXTS.*mycontext.yaml.*COLLECTIONS.*crowdsecurity/iptables.*"
rune -0 cscli hub list -a -o json rune -0 cscli hub list -a -o json
rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output) rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output)
rune -0 cscli hub list -a -o raw rune -0 cscli hub list -a -o raw
@ -107,6 +111,8 @@ teardown() {
assert_stderr --partial "Upgraded 0 postoverflows" assert_stderr --partial "Upgraded 0 postoverflows"
assert_stderr --partial "Upgrading scenarios" assert_stderr --partial "Upgrading scenarios"
assert_stderr --partial "Upgraded 0 scenarios" assert_stderr --partial "Upgraded 0 scenarios"
assert_stderr --partial "Upgrading contexts"
assert_stderr --partial "Upgraded 0 contexts"
assert_stderr --partial "Upgrading collections" assert_stderr --partial "Upgrading collections"
assert_stderr --partial "Upgraded 0 collections" assert_stderr --partial "Upgraded 0 collections"
@ -134,10 +140,11 @@ teardown() {
assert_line "parsers" assert_line "parsers"
assert_line "postoverflows" assert_line "postoverflows"
assert_line "scenarios" assert_line "scenarios"
assert_line "contexts"
assert_line "collections" assert_line "collections"
rune -0 cscli hub types -o human rune -0 cscli hub types -o human
rune -0 yq -o json <(output) rune -0 yq -o json <(output)
assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]' assert_json '["parsers","postoverflows","scenarios","contexts","appsec-configs","appsec-rules","collections"]'
rune -0 cscli hub types -o json rune -0 cscli hub types -o json
assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]' assert_json '["parsers","postoverflows","scenarios","contexts","appsec-configs","appsec-rules","collections"]'
} }

View file

@ -247,12 +247,14 @@ hub_purge_all() {
"$CONFIG_DIR"/collections/* \ "$CONFIG_DIR"/collections/* \
"$CONFIG_DIR"/parsers/*/* \ "$CONFIG_DIR"/parsers/*/* \
"$CONFIG_DIR"/scenarios/* \ "$CONFIG_DIR"/scenarios/* \
"$CONFIG_DIR"/postoverflows/* "$CONFIG_DIR"/postoverflows/* \
"$CONFIG_DIR"/contexts/*
rm -rf \ rm -rf \
"$CONFIG_DIR"/hub/collections/* \ "$CONFIG_DIR"/hub/collections/* \
"$CONFIG_DIR"/hub/parsers/*/* \ "$CONFIG_DIR"/hub/parsers/*/* \
"$CONFIG_DIR"/hub/scenarios/* \ "$CONFIG_DIR"/hub/scenarios/* \
"$CONFIG_DIR"/hub/postoverflows/* "$CONFIG_DIR"/hub/postoverflows/* \
"$CONFIG_DIR"/hub/contexts/*
local DATA_DIR local DATA_DIR
DATA_DIR=$(config_get .config_paths.data_dir) DATA_DIR=$(config_get .config_paths.data_dir)
# should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete), # should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete),