From 4acb4f8df31bc8fa977baf5f0176afe651a43959 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 7 Dec 2023 16:20:13 +0100 Subject: [PATCH] cwhub: context type (#2631) * add hub type "context" * cscli lapi: log.Fatal -> fmt.Errorf; lint * tests for context.yaml * load console context from hub * original & compiled context * deprecate "cscli lapi context delete" $ cscli lapi context delete Command "delete" is deprecated, please manually edit the context file. * cscli completion: add appsec-rules, appsec-configs, explain, hubtest --- cmd/crowdsec-cli/hubcontext.go | 40 +++++++ cmd/crowdsec-cli/lapi.go | 151 +++++++++++--------------- cmd/crowdsec-cli/main.go | 8 +- cmd/crowdsec-cli/support.go | 4 + cmd/crowdsec/crowdsec.go | 6 + docker/README.md | 2 + docker/docker_start.sh | 12 +- pkg/alertcontext/alertcontext.go | 10 +- pkg/alertcontext/config.go | 125 +++++++++++++++++++++ pkg/csconfig/crowdsec_service.go | 39 ++----- pkg/csconfig/crowdsec_service_test.go | 21 ++-- pkg/cwhub/item.go | 19 +++- pkg/cwhub/sync.go | 2 +- test/bats.mk | 2 +- test/bats/01_cscli.bats | 21 ++-- test/bats/09_context.bats | 95 ++++++++++++++++ test/bats/20_hub.bats | 15 ++- test/lib/setup_file.sh | 6 +- 18 files changed, 427 insertions(+), 151 deletions(-) create mode 100644 cmd/crowdsec-cli/hubcontext.go create mode 100644 pkg/alertcontext/config.go create mode 100644 test/bats/09_context.bats diff --git a/cmd/crowdsec-cli/hubcontext.go b/cmd/crowdsec-cli/hubcontext.go new file mode 100644 index 000000000..3fdd3aac5 --- /dev/null +++ b/cmd/crowdsec-cli/hubcontext.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewContextCLI() *itemCLI { + return &itemCLI{ + name: cwhub.CONTEXTS, + singular: "context", + oneOrMore: "context(s)", + help: cliHelp{ + example: `cscli contexts list -a +cscli contexts install crowdsecurity/yyy crowdsecurity/zzz +cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz +cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz +cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz +`, + }, + installHelp: cliHelp{ + example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`, + }, + removeHelp: cliHelp{ + example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`, + }, + upgradeHelp: cliHelp{ + example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`, + }, + inspectHelp: cliHelp{ + example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`, + }, + listHelp: cliHelp{ + example: `cscli contexts list +cscli contexts list -a +cscli contexts list crowdsecurity/yyy crowdsecurity/zzz + +List only enabled contexts unless "-a" or names are specified.`, + }, + } +} diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index 9b6900a8f..96da17913 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -2,10 +2,10 @@ package main import ( "context" + "errors" "fmt" "net/url" "os" - "slices" "sort" "strings" @@ -13,6 +13,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" + "slices" "github.com/crowdsecurity/go-cs-lib/version" @@ -26,26 +27,24 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/parser" ) -var LAPIURLPrefix string = "v1" +const LAPIURLPrefix = "v1" func runLapiStatus(cmd *cobra.Command, args []string) error { - var err error - password := strfmt.Password(csConfig.API.Client.Credentials.Password) apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL) login := csConfig.API.Client.Credentials.Login if err != nil { - log.Fatalf("parsing api url ('%s'): %s", apiurl, err) + return fmt.Errorf("parsing api url: %w", err) } hub, err := require.Hub(csConfig, nil) if err != nil { - log.Fatal(err) + return err } scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) if err != nil { - log.Fatalf("failed to get scenarios : %s", err) + return fmt.Errorf("failed to get scenarios: %w", err) } Client, err = apiclient.NewDefaultClient(apiurl, @@ -53,28 +52,27 @@ func runLapiStatus(cmd *cobra.Command, args []string) error { fmt.Sprintf("crowdsec/%s", version.String()), nil) if err != nil { - log.Fatalf("init default client: %s", err) + return fmt.Errorf("init default client: %w", err) } t := models.WatcherAuthRequest{ MachineID: &login, Password: &password, Scenarios: scenarios, } + log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath) log.Infof("Trying to authenticate with username %s on %s", login, apiurl) + _, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t) if err != nil { - log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err) - } else { - log.Infof("You can successfully interact with Local API (LAPI)") + return fmt.Errorf("failed to authenticate to Local API (LAPI): %w", err) } + log.Infof("You can successfully interact with Local API (LAPI)") return nil } func runLapiRegister(cmd *cobra.Command, args []string) error { - var err error - flags := cmd.Flags() apiURL, err := flags.GetString("url") @@ -95,16 +93,15 @@ func runLapiRegister(cmd *cobra.Command, args []string) error { if lapiUser == "" { lapiUser, err = generateID("") if err != nil { - log.Fatalf("unable to generate machine id: %s", err) + return fmt.Errorf("unable to generate machine id: %w", err) } } password := strfmt.Password(generatePassword(passwordLength)) if apiURL == "" { - if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" { - apiURL = csConfig.API.Client.Credentials.URL - } else { - log.Fatalf("No Local API URL. Please provide it in your configuration or with the -u parameter") + if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil || csConfig.API.Client.Credentials.URL == "" { + return fmt.Errorf("no Local API URL. Please provide it in your configuration or with the -u parameter") } + apiURL = csConfig.API.Client.Credentials.URL } /*URL needs to end with /, but user doesn't care*/ if !strings.HasSuffix(apiURL, "/") { @@ -116,7 +113,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error { } apiurl, err := url.Parse(apiURL) if err != nil { - log.Fatalf("parsing api url: %s", err) + return fmt.Errorf("parsing api url: %w", err) } _, err = apiclient.RegisterClient(&apiclient.Config{ MachineID: lapiUser, @@ -127,7 +124,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error { }, nil) if err != nil { - log.Fatalf("api client register: %s", err) + return fmt.Errorf("api client register: %w", err) } log.Printf("Successfully registered to Local API (LAPI)") @@ -147,12 +144,12 @@ func runLapiRegister(cmd *cobra.Command, args []string) error { } apiConfigDump, err := yaml.Marshal(apiCfg) if err != nil { - log.Fatalf("unable to marshal api credentials: %s", err) + return fmt.Errorf("unable to marshal api credentials: %w", err) } if dumpFile != "" { err = os.WriteFile(dumpFile, apiConfigDump, 0o600) if err != nil { - log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err) + return fmt.Errorf("write api credentials to '%s' failed: %w", dumpFile, err) } log.Printf("Local API credentials written to '%s'", dumpFile) } else { @@ -195,7 +192,7 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side } func NewLapiCmd() *cobra.Command { - var cmdLapi = &cobra.Command{ + cmdLapi := &cobra.Command{ Use: "lapi [action]", Short: "Manage interaction with Local API (LAPI)", Args: cobra.MinimumNArgs(1), @@ -221,6 +218,7 @@ func AddContext(key string, values []string) error { } if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok { csConfig.Crowdsec.ContextToSend[key] = make([]string, 0) + log.Infof("key '%s' added", key) } data := csConfig.Crowdsec.ContextToSend[key] @@ -247,11 +245,11 @@ func NewLapiContextCmd() *cobra.Command { if err := csConfig.LoadCrowdsec(); err != nil { fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath) if err.Error() != fileNotFoundMessage { - log.Fatalf("Unable to load CrowdSec Agent: %s", err) + return fmt.Errorf("unable to start CrowdSec agent: %w", err) } } if csConfig.DisableAgent { - log.Fatalf("Agent is disabled and lapi context can only be used on the agent") + return errors.New("agent is disabled and lapi context can only be used on the agent") } return nil @@ -271,12 +269,21 @@ cscli lapi context add --key file_source --value evt.Line.Src cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user `, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil { + return fmt.Errorf("while loading context: %w", err) + } + if keyToAdd != "" { if err := AddContext(keyToAdd, valuesToAdd); err != nil { - log.Fatalf(err.Error()) + return err } - return + return nil } for _, v := range valuesToAdd { @@ -284,9 +291,11 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user key := keySlice[len(keySlice)-1] value := []string{v} if err := AddContext(key, value); err != nil { - log.Fatalf(err.Error()) + return err } } + + return nil }, } cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send") @@ -298,19 +307,29 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user Use: "status", Short: "List context to send with alerts", DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil { + return fmt.Errorf("while loading context: %w", err) + } + if len(csConfig.Crowdsec.ContextToSend) == 0 { fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.") - return + return nil } dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend) if err != nil { - log.Fatalf("unable to show context status: %s", err) + return fmt.Errorf("unable to show context status: %w", err) } - fmt.Println(string(dump)) + fmt.Print(string(dump)) + return nil }, } cmdContext.AddCommand(cmdContextStatus) @@ -323,9 +342,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user cscli lapi context detect crowdsecurity/sshd-logs `, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - var err error - + RunE: func(cmd *cobra.Command, args []string) error { if !detectAll && len(args) == 0 { log.Infof("Please provide parsers to detect or --all flag.") printHelp(cmd) @@ -334,19 +351,18 @@ cscli lapi context detect crowdsecurity/sshd-logs // to avoid all the log.Info from the loaders functions log.SetLevel(log.WarnLevel) - err = exprhelpers.Init(nil) - if err != nil { - log.Fatalf("Failed to init expr helpers : %s", err) + if err := exprhelpers.Init(nil); err != nil { + return fmt.Errorf("failed to init expr helpers: %w", err) } hub, err := require.Hub(csConfig, nil) if err != nil { - log.Fatal(err) + return err } csParsers := parser.NewParsers(hub) if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil { - log.Fatalf("unable to load parsers: %s", err) + return fmt.Errorf("unable to load parsers: %w", err) } fieldByParsers := make(map[string][]string) @@ -366,7 +382,6 @@ cscli lapi context detect crowdsecurity/sshd-logs fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field) } } - } fmt.Printf("Acquisition :\n\n") @@ -399,59 +414,17 @@ cscli lapi context detect crowdsecurity/sshd-logs log.Errorf("parser '%s' not found, can't detect fields", parserNotFound) } } + + return nil }, } cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser") cmdContext.AddCommand(cmdContextDetect) - var keysToDelete []string - var valuesToDelete []string cmdContextDelete := &cobra.Command{ Use: "delete", - Short: "Delete context to send with alerts", - Example: `cscli lapi context delete --key source_ip -cscli lapi context delete --value evt.Line.Src - `, - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - if len(keysToDelete) == 0 && len(valuesToDelete) == 0 { - log.Fatalf("please provide at least a key or a value to delete") - } - - for _, key := range keysToDelete { - if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok { - delete(csConfig.Crowdsec.ContextToSend, key) - log.Infof("key '%s' has been removed", key) - } else { - log.Warningf("key '%s' doesn't exist", key) - } - } - - for _, value := range valuesToDelete { - valueFound := false - for key, context := range csConfig.Crowdsec.ContextToSend { - if slices.Contains(context, value) { - valueFound = true - csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context) - log.Infof("value '%s' has been removed from key '%s'", value, key) - } - if len(csConfig.Crowdsec.ContextToSend[key]) == 0 { - delete(csConfig.Crowdsec.ContextToSend, key) - } - } - if !valueFound { - log.Warningf("value '%s' not found", value) - } - } - - if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil { - log.Fatalf(err.Error()) - } - - }, + Deprecated: "please manually edit the context file.", } - cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete") - cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete") cmdContext.AddCommand(cmdContextDelete) return cmdContext @@ -459,6 +432,7 @@ cscli lapi context delete --value evt.Line.Src func detectStaticField(GrokStatics []parser.ExtraField) []string { ret := make([]string, 0) + for _, static := range GrokStatics { if static.Parsed != "" { fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed) @@ -487,7 +461,8 @@ func detectStaticField(GrokStatics []parser.ExtraField) []string { } func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string { - var ret = make([]string, 0) + ret := make([]string, 0) + if node.Grok.RunTimeRegexp != nil { for _, capturedField := range node.Grok.RunTimeRegexp.Names() { fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 7427289bb..31f5cb158 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -93,9 +93,10 @@ func initConfig() { } var validArgs = []string{ - "scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines", - "metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard", - "config", "completion", "version", "console", "notifications", "support", + "alerts", "appsec-configs", "appsec-rules", "bouncers", "capi", "collections", + "completion", "config", "console", "contexts", "dashboard", "decisions", "explain", + "hub", "hubtest", "lapi", "machines", "metrics", "notifications", "parsers", + "postoverflows", "scenarios", "simulation", "support", "version", } func prepender(filename string) string { @@ -246,6 +247,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewParserCLI().NewCommand()) rootCmd.AddCommand(NewScenarioCLI().NewCommand()) rootCmd.AddCommand(NewPostOverflowCLI().NewCommand()) + rootCmd.AddCommand(NewContextCLI().NewCommand()) rootCmd.AddCommand(NewAppsecConfigCLI().NewCommand()) rootCmd.AddCommand(NewAppsecRuleCLI().NewCommand()) diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 9d5efb7eb..dd705833f 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -37,6 +37,7 @@ const ( SUPPORT_OS_INFO_PATH = "osinfo.txt" SUPPORT_PARSERS_PATH = "hub/parsers.txt" SUPPORT_SCENARIOS_PATH = "hub/scenarios.txt" + SUPPORT_CONTEXTS_PATH = "hub/scenarios.txt" SUPPORT_COLLECTIONS_PATH = "hub/collections.txt" SUPPORT_POSTOVERFLOWS_PATH = "hub/postoverflows.txt" SUPPORT_BOUNCERS_PATH = "lapi/bouncers.txt" @@ -272,6 +273,7 @@ func (cli cliSupport) NewDumpCmd() *cobra.Command { - Installed parsers list - Installed scenarios list - Installed postoverflows list +- Installed context list - Bouncers list - Machines list - CAPI status @@ -321,6 +323,7 @@ cscli support dump -f /tmp/crowdsec-support.zip infos[SUPPORT_PARSERS_PATH] = []byte(err.Error()) infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error()) infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error()) + infos[SUPPORT_CONTEXTS_PATH] = []byte(err.Error()) infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error()) } @@ -356,6 +359,7 @@ cscli support dump -f /tmp/crowdsec-support.zip infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS) infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS) infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS) + infos[SUPPORT_CONTEXTS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS) infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS) } diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 1e0d54c07..774b9d381 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -14,6 +14,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/alertcontext" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" @@ -24,6 +25,10 @@ import ( func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) { var err error + if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil { + return nil, fmt.Errorf("while loading context: %w", err) + } + // Start loading configs csParsers := parser.NewParsers(hub) if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { @@ -41,6 +46,7 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er if err := LoadAcquisition(cConfig); err != nil { return nil, fmt.Errorf("while loading acquisition config: %w", err) } + return csParsers, nil } diff --git a/docker/README.md b/docker/README.md index 4d1182fa4..84e65ec07 100644 --- a/docker/README.md +++ b/docker/README.md @@ -320,10 +320,12 @@ config.yaml) each time the container is run. | `PARSERS` | | Parsers to install, separated by space | | `SCENARIOS` | | Scenarios to install, separated by space | | `POSTOVERFLOWS` | | Postoverflows to install, separated by space | +| `CONTEXTS` | | Context files to install, separated by space | | `DISABLE_COLLECTIONS` | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` | | `DISABLE_PARSERS` | | Parsers to remove, separated by space | | `DISABLE_SCENARIOS` | | Scenarios to remove, separated by space | | `DISABLE_POSTOVERFLOWS` | | Postoverflows to remove, separated by space | +| `DISABLE_POSTOVERFLOWS` | | Context files to remove, separated by space | | | | | | __Log verbosity__ | | | | `LEVEL_INFO` | false | Force INFO level for the container log | diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 61d695f2c..fe0a81f31 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -300,7 +300,7 @@ fi conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)' -## Install collections, parsers, scenarios & postoverflows +## Install hub items cscli hub update cscli_if_clean collections upgrade crowdsecurity/linux @@ -328,6 +328,11 @@ if [ "$POSTOVERFLOWS" != "" ]; then cscli_if_clean postoverflows install "$(difference "$POSTOVERFLOWS" "$DISABLE_POSTOVERFLOWS")" fi +if [ "$CONTEXTS" != "" ]; then + # shellcheck disable=SC2086 + cscli_if_clean contexts install "$(difference "$CONTEXTS" "$DISABLE_CONTEXTS")" +fi + ## Remove collections, parsers, scenarios & postoverflows if [ "$DISABLE_COLLECTIONS" != "" ]; then # shellcheck disable=SC2086 @@ -349,6 +354,11 @@ if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force fi +if [ "$DISABLE_CONTEXTS" != "" ]; then + # shellcheck disable=SC2086 + cscli_if_clean contexts remove "$DISABLE_CONTEXTS" --force +fi + ## Register bouncers via env for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do KEY=$(printf '%s' "${!BOUNCER}") diff --git a/pkg/alertcontext/alertcontext.go b/pkg/alertcontext/alertcontext.go index 8d305302c..7586e7cb4 100644 --- a/pkg/alertcontext/alertcontext.go +++ b/pkg/alertcontext/alertcontext.go @@ -63,13 +63,21 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error { } for key, values := range contextToSend { - alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0) + if _, ok := alertContext.ContextToSend[key]; !ok { + alertContext.ContextToSend[key] = make([]string, 0) + } + + if _, ok := alertContext.ContextToSendCompiled[key]; !ok { + alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0) + } + for _, value := range values { valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { return fmt.Errorf("compilation of '%s' context value failed: %v", value, err) } alertContext.ContextToSendCompiled[key] = append(alertContext.ContextToSendCompiled[key], valueCompiled) + alertContext.ContextToSend[key] = append(alertContext.ContextToSend[key], value) } } diff --git a/pkg/alertcontext/config.go b/pkg/alertcontext/config.go new file mode 100644 index 000000000..2305fb384 --- /dev/null +++ b/pkg/alertcontext/config.go @@ -0,0 +1,125 @@ +package alertcontext + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +// this file is here to avoid circular dependencies between the configuration and the hub + +// HubItemWrapper is a wrapper around a hub item to unmarshal only the context part +// because there are other fields like name etc. +type HubItemWrapper struct { + Context map[string][]string `yaml:"context"` +} + +// mergeContext adds the context from src to dest. +func mergeContext(dest map[string][]string, src map[string][]string) { + for k, v := range src { + if _, ok := dest[k]; !ok { + dest[k] = make([]string, 0) + } + for _, s := range v { + if !slices.Contains(dest[k], s) { + dest[k] = append(dest[k], s) + } + } + } +} + +// addContextFromItem merges the context from an item into the context to send to the console. +func addContextFromItem(toSend map[string][]string, item *cwhub.Item) error { + filePath := item.State.LocalPath + log.Tracef("loading console context from %s", filePath) + content, err := os.ReadFile(filePath) + if err != nil { + return err + } + + wrapper := &HubItemWrapper{} + + err = yaml.Unmarshal(content, wrapper) + if err != nil { + return fmt.Errorf("%s: %w", filePath, err) + } + + mergeContext(toSend, wrapper.Context) + + return nil +} + +// addContextFromFile merges the context from a file into the context to send to the console. +func addContextFromFile(toSend map[string][]string, filePath string) error { + log.Tracef("loading console context from %s", filePath) + content, err := os.ReadFile(filePath) + if err != nil { + return err + } + + newContext := make(map[string][]string, 0) + + err = yaml.Unmarshal(content, newContext) + if err != nil { + return fmt.Errorf("%s: %w", filePath, err) + } + + mergeContext(toSend, newContext) + + return nil +} + + +// LoadConsoleContext loads the context from the hub (if provided) and the file console_context_path. +func LoadConsoleContext(c *csconfig.Config, hub *cwhub.Hub) error { + c.Crowdsec.ContextToSend = make(map[string][]string, 0) + + if hub != nil { + items, err := hub.GetInstalledItems(cwhub.CONTEXTS) + if err != nil { + return err + } + + for _, item := range items { + // context in item files goes under the key 'context' + if err = addContextFromItem(c.Crowdsec.ContextToSend, item); err != nil { + return err + } + } + } + + ignoreMissing := false + + if c.Crowdsec.ConsoleContextPath != "" { + // if it's provided, it must exist + if _, err := os.Stat(c.Crowdsec.ConsoleContextPath); err != nil { + return fmt.Errorf("while checking console_context_path: %w", err) + } + } else { + c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml") + ignoreMissing = true + } + + if err := addContextFromFile(c.Crowdsec.ContextToSend, c.Crowdsec.ConsoleContextPath); err != nil { + if !ignoreMissing || !os.IsNotExist(err) { + return err + } + } + + feedback, err := json.Marshal(c.Crowdsec.ContextToSend) + if err != nil { + return fmt.Errorf("marshaling console context: %s", err) + } + + log.Debugf("console context to send: %s", feedback) + + return nil +} diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index c0b8b5a38..36d38cf74 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -108,8 +108,9 @@ func (c *Config) LoadCrowdsec() error { c.Crowdsec.OutputRoutinesCount = 1 } - var crowdsecCleanup = []*string{ + crowdsecCleanup := []*string{ &c.Crowdsec.AcquisitionFilePath, + &c.Crowdsec.ConsoleContextPath, } for _, k := range crowdsecCleanup { @@ -131,38 +132,10 @@ func (c *Config) LoadCrowdsec() error { c.Crowdsec.AcquisitionFiles[i] = f } - if err := c.LoadAPIClient(); err != nil { + if err = c.LoadAPIClient(); err != nil { return fmt.Errorf("loading api client: %s", err) } - c.Crowdsec.ContextToSend = make(map[string][]string, 0) - fallback := false - if c.Crowdsec.ConsoleContextPath == "" { - // fallback to default config file - c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml") - fallback = true - } - - f, err := filepath.Abs(c.Crowdsec.ConsoleContextPath) - if err != nil { - return fmt.Errorf("fail to get absolute path of %s: %s", c.Crowdsec.ConsoleContextPath, err) - } - - c.Crowdsec.ConsoleContextPath = f - yamlFile, err := os.ReadFile(c.Crowdsec.ConsoleContextPath) - if err != nil { - if fallback { - log.Debugf("Default context config file doesn't exist, will not use it") - } else { - return fmt.Errorf("failed to open context file: %s", err) - } - } else { - err = yaml.Unmarshal(yamlFile, c.Crowdsec.ContextToSend) - if err != nil { - return fmt.Errorf("unmarshaling labels console config file '%s': %s", c.Crowdsec.ConsoleContextPath, err) - } - } - return nil } @@ -170,10 +143,16 @@ func (c *CrowdsecServiceCfg) DumpContextConfigFile() error { var out []byte var err error + // XXX: MakeDirs + if out, err = yaml.Marshal(c.ContextToSend); err != nil { return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleContextPath, err) } + if err = os.MkdirAll(filepath.Dir(c.ConsoleContextPath), 0700); err != nil { + return fmt.Errorf("while creating directories for %s: %w", c.ConsoleContextPath, err) + } + if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil { return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleContextPath, err) } diff --git a/pkg/csconfig/crowdsec_service_test.go b/pkg/csconfig/crowdsec_service_test.go index e9d7e8de3..8d332271b 100644 --- a/pkg/csconfig/crowdsec_service_test.go +++ b/pkg/csconfig/crowdsec_service_test.go @@ -60,9 +60,10 @@ func TestLoadCrowdsec(t *testing.T) { ConsoleContextValueLength: 2500, AcquisitionFiles: []string{acquisFullPath}, SimulationFilePath: "./testdata/simulation.yaml", - ContextToSend: map[string][]string{ - "source_ip": {"evt.Parsed.source_ip"}, - }, + // context is loaded in pkg/alertcontext +// ContextToSend: map[string][]string{ +// "source_ip": {"evt.Parsed.source_ip"}, +// }, SimulationConfig: &SimulationConfig{ Simulation: ptr.Of(false), }, @@ -98,9 +99,10 @@ func TestLoadCrowdsec(t *testing.T) { OutputRoutinesCount: 1, ConsoleContextValueLength: 0, AcquisitionFiles: []string{acquisFullPath, acquisInDirFullPath}, - ContextToSend: map[string][]string{ - "source_ip": {"evt.Parsed.source_ip"}, - }, + // context is loaded in pkg/alertcontext +// ContextToSend: map[string][]string{ +// "source_ip": {"evt.Parsed.source_ip"}, +// }, SimulationFilePath: "./testdata/simulation.yaml", SimulationConfig: &SimulationConfig{ Simulation: ptr.Of(false), @@ -136,9 +138,10 @@ func TestLoadCrowdsec(t *testing.T) { ConsoleContextValueLength: 10, AcquisitionFiles: []string{}, SimulationFilePath: "", - ContextToSend: map[string][]string{ - "source_ip": {"evt.Parsed.source_ip"}, - }, + // context is loaded in pkg/alertcontext +// ContextToSend: map[string][]string{ +// "source_ip": {"evt.Parsed.source_ip"}, +// }, SimulationConfig: &SimulationConfig{ Simulation: ptr.Of(false), }, diff --git a/pkg/cwhub/item.go b/pkg/cwhub/item.go index 4d588da3a..7dbe3ebb3 100644 --- a/pkg/cwhub/item.go +++ b/pkg/cwhub/item.go @@ -16,6 +16,7 @@ const ( PARSERS = "parsers" POSTOVERFLOWS = "postoverflows" SCENARIOS = "scenarios" + CONTEXTS = "contexts" APPSEC_CONFIGS = "appsec-configs" APPSEC_RULES = "appsec-rules" ) @@ -29,7 +30,7 @@ const ( var ( // The order is important, as it is used to range over sub-items in collections. - ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS} + ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, CONTEXTS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS} ) type HubItems map[string]map[string]*Item @@ -120,6 +121,7 @@ type Item struct { PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` + Contexts []string `json:"contexts,omitempty" yaml:"contexts,omitempty"` AppsecConfigs []string `json:"appsec-configs,omitempty" yaml:"appsec-configs,omitempty"` AppsecRules []string `json:"appsec-rules,omitempty" yaml:"appsec-rules,omitempty"` } @@ -231,6 +233,15 @@ func (i *Item) SubItems() []*Item { sub = append(sub, s) } + for _, name := range i.Contexts { + s := i.hub.GetItem(CONTEXTS, name) + if s == nil { + continue + } + + sub = append(sub, s) + } + for _, name := range i.AppsecConfigs { s := i.hub.GetItem(APPSEC_CONFIGS, name) if s == nil { @@ -284,6 +295,12 @@ func (i *Item) logMissingSubItems() { } } + for _, subName := range i.Contexts { + if i.hub.GetItem(CONTEXTS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, CONTEXTS, i.Name) + } + } + for _, subName := range i.AppsecConfigs { if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil { log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name) diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index ed3abc2c5..18a93cef3 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -346,7 +346,7 @@ func (i *Item) checkSubItemVersions() error { // syncDir scans a directory for items, and updates the Hub state accordingly. func (h *Hub) syncDir(dir string) error { - // For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last + // For each, scan PARSERS, POSTOVERFLOWS... and COLLECTIONS last for _, scan := range ItemTypes { // cpath: top-level item directory, either downloaded or installed items. // i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ... diff --git a/test/bats.mk b/test/bats.mk index 35241b03e..cb452e2b8 100644 --- a/test/bats.mk +++ b/test/bats.mk @@ -83,7 +83,7 @@ bats-build: bats-environment # Create a reusable package with initial configuration + data bats-fixture: bats-check-requirements bats-update-tools - @echo "Creating functional test fixture..." + @echo "Creating functional test fixture." @$(TEST_DIR)/instance-data make # Remove the local crowdsec installation and the fixture config + data diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index 518590826..3f49a1299 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -252,19 +252,20 @@ teardown() { @test "cscli - malformed LAPI url" { LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path') - config_set "${LOCAL_API_CREDENTIALS}" '.url="https://127.0.0.1:-80"' + config_set "${LOCAL_API_CREDENTIALS}" '.url="http://127.0.0.1:-80"' - rune -1 cscli lapi status - assert_stderr --partial 'parsing api url' - assert_stderr --partial 'invalid port \":-80\" after host' + rune -1 cscli lapi status -o json + rune -0 jq -r '.msg' <(stderr) + assert_output 'parsing api url: parse "http://127.0.0.1:-80/": invalid port ":-80" after host' +} - rune -1 cscli alerts list - assert_stderr --partial 'parsing api url' - assert_stderr --partial 'invalid port \":-80\" after host' +@test "cscli - bad LAPI password" { + LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path') + config_set "${LOCAL_API_CREDENTIALS}" '.password="meh"' - rune -1 cscli decisions list - assert_stderr --partial 'parsing api url' - assert_stderr --partial 'invalid port \":-80\" after host' + rune -1 cscli lapi status -o json + rune -0 jq -r '.msg' <(stderr) + assert_output 'failed to authenticate to Local API (LAPI): API error: incorrect Username or Password' } @test "cscli metrics" { diff --git a/test/bats/09_context.bats b/test/bats/09_context.bats new file mode 100644 index 000000000..c56d3e773 --- /dev/null +++ b/test/bats/09_context.bats @@ -0,0 +1,95 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR + CONTEXT_YAML="$CONFIG_DIR/console/context.yaml" + export CONTEXT_YAML +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + config_set '.common.log_media="stdout"' + mkdir -p "$CONFIG_DIR/console" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "detect available context" { + rune -0 cscli lapi context detect -a + rune -0 yq -o json <(output) + assert_json '{"Acquisition":["evt.Line.Module","evt.Line.Raw","evt.Line.Src"]}' + + rune -0 cscli parsers install crowdsecurity/dateparse-enrich + rune -0 cscli lapi context detect crowdsecurity/dateparse-enrich + rune -0 yq -o json '.crowdsecurity/dateparse-enrich' <(output) + assert_json '["evt.MarshaledTime","evt.Meta.timestamp"]' +} + +@test "attempt to load from default context file, ignore if missing" { + rune -0 rm -f "$CONTEXT_YAML" + rune -0 "$CROWDSEC" -t --trace + assert_stderr --partial "loading console context from $CONTEXT_YAML" +} + +@test "error if context file is explicitly set but does not exist" { + config_set ".crowdsec_service.console_context_path=strenv(CONTEXT_YAML)" + rune -0 rm -f "$CONTEXT_YAML" + rune -1 "$CROWDSEC" -t + assert_stderr --partial "while checking console_context_path: stat $CONTEXT_YAML: no such file or directory" +} + +@test "context file is bad" { + echo "bad yaml" > "$CONTEXT_YAML" + rune -1 "$CROWDSEC" -t + assert_stderr --partial "while loading context: $CONTEXT_YAML: yaml: unmarshal errors" +} + +@test "context file is good" { + echo '{"source_ip":["evt.Parsed.source_ip"]}' > "$CONTEXT_YAML" + rune -0 "$CROWDSEC" -t --debug + assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}' +} + +@test "context file is from hub (local item)" { + mkdir -p "$CONFIG_DIR/contexts" + config_set "del(.crowdsec_service.console_context_path)" + echo '{"context":{"source_ip":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/foobar.yaml" + rune -0 "$CROWDSEC" -t --trace + assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/foobar.yaml" + assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}' +} + +@test "merge multiple contexts" { + mkdir -p "$CONFIG_DIR/contexts" + echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml" + echo '{"context":{"two":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/two.yaml" + rune -0 "$CROWDSEC" -t --trace + assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml" + assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/two.yaml" + assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip"],"two":["evt.Parsed.source_ip"]}' +} + +@test "merge contexts from hub and context.yaml file" { + mkdir -p "$CONFIG_DIR/contexts" + echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml" + echo '{"one":["evt.Parsed.source_ip_2"]}' > "$CONFIG_DIR/console/context.yaml" + rune -0 "$CROWDSEC" -t --trace + assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml" + assert_stderr --partial "loading console context from $CONFIG_DIR/console/context.yaml" + assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip","evt.Parsed.source_ip_2"]}' +} diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index 0b222dde0..010431a13 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -36,7 +36,7 @@ teardown() { rune -0 cscli hub list assert_output "No items to display" rune -0 cscli hub list -o json - assert_json '{"appsec-configs":[],"appsec-rules":[],parsers:[],scenarios:[],collections:[],postoverflows:[]}' + assert_json '{"appsec-configs":[],"appsec-rules":[],parsers:[],scenarios:[],collections:[],contexts:[],postoverflows:[]}' rune -0 cscli hub list -o raw assert_output 'name,status,version,description,type' @@ -47,6 +47,7 @@ teardown() { assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*" refute_output --partial 'POSTOVERFLOWS' refute_output --partial 'COLLECTIONS' + rune -0 cscli hub list -o json rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output) rune -0 cscli hub list -o raw @@ -55,8 +56,11 @@ teardown() { refute_output --partial 'crowdsecurity/iptables' # all items + mkdir -p "$CONFIG_DIR/contexts" + # there are no contexts yet, so we create a local one + touch "$CONFIG_DIR/contexts/mycontext.yaml" rune -0 cscli hub list -a - assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*crowdsecurity/iptables.*" + assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*CONTEXTS.*mycontext.yaml.*COLLECTIONS.*crowdsecurity/iptables.*" rune -0 cscli hub list -a -o json rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output) rune -0 cscli hub list -a -o raw @@ -107,6 +111,8 @@ teardown() { assert_stderr --partial "Upgraded 0 postoverflows" assert_stderr --partial "Upgrading scenarios" assert_stderr --partial "Upgraded 0 scenarios" + assert_stderr --partial "Upgrading contexts" + assert_stderr --partial "Upgraded 0 contexts" assert_stderr --partial "Upgrading collections" assert_stderr --partial "Upgraded 0 collections" @@ -134,10 +140,11 @@ teardown() { assert_line "parsers" assert_line "postoverflows" assert_line "scenarios" + assert_line "contexts" assert_line "collections" rune -0 cscli hub types -o human rune -0 yq -o json <(output) - assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]' + assert_json '["parsers","postoverflows","scenarios","contexts","appsec-configs","appsec-rules","collections"]' rune -0 cscli hub types -o json - assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]' + assert_json '["parsers","postoverflows","scenarios","contexts","appsec-configs","appsec-rules","collections"]' } diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index a70d14be3..d841bbe7c 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -247,12 +247,14 @@ hub_purge_all() { "$CONFIG_DIR"/collections/* \ "$CONFIG_DIR"/parsers/*/* \ "$CONFIG_DIR"/scenarios/* \ - "$CONFIG_DIR"/postoverflows/* + "$CONFIG_DIR"/postoverflows/* \ + "$CONFIG_DIR"/contexts/* rm -rf \ "$CONFIG_DIR"/hub/collections/* \ "$CONFIG_DIR"/hub/parsers/*/* \ "$CONFIG_DIR"/hub/scenarios/* \ - "$CONFIG_DIR"/hub/postoverflows/* + "$CONFIG_DIR"/hub/postoverflows/* \ + "$CONFIG_DIR"/hub/contexts/* local DATA_DIR DATA_DIR=$(config_get .config_paths.data_dir) # should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete),