diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index 0a8f6c98b..d2858c7be 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -270,6 +270,15 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil { + return fmt.Errorf("while loading context: %w", err) + } + if keyToAdd != "" { if err := AddContext(keyToAdd, valuesToAdd); err != nil { return err @@ -299,6 +308,15 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user Short: "List context to send with alerts", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil { + return fmt.Errorf("while loading context: %w", err) + } + if len(csConfig.Crowdsec.ContextToSend) == 0 { fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.") return nil @@ -309,7 +327,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user return fmt.Errorf("unable to show context status: %w", err) } - fmt.Println(string(dump)) + fmt.Print(string(dump)) return nil }, @@ -413,6 +431,12 @@ cscli lapi context delete --value evt.Line.Src `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { + // pass a nil hub to load only from console/context.yaml + + if err := alertcontext.LoadConsoleContext(csConfig, nil); err != nil { + return fmt.Errorf("while loading context: %w", err) + } + if len(keysToDelete) == 0 && len(valuesToDelete) == 0 { return errors.New("please provide at least a key or a value to delete") } diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 1e0d54c07..774b9d381 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -14,6 +14,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/alertcontext" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" @@ -24,6 +25,10 @@ import ( func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) { var err error + if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil { + return nil, fmt.Errorf("while loading context: %w", err) + } + // Start loading configs csParsers := parser.NewParsers(hub) if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { @@ -41,6 +46,7 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er if err := LoadAcquisition(cConfig); err != nil { return nil, fmt.Errorf("while loading acquisition config: %w", err) } + return csParsers, nil } diff --git a/pkg/alertcontext/config.go b/pkg/alertcontext/config.go new file mode 100644 index 000000000..2305fb384 --- /dev/null +++ b/pkg/alertcontext/config.go @@ -0,0 +1,125 @@ +package alertcontext + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +// this file is here to avoid circular dependencies between the configuration and the hub + +// HubItemWrapper is a wrapper around a hub item to unmarshal only the context part +// because there are other fields like name etc. +type HubItemWrapper struct { + Context map[string][]string `yaml:"context"` +} + +// mergeContext adds the context from src to dest. +func mergeContext(dest map[string][]string, src map[string][]string) { + for k, v := range src { + if _, ok := dest[k]; !ok { + dest[k] = make([]string, 0) + } + for _, s := range v { + if !slices.Contains(dest[k], s) { + dest[k] = append(dest[k], s) + } + } + } +} + +// addContextFromItem merges the context from an item into the context to send to the console. +func addContextFromItem(toSend map[string][]string, item *cwhub.Item) error { + filePath := item.State.LocalPath + log.Tracef("loading console context from %s", filePath) + content, err := os.ReadFile(filePath) + if err != nil { + return err + } + + wrapper := &HubItemWrapper{} + + err = yaml.Unmarshal(content, wrapper) + if err != nil { + return fmt.Errorf("%s: %w", filePath, err) + } + + mergeContext(toSend, wrapper.Context) + + return nil +} + +// addContextFromFile merges the context from a file into the context to send to the console. +func addContextFromFile(toSend map[string][]string, filePath string) error { + log.Tracef("loading console context from %s", filePath) + content, err := os.ReadFile(filePath) + if err != nil { + return err + } + + newContext := make(map[string][]string, 0) + + err = yaml.Unmarshal(content, newContext) + if err != nil { + return fmt.Errorf("%s: %w", filePath, err) + } + + mergeContext(toSend, newContext) + + return nil +} + + +// LoadConsoleContext loads the context from the hub (if provided) and the file console_context_path. +func LoadConsoleContext(c *csconfig.Config, hub *cwhub.Hub) error { + c.Crowdsec.ContextToSend = make(map[string][]string, 0) + + if hub != nil { + items, err := hub.GetInstalledItems(cwhub.CONTEXTS) + if err != nil { + return err + } + + for _, item := range items { + // context in item files goes under the key 'context' + if err = addContextFromItem(c.Crowdsec.ContextToSend, item); err != nil { + return err + } + } + } + + ignoreMissing := false + + if c.Crowdsec.ConsoleContextPath != "" { + // if it's provided, it must exist + if _, err := os.Stat(c.Crowdsec.ConsoleContextPath); err != nil { + return fmt.Errorf("while checking console_context_path: %w", err) + } + } else { + c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml") + ignoreMissing = true + } + + if err := addContextFromFile(c.Crowdsec.ContextToSend, c.Crowdsec.ConsoleContextPath); err != nil { + if !ignoreMissing || !os.IsNotExist(err) { + return err + } + } + + feedback, err := json.Marshal(c.Crowdsec.ContextToSend) + if err != nil { + return fmt.Errorf("marshaling console context: %s", err) + } + + log.Debugf("console context to send: %s", feedback) + + return nil +} diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index 839cda617..36d38cf74 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -1,7 +1,6 @@ package csconfig import ( - "encoding/json" "fmt" "os" "path/filepath" @@ -137,55 +136,23 @@ func (c *Config) LoadCrowdsec() error { return fmt.Errorf("loading api client: %s", err) } - if c.Crowdsec.ConsoleContextPath != "" { - // if it's provided, it must exist - if _, err = os.Stat(c.Crowdsec.ConsoleContextPath); err != nil { - return fmt.Errorf("while checking console_context_path: %w", err) - } - } else { - c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml") - } - - c.Crowdsec.ContextToSend, err = buildContextToSend(c) - if err != nil { - return err - } - return nil } -func buildContextToSend(c *Config) (map[string][]string, error) { - ret := make(map[string][]string, 0) - - log.Tracef("loading console context from %s", c.Crowdsec.ConsoleContextPath) - content, err := os.ReadFile(c.Crowdsec.ConsoleContextPath) - if err != nil && !os.IsNotExist(err) { - return nil, fmt.Errorf("failed to open context file: %s", err) - } - - err = yaml.Unmarshal(content, ret) - if err != nil { - return nil, fmt.Errorf("while loading context from %s: %s", c.Crowdsec.ConsoleContextPath, err) - } - - feedback, err := json.Marshal(ret) - if err != nil { - return nil, fmt.Errorf("marshaling console context: %s", err) - } - - log.Debugf("console context to send: %s", feedback) - - return ret, nil -} - func (c *CrowdsecServiceCfg) DumpContextConfigFile() error { var out []byte var err error + // XXX: MakeDirs + if out, err = yaml.Marshal(c.ContextToSend); err != nil { return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleContextPath, err) } + if err = os.MkdirAll(filepath.Dir(c.ConsoleContextPath), 0700); err != nil { + return fmt.Errorf("while creating directories for %s: %w", c.ConsoleContextPath, err) + } + if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil { return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleContextPath, err) } diff --git a/pkg/csconfig/crowdsec_service_test.go b/pkg/csconfig/crowdsec_service_test.go index e9d7e8de3..8d332271b 100644 --- a/pkg/csconfig/crowdsec_service_test.go +++ b/pkg/csconfig/crowdsec_service_test.go @@ -60,9 +60,10 @@ func TestLoadCrowdsec(t *testing.T) { ConsoleContextValueLength: 2500, AcquisitionFiles: []string{acquisFullPath}, SimulationFilePath: "./testdata/simulation.yaml", - ContextToSend: map[string][]string{ - "source_ip": {"evt.Parsed.source_ip"}, - }, + // context is loaded in pkg/alertcontext +// ContextToSend: map[string][]string{ +// "source_ip": {"evt.Parsed.source_ip"}, +// }, SimulationConfig: &SimulationConfig{ Simulation: ptr.Of(false), }, @@ -98,9 +99,10 @@ func TestLoadCrowdsec(t *testing.T) { OutputRoutinesCount: 1, ConsoleContextValueLength: 0, AcquisitionFiles: []string{acquisFullPath, acquisInDirFullPath}, - ContextToSend: map[string][]string{ - "source_ip": {"evt.Parsed.source_ip"}, - }, + // context is loaded in pkg/alertcontext +// ContextToSend: map[string][]string{ +// "source_ip": {"evt.Parsed.source_ip"}, +// }, SimulationFilePath: "./testdata/simulation.yaml", SimulationConfig: &SimulationConfig{ Simulation: ptr.Of(false), @@ -136,9 +138,10 @@ func TestLoadCrowdsec(t *testing.T) { ConsoleContextValueLength: 10, AcquisitionFiles: []string{}, SimulationFilePath: "", - ContextToSend: map[string][]string{ - "source_ip": {"evt.Parsed.source_ip"}, - }, + // context is loaded in pkg/alertcontext +// ContextToSend: map[string][]string{ +// "source_ip": {"evt.Parsed.source_ip"}, +// }, SimulationConfig: &SimulationConfig{ Simulation: ptr.Of(false), }, diff --git a/test/bats/09_context.bats b/test/bats/09_context.bats index 8b9ddf5a5..c56d3e773 100644 --- a/test/bats/09_context.bats +++ b/test/bats/09_context.bats @@ -29,6 +29,17 @@ teardown() { #---------- +@test "detect available context" { + rune -0 cscli lapi context detect -a + rune -0 yq -o json <(output) + assert_json '{"Acquisition":["evt.Line.Module","evt.Line.Raw","evt.Line.Src"]}' + + rune -0 cscli parsers install crowdsecurity/dateparse-enrich + rune -0 cscli lapi context detect crowdsecurity/dateparse-enrich + rune -0 yq -o json '.crowdsecurity/dateparse-enrich' <(output) + assert_json '["evt.MarshaledTime","evt.Meta.timestamp"]' +} + @test "attempt to load from default context file, ignore if missing" { rune -0 rm -f "$CONTEXT_YAML" rune -0 "$CROWDSEC" -t --trace @@ -36,7 +47,7 @@ teardown() { } @test "error if context file is explicitly set but does not exist" { - config_set ".crowdsec_service.console_context_path=\"$CONTEXT_YAML\"" + config_set ".crowdsec_service.console_context_path=strenv(CONTEXT_YAML)" rune -0 rm -f "$CONTEXT_YAML" rune -1 "$CROWDSEC" -t assert_stderr --partial "while checking console_context_path: stat $CONTEXT_YAML: no such file or directory" @@ -45,7 +56,7 @@ teardown() { @test "context file is bad" { echo "bad yaml" > "$CONTEXT_YAML" rune -1 "$CROWDSEC" -t - assert_stderr --partial "while loading context from $CONTEXT_YAML: yaml: unmarshal errors" + assert_stderr --partial "while loading context: $CONTEXT_YAML: yaml: unmarshal errors" } @test "context file is good" { @@ -53,3 +64,32 @@ teardown() { rune -0 "$CROWDSEC" -t --debug assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}' } + +@test "context file is from hub (local item)" { + mkdir -p "$CONFIG_DIR/contexts" + config_set "del(.crowdsec_service.console_context_path)" + echo '{"context":{"source_ip":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/foobar.yaml" + rune -0 "$CROWDSEC" -t --trace + assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/foobar.yaml" + assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}' +} + +@test "merge multiple contexts" { + mkdir -p "$CONFIG_DIR/contexts" + echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml" + echo '{"context":{"two":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/two.yaml" + rune -0 "$CROWDSEC" -t --trace + assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml" + assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/two.yaml" + assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip"],"two":["evt.Parsed.source_ip"]}' +} + +@test "merge contexts from hub and context.yaml file" { + mkdir -p "$CONFIG_DIR/contexts" + echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml" + echo '{"one":["evt.Parsed.source_ip_2"]}' > "$CONFIG_DIR/console/context.yaml" + rune -0 "$CROWDSEC" -t --trace + assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml" + assert_stderr --partial "loading console context from $CONFIG_DIR/console/context.yaml" + assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip","evt.Parsed.source_ip_2"]}' +}