load console context from hub

cmd/crowdsec-cli/lapi.go

@@ -270,6 +270,15 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 		`,
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
+				return err
+			}
+
+			if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
+				return fmt.Errorf("while loading context: %w", err)
+			}
+
 			if keyToAdd != "" {
 				if err := AddContext(keyToAdd, valuesToAdd); err != nil {
 					return err
@@ -299,6 +308,15 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 		Short:             "List context to send with alerts",
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
+				return err
+			}
+
+			if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
+				return fmt.Errorf("while loading context: %w", err)
+			}
+
 			if len(csConfig.Crowdsec.ContextToSend) == 0 {
 				fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
 				return nil
@@ -309,7 +327,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 				return fmt.Errorf("unable to show context status: %w", err)
 			}
 
-			fmt.Println(string(dump))
+			fmt.Print(string(dump))
 
 			return nil
 		},
@@ -413,6 +431,12 @@ cscli lapi context delete --value evt.Line.Src
 		`,
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
+			// pass a nil hub to load only from console/context.yaml
+
+			if err := alertcontext.LoadConsoleContext(csConfig, nil); err != nil {
+				return fmt.Errorf("while loading context: %w", err)
+			}
+
 			if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
 				return errors.New("please provide at least a key or a value to delete")
 			}

cmd/crowdsec/crowdsec.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@@ -24,6 +25,10 @@ import (
 func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
 	var err error
 
+	if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
+		return nil, fmt.Errorf("while loading context: %w", err)
+	}
+
 	// Start loading configs
 	csParsers := parser.NewParsers(hub)
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
@@ -41,6 +46,7 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er
 	if err := LoadAcquisition(cConfig); err != nil {
 		return nil, fmt.Errorf("while loading acquisition config: %w", err)
 	}
+
 	return csParsers, nil
 }
 

pkg/alertcontext/config.go

@@ -0,0 +1,125 @@
+package alertcontext
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
+
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+// this file is here to avoid circular dependencies between the configuration and the hub
+
+// HubItemWrapper is a wrapper around a hub item to unmarshal only the context part
+// because there are other fields like name etc.
+type HubItemWrapper struct {
+	Context map[string][]string `yaml:"context"`
+}
+
+// mergeContext adds the context from src to dest.
+func mergeContext(dest map[string][]string, src map[string][]string) {
+	for k, v := range src {
+		if _, ok := dest[k]; !ok {
+			dest[k] = make([]string, 0)
+		}
+		for _, s := range v {
+			if !slices.Contains(dest[k], s) {
+				dest[k] = append(dest[k], s)
+			}
+		}
+	}
+}
+
+// addContextFromItem merges the context from an item into the context to send to the console.
+func addContextFromItem(toSend map[string][]string, item *cwhub.Item) error {
+	filePath := item.State.LocalPath
+	log.Tracef("loading console context from %s", filePath)
+	content, err := os.ReadFile(filePath)
+	if err != nil {
+		return err
+	}
+
+	wrapper := &HubItemWrapper{}
+
+	err = yaml.Unmarshal(content, wrapper)
+	if err != nil {
+		return fmt.Errorf("%s: %w", filePath, err)
+	}
+
+	mergeContext(toSend, wrapper.Context)
+
+	return nil
+}
+
+// addContextFromFile merges the context from a file into the context to send to the console.
+func addContextFromFile(toSend map[string][]string, filePath string) error {
+	log.Tracef("loading console context from %s", filePath)
+	content, err := os.ReadFile(filePath)
+	if err != nil {
+		return err
+	}
+
+	newContext := make(map[string][]string, 0)
+
+	err = yaml.Unmarshal(content, newContext)
+	if err != nil {
+		return fmt.Errorf("%s: %w", filePath, err)
+	}
+
+	mergeContext(toSend, newContext)
+
+	return nil
+}
+
+
+// LoadConsoleContext loads the context from the hub (if provided) and the file console_context_path.
+func LoadConsoleContext(c *csconfig.Config, hub *cwhub.Hub) error {
+	c.Crowdsec.ContextToSend = make(map[string][]string, 0)
+
+	if hub != nil {
+		items, err := hub.GetInstalledItems(cwhub.CONTEXTS)
+		if err != nil {
+			return err
+		}
+
+		for _, item := range items {
+			// context in item files goes under the key 'context'
+			if err = addContextFromItem(c.Crowdsec.ContextToSend, item); err != nil {
+				return err
+			}
+		}
+	}
+
+	ignoreMissing := false
+
+	if c.Crowdsec.ConsoleContextPath != "" {
+		// if it's provided, it must exist
+		if _, err := os.Stat(c.Crowdsec.ConsoleContextPath); err != nil {
+			return fmt.Errorf("while checking console_context_path: %w", err)
+		}
+	} else {
+		c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
+		ignoreMissing = true
+	}
+
+	if err := addContextFromFile(c.Crowdsec.ContextToSend, c.Crowdsec.ConsoleContextPath); err != nil {
+		if !ignoreMissing || !os.IsNotExist(err) {
+			return err
+		}
+	}
+
+	feedback, err := json.Marshal(c.Crowdsec.ContextToSend)
+	if err != nil {
+		return fmt.Errorf("marshaling console context: %s", err)
+	}
+
+	log.Debugf("console context to send: %s", feedback)
+
+	return nil
+}

pkg/csconfig/crowdsec_service.go

@@ -1,7 +1,6 @@
 package csconfig
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -137,55 +136,23 @@ func (c *Config) LoadCrowdsec() error {
 		return fmt.Errorf("loading api client: %s", err)
 	}
 
-	if c.Crowdsec.ConsoleContextPath != "" {
-		// if it's provided, it must exist
-		if _, err = os.Stat(c.Crowdsec.ConsoleContextPath); err != nil {
-			return fmt.Errorf("while checking console_context_path: %w", err)
-		}
-	} else {
-		c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
-	}
-
-	c.Crowdsec.ContextToSend, err = buildContextToSend(c)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 
-func buildContextToSend(c *Config) (map[string][]string, error) {
-	ret := make(map[string][]string, 0)
-
-	log.Tracef("loading console context from %s", c.Crowdsec.ConsoleContextPath)
-	content, err := os.ReadFile(c.Crowdsec.ConsoleContextPath)
-	if err != nil && !os.IsNotExist(err) {
-		return nil, fmt.Errorf("failed to open context file: %s", err)
-	}
-
-	err = yaml.Unmarshal(content, ret)
-	if err != nil {
-		return nil, fmt.Errorf("while loading context from %s: %s", c.Crowdsec.ConsoleContextPath, err)
-	}
-
-	feedback, err := json.Marshal(ret)
-	if err != nil {
-		return nil, fmt.Errorf("marshaling console context: %s", err)
-	}
-
-	log.Debugf("console context to send: %s", feedback)
-
-	return ret, nil
-}
-
 func (c *CrowdsecServiceCfg) DumpContextConfigFile() error {
 	var out []byte
 	var err error
 
+	// XXX: MakeDirs
+
 	if out, err = yaml.Marshal(c.ContextToSend); err != nil {
 		return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleContextPath, err)
 	}
 
+	if err = os.MkdirAll(filepath.Dir(c.ConsoleContextPath), 0700); err != nil {
+		return fmt.Errorf("while creating directories for %s: %w", c.ConsoleContextPath, err)
+	}
+
 	if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil {
 		return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleContextPath, err)
 	}

pkg/csconfig/crowdsec_service_test.go

@@ -60,9 +60,10 @@ func TestLoadCrowdsec(t *testing.T) {
 				ConsoleContextValueLength: 2500,
 				AcquisitionFiles:          []string{acquisFullPath},
 				SimulationFilePath:        "./testdata/simulation.yaml",
-				ContextToSend: map[string][]string{
+				// context is loaded in pkg/alertcontext
-					"source_ip": {"evt.Parsed.source_ip"},
+//				ContextToSend: map[string][]string{
-				},
+//					"source_ip": {"evt.Parsed.source_ip"},
+//				},
 				SimulationConfig: &SimulationConfig{
 					Simulation: ptr.Of(false),
 				},
@@ -98,9 +99,10 @@ func TestLoadCrowdsec(t *testing.T) {
 				OutputRoutinesCount:       1,
 				ConsoleContextValueLength: 0,
 				AcquisitionFiles:          []string{acquisFullPath, acquisInDirFullPath},
-				ContextToSend: map[string][]string{
+				// context is loaded in pkg/alertcontext
-					"source_ip": {"evt.Parsed.source_ip"},
+//				ContextToSend: map[string][]string{
-				},
+//					"source_ip": {"evt.Parsed.source_ip"},
+//				},
 				SimulationFilePath: "./testdata/simulation.yaml",
 				SimulationConfig: &SimulationConfig{
 					Simulation: ptr.Of(false),
@@ -136,9 +138,10 @@ func TestLoadCrowdsec(t *testing.T) {
 				ConsoleContextValueLength: 10,
 				AcquisitionFiles:          []string{},
 				SimulationFilePath:        "",
-				ContextToSend: map[string][]string{
+				// context is loaded in pkg/alertcontext
-					"source_ip": {"evt.Parsed.source_ip"},
+//				ContextToSend: map[string][]string{
-				},
+//					"source_ip": {"evt.Parsed.source_ip"},
+//				},
 				SimulationConfig: &SimulationConfig{
 					Simulation: ptr.Of(false),
 				},

test/bats/09_context.bats

@@ -29,6 +29,17 @@ teardown() {
 
 #----------
 
+@test "detect available context" {
+    rune -0 cscli lapi context detect -a
+    rune -0 yq -o json <(output)
+    assert_json '{"Acquisition":["evt.Line.Module","evt.Line.Raw","evt.Line.Src"]}'
+
+    rune -0 cscli parsers install crowdsecurity/dateparse-enrich
+    rune -0 cscli lapi context detect crowdsecurity/dateparse-enrich
+    rune -0 yq -o json '.crowdsecurity/dateparse-enrich' <(output)
+    assert_json '["evt.MarshaledTime","evt.Meta.timestamp"]'
+}
+
 @test "attempt to load from default context file, ignore if missing" {
     rune -0 rm -f "$CONTEXT_YAML"
     rune -0 "$CROWDSEC" -t --trace
@@ -36,7 +47,7 @@ teardown() {
 }
 
 @test "error if context file is explicitly set but does not exist" {
-    config_set ".crowdsec_service.console_context_path=\"$CONTEXT_YAML\""
+    config_set ".crowdsec_service.console_context_path=strenv(CONTEXT_YAML)"
     rune -0 rm -f "$CONTEXT_YAML"
     rune -1 "$CROWDSEC" -t
     assert_stderr --partial "while checking console_context_path: stat $CONTEXT_YAML: no such file or directory"
@@ -45,7 +56,7 @@ teardown() {
 @test "context file is bad" {
     echo "bad yaml" > "$CONTEXT_YAML"
     rune -1 "$CROWDSEC" -t
-    assert_stderr --partial "while loading context from $CONTEXT_YAML: yaml: unmarshal errors"
+    assert_stderr --partial "while loading context: $CONTEXT_YAML: yaml: unmarshal errors"
 }
 
 @test "context file is good" {
@@ -53,3 +64,32 @@ teardown() {
     rune -0 "$CROWDSEC" -t --debug
     assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
 }
+
+@test "context file is from hub (local item)" {
+    mkdir -p "$CONFIG_DIR/contexts"
+    config_set "del(.crowdsec_service.console_context_path)"
+    echo '{"context":{"source_ip":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/foobar.yaml"
+    rune -0 "$CROWDSEC" -t --trace
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/foobar.yaml"
+    assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
+}
+
+@test "merge multiple contexts" {
+    mkdir -p "$CONFIG_DIR/contexts"
+    echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
+    echo '{"context":{"two":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/two.yaml"
+    rune -0 "$CROWDSEC" -t --trace
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/two.yaml"
+    assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip"],"two":["evt.Parsed.source_ip"]}'
+}
+
+@test "merge contexts from hub and context.yaml file" {
+    mkdir -p "$CONFIG_DIR/contexts"
+    echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
+    echo '{"one":["evt.Parsed.source_ip_2"]}' > "$CONFIG_DIR/console/context.yaml"
+    rune -0 "$CROWDSEC" -t --trace
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
+    assert_stderr --partial "loading console context from $CONFIG_DIR/console/context.yaml"
+    assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip","evt.Parsed.source_ip_2"]}'
+}