beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

load console context from hub

Browse source
This commit is contained in:
Marco Mariani 2023-12-01 15:19:03 +01:00
parent c1a04ead79
commit eb1bea26cd
6 changed files with 216 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
cmd/crowdsec-cli/lapi.go
View file

@ -270,6 +270,15 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if keyToAdd != "" {
if err := AddContext(keyToAdd, valuesToAdd); err != nil {
return err
@ -299,6 +308,15 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
Short: "List context to send with alerts",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if len(csConfig.Crowdsec.ContextToSend) == 0 {
fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
return nil
@ -309,7 +327,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
return fmt.Errorf("unable to show context status: %w", err)
}
fmt.Println(string(dump))
fmt.Print(string(dump))
return nil
},
@ -413,6 +431,12 @@ cscli lapi context delete --value evt.Line.Src
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
// pass a nil hub to load only from console/context.yaml
if err := alertcontext.LoadConsoleContext(csConfig, nil); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
return errors.New("please provide at least a key or a value to delete")
}

6
cmd/crowdsec/crowdsec.go
View file

@ -14,6 +14,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@ -24,6 +25,10 @@ import (
func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
var err error
if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
return nil, fmt.Errorf("while loading context: %w", err)
}
// Start loading configs
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
@ -41,6 +46,7 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er
if err := LoadAcquisition(cConfig); err != nil {
return nil, fmt.Errorf("while loading acquisition config: %w", err)
}
return csParsers, nil
}

125
pkg/alertcontext/config.go Normal file
View file

@ -0,0 +1,125 @@
package alertcontext
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// this file is here to avoid circular dependencies between the configuration and the hub
// HubItemWrapper is a wrapper around a hub item to unmarshal only the context part
// because there are other fields like name etc.
type HubItemWrapper struct {
Context map[string][]string `yaml:"context"`
}
// mergeContext adds the context from src to dest.
func mergeContext(dest map[string][]string, src map[string][]string) {
for k, v := range src {
if _, ok := dest[k]; !ok {
dest[k] = make([]string, 0)
}
for _, s := range v {
if !slices.Contains(dest[k], s) {
dest[k] = append(dest[k], s)
}
}
}
}
// addContextFromItem merges the context from an item into the context to send to the console.
func addContextFromItem(toSend map[string][]string, item *cwhub.Item) error {
filePath := item.State.LocalPath
log.Tracef("loading console context from %s", filePath)
content, err := os.ReadFile(filePath)
if err != nil {
return err
}
wrapper := &HubItemWrapper{}
err = yaml.Unmarshal(content, wrapper)
if err != nil {
return fmt.Errorf("%s: %w", filePath, err)
}
mergeContext(toSend, wrapper.Context)
return nil
}
// addContextFromFile merges the context from a file into the context to send to the console.
func addContextFromFile(toSend map[string][]string, filePath string) error {
log.Tracef("loading console context from %s", filePath)
content, err := os.ReadFile(filePath)
if err != nil {
return err
}
newContext := make(map[string][]string, 0)
err = yaml.Unmarshal(content, newContext)
if err != nil {
return fmt.Errorf("%s: %w", filePath, err)
}
mergeContext(toSend, newContext)
return nil
}
// LoadConsoleContext loads the context from the hub (if provided) and the file console_context_path.
func LoadConsoleContext(c *csconfig.Config, hub *cwhub.Hub) error {
c.Crowdsec.ContextToSend = make(map[string][]string, 0)
if hub != nil {
items, err := hub.GetInstalledItems(cwhub.CONTEXTS)
if err != nil {
return err
}
for _, item := range items {
// context in item files goes under the key 'context'
if err = addContextFromItem(c.Crowdsec.ContextToSend, item); err != nil {
return err
}
}
}
ignoreMissing := false
if c.Crowdsec.ConsoleContextPath != "" {
// if it's provided, it must exist
if _, err := os.Stat(c.Crowdsec.ConsoleContextPath); err != nil {
return fmt.Errorf("while checking console_context_path: %w", err)
}
} else {
c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
ignoreMissing = true
}
if err := addContextFromFile(c.Crowdsec.ContextToSend, c.Crowdsec.ConsoleContextPath); err != nil {
if !ignoreMissing || !os.IsNotExist(err) {
return err
}
}
feedback, err := json.Marshal(c.Crowdsec.ContextToSend)
if err != nil {
return fmt.Errorf("marshaling console context: %s", err)
}
log.Debugf("console context to send: %s", feedback)
return nil
}

45
pkg/csconfig/crowdsec_service.go
View file

@ -1,7 +1,6 @@
package csconfig
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
@ -137,55 +136,23 @@ func (c *Config) LoadCrowdsec() error {
return fmt.Errorf("loading api client: %s", err)
}
if c.Crowdsec.ConsoleContextPath != "" {
// if it's provided, it must exist
if _, err = os.Stat(c.Crowdsec.ConsoleContextPath); err != nil {
return fmt.Errorf("while checking console_context_path: %w", err)
}
} else {
c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
}
c.Crowdsec.ContextToSend, err = buildContextToSend(c)
if err != nil {
return err
}
return nil
}
func buildContextToSend(c *Config) (map[string][]string, error) {
ret := make(map[string][]string, 0)
log.Tracef("loading console context from %s", c.Crowdsec.ConsoleContextPath)
content, err := os.ReadFile(c.Crowdsec.ConsoleContextPath)
if err != nil && !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to open context file: %s", err)
}
err = yaml.Unmarshal(content, ret)
if err != nil {
return nil, fmt.Errorf("while loading context from %s: %s", c.Crowdsec.ConsoleContextPath, err)
}
feedback, err := json.Marshal(ret)
if err != nil {
return nil, fmt.Errorf("marshaling console context: %s", err)
}
log.Debugf("console context to send: %s", feedback)
return ret, nil
}
func (c *CrowdsecServiceCfg) DumpContextConfigFile() error {
var out []byte
var err error
// XXX: MakeDirs
if out, err = yaml.Marshal(c.ContextToSend); err != nil {
return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleContextPath, err)
}
if err = os.MkdirAll(filepath.Dir(c.ConsoleContextPath), 0700); err != nil {
return fmt.Errorf("while creating directories for %s: %w", c.ConsoleContextPath, err)
}
if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil {
return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleContextPath, err)
}

21
pkg/csconfig/crowdsec_service_test.go
View file

@ -60,9 +60,10 @@ func TestLoadCrowdsec(t *testing.T) {
ConsoleContextValueLength: 2500,
AcquisitionFiles: []string{acquisFullPath},
SimulationFilePath: "./testdata/simulation.yaml",
ContextToSend: map[string][]string{
"source_ip": {"evt.Parsed.source_ip"},
},
// context is loaded in pkg/alertcontext
// ContextToSend: map[string][]string{
// "source_ip": {"evt.Parsed.source_ip"},
// },
SimulationConfig: &SimulationConfig{
Simulation: ptr.Of(false),
},
@ -98,9 +99,10 @@ func TestLoadCrowdsec(t *testing.T) {
OutputRoutinesCount: 1,
ConsoleContextValueLength: 0,
AcquisitionFiles: []string{acquisFullPath, acquisInDirFullPath},
ContextToSend: map[string][]string{
"source_ip": {"evt.Parsed.source_ip"},
},
// context is loaded in pkg/alertcontext
// ContextToSend: map[string][]string{
// "source_ip": {"evt.Parsed.source_ip"},
// },
SimulationFilePath: "./testdata/simulation.yaml",
SimulationConfig: &SimulationConfig{
Simulation: ptr.Of(false),
@ -136,9 +138,10 @@ func TestLoadCrowdsec(t *testing.T) {
ConsoleContextValueLength: 10,
AcquisitionFiles: []string{},
SimulationFilePath: "",
ContextToSend: map[string][]string{
"source_ip": {"evt.Parsed.source_ip"},
},
// context is loaded in pkg/alertcontext
// ContextToSend: map[string][]string{
// "source_ip": {"evt.Parsed.source_ip"},
// },
SimulationConfig: &SimulationConfig{
Simulation: ptr.Of(false),
},

44
test/bats/09_context.bats
View file

@ -29,6 +29,17 @@ teardown() {
#----------
@test "detect available context" {
rune -0 cscli lapi context detect -a
rune -0 yq -o json <(output)
assert_json '{"Acquisition":["evt.Line.Module","evt.Line.Raw","evt.Line.Src"]}'
rune -0 cscli parsers install crowdsecurity/dateparse-enrich
rune -0 cscli lapi context detect crowdsecurity/dateparse-enrich
rune -0 yq -o json '.crowdsecurity/dateparse-enrich' <(output)
assert_json '["evt.MarshaledTime","evt.Meta.timestamp"]'
}
@test "attempt to load from default context file, ignore if missing" {
rune -0 rm -f "$CONTEXT_YAML"
rune -0 "$CROWDSEC" -t --trace
@ -36,7 +47,7 @@ teardown() {
}
@test "error if context file is explicitly set but does not exist" {
config_set ".crowdsec_service.console_context_path=\"$CONTEXT_YAML\""
config_set ".crowdsec_service.console_context_path=strenv(CONTEXT_YAML)"
rune -0 rm -f "$CONTEXT_YAML"
rune -1 "$CROWDSEC" -t
assert_stderr --partial "while checking console_context_path: stat $CONTEXT_YAML: no such file or directory"
@ -45,7 +56,7 @@ teardown() {
@test "context file is bad" {
echo "bad yaml" > "$CONTEXT_YAML"
rune -1 "$CROWDSEC" -t
assert_stderr --partial "while loading context from $CONTEXT_YAML: yaml: unmarshal errors"
assert_stderr --partial "while loading context: $CONTEXT_YAML: yaml: unmarshal errors"
}
@test "context file is good" {
@ -53,3 +64,32 @@ teardown() {
rune -0 "$CROWDSEC" -t --debug
assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
}
@test "context file is from hub (local item)" {
mkdir -p "$CONFIG_DIR/contexts"
config_set "del(.crowdsec_service.console_context_path)"
echo '{"context":{"source_ip":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/foobar.yaml"
rune -0 "$CROWDSEC" -t --trace
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/foobar.yaml"
assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
}
@test "merge multiple contexts" {
mkdir -p "$CONFIG_DIR/contexts"
echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
echo '{"context":{"two":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/two.yaml"
rune -0 "$CROWDSEC" -t --trace
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/two.yaml"
assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip"],"two":["evt.Parsed.source_ip"]}'
}
@test "merge contexts from hub and context.yaml file" {
mkdir -p "$CONFIG_DIR/contexts"
echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
echo '{"one":["evt.Parsed.source_ip_2"]}' > "$CONFIG_DIR/console/context.yaml"
rune -0 "$CROWDSEC" -t --trace
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
assert_stderr --partial "loading console context from $CONFIG_DIR/console/context.yaml"
assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip","evt.Parsed.source_ip_2"]}'
}