beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Alert context (#1895)

Browse source 
Co-authored-by: bui <thibault@crowdsec.net>
This commit is contained in:
AlteredCoder 2023-01-04 16:50:02 +01:00 committed by GitHub
parent dfc4126384
commit 185f9ad541
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 1201 additions and 142 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
cmd/crowdsec-cli/alerts.go
View file

@ -7,6 +7,7 @@ import (
"fmt"
"net/url"
"os"
"sort"
"strconv"
"strings"
@ -112,6 +113,29 @@ func DisplayOneAlert(alert *models.Alert, withDetail bool) error {
alertDecisionsTable(color.Output, alert)
if len(alert.Meta) > 0 {
fmt.Printf("\n - Context :\n")
sort.Slice(alert.Meta, func(i, j int) bool {
return alert.Meta[i].Key < alert.Meta[j].Key
})
table := newTable(color.Output)
table.SetRowLines(false)
table.SetHeaders("Key", "Value")
for _, meta := range alert.Meta {
var valSlice []string
if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil {
return fmt.Errorf("unknown context value type '%s' : %s", meta.Value, err)
}
for _, value := range valSlice {
table.AddRow(
meta.Key,
value,
)
}
}
table.Render()
}
if withDetail {
fmt.Printf("\n - Events :\n")
for _, event := range alert.Events {

40
cmd/crowdsec-cli/console.go
View file

@ -46,7 +46,7 @@ func NewConsoleCmd() *cobra.Command {
log.Fatalf("No configuration for Central API (CAPI) in '%s'", *csConfig.FilePath)
}
if csConfig.API.Server.OnlineClient.Credentials == nil {
log.Fatal("You must configure Central API (CAPI) with `cscli capi register` before enrolling your instance")
log.Fatal("You must configure Central API (CAPI) with `cscli capi register` before accessing console features.")
}
return nil
},
@ -129,9 +129,9 @@ After running this command your will need to validate the enrollment in the weba
var enableAll, disableAll bool
cmdEnable := &cobra.Command{
Use: "enable [feature-flag]",
Short: "Enable a feature flag",
Example: "enable tainted",
Use: "enable [option]",
Short: "Enable a console option",
Example: "sudo cscli console enable tainted",
Long: `
Enable given information push to the central API. Allows to empower the console`,
ValidArgs: csconfig.CONSOLE_CONFIGS,
@ -153,13 +153,13 @@ Enable given information push to the central API. Allows to empower the console`
log.Infof(ReloadMessage())
},
}
cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all feature flags")
cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
cmdConsole.AddCommand(cmdEnable)
cmdDisable := &cobra.Command{
Use: "disable [feature-flag]",
Short: "Disable a feature flag",
Example: "disable tainted",
Use: "disable [option]",
Short: "Disable a console option",
Example: "sudo cscli console disable tainted",
Long: `
Disable given information push to the central API.`,
ValidArgs: csconfig.CONSOLE_CONFIGS,
@ -183,13 +183,13 @@ Disable given information push to the central API.`,
log.Infof(ReloadMessage())
},
}
cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Enable all feature flags")
cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
cmdConsole.AddCommand(cmdDisable)
cmdConsoleStatus := &cobra.Command{
Use: "status [feature-flag]",
Short: "Shows status of one or all feature flags",
Example: "status tainted",
Use: "status [option]",
Short: "Shows status of one or all console options",
Example: `sudo cscli console status`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
switch csConfig.Cscli.Output {
@ -212,6 +212,7 @@ Disable given information push to the central API.`,
{"share_manual_decisions", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareManualDecisions)},
{"share_custom", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios)},
{"share_tainted", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios)},
{"share_context", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareContext)},
}
for _, row := range rows {
err = csvwriter.Write(row)
@ -223,8 +224,8 @@ Disable given information push to the central API.`,
}
},
}
cmdConsole.AddCommand(cmdConsoleStatus)
return cmdConsole
}
@ -270,6 +271,19 @@ func SetConsoleOpts(args []string, wanted bool) {
log.Infof("%s set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted)
csConfig.API.Server.ConsoleConfig.ShareManualDecisions = types.BoolPtr(wanted)
}
case csconfig.SEND_CONTEXT:
/*for each flag check if it's already set before setting it*/
if csConfig.API.Server.ConsoleConfig.ShareContext != nil {
if *csConfig.API.Server.ConsoleConfig.ShareContext == wanted {
log.Infof("%s already set to %t", csconfig.SEND_CONTEXT, wanted)
} else {
log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted)
*csConfig.API.Server.ConsoleConfig.ShareContext = wanted
}
} else {
log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted)
csConfig.API.Server.ConsoleConfig.ShareContext = types.BoolPtr(wanted)
}
default:
log.Fatalf("unknown flag %s", arg)
}

6
cmd/crowdsec-cli/console_table.go
View file

@ -41,6 +41,12 @@ func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) {
}
t.AddRow(option, activated, "Send alerts from tainted scenarios to the console")
case csconfig.SEND_CONTEXT:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareContext {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send context with alerts to the console")
}
}

357
cmd/crowdsec-cli/lapi.go
View file

@ -5,6 +5,7 @@ import (
"fmt"
"net/url"
"os"
"sort"
"strings"
"github.com/go-openapi/strfmt"
@ -13,16 +14,20 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
var LAPIURLPrefix string = "v1"
func runLapiStatus (cmd *cobra.Command, args []string) error {
func runLapiStatus(cmd *cobra.Command, args []string) error {
var err error
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
@ -68,7 +73,6 @@ func runLapiStatus (cmd *cobra.Command, args []string) error {
return nil
}
func runLapiRegister(cmd *cobra.Command, args []string) error {
var err error
@ -160,7 +164,6 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
return nil
}
func NewLapiStatusCmd() *cobra.Command {
cmdLapiStatus := &cobra.Command{
Use: "status",
@ -173,7 +176,6 @@ func NewLapiStatusCmd() *cobra.Command {
return cmdLapiStatus
}
func NewLapiRegisterCmd() *cobra.Command {
cmdLapiRegister := &cobra.Command{
Use: "register",
@ -182,7 +184,7 @@ func NewLapiRegisterCmd() *cobra.Command {
Keep in mind the machine needs to be validated by an administrator on LAPI side to be effective.`,
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: runLapiRegister,
RunE: runLapiRegister,
}
flags := cmdLapiRegister.Flags()
@ -193,7 +195,6 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
return cmdLapiRegister
}
func NewLapiCmd() *cobra.Command {
var cmdLapi = &cobra.Command{
Use: "lapi [action]",
@ -210,6 +211,350 @@ func NewLapiCmd() *cobra.Command {
cmdLapi.AddCommand(NewLapiRegisterCmd())
cmdLapi.AddCommand(NewLapiStatusCmd())
cmdLapi.AddCommand(NewLapiContextCmd())
return cmdLapi
}
func NewLapiContextCmd() *cobra.Command {
cmdContext := &cobra.Command{
Use: "context [command]",
Short: "Manage context to send with alerts",
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadCrowdsec(); err != nil {
fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
if err.Error() != fileNotFoundMessage {
log.Fatalf("Unable to load CrowdSec Agent: %s", err)
}
}
if csConfig.DisableAgent {
log.Fatalf("Agent is disabled and lapi context can only be used on the agent")
}
return nil
},
Run: func(cmd *cobra.Command, args []string) {
printHelp(cmd)
},
}
var keyToAdd string
var valuesToAdd []string
cmdContextAdd := &cobra.Command{
Use: "add",
Short: "Add context to send with alerts. You must specify the output key with the expr value you want",
Example: `cscli lapi context add --key source_ip --value evt.Meta.source_ip
cscli lapi context add --key file_source --value evt.Line.Src
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := alertcontext.ValidateContextExpr(keyToAdd, valuesToAdd); err != nil {
log.Fatalf("invalid context configuration :%s", err)
}
if _, ok := csConfig.Crowdsec.ContextToSend[keyToAdd]; !ok {
csConfig.Crowdsec.ContextToSend[keyToAdd] = make([]string, 0)
log.Infof("key '%s' added", keyToAdd)
}
data := csConfig.Crowdsec.ContextToSend[keyToAdd]
for _, val := range valuesToAdd {
if !inSlice(val, data) {
log.Infof("value '%s' added to key '%s'", val, keyToAdd)
data = append(data, val)
}
csConfig.Crowdsec.ContextToSend[keyToAdd] = data
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
}
},
}
cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
cmdContextAdd.MarkFlagRequired("key")
cmdContextAdd.MarkFlagRequired("value")
cmdContext.AddCommand(cmdContextAdd)
cmdContextStatus := &cobra.Command{
Use: "status",
Short: "List context to send with alerts",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if len(csConfig.Crowdsec.ContextToSend) == 0 {
fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
return
}
dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
if err != nil {
log.Fatalf("unable to show context status: %s", err)
}
fmt.Println(string(dump))
},
}
cmdContext.AddCommand(cmdContextStatus)
var detectAll bool
cmdContextDetect := &cobra.Command{
Use: "detect",
Short: "Detect available fields from the installed parsers",
Example: `cscli lapi context detect --all
cscli lapi context detect crowdsecurity/sshd-logs
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
var err error
if !detectAll && len(args) == 0 {
log.Infof("Please provide parsers to detect or --all flag.")
printHelp(cmd)
}
// to avoid all the log.Info from the loaders functions
log.SetLevel(log.ErrorLevel)
err = exprhelpers.Init(nil)
if err != nil {
log.Fatalf("Failed to init expr helpers : %s", err)
}
// Populate cwhub package tools
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
}
csParsers := parser.NewParsers()
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
log.Fatalf("unable to load parsers: %s", err)
}
fieldByParsers := make(map[string][]string)
for _, node := range csParsers.Nodes {
if !detectAll && !inSlice(node.Name, args) {
continue
}
if !detectAll {
args = removeFromSlice(node.Name, args)
}
fieldByParsers[node.Name] = make([]string, 0)
fieldByParsers[node.Name] = detectNode(node, *csParsers.Ctx)
subNodeFields := detectSubNode(node, *csParsers.Ctx)
for _, field := range subNodeFields {
if !inSlice(field, fieldByParsers[node.Name]) {
fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
}
}
}
fmt.Printf("Acquisition :\n\n")
fmt.Printf(" - evt.Line.Module\n")
fmt.Printf(" - evt.Line.Raw\n")
fmt.Printf(" - evt.Line.Src\n")
fmt.Println()
parsersKey := make([]string, 0)
for k := range fieldByParsers {
parsersKey = append(parsersKey, k)
}
sort.Strings(parsersKey)
for _, k := range parsersKey {
if len(fieldByParsers[k]) == 0 {
continue
}
fmt.Printf("%s :\n\n", k)
values := fieldByParsers[k]
sort.Strings(values)
for _, value := range values {
fmt.Printf(" - %s\n", value)
}
fmt.Println()
}
if len(args) > 0 {
for _, parserNotFound := range args {
log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
}
}
},
}
cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
cmdContext.AddCommand(cmdContextDetect)
var keysToDelete []string
var valuesToDelete []string
cmdContextDelete := &cobra.Command{
Use: "delete",
Short: "Delete context to send with alerts",
Example: `cscli lapi context delete --key source_ip
cscli lapi context delete --value evt.Line.Src
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
log.Fatalf("please provide at least a key or a value to delete")
}
for _, key := range keysToDelete {
if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
delete(csConfig.Crowdsec.ContextToSend, key)
log.Infof("key '%s' has been removed", key)
} else {
log.Warningf("key '%s' doesn't exist", key)
}
}
for _, value := range valuesToDelete {
valueFound := false
for key, context := range csConfig.Crowdsec.ContextToSend {
if inSlice(value, context) {
valueFound = true
csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
log.Infof("value '%s' has been removed from key '%s'", value, key)
}
if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
delete(csConfig.Crowdsec.ContextToSend, key)
}
}
if !valueFound {
log.Warningf("value '%s' not found", value)
}
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
}
},
}
cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
cmdContext.AddCommand(cmdContextDelete)
return cmdContext
}
func detectStaticField(GrokStatics []types.ExtraField) []string {
ret := make([]string, 0)
for _, static := range GrokStatics {
if static.Parsed != "" {
fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
if static.Meta != "" {
fieldName := fmt.Sprintf("evt.Meta.%s", static.Meta)
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
if static.TargetByName != "" {
fieldName := static.TargetByName
if !strings.HasPrefix(fieldName, "evt.") {
fieldName = "evt." + fieldName
}
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
}
return ret
}
func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
var ret = make([]string, 0)
if node.Grok.RunTimeRegexp != nil {
for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
}
if node.Grok.RegexpName != "" {
grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName)
if err != nil {
log.Warningf("Can't get subgrok: %s", err)
}
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
}
if len(node.Grok.Statics) > 0 {
staticsField := detectStaticField(node.Grok.Statics)
for _, staticField := range staticsField {
if !inSlice(staticField, ret) {
ret = append(ret, staticField)
}
}
}
if len(node.Statics) > 0 {
staticsField := detectStaticField(node.Statics)
for _, staticField := range staticsField {
if !inSlice(staticField, ret) {
ret = append(ret, staticField)
}
}
}
return ret
}
func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
var ret = make([]string, 0)
for _, subnode := range node.LeavesNodes {
if subnode.Grok.RunTimeRegexp != nil {
for _, capturedField := range subnode.Grok.RunTimeRegexp.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
}
if subnode.Grok.RegexpName != "" {
grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName)
if err != nil {
log.Warningf("Can't get subgrok: %s", err)
}
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
}
if len(subnode.Grok.Statics) > 0 {
staticsField := detectStaticField(subnode.Grok.Statics)
for _, staticField := range staticsField {
if !inSlice(staticField, ret) {
ret = append(ret, staticField)
}
}
}
if len(subnode.Statics) > 0 {
staticsField := detectStaticField(subnode.Statics)
for _, staticField := range staticsField {
if !inSlice(staticField, ret) {
ret = append(ret, staticField)
}
}
}
}
return ret
}

25
cmd/crowdsec-cli/utils.go
View file

@ -748,3 +748,28 @@ func getDBClient() (*database.Client, error) {
}
return ret, nil
}
func removeFromSlice(val string, slice []string) []string {
var i int
var value string
valueFound := false
// get the index
for i, value = range slice {
if value == val {
valueFound = true
break
}
}
if valueFound {
slice[i] = slice[len(slice)-1]
slice[len(slice)-1] = ""
slice = slice[:len(slice)-1]
}
return slice
}

2
cmd/crowdsec/crowdsec.go
View file

@ -28,7 +28,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
}
// Start loading configs
csParsers := newParsers()
csParsers := parser.NewParsers()
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
}

40
cmd/crowdsec/main.go
View file

@ -7,7 +7,6 @@ import (
"os"
"path/filepath"
"runtime"
"sort"
"strings"
"time"
@ -72,45 +71,6 @@ type Flags struct {
type labelsMap map[string]string
// Return new parsers
// nodes and povfwnodes are already initialized in parser.LoadStages
func newParsers() *parser.Parsers {
parsers := &parser.Parsers{
Ctx: &parser.UnixParserCtx{},
Povfwctx: &parser.UnixParserCtx{},
StageFiles: make([]parser.Stagefile, 0),
PovfwStageFiles: make([]parser.Stagefile, 0),
}
for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
for _, hubParserItem := range cwhub.GetItemMap(itemType) {
if hubParserItem.Installed {
stagefile := parser.Stagefile{
Filename: hubParserItem.LocalPath,
Stage: hubParserItem.Stage,
}
if itemType == cwhub.PARSERS {
parsers.StageFiles = append(parsers.StageFiles, stagefile)
}
if itemType == cwhub.PARSERS_OVFLW {
parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
}
}
}
}
if parsers.StageFiles != nil {
sort.Slice(parsers.StageFiles, func(i, j int) bool {
return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
})
}
if parsers.PovfwStageFiles != nil {
sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
})
}
return parsers
}
func LoadBuckets(cConfig *csconfig.Config) error {
var (
err error

1
config/config.yaml
View file

@ -16,6 +16,7 @@ config_paths:
notification_dir: /etc/crowdsec/notifications/
plugin_dir: /usr/local/lib/crowdsec/plugins/
crowdsec_service:
#console_context_path: /etc/crowdsec/console/context.yaml
acquisition_path: /etc/crowdsec/acquis.yaml
acquisition_dir: /etc/crowdsec/acquis.d
parser_routines: 1

1
config/config_win.yaml
View file

@ -13,6 +13,7 @@ config_paths:
plugin_dir: C:\ProgramData\CrowdSec\plugins\
notification_dir: C:\ProgramData\CrowdSec\config\notifications\
crowdsec_service:
#console_context_path: C:\ProgramData\CrowdSec\console\context.yaml
acquisition_path: C:\ProgramData\CrowdSec\config\acquis.yaml
parser_routines: 1
cscli:

1
config/console.yaml
View file

@ -1,3 +1,4 @@
share_manual_decisions: false
share_custom: true
share_tainted: true
share_context: false

0
config/context.yaml Normal file
View file

3
debian/rules vendored
View file

@ -28,7 +28,7 @@ override_dh_auto_install:
mkdir -p debian/crowdsec/usr/share/crowdsec
mkdir -p debian/crowdsec/etc/crowdsec/hub/
mkdir -p debian/crowdsec/usr/share/crowdsec/config
mkdir -p debian/crowdsec/etc/crowdsec/console/
mkdir -p debian/crowdsec/usr/lib/crowdsec/plugins/
mkdir -p debian/crowdsec/etc/crowdsec/notifications/
@ -44,6 +44,7 @@ override_dh_auto_install:
install -m 600 config/config.yaml debian/crowdsec/etc/crowdsec/config.yaml
cp config/simulation.yaml debian/crowdsec/etc/crowdsec/simulation.yaml
cp config/profiles.yaml debian/crowdsec/etc/crowdsec/profiles.yaml
cp config/context.yaml debian/crowdsec/etc/crowdsec/console/context.yaml
cp config/console.yaml debian/crowdsec/etc/crowdsec/console.yaml
cp -a config/patterns debian/crowdsec/etc/crowdsec

157
pkg/alertcontext/alertcontext.go Normal file
View file

@ -0,0 +1,157 @@
package alertcontext
import (
"encoding/json"
"fmt"
"strconv"
"github.com/antonmedv/expr"
"github.com/antonmedv/expr/vm"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
)
const (
maxContextValueLen = 4000
)
var (
alertContext = Context{}
)
type Context struct {
ContextToSend map[string][]string
ContextValueLen int
ContextToSendCompiled map[string][]*vm.Program
Log *log.Logger
}
func ValidateContextExpr(key string, expressions []string) error {
for _, expression := range expressions {
_, err := expr.Compile(expression, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
if err != nil {
return fmt.Errorf("compilation of '%s' failed: %v", expression, err)
}
}
return nil
}
func NewAlertContext(contextToSend map[string][]string, valueLength int) error {
var clog = log.New()
if err := types.ConfigureLogger(clog); err != nil {
return fmt.Errorf("couldn't create logger for alert context: %s", err)
}
if valueLength == 0 {
clog.Debugf("No console context value length provided, using default: %d", maxContextValueLen)
valueLength = maxContextValueLen
}
if valueLength > maxContextValueLen {
clog.Debugf("Provided console context value length (%d) is higher than the maximum, using default: %d", valueLength, maxContextValueLen)
valueLength = maxContextValueLen
}
alertContext = Context{
ContextToSend: contextToSend,
ContextValueLen: valueLength,
Log: clog,
ContextToSendCompiled: make(map[string][]*vm.Program),
}
for key, values := range contextToSend {
alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0)
for _, value := range values {
valueCompiled, err := expr.Compile(value, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
if err != nil {
return fmt.Errorf("compilation of '%s' context value failed: %v", value, err)
}
alertContext.ContextToSendCompiled[key] = append(alertContext.ContextToSendCompiled[key], valueCompiled)
}
}
return nil
}
func truncate(values []string, contextValueLen int) (string, error) {
var ret string
valueByte, err := json.Marshal(values)
if err != nil {
return "", fmt.Errorf("unable to dump metas: %s", err)
}
ret = string(valueByte)
for {
if len(ret) <= contextValueLen {
break
}
// if there is only 1 value left and that the size is too big, truncate it
if len(values) == 1 {
valueToTruncate := values[0]
half := len(valueToTruncate) / 2
lastValueTruncated := valueToTruncate[:half] + "..."
values = values[:len(values)-1]
values = append(values, lastValueTruncated)
} else {
// if there is multiple value inside, just remove the last one
values = values[:len(values)-1]
}
valueByte, err = json.Marshal(values)
if err != nil {
return "", fmt.Errorf("unable to dump metas: %s", err)
}
ret = string(valueByte)
}
return ret, nil
}
func EventToContext(events []types.Event) (models.Meta, []error) {
var errors []error
metas := make([]*models.MetaItems0, 0)
tmpContext := make(map[string][]string)
for _, evt := range events {
for key, values := range alertContext.ContextToSendCompiled {
if _, ok := tmpContext[key]; !ok {
tmpContext[key] = make([]string, 0)
}
for _, value := range values {
var val string
output, err := expr.Run(value, exprhelpers.GetExprEnv(map[string]interface{}{"evt": evt}))
if err != nil {
errors = append(errors, fmt.Errorf("failed to get value for %s : %v", key, err))
continue
}
switch out := output.(type) {
case string:
val = out
case int:
val = strconv.Itoa(out)
default:
errors = append(errors, fmt.Errorf("unexpected return type for %s : %T", key, output))
continue
}
if val != "" && !types.InSlice(val, tmpContext[key]) {
tmpContext[key] = append(tmpContext[key], val)
}
}
}
}
for key, values := range tmpContext {
if len(values) == 0 {
continue
}
valueStr, err := truncate(values, alertContext.ContextValueLen)
if err != nil {
log.Warningf(err.Error())
}
meta := models.MetaItems0{
Key: key,
Value: valueStr,
}
metas = append(metas, &meta)
}
ret := models.Meta(metas)
return ret, errors
}

201
pkg/alertcontext/alertcontext_test.go Normal file
View file

@ -0,0 +1,201 @@
package alertcontext
import (
"fmt"
"testing"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/stretchr/testify/assert"
)
func TestNewAlertContext(t *testing.T) {
tests := []struct {
name string
contextToSend map[string][]string
valueLength int
expectedErr error
}{
{
name: "basic config test",
contextToSend: map[string][]string{
"test": []string{"evt.Parsed.source_ip"},
},
valueLength: 100,
expectedErr: nil,
},
}
for _, test := range tests {
fmt.Printf("Running test '%s'\n", test.name)
err := NewAlertContext(test.contextToSend, test.valueLength)
assert.ErrorIs(t, err, test.expectedErr)
}
}
func TestEventToContext(t *testing.T) {
tests := []struct {
name string
contextToSend map[string][]string
valueLength int
events []types.Event
expectedResult models.Meta
}{
{
name: "basic test",
contextToSend: map[string][]string{
"source_ip": []string{"evt.Parsed.source_ip"},
"nonexistent_field": []string{"evt.Parsed.nonexist"},
},
valueLength: 100,
events: []types.Event{
{
Parsed: map[string]string{
"source_ip": "1.2.3.4",
"source_machine": "mymachine",
},
},
},
expectedResult: []*models.MetaItems0{
{
Key: "source_ip",
Value: "[\"1.2.3.4\"]",
},
},
},
{
name: "test many events",
contextToSend: map[string][]string{
"source_ip": []string{"evt.Parsed.source_ip"},
"source_machine": []string{"evt.Parsed.source_machine"},
"cve": []string{"evt.Parsed.cve"},
},
valueLength: 100,
events: []types.Event{
{
Parsed: map[string]string{
"source_ip": "1.2.3.4",
"source_machine": "mymachine",
"cve": "CVE-2022-1234",
},
},
{
Parsed: map[string]string{
"source_ip": "1.2.3.4",
"source_machine": "mymachine",
"cve": "CVE-2022-1235",
},
},
{
Parsed: map[string]string{
"source_ip": "1.2.3.4",
"source_machine": "mymachine",
"cve": "CVE-2022-125",
},
},
},
expectedResult: []*models.MetaItems0{
{
Key: "source_ip",
Value: "[\"1.2.3.4\"]",
},
{
Key: "source_machine",
Value: "[\"mymachine\"]",
},
{
Key: "cve",
Value: "[\"CVE-2022-1234\",\"CVE-2022-1235\",\"CVE-2022-125\"]",
},
},
},
{
name: "test many events with result above max length (need truncate, keep only 2 on 3 elements)",
contextToSend: map[string][]string{
"source_ip": []string{"evt.Parsed.source_ip"},
"source_machine": []string{"evt.Parsed.source_machine"},
"uri": []string{"evt.Parsed.uri"},
},
valueLength: 100,
events: []types.Event{
{
Parsed: map[string]string{
"source_ip": "1.2.3.4",
"source_machine": "mymachine",
"uri": "/test/test/test/../../../../../../../../",
},
},
{
Parsed: map[string]string{
"source_ip": "1.2.3.4",
"source_machine": "mymachine",
"uri": "/admin/admin/admin/../../../../../../../../",
},
},
{
Parsed: map[string]string{
"source_ip": "1.2.3.4",
"source_machine": "mymachine",
"uri": "/login/login/login/../../../../../../../../../../../",
},
},
},
expectedResult: []*models.MetaItems0{
{
Key: "source_ip",
Value: "[\"1.2.3.4\"]",
},
{
Key: "source_machine",
Value: "[\"mymachine\"]",
},
{
Key: "uri",
Value: "[\"/test/test/test/../../../../../../../../\",\"/admin/admin/admin/../../../../../../../../\"]",
},
},
},
{
name: "test one events with result above max length (need truncate on one element)",
contextToSend: map[string][]string{
"source_ip": []string{"evt.Parsed.source_ip"},
"source_machine": []string{"evt.Parsed.source_machine"},
"uri": []string{"evt.Parsed.uri"},
},
valueLength: 100,
events: []types.Event{
{
Parsed: map[string]string{
"source_ip": "1.2.3.4",
"source_machine": "mymachine",
"uri": "/test/test/test/../../../../.should_truncate_just_after_this/../../../..../../../../../../../../../../../../../../../end",
},
},
},
expectedResult: []*models.MetaItems0{
{
Key: "source_machine",
Value: "[\"mymachine\"]",
},
{
Key: "uri",
Value: "[\"/test/test/test/../../../../.should_truncate_just_after_this...\"]",
},
{
Key: "source_ip",
Value: "[\"1.2.3.4\"]",
},
},
},
}
for _, test := range tests {
fmt.Printf("Running test '%s'\n", test.name)
err := NewAlertContext(test.contextToSend, test.valueLength)
assert.ErrorIs(t, err, nil)
metas, _ := EventToContext(test.events)
assert.ElementsMatch(t, test.expectedResult, metas)
}
}

19
pkg/apiserver/apic.go
View file

@ -85,8 +85,8 @@ func (a *apic) FetchScenariosListFromDB() ([]string, error) {
return scenarios, nil
}
func alertToSignal(alert *models.Alert, scenarioTrust string) *models.AddSignalsRequestItem {
return &models.AddSignalsRequestItem{
func alertToSignal(alert *models.Alert, scenarioTrust string, shareContext bool) *models.AddSignalsRequestItem {
signal := &models.AddSignalsRequestItem{
Message: alert.Message,
Scenario: alert.Scenario,
ScenarioHash: alert.ScenarioHash,
@ -96,8 +96,19 @@ func alertToSignal(alert *models.Alert, scenarioTrust string) *models.AddSignals
StopAt: alert.StopAt,
CreatedAt: alert.CreatedAt,
MachineID: alert.MachineID,
ScenarioTrust: &scenarioTrust,
ScenarioTrust: scenarioTrust,
}
if shareContext {
signal.Context = make([]*models.AddSignalsRequestItemContextItems0, 0)
for _, meta := range alert.Meta {
contextItem := models.AddSignalsRequestItemContextItems0{
Key: meta.Key,
Value: meta.Value,
}
signal.Context = append(signal.Context, &contextItem)
}
}
return signal
}
func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, consoleConfig *csconfig.ConsoleConfig) (*apic, error) {
@ -176,7 +187,7 @@ func (a *apic) Push() error {
var signals []*models.AddSignalsRequestItem
for _, alert := range alerts {
if ok := shouldShareAlert(alert, a.consoleConfig); ok {
signals = append(signals, alertToSignal(alert, getScenarioTrustOfAlert(alert)))
signals = append(signals, alertToSignal(alert, getScenarioTrustOfAlert(alert), *a.consoleConfig.ShareContext))
}
}
a.mu.Lock()

1
pkg/apiserver/apic_test.go
View file

@ -58,6 +58,7 @@ func getAPIC(t *testing.T) *apic {
ShareManualDecisions: types.BoolPtr(false),
ShareTaintedScenarios: types.BoolPtr(false),
ShareCustomScenarios: types.BoolPtr(false),
ShareContext: types.BoolPtr(false),
},
}
}

1
pkg/csconfig/api_test.go
View file

@ -213,6 +213,7 @@ func TestLoadAPIServer(t *testing.T) {
ShareManualDecisions: types.BoolPtr(false),
ShareTaintedScenarios: types.BoolPtr(true),
ShareCustomScenarios: types.BoolPtr(true),
ShareContext: types.BoolPtr(false),
},
LogDir: LogDirFullPath,
LogMedia: "stdout",

11
pkg/csconfig/console.go
View file

@ -14,9 +14,10 @@ const (
SEND_CUSTOM_SCENARIOS = "custom"
SEND_TAINTED_SCENARIOS = "tainted"
SEND_MANUAL_SCENARIOS = "manual"
SEND_CONTEXT = "context"
)
var CONSOLE_CONFIGS = []string{SEND_CUSTOM_SCENARIOS, SEND_MANUAL_SCENARIOS, SEND_TAINTED_SCENARIOS}
var CONSOLE_CONFIGS = []string{SEND_CUSTOM_SCENARIOS, SEND_MANUAL_SCENARIOS, SEND_TAINTED_SCENARIOS, SEND_CONTEXT}
var DefaultConsoleConfigFilePath = DefaultConfigPath("console.yaml")
@ -24,6 +25,7 @@ type ConsoleConfig struct {
ShareManualDecisions *bool `yaml:"share_manual_decisions"`
ShareTaintedScenarios *bool `yaml:"share_tainted"`
ShareCustomScenarios *bool `yaml:"share_custom"`
ShareContext *bool `yaml:"share_context"`
}
func (c *LocalApiServerCfg) LoadConsoleConfig() error {
@ -33,6 +35,7 @@ func (c *LocalApiServerCfg) LoadConsoleConfig() error {
c.ConsoleConfig.ShareCustomScenarios = types.BoolPtr(true)
c.ConsoleConfig.ShareTaintedScenarios = types.BoolPtr(true)
c.ConsoleConfig.ShareManualDecisions = types.BoolPtr(false)
c.ConsoleConfig.ShareContext = types.BoolPtr(false)
return nil
}
@ -57,6 +60,12 @@ func (c *LocalApiServerCfg) LoadConsoleConfig() error {
log.Debugf("no share_manual scenarios found, setting to false")
c.ConsoleConfig.ShareManualDecisions = types.BoolPtr(false)
}
if c.ConsoleConfig.ShareContext == nil {
log.Debugf("no 'context' found, setting to false")
c.ConsoleConfig.ShareContext = types.BoolPtr(false)
}
log.Debugf("Console configuration '%s' loaded successfully", c.ConsoleConfigPath)
return nil

84
pkg/csconfig/crowdsec_service.go
View file

@ -7,31 +7,34 @@ import (
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
// CrowdsecServiceCfg contains the location of parsers/scenarios/... and acquisition files
type CrowdsecServiceCfg struct {
Enable *bool `yaml:"enable"`
AcquisitionFilePath string `yaml:"acquisition_path,omitempty"`
AcquisitionDirPath string `yaml:"acquisition_dir,omitempty"`
Enable *bool `yaml:"enable"`
AcquisitionFilePath string `yaml:"acquisition_path,omitempty"`
AcquisitionDirPath string `yaml:"acquisition_dir,omitempty"`
ConsoleContextPath string `yaml:"console_context_path"`
ConsoleContextValueLength int `yaml:"console_context_value_length"`
AcquisitionFiles []string `yaml:"-"`
ParserRoutinesCount int `yaml:"parser_routines"`
BucketsRoutinesCount int `yaml:"buckets_routines"`
OutputRoutinesCount int `yaml:"output_routines"`
SimulationConfig *SimulationConfig `yaml:"-"`
LintOnly bool `yaml:"-"` // if set to true, exit after loading configs
BucketStateFile string `yaml:"state_input_file,omitempty"` // if we need to unserialize buckets at start
BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode
AcquisitionFiles []string `yaml:"-"`
ParserRoutinesCount int `yaml:"parser_routines"`
BucketsRoutinesCount int `yaml:"buckets_routines"`
OutputRoutinesCount int `yaml:"output_routines"`
SimulationConfig *SimulationConfig `yaml:"-"`
LintOnly bool `yaml:"-"` // if set to true, exit after loading configs
BucketStateFile string `yaml:"state_input_file,omitempty"` // if we need to unserialize buckets at start
BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode
HubDir string `yaml:"-"`
DataDir string `yaml:"-"`
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
SimulationFilePath string `yaml:"-"`
HubDir string `yaml:"-"`
DataDir string `yaml:"-"`
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
SimulationFilePath string `yaml:"-"`
ContextToSend map[string][]string `yaml:"-"`
}
func (c *Config) LoadCrowdsec() error {
@ -152,5 +155,50 @@ func (c *Config) LoadCrowdsec() error {
return errors.Wrap(err, "while loading hub")
}
c.Crowdsec.ContextToSend = make(map[string][]string, 0)
fallback := false
if c.Crowdsec.ConsoleContextPath == "" {
// fallback to default config file
c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml")
fallback = true
}
f, err := filepath.Abs(c.Crowdsec.ConsoleContextPath)
if err != nil {
return fmt.Errorf("fail to get absolute path of %s: %s", c.Crowdsec.ConsoleContextPath, err)
}
c.Crowdsec.ConsoleContextPath = f
yamlFile, err := os.ReadFile(c.Crowdsec.ConsoleContextPath)
if err != nil {
if fallback {
log.Debugf("Default context config file doesn't exist, will not use it")
} else {
return fmt.Errorf("failed to open context file: %s", err)
}
} else {
err = yaml.Unmarshal(yamlFile, c.Crowdsec.ContextToSend)
if err != nil {
return fmt.Errorf("unmarshaling labels console config file '%s': %s", c.Crowdsec.ConsoleContextPath, err)
}
}
return nil
}
func (c *CrowdsecServiceCfg) DumpContextConfigFile() error {
var out []byte
var err error
if out, err = yaml.Marshal(c.ContextToSend); err != nil {
return errors.Wrapf(err, "while marshaling ConsoleConfig (for %s)", c.ConsoleContextPath)
}
if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil {
return errors.Wrapf(err, "while dumping console config to %s", c.ConsoleContextPath)
}
log.Infof("%s file saved", c.ConsoleContextPath)
return nil
}

103
pkg/csconfig/crowdsec_service_test.go
View file

@ -33,6 +33,9 @@ func TestLoadCrowdsec(t *testing.T) {
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
require.NoError(t, err)
contextFileFullPath, err := filepath.Abs("./tests/context.yaml")
require.NoError(t, err)
tests := []struct {
name string
input *Config
@ -53,23 +56,30 @@ func TestLoadCrowdsec(t *testing.T) {
},
},
Crowdsec: &CrowdsecServiceCfg{
AcquisitionFilePath: "./tests/acquis.yaml",
SimulationFilePath: "./tests/simulation.yaml",
AcquisitionFilePath: "./tests/acquis.yaml",
SimulationFilePath: "./tests/simulation.yaml",
ConsoleContextPath: "./tests/context.yaml",
ConsoleContextValueLength: 2500,
},
},
expectedResult: &CrowdsecServiceCfg{
Enable: types.BoolPtr(true),
AcquisitionDirPath: "",
AcquisitionFilePath: acquisFullPath,
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
AcquisitionFiles: []string{acquisFullPath},
SimulationFilePath: "./tests/simulation.yaml",
Enable: types.BoolPtr(true),
AcquisitionDirPath: "",
ConsoleContextPath: contextFileFullPath,
AcquisitionFilePath: acquisFullPath,
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
ConsoleContextValueLength: 2500,
AcquisitionFiles: []string{acquisFullPath},
SimulationFilePath: "./tests/simulation.yaml",
ContextToSend: map[string][]string{
"source_ip": {"evt.Parsed.source_ip"},
},
SimulationConfig: &SimulationConfig{
Simulation: &falseBoolPtr,
},
@ -92,21 +102,27 @@ func TestLoadCrowdsec(t *testing.T) {
AcquisitionFilePath: "./tests/acquis.yaml",
AcquisitionDirPath: "./tests/acquis/",
SimulationFilePath: "./tests/simulation.yaml",
ConsoleContextPath: "./tests/context.yaml",
},
},
expectedResult: &CrowdsecServiceCfg{
Enable: types.BoolPtr(true),
AcquisitionDirPath: acquisDirFullPath,
AcquisitionFilePath: acquisFullPath,
ConfigDir: configDirFullPath,
HubIndexFile: hubIndexFileFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
AcquisitionFiles: []string{acquisFullPath, acquisInDirFullPath},
SimulationFilePath: "./tests/simulation.yaml",
Enable: types.BoolPtr(true),
AcquisitionDirPath: acquisDirFullPath,
AcquisitionFilePath: acquisFullPath,
ConsoleContextPath: contextFileFullPath,
ConfigDir: configDirFullPath,
HubIndexFile: hubIndexFileFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
ConsoleContextValueLength: 0,
AcquisitionFiles: []string{acquisFullPath, acquisInDirFullPath},
ContextToSend: map[string][]string{
"source_ip": {"evt.Parsed.source_ip"},
},
SimulationFilePath: "./tests/simulation.yaml",
SimulationConfig: &SimulationConfig{
Simulation: &falseBoolPtr,
},
@ -125,21 +141,29 @@ func TestLoadCrowdsec(t *testing.T) {
CredentialsFilePath: "./tests/lapi-secrets.yaml",
},
},
Crowdsec: &CrowdsecServiceCfg{},
Crowdsec: &CrowdsecServiceCfg{
ConsoleContextPath: contextFileFullPath,
ConsoleContextValueLength: 10,
},
},
expectedResult: &CrowdsecServiceCfg{
Enable: types.BoolPtr(true),
AcquisitionDirPath: "",
AcquisitionFilePath: "",
ConfigDir: configDirFullPath,
HubIndexFile: hubIndexFileFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
AcquisitionFiles: []string{},
SimulationFilePath: "",
Enable: types.BoolPtr(true),
AcquisitionDirPath: "",
AcquisitionFilePath: "",
ConfigDir: configDirFullPath,
HubIndexFile: hubIndexFileFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
ConsoleContextPath: contextFileFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
ConsoleContextValueLength: 10,
AcquisitionFiles: []string{},
SimulationFilePath: "",
ContextToSend: map[string][]string{
"source_ip": {"evt.Parsed.source_ip"},
},
SimulationConfig: &SimulationConfig{
Simulation: &falseBoolPtr,
},
@ -159,6 +183,7 @@ func TestLoadCrowdsec(t *testing.T) {
},
},
Crowdsec: &CrowdsecServiceCfg{
ConsoleContextPath: "",
AcquisitionFilePath: "./tests/acquis_not_exist.yaml",
},
},

2
pkg/csconfig/tests/context.yaml Normal file
View file

@ -0,0 +1,2 @@
source_ip:
- evt.Parsed.source_ip

8
pkg/leakybucket/manager_load.go
View file

@ -11,6 +11,8 @@ import (
"sync"
"time"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
@ -225,6 +227,11 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
ret = append(ret, bucketFactory)
}
}
if err := alertcontext.NewAlertContext(cscfg.ContextToSend, cscfg.ConsoleContextValueLength); err != nil {
return nil, nil, fmt.Errorf("unable to load alert context: %s", err)
}
log.Warningf("Loaded %d scenarios", len(ret))
return ret, response, nil
}
@ -349,6 +356,7 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error {
return fmt.Errorf("invalid bucket from %s : %v", bucketFactory.Filename, err)
}
bucketFactory.tomb = tomb
return nil
}

14
pkg/leakybucket/overflows.go
View file

@ -6,6 +6,7 @@ import (
"sort"
"strconv"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/davecgh/go-spew/spew"
@ -17,7 +18,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
)
//SourceFromEvent extracts and formats a valid models.Source object from an Event
// SourceFromEvent extracts and formats a valid models.Source object from an Event
func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, error) {
srcs := make(map[string]models.Source)
/*if it's already an overflow, we have properly formatted sources.
@ -160,7 +161,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e
return srcs, nil
}
//EventsFromQueue iterates the queue to collect & prepare meta-datas from alert
// EventsFromQueue iterates the queue to collect & prepare meta-datas from alert
func EventsFromQueue(queue *Queue) []*models.Event {
events := []*models.Event{}
@ -207,7 +208,7 @@ func EventsFromQueue(queue *Queue) []*models.Event {
return events
}
//alertFormatSource iterates over the queue to collect sources
// alertFormatSource iterates over the queue to collect sources
func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, string, error) {
var sources map[string]models.Source = make(map[string]models.Source)
var source_type string
@ -233,7 +234,7 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st
return sources, source_type, nil
}
//NewAlert will generate a RuntimeAlert and its APIAlert(s) from a bucket that overflowed
// NewAlert will generate a RuntimeAlert and its APIAlert(s) from a bucket that overflowed
func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) {
var runtimeAlert types.RuntimeAlert
@ -293,6 +294,11 @@ func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) {
*apiAlert.Message = fmt.Sprintf("%s %s performed '%s' (%d events over %s) at %s", source_scope, sourceStr, leaky.Name, leaky.Total_count, leaky.Ovflw_ts.Sub(leaky.First_ts), leaky.Last_ts)
//Get the events from Leaky/Queue
apiAlert.Events = EventsFromQueue(queue)
var warnings []error
apiAlert.Meta, warnings = alertcontext.EventToContext(leaky.Queue.GetQueue())
for _, w := range warnings {
log.Warningf("while extracting context from bucket %s : %s", leaky.Name, w)
}
//Loop over the Sources and generate appropriate number of ApiAlerts
for _, srcValue := range sources {

117
pkg/models/add_signals_request_item.go
View file

@ -7,6 +7,7 @@ package models
import (
"context"
"strconv"
"github.com/go-openapi/errors"
"github.com/go-openapi/strfmt"
@ -19,6 +20,12 @@ import (
// swagger:model AddSignalsRequestItem
type AddSignalsRequestItem struct {
// alert id
AlertID int64 `json:"alert_id,omitempty"`
// context
Context []*AddSignalsRequestItemContextItems0 `json:"context"`
// created at
CreatedAt string `json:"created_at,omitempty"`
@ -38,8 +45,7 @@ type AddSignalsRequestItem struct {
ScenarioHash *string `json:"scenario_hash"`
// scenario trust
// Required: true
ScenarioTrust *string `json:"scenario_trust"`
ScenarioTrust string `json:"scenario_trust,omitempty"`
// scenario version
// Required: true
@ -62,6 +68,10 @@ type AddSignalsRequestItem struct {
func (m *AddSignalsRequestItem) Validate(formats strfmt.Registry) error {
var res []error
if err := m.validateContext(formats); err != nil {
res = append(res, err)
}
if err := m.validateMessage(formats); err != nil {
res = append(res, err)
}
@ -74,10 +84,6 @@ func (m *AddSignalsRequestItem) Validate(formats strfmt.Registry) error {
res = append(res, err)
}
if err := m.validateScenarioTrust(formats); err != nil {
res = append(res, err)
}
if err := m.validateScenarioVersion(formats); err != nil {
res = append(res, err)
}
@ -100,6 +106,32 @@ func (m *AddSignalsRequestItem) Validate(formats strfmt.Registry) error {
return nil
}
func (m *AddSignalsRequestItem) validateContext(formats strfmt.Registry) error {
if swag.IsZero(m.Context) { // not required
return nil
}
for i := 0; i < len(m.Context); i++ {
if swag.IsZero(m.Context[i]) { // not required
continue
}
if m.Context[i] != nil {
if err := m.Context[i].Validate(formats); err != nil {
if ve, ok := err.(*errors.Validation); ok {
return ve.ValidateName("context" + "." + strconv.Itoa(i))
} else if ce, ok := err.(*errors.CompositeError); ok {
return ce.ValidateName("context" + "." + strconv.Itoa(i))
}
return err
}
}
}
return nil
}
func (m *AddSignalsRequestItem) validateMessage(formats strfmt.Registry) error {
if err := validate.Required("message", "body", m.Message); err != nil {
@ -127,15 +159,6 @@ func (m *AddSignalsRequestItem) validateScenarioHash(formats strfmt.Registry) er
return nil
}
func (m *AddSignalsRequestItem) validateScenarioTrust(formats strfmt.Registry) error {
if err := validate.Required("scenario_trust", "body", m.ScenarioTrust); err != nil {
return err
}
return nil
}
func (m *AddSignalsRequestItem) validateScenarioVersion(formats strfmt.Registry) error {
if err := validate.Required("scenario_version", "body", m.ScenarioVersion); err != nil {
@ -187,6 +210,10 @@ func (m *AddSignalsRequestItem) validateStopAt(formats strfmt.Registry) error {
func (m *AddSignalsRequestItem) ContextValidate(ctx context.Context, formats strfmt.Registry) error {
var res []error
if err := m.contextValidateContext(ctx, formats); err != nil {
res = append(res, err)
}
if err := m.contextValidateSource(ctx, formats); err != nil {
res = append(res, err)
}
@ -197,6 +224,26 @@ func (m *AddSignalsRequestItem) ContextValidate(ctx context.Context, formats str
return nil
}
func (m *AddSignalsRequestItem) contextValidateContext(ctx context.Context, formats strfmt.Registry) error {
for i := 0; i < len(m.Context); i++ {
if m.Context[i] != nil {
if err := m.Context[i].ContextValidate(ctx, formats); err != nil {
if ve, ok := err.(*errors.Validation); ok {
return ve.ValidateName("context" + "." + strconv.Itoa(i))
} else if ce, ok := err.(*errors.CompositeError); ok {
return ce.ValidateName("context" + "." + strconv.Itoa(i))
}
return err
}
}
}
return nil
}
func (m *AddSignalsRequestItem) contextValidateSource(ctx context.Context, formats strfmt.Registry) error {
if m.Source != nil {
@ -230,3 +277,43 @@ func (m *AddSignalsRequestItem) UnmarshalBinary(b []byte) error {
*m = res
return nil
}
// AddSignalsRequestItemContextItems0 add signals request item context items0
//
// swagger:model AddSignalsRequestItemContextItems0
type AddSignalsRequestItemContextItems0 struct {
// key
Key string `json:"key,omitempty"`
// value
Value string `json:"value,omitempty"`
}
// Validate validates this add signals request item context items0
func (m *AddSignalsRequestItemContextItems0) Validate(formats strfmt.Registry) error {
return nil
}
// ContextValidate validates this add signals request item context items0 based on context it is used
func (m *AddSignalsRequestItemContextItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error {
return nil
}
// MarshalBinary interface implementation
func (m *AddSignalsRequestItemContextItems0) MarshalBinary() ([]byte, error) {
if m == nil {
return nil, nil
}
return swag.WriteJSON(m)
}
// UnmarshalBinary interface implementation
func (m *AddSignalsRequestItemContextItems0) UnmarshalBinary(b []byte) error {
var res AddSignalsRequestItemContextItems0
if err := swag.ReadJSON(b, &res); err != nil {
return err
}
*m = res
return nil
}

41
pkg/parser/unix_parser.go
View file

@ -4,9 +4,11 @@ import (
"fmt"
"os"
"path"
"sort"
"strings"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/grokky"
log "github.com/sirupsen/logrus"
@ -50,6 +52,45 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) {
return &r, nil
}
// Return new parsers
// nodes and povfwnodes are already initialized in parser.LoadStages
func NewParsers() *Parsers {
parsers := &Parsers{
Ctx: &UnixParserCtx{},
Povfwctx: &UnixParserCtx{},
StageFiles: make([]Stagefile, 0),
PovfwStageFiles: make([]Stagefile, 0),
}
for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
for _, hubParserItem := range cwhub.GetItemMap(itemType) {
if hubParserItem.Installed {
stagefile := Stagefile{
Filename: hubParserItem.LocalPath,
Stage: hubParserItem.Stage,
}
if itemType == cwhub.PARSERS {
parsers.StageFiles = append(parsers.StageFiles, stagefile)
}
if itemType == cwhub.PARSERS_OVFLW {
parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
}
}
}
}
if parsers.StageFiles != nil {
sort.Slice(parsers.StageFiles, func(i, j int) bool {
return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
})
}
if parsers.PovfwStageFiles != nil {
sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
})
}
return parsers
}
func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
var err error

3
rpm/SPECS/crowdsec.spec
View file

@ -45,6 +45,7 @@ sed -i "s#/usr/local/lib/crowdsec/plugins/#%{_libdir}/%{name}/plugins/#g" config
rm -rf %{buildroot}
mkdir -p %{buildroot}/etc/crowdsec/hub
mkdir -p %{buildroot}/etc/crowdsec/patterns
mkdir -p %{buildroot}/etc/crowdsec/console/
mkdir -p %{buildroot}%{_sharedstatedir}/%{name}/data
mkdir -p %{buildroot}%{_presetdir}
@ -62,6 +63,7 @@ install -m 600 -D config/config.yaml %{buildroot}%{_sysconfdir}/crowdsec
install -m 644 -D config/simulation.yaml %{buildroot}%{_sysconfdir}/crowdsec
install -m 644 -D config/profiles.yaml %{buildroot}%{_sysconfdir}/crowdsec
install -m 644 -D config/console.yaml %{buildroot}%{_sysconfdir}/crowdsec
install -m 644 -D config/context.yaml %{buildroot}%{_sysconfdir}/crowdsec/console/
install -m 750 -D config/%{name}.cron.daily %{buildroot}%{_sysconfdir}/cron.daily/%{name}
install -m 644 -D %{SOURCE1} %{buildroot}%{_presetdir}
@ -115,6 +117,7 @@ rm -rf %{buildroot}
%config(noreplace) %{_sysconfdir}/%{name}/simulation.yaml
%config(noreplace) %{_sysconfdir}/%{name}/profiles.yaml
%config(noreplace) %{_sysconfdir}/%{name}/console.yaml
%config(noreplace) %{_sysconfdir}/%{name}/console/context.yaml
%config(noreplace) %{_presetdir}/80-%{name}.preset
%config(noreplace) %{_sysconfdir}/%{name}/notifications/http.yaml
%config(noreplace) %{_sysconfdir}/%{name}/notifications/slack.yaml

67
tests/bats/81_alert_context.bats Normal file
View file

@ -0,0 +1,67 @@
#!/usr/bin/env bats
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
set -u
fake_log() {
for _ in $(seq 1 6); do
echo "$(LC_ALL=C date '+%b %d %H:%M:%S ')"'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424'
done
}
setup_file() {
load "../lib/setup_file.sh"
}
teardown_file() {
load "../lib/teardown_file.sh"
}
setup() {
load "../lib/setup.sh"
./instance-data load
}
teardown() {
./instance-crowdsec stop
}
#----------
@test "$FILE 1.1.1.172 has context" {
tmpfile=$(TMPDIR="${BATS_TEST_TMPDIR}" mktemp)
touch "${tmpfile}"
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
cat <<-EOT >"${ACQUIS_YAML}"
filename: $tmpfile
labels:
type: syslog
EOT
CONTEXT_YAML=$(config_get '.crowdsec_service.console_context_path')
cat <<-EOT >"${CONTEXT_YAML}"
target_user:
- evt.Parsed.sshd_invalid_user
source_ip:
- evt.Parsed.sshd_client_ip
source_host:
- evt.Meta.machine
EOT
./instance-crowdsec start
sleep 2
fake_log >>"${tmpfile}"
sleep 2
rm -f -- "${tmpfile}"
run -0 cscli alerts list -o json
run -0 jq '.[0].id' <(output)
ALERT_ID="$output"
run -0 cscli alerts inspect "$ALERT_ID" -o json
run -0 jq -c '.meta | sort_by(.key) | map([.key,.value])' <(output)
assert_json '[["source_host","[\"sd-126005\"]"],["source_ip","[\"1.1.1.172\"]"],["target_user","[\"netflix\"]"]]'
}

4
tests/lib/config/config-local
View file

@ -61,6 +61,8 @@ config_generate() {
../config/online_api_credentials.yaml \
"${CONFIG_DIR}/"
cp ../config/context.yaml "${CONFIG_DIR}/console/"
# the default acquis file contains files that are not readable by everyone
touch "$LOG_DIR/empty.log"
cat <<-EOT >"$CONFIG_DIR/acquis.yaml"
@ -94,6 +96,7 @@ config_generate() {
.api.client.credentials_path=strenv(CONFIG_DIR)+"/local_api_credentials.yaml" |
.api.server.profiles_path=strenv(CONFIG_DIR)+"/profiles.yaml" |
.api.server.console_path=strenv(CONFIG_DIR)+"/console.yaml" |
.crowdsec_service.console_context_path=strenv(CONFIG_DIR) + "/console/context.yaml" |
.api.server.online_client.credentials_path=strenv(CONFIG_DIR)+"/online_api_credentials.yaml"
' ../config/config.yaml >"${CONFIG_DIR}/config.yaml"
}
@ -107,6 +110,7 @@ make_init_data() {
mkdir -p "${CONFIG_DIR}/notifications"
mkdir -p "${CONFIG_DIR}/hub"
mkdir -p "${CONFIG_DIR}/patterns"
mkdir -p "${CONFIG_DIR}/console"
cp -a "../config/patterns" "${CONFIG_DIR}/"
config_generate
# XXX errors from instance-db should be reported...

7
windows/installer/product.wxs
View file

@ -54,6 +54,11 @@
<PermissionEx Sddl="D:PAI(A;;FA;;;SY)(A;;FA;;;BA)"/>
</File>
</Component>
<Directory Id="ConsoleDir" Name="console">
<Component Id="ConsoleContextFile" Guid="f146e12a-8f02-4129-9029-577807966e92">
<File Id="context.yaml" Source="config\context.yaml" />
</Component>
</Directory>
<Component Id="OnlineCreds" Guid="a652a6cb-d464-40b1-8f50-78dce0135d20">
<File Id="online_api_credentials.yaml" Source="config\online_api_credentials.yaml">
<PermissionEx Sddl="D:PAI(A;;FA;;;SY)(A;;FA;;;BA)"/>
@ -163,6 +168,8 @@
<ComponentRef Id="NotifConfig" />
<ComponentRef Id="CreateCrowdsecPluginsDir"/>
<ComponentRef Id="CreateCrowdsecDataDir" />
<ComponentRef Id="ConsoleDir"/>
<ComponentRef Id="ConsoleContextFile"/>
<ComponentRef Id="Csconfig_lapi" />
<ComponentRef Id="Csconfig_no_lapi" />
<ComponentGroupRef Id="CrowdsecPatterns" />

3
wizard.sh
View file

@ -28,6 +28,7 @@ CROWDSEC_CONFIG_PATH="${CROWDSEC_PATH}"
CROWDSEC_LOG_FILE="/var/log/crowdsec.log"
LAPI_LOG_FILE="/var/log/crowdsec_api.log"
CROWDSEC_PLUGIN_DIR="${CROWDSEC_USR_DIR}/plugins"
CROWDSEC_CONSOLE_DIR="${CROWDSEC_PATH}/console"
CROWDSEC_BIN="./cmd/crowdsec/crowdsec"
CSCLI_BIN="./cmd/crowdsec-cli/cscli"
@ -403,6 +404,7 @@ install_crowdsec() {
mkdir -p "${CROWDSEC_CONFIG_PATH}/postoverflows" || exit
mkdir -p "${CROWDSEC_CONFIG_PATH}/collections" || exit
mkdir -p "${CROWDSEC_CONFIG_PATH}/patterns" || exit
mkdir -p "${CROWDSEC_CONSOLE_DIR}" || exit
#tmp
mkdir -p /tmp/data
@ -419,6 +421,7 @@ install_crowdsec() {
install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 644 -D ./config/"${CONSOLE_FILE}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 644 -D ./config/context.yaml "${CROWDSEC_CONSOLE_DIR}" 1> /dev/null || exit
DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" || log_fatal "unable to generate user configuration file"
if [[ ${DOCKER_MODE} == "false" ]]; then