Compare commits

...

10 commits

Author SHA1 Message Date
Sebastien Blot 5f0044d276
close geoip db on shutdown 2024-04-17 09:54:55 +02:00
Sebastien Blot bdc62d3715
only ignore rules without a message 2024-04-17 09:46:43 +02:00
Sebastien Blot 927f2e4c48
lint 2024-04-15 23:57:02 +02:00
Sebastien Blot 170e5e8dd8
perform geoip enrich on appsec alerts 2024-04-15 22:31:08 +02:00
Sebastien Blot 127969d325
add geoip enrich expr helpers 2024-04-15 22:06:59 +02:00
Sebastien Blot 81976c6982
also log rule message to meta 2024-04-15 18:33:24 +02:00
Sebastien Blot b370925967
fix lint 2024-04-15 18:23:06 +02:00
Sebastien Blot 7447b8bf04
better handling of multiple matched zones 2024-04-15 14:06:11 +02:00
Sebastien Blot f6038feabe
Merge branch 'master' into appsec-properly-populate-event 2024-04-15 13:40:01 +02:00
Sebastien Blot c71cb4bcda
wip 2024-03-25 17:37:05 +01:00
17 changed files with 275 additions and 118 deletions

View file

@ -19,6 +19,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -32,6 +33,13 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, []
return nil, nil, fmt.Errorf("while loading context: %w", err)
}
err = exprhelpers.GeoIPInit(hub.GetDataDir())
if err != nil {
//GeoIP databases are not mandatory, do not make crowdsec fail if they are not present
log.Warnf("unable to initialize GeoIP: %s", err)
}
// Start loading configs
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {

View file

@ -177,6 +177,9 @@ func ShutdownCrowdsecRoutines() error {
// He's dead, Jim.
crowdsecTomb.Kill(nil)
// close the potential geoips reader we have to avoid leaking ressources on reload
exprhelpers.GeoIPClose()
return reterr
}

View file

@ -3,18 +3,32 @@ package appsecacquisition
import (
"encoding/json"
"fmt"
"net"
"time"
"github.com/crowdsecurity/coraza/v3/collection"
"github.com/crowdsecurity/coraza/v3/types/variables"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/oschwald/geoip2-golang"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)
func appendMeta(meta models.Meta, key string, value string) models.Meta {
if value == "" {
return meta
}
meta = append(meta, &models.MetaItems0{
Key: key,
Value: value,
})
return meta
}
func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
//if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI
if !inEvt.Appsec.HasInBandMatches {
@ -29,12 +43,40 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
Scope: ptr.Of(types.Ip),
}
asndata, err := exprhelpers.GeoIPASNEnrich(inEvt.Parsed["source_ip"])
if err != nil {
log.Errorf("Unable to enrich ip '%s'", inEvt.Parsed["source_ip"])
} else if asndata != nil {
record := asndata.(*geoip2.ASN)
source.AsName = record.AutonomousSystemOrganization
source.AsNumber = fmt.Sprintf("%d", record.AutonomousSystemNumber)
}
cityData, err := exprhelpers.GeoIPEnrich(inEvt.Parsed["source_ip"])
if err != nil {
log.Errorf("Unable to enrich ip '%s'", inEvt.Parsed["source_ip"])
} else if cityData != nil {
record := cityData.(*geoip2.City)
source.Cn = record.Country.IsoCode
source.Latitude = float32(record.Location.Latitude)
source.Longitude = float32(record.Location.Longitude)
}
rangeData, err := exprhelpers.GeoIPRangeEnrich(inEvt.Parsed["source_ip"])
if err != nil {
log.Errorf("Unable to enrich ip '%s'", inEvt.Parsed["source_ip"])
} else if rangeData != nil {
record := rangeData.(*net.IPNet)
source.Range = record.String()
}
evt.Overflow.Sources = make(map[string]models.Source)
evt.Overflow.Sources["ip"] = source
alert := models.Alert{}
alert.Capacity = ptr.Of(int32(1))
alert.Events = make([]*models.Event, 0)
alert.Events = make([]*models.Event, len(evt.Appsec.GetRuleIDs()))
alert.Meta = make(models.Meta, 0)
for _, key := range []string{"target_uri", "method"} {
@ -64,7 +106,38 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
}
}
alert.EventsCount = ptr.Of(int32(1))
now := ptr.Of(time.Now().UTC().Format(time.RFC3339))
for _, matched_rule := range inEvt.Appsec.MatchedRules {
evtRule := models.Event{}
evtRule.Timestamp = now
evtRule.Meta = make(models.Meta, 0)
for _, key := range []string{"id", "name", "method", "uri", "matched_zones", "msg"} {
switch value := matched_rule[key].(type) {
case string:
evtRule.Meta = appendMeta(evtRule.Meta, key, value)
case int:
evtRule.Meta = appendMeta(evtRule.Meta, key, fmt.Sprintf("%d", value))
case []string:
for _, v := range value {
evtRule.Meta = appendMeta(evtRule.Meta, key, v)
}
case []int:
for _, v := range value {
evtRule.Meta = appendMeta(evtRule.Meta, key, fmt.Sprintf("%d", v))
}
default:
evtRule.Meta = appendMeta(evtRule.Meta, key, fmt.Sprintf("%v", value))
}
}
alert.Events = append(alert.Events, &evtRule)
}
alert.EventsCount = ptr.Of(int32(len(evt.Appsec.MatchedRules)))
alert.Leakspeed = ptr.Of("")
alert.Scenario = ptr.Of(inEvt.Appsec.MatchedRules.GetName())
alert.ScenarioHash = ptr.Of(inEvt.Appsec.MatchedRules.GetHash())
@ -200,7 +273,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR
})
for _, rule := range req.Tx.MatchedRules() {
if rule.Message() == "" || rule.DisruptiveAction() == "pass" || rule.DisruptiveAction() == "allow" {
if rule.Message() == "" {
r.logger.Tracef("discarding rule %d (action: %s)", rule.Rule().ID(), rule.DisruptiveAction())
continue
}

View file

@ -1,9 +1,11 @@
package exprhelpers
import (
"net"
"time"
"github.com/crowdsecurity/crowdsec/pkg/cticlient"
"github.com/oschwald/geoip2-golang"
)
type exprCustomFunc struct {
@ -455,6 +457,27 @@ var exprFuncs = []exprCustomFunc{
new(func(string) bool),
},
},
{
name: "GeoIPEnrich",
function: GeoIPEnrich,
signature: []interface{}{
new(func(string) *geoip2.City),
},
},
{
name: "GeoIPASNEnrich",
function: GeoIPASNEnrich,
signature: []interface{}{
new(func(string) *geoip2.ASN),
},
},
{
name: "GeoIPRangeEnrich",
function: GeoIPRangeEnrich,
signature: []interface{}{
new(func(string) *net.IPNet),
},
},
}
//go 1.20 "CutPrefix": strings.CutPrefix,

63
pkg/exprhelpers/geoip.go Normal file
View file

@ -0,0 +1,63 @@
package exprhelpers
import (
"net"
)
func GeoIPEnrich(params ...any) (any, error) {
if geoIPCityReader == nil {
return nil, nil
}
ip := params[0].(string)
parsedIP := net.ParseIP(ip)
city, err := geoIPCityReader.City(parsedIP)
if err != nil {
return nil, err
}
return city, nil
}
func GeoIPASNEnrich(params ...any) (any, error) {
if geoIPASNReader == nil {
return nil, nil
}
ip := params[0].(string)
parsedIP := net.ParseIP(ip)
asn, err := geoIPASNReader.ASN(parsedIP)
if err != nil {
return nil, err
}
return asn, nil
}
func GeoIPRangeEnrich(params ...any) (any, error) {
if geoIPRangeReader == nil {
return nil, nil
}
ip := params[0].(string)
var dummy interface{}
parsedIP := net.ParseIP(ip)
rangeIP, ok, err := geoIPRangeReader.LookupNetwork(parsedIP, &dummy)
if err != nil {
return nil, err
}
if !ok {
return nil, nil
}
return rangeIP, nil
}

View file

@ -20,6 +20,8 @@ import (
"github.com/c-robinson/iplib"
"github.com/cespare/xxhash/v2"
"github.com/davecgh/go-spew/spew"
"github.com/oschwald/geoip2-golang"
"github.com/oschwald/maxminddb-golang"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"github.com/umahmood/haversine"
@ -55,6 +57,10 @@ var exprFunctionOptions []expr.Option
var keyValuePattern = regexp.MustCompile(`(?P<key>[^=\s]+)=(?:"(?P<quoted_value>[^"\\]*(?:\\.[^"\\]*)*)"|(?P<value>[^=\s]+)|\s*)`)
var geoIPCityReader *geoip2.Reader
var geoIPASNReader *geoip2.Reader
var geoIPRangeReader *maxminddb.Reader
func GetExprOptions(ctx map[string]interface{}) []expr.Option {
if len(exprFunctionOptions) == 0 {
exprFunctionOptions = []expr.Option{}
@ -72,6 +78,42 @@ func GetExprOptions(ctx map[string]interface{}) []expr.Option {
return ret
}
func GeoIPInit(datadir string) error {
var err error
geoIPCityReader, err = geoip2.Open(filepath.Join(datadir, "GeoLite2-City.mmdb"))
if err != nil {
log.Errorf("unable to open GeoLite2-City.mmdb : %s", err)
return err
}
geoIPASNReader, err = geoip2.Open(filepath.Join(datadir, "GeoLite2-ASN.mmdb"))
if err != nil {
log.Errorf("unable to open GeoLite2-ASN.mmdb : %s", err)
return err
}
geoIPRangeReader, err = maxminddb.Open(filepath.Join(datadir, "GeoLite2-ASN.mmdb"))
if err != nil {
log.Errorf("unable to open GeoLite2-ASN.mmdb : %s", err)
return err
}
return nil
}
func GeoIPClose() {
if geoIPCityReader != nil {
geoIPCityReader.Close()
}
if geoIPASNReader != nil {
geoIPASNReader.Close()
}
if geoIPRangeReader != nil {
geoIPRangeReader.Close()
}
}
func Init(databaseClient *database.Client) error {
dataFile = make(map[string][]string)
dataFileRegex = make(map[string][]*regexp.Regexp)

View file

@ -7,7 +7,7 @@ import (
)
/* should be part of a package shared with enrich/geoip.go */
type EnrichFunc func(string, *types.Event, interface{}, *log.Entry) (map[string]string, error)
type EnrichFunc func(string, *types.Event, *log.Entry) (map[string]string, error)
type InitFunc func(map[string]string) (interface{}, error)
type EnricherCtx struct {
@ -16,59 +16,42 @@ type EnricherCtx struct {
type Enricher struct {
Name string
InitFunc InitFunc
EnrichFunc EnrichFunc
Ctx interface{}
}
/* mimic plugin loading */
func Loadplugin(path string) (EnricherCtx, error) {
func Loadplugin() (EnricherCtx, error) {
enricherCtx := EnricherCtx{}
enricherCtx.Registered = make(map[string]*Enricher)
enricherConfig := map[string]string{"datadir": path}
EnrichersList := []*Enricher{
{
Name: "GeoIpCity",
InitFunc: GeoIPCityInit,
EnrichFunc: GeoIpCity,
},
{
Name: "GeoIpASN",
InitFunc: GeoIPASNInit,
EnrichFunc: GeoIpASN,
},
{
Name: "IpToRange",
InitFunc: IpToRangeInit,
EnrichFunc: IpToRange,
},
{
Name: "reverse_dns",
InitFunc: reverseDNSInit,
EnrichFunc: reverse_dns,
},
{
Name: "ParseDate",
InitFunc: parseDateInit,
EnrichFunc: ParseDate,
},
{
Name: "UnmarshalJSON",
InitFunc: unmarshalInit,
EnrichFunc: unmarshalJSON,
},
}
for _, enricher := range EnrichersList {
log.Debugf("Initiating enricher '%s'", enricher.Name)
pluginCtx, err := enricher.InitFunc(enricherConfig)
if err != nil {
log.Errorf("unable to register plugin '%s': %v", enricher.Name, err)
continue
}
enricher.Ctx = pluginCtx
log.Infof("Successfully registered enricher '%s'", enricher.Name)
enricherCtx.Registered[enricher.Name] = enricher
}

View file

@ -56,7 +56,7 @@ func GenDateParse(date string) (string, time.Time) {
return "", time.Time{}
}
func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[string]string, error) {
func ParseDate(in string, p *types.Event, plog *log.Entry) (map[string]string, error) {
var ret = make(map[string]string)
var strDate string
@ -105,7 +105,3 @@ func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[s
return ret, nil
}
func parseDateInit(cfg map[string]string) (interface{}, error) {
return nil, nil
}

View file

@ -48,7 +48,7 @@ func TestDateParse(t *testing.T) {
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
strTime, err := ParseDate(tt.evt.StrTime, &tt.evt, nil, logger)
strTime, err := ParseDate(tt.evt.StrTime, &tt.evt, logger)
cstest.RequireErrorContains(t, err, tt.expectedErr)
if tt.expectedErr != "" {
return

View file

@ -11,7 +11,7 @@ import (
/* All plugins must export a list of function pointers for exported symbols */
//var ExportedFuncs = []string{"reverse_dns"}
func reverse_dns(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
func reverse_dns(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string)
if field == "" {
return nil, nil
@ -25,7 +25,3 @@ func reverse_dns(field string, p *types.Event, ctx interface{}, plog *log.Entry)
ret["reverse_dns"] = rets[0]
return ret, nil
}
func reverseDNSInit(cfg map[string]string) (interface{}, error) {
return nil, nil
}

View file

@ -6,53 +6,53 @@ import (
"strconv"
"github.com/oschwald/geoip2-golang"
"github.com/oschwald/maxminddb-golang"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func IpToRange(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
var dummy interface{}
ret := make(map[string]string)
if field == "" {
return nil, nil
}
ip := net.ParseIP(field)
if ip == nil {
plog.Infof("Can't parse ip %s, no range enrich", field)
return nil, nil
}
net, ok, err := ctx.(*maxminddb.Reader).LookupNetwork(ip, &dummy)
if err != nil {
plog.Errorf("Failed to fetch network for %s : %v", ip.String(), err)
return nil, nil
}
if !ok {
plog.Debugf("Unable to find range of %s", ip.String())
return nil, nil
}
ret["SourceRange"] = net.String()
return ret, nil
}
func GeoIpASN(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string)
func IpToRange(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
if field == "" {
return nil, nil
}
ip := net.ParseIP(field)
if ip == nil {
plog.Infof("Can't parse ip %s, no ASN enrich", ip)
return nil, nil
}
record, err := ctx.(*geoip2.Reader).ASN(ip)
r, err := exprhelpers.GeoIPRangeEnrich(field)
if err != nil {
plog.Errorf("Unable to enrich ip '%s'", field)
return nil, nil //nolint:nilerr
}
if r == nil {
plog.Warnf("No range found for ip '%s'", field)
return nil, nil
}
record := r.(*net.IPNet)
ret := make(map[string]string)
ret["SourceRange"] = record.String()
return ret, nil
}
func GeoIpASN(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
if field == "" {
return nil, nil
}
r, err := exprhelpers.GeoIPASNEnrich(field)
if err != nil {
plog.Errorf("Unable to enrich ip '%s'", field)
return nil, nil //nolint:nilerr
}
record := r.(*geoip2.ASN)
ret := make(map[string]string)
ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
ret["ASNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
ret["ASNOrg"] = record.AutonomousSystemOrganization
@ -62,21 +62,21 @@ func GeoIpASN(field string, p *types.Event, ctx interface{}, plog *log.Entry) (m
return ret, nil
}
func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string)
func GeoIpCity(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
if field == "" {
return nil, nil
}
ip := net.ParseIP(field)
if ip == nil {
plog.Infof("Can't parse ip %s, no City enrich", ip)
return nil, nil
}
record, err := ctx.(*geoip2.Reader).City(ip)
r, err := exprhelpers.GeoIPEnrich(field)
if err != nil {
plog.Debugf("Unable to enrich ip '%s'", ip)
plog.Errorf("Unable to enrich ip '%s'", field)
return nil, nil //nolint:nilerr
}
record := r.(*geoip2.City)
ret := make(map[string]string)
if record.Country.IsoCode != "" {
ret["IsoCode"] = record.Country.IsoCode
ret["IsInEU"] = strconv.FormatBool(record.Country.IsInEuropeanUnion)
@ -88,7 +88,7 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (
ret["IsInEU"] = strconv.FormatBool(record.RepresentedCountry.IsInEuropeanUnion)
} else {
ret["IsoCode"] = ""
ret["IsInEU"] = strconv.FormatBool(false)
ret["IsInEU"] = "false"
}
ret["Latitude"] = fmt.Sprintf("%f", record.Location.Latitude)
@ -98,33 +98,3 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (
return ret, nil
}
func GeoIPCityInit(cfg map[string]string) (interface{}, error) {
dbCityReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return dbCityReader, nil
}
func GeoIPASNInit(cfg map[string]string) (interface{}, error) {
dbASReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return dbASReader, nil
}
func IpToRangeInit(cfg map[string]string) (interface{}, error) {
ipToRangeReader, err := maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return ipToRangeReader, nil
}

View file

@ -8,7 +8,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
func unmarshalJSON(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
err := json.Unmarshal([]byte(p.Line.Raw), &p.Unmarshaled)
if err != nil {
plog.Errorf("could not unmarshal JSON: %s", err)
@ -17,7 +17,3 @@ func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entr
plog.Tracef("unmarshaled JSON: %+v", p.Unmarshaled)
return nil, nil
}
func unmarshalInit(cfg map[string]string) (interface{}, error) {
return nil, nil
}

View file

@ -64,7 +64,7 @@ type Node struct {
Data []*types.DataSource `yaml:"data,omitempty"`
}
func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
func (n *Node) validate(ectx EnricherCtx) error {
//stage is being set automagically
if n.Stage == "" {
@ -563,7 +563,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
return fmt.Errorf("Node is empty")
}
if err := n.validate(pctx, ectx); err != nil {
if err := n.validate(ectx); err != nil {
return err
}

View file

@ -56,7 +56,7 @@ func TestParserConfigs(t *testing.T) {
t.Fatalf("Compile: (%d/%d) expected error", idx+1, len(CfgTests))
}
err = CfgTests[idx].NodeCfg.validate(pctx, EnricherCtx{})
err = CfgTests[idx].NodeCfg.validate(EnricherCtx{})
if CfgTests[idx].Valid == true && err != nil {
t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
}

View file

@ -152,7 +152,11 @@ func prepTests() (*UnixParserCtx, EnricherCtx, error) {
//Load enrichment
datadir := "./test_data/"
ectx, err = Loadplugin(datadir)
err = exprhelpers.GeoIPInit(datadir)
if err != nil {
log.Fatalf("unable to initialize GeoIP: %s", err)
}
ectx, err = Loadplugin()
if err != nil {
log.Fatalf("failed to load plugin geoip : %v", err)
}

View file

@ -155,7 +155,7 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error {
/*still way too hackish, but : inject all the results in enriched, and */
if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok {
clog.Tracef("Found method '%s'", static.Method)
ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx, n.Logger.WithField("method", static.Method))
ret, err := enricherPlugin.EnrichFunc(value, event, n.Logger.WithField("method", static.Method))
if err != nil {
clog.Errorf("method '%s' returned an error : %v", static.Method, err)
}

View file

@ -117,7 +117,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
*/
log.Infof("Loading enrich plugins")
parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir)
parsers.EnricherCtx, err = Loadplugin()
if err != nil {
return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
}