add geoip enrich expr helpers

This commit is contained in:
Sebastien Blot 2024-04-15 22:06:59 +02:00
parent 81976c6982
commit 127969d325
No known key found for this signature in database
GPG key ID: DFC2902F40449F6A
15 changed files with 184 additions and 115 deletions

View file

@ -19,6 +19,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
@ -32,6 +33,13 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, []
return nil, nil, fmt.Errorf("while loading context: %w", err) return nil, nil, fmt.Errorf("while loading context: %w", err)
} }
err = exprhelpers.GeoIPInit(hub.GetDataDir())
if err != nil {
//GeoIP databases are not mandatory, do not make crowdsec fail if they are not present
log.Warnf("unable to initialize GeoIP: %s", err)
}
// Start loading configs // Start loading configs
csParsers := parser.NewParsers(hub) csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {

View file

@ -1,9 +1,11 @@
package exprhelpers package exprhelpers
import ( import (
"net"
"time" "time"
"github.com/crowdsecurity/crowdsec/pkg/cticlient" "github.com/crowdsecurity/crowdsec/pkg/cticlient"
"github.com/oschwald/geoip2-golang"
) )
type exprCustomFunc struct { type exprCustomFunc struct {
@ -455,6 +457,27 @@ var exprFuncs = []exprCustomFunc{
new(func(string) bool), new(func(string) bool),
}, },
}, },
{
name: "GeoIPEnrich",
function: GeoIPEnrich,
signature: []interface{}{
new(func(string) *geoip2.City),
},
},
{
name: "GeoIPASNEnrich",
function: GeoIPASNEnrich,
signature: []interface{}{
new(func(string) *geoip2.ASN),
},
},
{
name: "GeoIPRangeEnrich",
function: GeoIPRangeEnrich,
signature: []interface{}{
new(func(string) *net.IPNet),
},
},
} }
//go 1.20 "CutPrefix": strings.CutPrefix, //go 1.20 "CutPrefix": strings.CutPrefix,

63
pkg/exprhelpers/geoip.go Normal file
View file

@ -0,0 +1,63 @@
package exprhelpers
import (
"net"
)
func GeoIPEnrich(params ...any) (any, error) {
if geoIPCityReader == nil {
return nil, nil
}
ip := params[0].(string)
parsedIP := net.ParseIP(ip)
city, err := geoIPCityReader.City(parsedIP)
if err != nil {
return nil, err
}
return city, nil
}
func GeoIPASNEnrich(params ...any) (any, error) {
if geoIPASNReader == nil {
return nil, nil
}
ip := params[0].(string)
parsedIP := net.ParseIP(ip)
asn, err := geoIPASNReader.ASN(parsedIP)
if err != nil {
return nil, err
}
return asn, nil
}
func GeoIPRangeEnrich(params ...any) (any, error) {
if geoIPRangeReader == nil {
return nil, nil
}
ip := params[0].(string)
var dummy interface{}
parsedIP := net.ParseIP(ip)
rangeIP, ok, err := geoIPRangeReader.LookupNetwork(parsedIP, &dummy)
if err != nil {
return nil, err
}
if !ok {
return nil, nil
}
return rangeIP, nil
}

View file

@ -20,6 +20,8 @@ import (
"github.com/c-robinson/iplib" "github.com/c-robinson/iplib"
"github.com/cespare/xxhash/v2" "github.com/cespare/xxhash/v2"
"github.com/davecgh/go-spew/spew" "github.com/davecgh/go-spew/spew"
"github.com/oschwald/geoip2-golang"
"github.com/oschwald/maxminddb-golang"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/umahmood/haversine" "github.com/umahmood/haversine"
@ -55,6 +57,10 @@ var exprFunctionOptions []expr.Option
var keyValuePattern = regexp.MustCompile(`(?P<key>[^=\s]+)=(?:"(?P<quoted_value>[^"\\]*(?:\\.[^"\\]*)*)"|(?P<value>[^=\s]+)|\s*)`) var keyValuePattern = regexp.MustCompile(`(?P<key>[^=\s]+)=(?:"(?P<quoted_value>[^"\\]*(?:\\.[^"\\]*)*)"|(?P<value>[^=\s]+)|\s*)`)
var geoIPCityReader *geoip2.Reader
var geoIPASNReader *geoip2.Reader
var geoIPRangeReader *maxminddb.Reader
func GetExprOptions(ctx map[string]interface{}) []expr.Option { func GetExprOptions(ctx map[string]interface{}) []expr.Option {
if len(exprFunctionOptions) == 0 { if len(exprFunctionOptions) == 0 {
exprFunctionOptions = []expr.Option{} exprFunctionOptions = []expr.Option{}
@ -72,6 +78,30 @@ func GetExprOptions(ctx map[string]interface{}) []expr.Option {
return ret return ret
} }
func GeoIPInit(datadir string) error {
var err error
geoIPCityReader, err = geoip2.Open(filepath.Join(datadir, "GeoLite2-City.mmdb"))
if err != nil {
log.Errorf("unable to open GeoLite2-City.mmdb : %s", err)
return err
}
geoIPASNReader, err = geoip2.Open(filepath.Join(datadir, "GeoLite2-ASN.mmdb"))
if err != nil {
log.Errorf("unable to open GeoLite2-ASN.mmdb : %s", err)
return err
}
geoIPRangeReader, err = maxminddb.Open(filepath.Join(datadir, "GeoLite2-ASN.mmdb"))
if err != nil {
log.Errorf("unable to open GeoLite2-ASN.mmdb : %s", err)
return err
}
return nil
}
func Init(databaseClient *database.Client) error { func Init(databaseClient *database.Client) error {
dataFile = make(map[string][]string) dataFile = make(map[string][]string)
dataFileRegex = make(map[string][]*regexp.Regexp) dataFileRegex = make(map[string][]*regexp.Regexp)

View file

@ -7,7 +7,7 @@ import (
) )
/* should be part of a package shared with enrich/geoip.go */ /* should be part of a package shared with enrich/geoip.go */
type EnrichFunc func(string, *types.Event, interface{}, *log.Entry) (map[string]string, error) type EnrichFunc func(string, *types.Event, *log.Entry) (map[string]string, error)
type InitFunc func(map[string]string) (interface{}, error) type InitFunc func(map[string]string) (interface{}, error)
type EnricherCtx struct { type EnricherCtx struct {
@ -16,59 +16,42 @@ type EnricherCtx struct {
type Enricher struct { type Enricher struct {
Name string Name string
InitFunc InitFunc
EnrichFunc EnrichFunc EnrichFunc EnrichFunc
Ctx interface{}
} }
/* mimic plugin loading */ /* mimic plugin loading */
func Loadplugin(path string) (EnricherCtx, error) { func Loadplugin() (EnricherCtx, error) {
enricherCtx := EnricherCtx{} enricherCtx := EnricherCtx{}
enricherCtx.Registered = make(map[string]*Enricher) enricherCtx.Registered = make(map[string]*Enricher)
enricherConfig := map[string]string{"datadir": path}
EnrichersList := []*Enricher{ EnrichersList := []*Enricher{
{ {
Name: "GeoIpCity", Name: "GeoIpCity",
InitFunc: GeoIPCityInit,
EnrichFunc: GeoIpCity, EnrichFunc: GeoIpCity,
}, },
{ {
Name: "GeoIpASN", Name: "GeoIpASN",
InitFunc: GeoIPASNInit,
EnrichFunc: GeoIpASN, EnrichFunc: GeoIpASN,
}, },
{ {
Name: "IpToRange", Name: "IpToRange",
InitFunc: IpToRangeInit,
EnrichFunc: IpToRange, EnrichFunc: IpToRange,
}, },
{ {
Name: "reverse_dns", Name: "reverse_dns",
InitFunc: reverseDNSInit,
EnrichFunc: reverse_dns, EnrichFunc: reverse_dns,
}, },
{ {
Name: "ParseDate", Name: "ParseDate",
InitFunc: parseDateInit,
EnrichFunc: ParseDate, EnrichFunc: ParseDate,
}, },
{ {
Name: "UnmarshalJSON", Name: "UnmarshalJSON",
InitFunc: unmarshalInit,
EnrichFunc: unmarshalJSON, EnrichFunc: unmarshalJSON,
}, },
} }
for _, enricher := range EnrichersList { for _, enricher := range EnrichersList {
log.Debugf("Initiating enricher '%s'", enricher.Name)
pluginCtx, err := enricher.InitFunc(enricherConfig)
if err != nil {
log.Errorf("unable to register plugin '%s': %v", enricher.Name, err)
continue
}
enricher.Ctx = pluginCtx
log.Infof("Successfully registered enricher '%s'", enricher.Name) log.Infof("Successfully registered enricher '%s'", enricher.Name)
enricherCtx.Registered[enricher.Name] = enricher enricherCtx.Registered[enricher.Name] = enricher
} }

View file

@ -56,7 +56,7 @@ func GenDateParse(date string) (string, time.Time) {
return "", time.Time{} return "", time.Time{}
} }
func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[string]string, error) { func ParseDate(in string, p *types.Event, plog *log.Entry) (map[string]string, error) {
var ret = make(map[string]string) var ret = make(map[string]string)
var strDate string var strDate string
@ -105,7 +105,3 @@ func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[s
return ret, nil return ret, nil
} }
func parseDateInit(cfg map[string]string) (interface{}, error) {
return nil, nil
}

View file

@ -48,7 +48,7 @@ func TestDateParse(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
tt := tt tt := tt
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
strTime, err := ParseDate(tt.evt.StrTime, &tt.evt, nil, logger) strTime, err := ParseDate(tt.evt.StrTime, &tt.evt, logger)
cstest.RequireErrorContains(t, err, tt.expectedErr) cstest.RequireErrorContains(t, err, tt.expectedErr)
if tt.expectedErr != "" { if tt.expectedErr != "" {
return return

View file

@ -11,7 +11,7 @@ import (
/* All plugins must export a list of function pointers for exported symbols */ /* All plugins must export a list of function pointers for exported symbols */
//var ExportedFuncs = []string{"reverse_dns"} //var ExportedFuncs = []string{"reverse_dns"}
func reverse_dns(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { func reverse_dns(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string) ret := make(map[string]string)
if field == "" { if field == "" {
return nil, nil return nil, nil
@ -25,7 +25,3 @@ func reverse_dns(field string, p *types.Event, ctx interface{}, plog *log.Entry)
ret["reverse_dns"] = rets[0] ret["reverse_dns"] = rets[0]
return ret, nil return ret, nil
} }
func reverseDNSInit(cfg map[string]string) (interface{}, error) {
return nil, nil
}

View file

@ -6,53 +6,53 @@ import (
"strconv" "strconv"
"github.com/oschwald/geoip2-golang" "github.com/oschwald/geoip2-golang"
"github.com/oschwald/maxminddb-golang"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
) )
func IpToRange(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { func IpToRange(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
var dummy interface{}
ret := make(map[string]string)
if field == "" {
return nil, nil
}
ip := net.ParseIP(field)
if ip == nil {
plog.Infof("Can't parse ip %s, no range enrich", field)
return nil, nil
}
net, ok, err := ctx.(*maxminddb.Reader).LookupNetwork(ip, &dummy)
if err != nil {
plog.Errorf("Failed to fetch network for %s : %v", ip.String(), err)
return nil, nil
}
if !ok {
plog.Debugf("Unable to find range of %s", ip.String())
return nil, nil
}
ret["SourceRange"] = net.String()
return ret, nil
}
func GeoIpASN(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string)
if field == "" { if field == "" {
return nil, nil return nil, nil
} }
ip := net.ParseIP(field) r, err := exprhelpers.GeoIPRangeEnrich(field)
if ip == nil {
plog.Infof("Can't parse ip %s, no ASN enrich", ip)
return nil, nil
}
record, err := ctx.(*geoip2.Reader).ASN(ip)
if err != nil { if err != nil {
plog.Errorf("Unable to enrich ip '%s'", field) plog.Errorf("Unable to enrich ip '%s'", field)
return nil, nil //nolint:nilerr return nil, nil //nolint:nilerr
} }
if r == nil {
plog.Warnf("No range found for ip '%s'", field)
return nil, nil //nolint:nilerr
}
record := r.(*net.IPNet)
ret := make(map[string]string)
ret["SourceRange"] = record.String()
return ret, nil
}
func GeoIpASN(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
if field == "" {
return nil, nil
}
r, err := exprhelpers.GeoIPASNEnrich(field)
if err != nil {
plog.Errorf("Unable to enrich ip '%s'", field)
return nil, nil //nolint:nilerr
}
record := r.(*geoip2.ASN)
ret := make(map[string]string)
ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber) ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
ret["ASNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber) ret["ASNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
ret["ASNOrg"] = record.AutonomousSystemOrganization ret["ASNOrg"] = record.AutonomousSystemOrganization
@ -62,21 +62,21 @@ func GeoIpASN(field string, p *types.Event, ctx interface{}, plog *log.Entry) (m
return ret, nil return ret, nil
} }
func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { func GeoIpCity(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string)
if field == "" { if field == "" {
return nil, nil return nil, nil
} }
ip := net.ParseIP(field)
if ip == nil { r, err := exprhelpers.GeoIPEnrich(field)
plog.Infof("Can't parse ip %s, no City enrich", ip)
return nil, nil
}
record, err := ctx.(*geoip2.Reader).City(ip)
if err != nil { if err != nil {
plog.Debugf("Unable to enrich ip '%s'", ip) plog.Errorf("Unable to enrich ip '%s'", field)
return nil, nil //nolint:nilerr return nil, nil //nolint:nilerr
} }
record := r.(*geoip2.City)
ret := make(map[string]string)
if record.Country.IsoCode != "" { if record.Country.IsoCode != "" {
ret["IsoCode"] = record.Country.IsoCode ret["IsoCode"] = record.Country.IsoCode
ret["IsInEU"] = strconv.FormatBool(record.Country.IsInEuropeanUnion) ret["IsInEU"] = strconv.FormatBool(record.Country.IsInEuropeanUnion)
@ -88,7 +88,7 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (
ret["IsInEU"] = strconv.FormatBool(record.RepresentedCountry.IsInEuropeanUnion) ret["IsInEU"] = strconv.FormatBool(record.RepresentedCountry.IsInEuropeanUnion)
} else { } else {
ret["IsoCode"] = "" ret["IsoCode"] = ""
ret["IsInEU"] = strconv.FormatBool(false) ret["IsInEU"] = "false"
} }
ret["Latitude"] = fmt.Sprintf("%f", record.Location.Latitude) ret["Latitude"] = fmt.Sprintf("%f", record.Location.Latitude)
@ -98,33 +98,3 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (
return ret, nil return ret, nil
} }
func GeoIPCityInit(cfg map[string]string) (interface{}, error) {
dbCityReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return dbCityReader, nil
}
func GeoIPASNInit(cfg map[string]string) (interface{}, error) {
dbASReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return dbASReader, nil
}
func IpToRangeInit(cfg map[string]string) (interface{}, error) {
ipToRangeReader, err := maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return ipToRangeReader, nil
}

View file

@ -8,7 +8,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
) )
func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { func unmarshalJSON(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
err := json.Unmarshal([]byte(p.Line.Raw), &p.Unmarshaled) err := json.Unmarshal([]byte(p.Line.Raw), &p.Unmarshaled)
if err != nil { if err != nil {
plog.Errorf("could not unmarshal JSON: %s", err) plog.Errorf("could not unmarshal JSON: %s", err)
@ -17,7 +17,3 @@ func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entr
plog.Tracef("unmarshaled JSON: %+v", p.Unmarshaled) plog.Tracef("unmarshaled JSON: %+v", p.Unmarshaled)
return nil, nil return nil, nil
} }
func unmarshalInit(cfg map[string]string) (interface{}, error) {
return nil, nil
}

View file

@ -64,7 +64,7 @@ type Node struct {
Data []*types.DataSource `yaml:"data,omitempty"` Data []*types.DataSource `yaml:"data,omitempty"`
} }
func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error { func (n *Node) validate(ectx EnricherCtx) error {
//stage is being set automagically //stage is being set automagically
if n.Stage == "" { if n.Stage == "" {
@ -563,7 +563,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
return fmt.Errorf("Node is empty") return fmt.Errorf("Node is empty")
} }
if err := n.validate(pctx, ectx); err != nil { if err := n.validate(ectx); err != nil {
return err return err
} }

View file

@ -56,7 +56,7 @@ func TestParserConfigs(t *testing.T) {
t.Fatalf("Compile: (%d/%d) expected error", idx+1, len(CfgTests)) t.Fatalf("Compile: (%d/%d) expected error", idx+1, len(CfgTests))
} }
err = CfgTests[idx].NodeCfg.validate(pctx, EnricherCtx{}) err = CfgTests[idx].NodeCfg.validate(EnricherCtx{})
if CfgTests[idx].Valid == true && err != nil { if CfgTests[idx].Valid == true && err != nil {
t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err) t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
} }

View file

@ -152,7 +152,11 @@ func prepTests() (*UnixParserCtx, EnricherCtx, error) {
//Load enrichment //Load enrichment
datadir := "./test_data/" datadir := "./test_data/"
ectx, err = Loadplugin(datadir) err = exprhelpers.GeoIPInit(datadir)
if err != nil {
log.Fatalf("unable to initialize GeoIP: %s", err)
}
ectx, err = Loadplugin()
if err != nil { if err != nil {
log.Fatalf("failed to load plugin geoip : %v", err) log.Fatalf("failed to load plugin geoip : %v", err)
} }

View file

@ -155,7 +155,7 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error {
/*still way too hackish, but : inject all the results in enriched, and */ /*still way too hackish, but : inject all the results in enriched, and */
if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok { if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok {
clog.Tracef("Found method '%s'", static.Method) clog.Tracef("Found method '%s'", static.Method)
ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx, n.Logger.WithField("method", static.Method)) ret, err := enricherPlugin.EnrichFunc(value, event, n.Logger.WithField("method", static.Method))
if err != nil { if err != nil {
clog.Errorf("method '%s' returned an error : %v", static.Method, err) clog.Errorf("method '%s' returned an error : %v", static.Method, err)
} }

View file

@ -117,7 +117,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
*/ */
log.Infof("Loading enrich plugins") log.Infof("Loading enrich plugins")
parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir) parsers.EnricherCtx, err = Loadplugin()
if err != nil { if err != nil {
return parsers, fmt.Errorf("failed to load enrich plugin : %v", err) return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
} }