beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Generic dateparse approach (#1669)

Browse source 
* Allow any parser to suggest a format string for the date to be parsed.

* allow the enricher functions to get the parser's logger so they can inherit the level
This commit is contained in:
Thibault "bui" Koechlin 2022-07-28 16:41:41 +02:00 committed by GitHub
parent 1fc29d094f
commit 866c200c31
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 129 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pkg/parser/enrich.go
View file

@ -6,7 +6,7 @@ import (
)
/* should be part of a packaged shared with enrich/geoip.go */
type EnrichFunc func(string, *types.Event, interface{}) (map[string]string, error)
type EnrichFunc func(string, *types.Event, interface{}, *log.Entry) (map[string]string, error)
type InitFunc func(map[string]string) (interface{}, error)
type EnricherCtx struct {

73
pkg/parser/enrich_date.go
View file

@ -7,6 +7,23 @@ import (
log "github.com/sirupsen/logrus"
)
func parseDateWithFormat(date, format string) (string, time.Time) {
t, err := time.Parse(format, date)
if err == nil && !t.IsZero() {
//if the year isn't set, set it to current date :)
if t.Year() == 0 {
t = t.AddDate(time.Now().UTC().Year(), 0, 0)
}
retstr, err := t.MarshalText()
if err != nil {
log.Warningf("Failed marshaling '%v'", t)
return "", time.Time{}
}
return string(retstr), t
}
return "", time.Time{}
}
func GenDateParse(date string) (string, time.Time) {
var (
layouts = [...]string{
@ -29,40 +46,44 @@ func GenDateParse(date string) (string, time.Time) {
)
for _, dateFormat := range layouts {
t, err := time.Parse(dateFormat, date)
if err == nil && !t.IsZero() {
//if the year isn't set, set it to current date :)
if t.Year() == 0 {
t = t.AddDate(time.Now().UTC().Year(), 0, 0)
}
retstr, err := t.MarshalText()
if err != nil {
log.Warningf("Failed marshaling '%v'", t)
continue
}
return string(retstr), t
retstr, parsedDate := parseDateWithFormat(date, dateFormat)
if !parsedDate.IsZero() {
return retstr, parsedDate
}
}
return "", time.Time{}
}
func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[string]string, error) {
var ret map[string]string = make(map[string]string)
var strDate string
var parsedDate time.Time
if p.StrTimeFormat != "" {
strDate, parsedDate = parseDateWithFormat(in, p.StrTimeFormat)
if !parsedDate.IsZero() {
ret["MarshaledTime"] = strDate
return ret, nil
} else {
plog.Debugf("unable to parse '%s' with layout '%s'", in, p.StrTimeFormat)
}
}
strDate, parsedDate = GenDateParse(in)
if !parsedDate.IsZero() {
ret["MarshaledTime"] = strDate
return ret, nil
}
plog.Debugf("no suitable date format found for '%s', falling back to now", in)
now := time.Now().UTC()
retstr, err := now.MarshalText()
if err != nil {
log.Warning("Failed marshaling current time")
return "", time.Time{}
plog.Warning("Failed marshaling current time")
return ret, err
}
return string(retstr), now
}
ret["MarshaledTime"] = string(retstr)
func ParseDate(in string, p *types.Event, x interface{}) (map[string]string, error) {
var ret map[string]string = make(map[string]string)
tstr, tbin := GenDateParse(in)
if !tbin.IsZero() {
ret["MarshaledTime"] = tstr
return ret, nil
}
return nil, nil
return ret, nil
}
func parseDateInit(cfg map[string]string) (interface{}, error) {

65
pkg/parser/enrich_date_test.go Normal file
View file

@ -0,0 +1,65 @@
package parser
import (
"testing"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
)
func TestDateParse(t *testing.T) {
tests := []struct {
name string
evt types.Event
expected_err *error
expected_strTime *string
}{
{
name: "RFC3339",
evt: types.Event{
StrTime: "2019-10-12T07:20:50.52Z",
},
expected_err: nil,
expected_strTime: types.StrPtr("2019-10-12T07:20:50.52Z"),
},
{
name: "02/Jan/2006:15:04:05 -0700",
evt: types.Event{
StrTime: "02/Jan/2006:15:04:05 -0700",
},
expected_err: nil,
expected_strTime: types.StrPtr("2006-01-02T15:04:05-07:00"),
},
{
name: "Dec 17 08:17:43",
evt: types.Event{
StrTime: "2011 X 17 zz 08X17X43 oneone Dec",
StrTimeFormat: "2006 X 2 zz 15X04X05 oneone Jan",
},
expected_err: nil,
expected_strTime: types.StrPtr("2011-12-17T08:17:43Z"),
},
}
logger := log.WithFields(log.Fields{
"test": "test",
})
for idx, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
strTime, err := ParseDate(tt.evt.StrTime, &tt.evt, nil, logger)
if tt.expected_err != nil {
if err != *tt.expected_err {
t.Errorf("%s: expected error %v, got %v", tt.name, tt.expected_err, err)
}
} else if err != nil {
t.Errorf("%s: expected no error, got %v", tt.name, err)
}
if err != nil {
return
}
if tt.expected_strTime != nil && strTime["MarshaledTime"] != *tt.expected_strTime {
t.Errorf("%d: expected strTime %s, got %s", idx, *tt.expected_strTime, strTime["MarshaledTime"])
}
})
}
}

4
pkg/parser/enrich_dns.go
View file

@ -11,14 +11,14 @@ import (
/* All plugins must export a list of function pointers for exported symbols */
//var ExportedFuncs = []string{"reverse_dns"}
func reverse_dns(field string, p *types.Event, ctx interface{}) (map[string]string, error) {
func reverse_dns(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string)
if field == "" {
return nil, nil
}
rets, err := net.LookupAddr(field)
if err != nil {
log.Debugf("failed to resolve '%s'", field)
plog.Debugf("failed to resolve '%s'", field)
return nil, nil
}
//When using the host C library resolver, at most one result will be returned. To bypass the host resolver, use a custom Resolver.

24
pkg/parser/enrich_geoip.go
View file

@ -12,7 +12,7 @@ import (
"github.com/oschwald/maxminddb-golang"
)
func IpToRange(field string, p *types.Event, ctx interface{}) (map[string]string, error) {
func IpToRange(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
var dummy interface{}
ret := make(map[string]string)
@ -21,23 +21,23 @@ func IpToRange(field string, p *types.Event, ctx interface{}) (map[string]string
}
ip := net.ParseIP(field)
if ip == nil {
log.Infof("Can't parse ip %s, no range enrich", field)
plog.Infof("Can't parse ip %s, no range enrich", field)
return nil, nil
}
net, ok, err := ctx.(*maxminddb.Reader).LookupNetwork(ip, &dummy)
if err != nil {
log.Errorf("Failed to fetch network for %s : %v", ip.String(), err)
plog.Errorf("Failed to fetch network for %s : %v", ip.String(), err)
return nil, nil
}
if !ok {
log.Debugf("Unable to find range of %s", ip.String())
plog.Debugf("Unable to find range of %s", ip.String())
return nil, nil
}
ret["SourceRange"] = net.String()
return ret, nil
}
func GeoIpASN(field string, p *types.Event, ctx interface{}) (map[string]string, error) {
func GeoIpASN(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string)
if field == "" {
return nil, nil
@ -45,36 +45,36 @@ func GeoIpASN(field string, p *types.Event, ctx interface{}) (map[string]string,
ip := net.ParseIP(field)
if ip == nil {
log.Infof("Can't parse ip %s, no ASN enrich", ip)
plog.Infof("Can't parse ip %s, no ASN enrich", ip)
return nil, nil
}
record, err := ctx.(*geoip2.Reader).ASN(ip)
if err != nil {
log.Errorf("Unable to enrich ip '%s'", field)
plog.Errorf("Unable to enrich ip '%s'", field)
return nil, nil
}
ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
ret["ASNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
ret["ASNOrg"] = record.AutonomousSystemOrganization
log.Tracef("geoip ASN %s -> %s, %s", field, ret["ASNNumber"], ret["ASNOrg"])
plog.Tracef("geoip ASN %s -> %s, %s", field, ret["ASNNumber"], ret["ASNOrg"])
return ret, nil
}
func GeoIpCity(field string, p *types.Event, ctx interface{}) (map[string]string, error) {
func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string)
if field == "" {
return nil, nil
}
ip := net.ParseIP(field)
if ip == nil {
log.Infof("Can't parse ip %s, no City enrich", ip)
plog.Infof("Can't parse ip %s, no City enrich", ip)
return nil, nil
}
record, err := ctx.(*geoip2.Reader).City(ip)
if err != nil {
log.Debugf("Unable to enrich ip '%s'", ip)
plog.Debugf("Unable to enrich ip '%s'", ip)
return nil, nil
}
if record.Country.IsoCode != "" {
@ -94,7 +94,7 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}) (map[string]string
ret["Latitude"] = fmt.Sprintf("%f", record.Location.Latitude)
ret["Longitude"] = fmt.Sprintf("%f", record.Location.Longitude)
log.Tracef("geoip City %s -> %s, %s", field, ret["IsoCode"], ret["IsInEU"])
plog.Tracef("geoip City %s -> %s, %s", field, ret["IsoCode"], ret["IsInEU"])
return ret, nil
}

2
pkg/parser/runtime.go
View file

@ -157,7 +157,7 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er
/*still way too hackish, but : inject all the results in enriched, and */
if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok {
clog.Tracef("Found method '%s'", static.Method)
ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx)
ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx, n.Logger)
if err != nil {
clog.Errorf("method '%s' returned an error : %v", static.Method, err)
}

1
pkg/types/event.go
View file

@ -34,6 +34,7 @@ type Event struct {
Overflow RuntimeAlert `yaml:"Overflow,omitempty" json:"Alert,omitempty"`
Time time.Time `yaml:"Time,omitempty" json:"Time,omitempty"` //parsed time `json:"-"` ``
StrTime string `yaml:"StrTime,omitempty" json:"StrTime,omitempty"`
StrTimeFormat string `yaml:"StrTimeFormat,omitempty" json:"StrTimeFormat,omitempty"`
MarshaledTime string `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"`
Process bool `yaml:"Process,omitempty" json:"Process,omitempty"` //can be set to false to avoid processing line
/* Meta is the only part that will make it to the API - it should be normalized */