diff --git a/pkg/parser/enrich.go b/pkg/parser/enrich.go index 43331c63c..2cc56aff7 100644 --- a/pkg/parser/enrich.go +++ b/pkg/parser/enrich.go @@ -6,7 +6,7 @@ import ( ) /* should be part of a packaged shared with enrich/geoip.go */ -type EnrichFunc func(string, *types.Event, interface{}) (map[string]string, error) +type EnrichFunc func(string, *types.Event, interface{}, *log.Entry) (map[string]string, error) type InitFunc func(map[string]string) (interface{}, error) type EnricherCtx struct { diff --git a/pkg/parser/enrich_date.go b/pkg/parser/enrich_date.go index 6fe1b12f9..948e1764c 100644 --- a/pkg/parser/enrich_date.go +++ b/pkg/parser/enrich_date.go @@ -7,6 +7,23 @@ import ( log "github.com/sirupsen/logrus" ) +func parseDateWithFormat(date, format string) (string, time.Time) { + t, err := time.Parse(format, date) + if err == nil && !t.IsZero() { + //if the year isn't set, set it to current date :) + if t.Year() == 0 { + t = t.AddDate(time.Now().UTC().Year(), 0, 0) + } + retstr, err := t.MarshalText() + if err != nil { + log.Warningf("Failed marshaling '%v'", t) + return "", time.Time{} + } + return string(retstr), t + } + return "", time.Time{} +} + func GenDateParse(date string) (string, time.Time) { var ( layouts = [...]string{ @@ -29,40 +46,44 @@ func GenDateParse(date string) (string, time.Time) { ) for _, dateFormat := range layouts { - t, err := time.Parse(dateFormat, date) - if err == nil && !t.IsZero() { - //if the year isn't set, set it to current date :) - if t.Year() == 0 { - t = t.AddDate(time.Now().UTC().Year(), 0, 0) - } - retstr, err := t.MarshalText() - if err != nil { - log.Warningf("Failed marshaling '%v'", t) - continue - } - return string(retstr), t + retstr, parsedDate := parseDateWithFormat(date, dateFormat) + if !parsedDate.IsZero() { + return retstr, parsedDate } } + return "", time.Time{} +} +func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[string]string, error) { + + var ret map[string]string = make(map[string]string) + var strDate string + var parsedDate time.Time + + if p.StrTimeFormat != "" { + strDate, parsedDate = parseDateWithFormat(in, p.StrTimeFormat) + if !parsedDate.IsZero() { + ret["MarshaledTime"] = strDate + return ret, nil + } else { + plog.Debugf("unable to parse '%s' with layout '%s'", in, p.StrTimeFormat) + } + } + strDate, parsedDate = GenDateParse(in) + if !parsedDate.IsZero() { + ret["MarshaledTime"] = strDate + return ret, nil + } + plog.Debugf("no suitable date format found for '%s', falling back to now", in) now := time.Now().UTC() retstr, err := now.MarshalText() if err != nil { - log.Warning("Failed marshaling current time") - return "", time.Time{} + plog.Warning("Failed marshaling current time") + return ret, err } - return string(retstr), now -} + ret["MarshaledTime"] = string(retstr) -func ParseDate(in string, p *types.Event, x interface{}) (map[string]string, error) { - - var ret map[string]string = make(map[string]string) - tstr, tbin := GenDateParse(in) - if !tbin.IsZero() { - ret["MarshaledTime"] = tstr - return ret, nil - } - - return nil, nil + return ret, nil } func parseDateInit(cfg map[string]string) (interface{}, error) { diff --git a/pkg/parser/enrich_date_test.go b/pkg/parser/enrich_date_test.go new file mode 100644 index 000000000..a5480c2ff --- /dev/null +++ b/pkg/parser/enrich_date_test.go @@ -0,0 +1,65 @@ +package parser + +import ( + "testing" + + "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" +) + +func TestDateParse(t *testing.T) { + tests := []struct { + name string + evt types.Event + expected_err *error + expected_strTime *string + }{ + { + name: "RFC3339", + evt: types.Event{ + StrTime: "2019-10-12T07:20:50.52Z", + }, + expected_err: nil, + expected_strTime: types.StrPtr("2019-10-12T07:20:50.52Z"), + }, + { + name: "02/Jan/2006:15:04:05 -0700", + evt: types.Event{ + StrTime: "02/Jan/2006:15:04:05 -0700", + }, + expected_err: nil, + expected_strTime: types.StrPtr("2006-01-02T15:04:05-07:00"), + }, + { + name: "Dec 17 08:17:43", + evt: types.Event{ + StrTime: "2011 X 17 zz 08X17X43 oneone Dec", + StrTimeFormat: "2006 X 2 zz 15X04X05 oneone Jan", + }, + expected_err: nil, + expected_strTime: types.StrPtr("2011-12-17T08:17:43Z"), + }, + } + + logger := log.WithFields(log.Fields{ + "test": "test", + }) + for idx, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strTime, err := ParseDate(tt.evt.StrTime, &tt.evt, nil, logger) + if tt.expected_err != nil { + if err != *tt.expected_err { + t.Errorf("%s: expected error %v, got %v", tt.name, tt.expected_err, err) + } + } else if err != nil { + t.Errorf("%s: expected no error, got %v", tt.name, err) + } + if err != nil { + return + } + if tt.expected_strTime != nil && strTime["MarshaledTime"] != *tt.expected_strTime { + t.Errorf("%d: expected strTime %s, got %s", idx, *tt.expected_strTime, strTime["MarshaledTime"]) + } + }) + } +} diff --git a/pkg/parser/enrich_dns.go b/pkg/parser/enrich_dns.go index d568a00af..d333f40ad 100644 --- a/pkg/parser/enrich_dns.go +++ b/pkg/parser/enrich_dns.go @@ -11,14 +11,14 @@ import ( /* All plugins must export a list of function pointers for exported symbols */ //var ExportedFuncs = []string{"reverse_dns"} -func reverse_dns(field string, p *types.Event, ctx interface{}) (map[string]string, error) { +func reverse_dns(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { ret := make(map[string]string) if field == "" { return nil, nil } rets, err := net.LookupAddr(field) if err != nil { - log.Debugf("failed to resolve '%s'", field) + plog.Debugf("failed to resolve '%s'", field) return nil, nil } //When using the host C library resolver, at most one result will be returned. To bypass the host resolver, use a custom Resolver. diff --git a/pkg/parser/enrich_geoip.go b/pkg/parser/enrich_geoip.go index baf295a43..0f96d4fb9 100644 --- a/pkg/parser/enrich_geoip.go +++ b/pkg/parser/enrich_geoip.go @@ -12,7 +12,7 @@ import ( "github.com/oschwald/maxminddb-golang" ) -func IpToRange(field string, p *types.Event, ctx interface{}) (map[string]string, error) { +func IpToRange(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { var dummy interface{} ret := make(map[string]string) @@ -21,23 +21,23 @@ func IpToRange(field string, p *types.Event, ctx interface{}) (map[string]string } ip := net.ParseIP(field) if ip == nil { - log.Infof("Can't parse ip %s, no range enrich", field) + plog.Infof("Can't parse ip %s, no range enrich", field) return nil, nil } net, ok, err := ctx.(*maxminddb.Reader).LookupNetwork(ip, &dummy) if err != nil { - log.Errorf("Failed to fetch network for %s : %v", ip.String(), err) + plog.Errorf("Failed to fetch network for %s : %v", ip.String(), err) return nil, nil } if !ok { - log.Debugf("Unable to find range of %s", ip.String()) + plog.Debugf("Unable to find range of %s", ip.String()) return nil, nil } ret["SourceRange"] = net.String() return ret, nil } -func GeoIpASN(field string, p *types.Event, ctx interface{}) (map[string]string, error) { +func GeoIpASN(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { ret := make(map[string]string) if field == "" { return nil, nil @@ -45,36 +45,36 @@ func GeoIpASN(field string, p *types.Event, ctx interface{}) (map[string]string, ip := net.ParseIP(field) if ip == nil { - log.Infof("Can't parse ip %s, no ASN enrich", ip) + plog.Infof("Can't parse ip %s, no ASN enrich", ip) return nil, nil } record, err := ctx.(*geoip2.Reader).ASN(ip) if err != nil { - log.Errorf("Unable to enrich ip '%s'", field) + plog.Errorf("Unable to enrich ip '%s'", field) return nil, nil } ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber) ret["ASNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber) ret["ASNOrg"] = record.AutonomousSystemOrganization - log.Tracef("geoip ASN %s -> %s, %s", field, ret["ASNNumber"], ret["ASNOrg"]) + plog.Tracef("geoip ASN %s -> %s, %s", field, ret["ASNNumber"], ret["ASNOrg"]) return ret, nil } -func GeoIpCity(field string, p *types.Event, ctx interface{}) (map[string]string, error) { +func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { ret := make(map[string]string) if field == "" { return nil, nil } ip := net.ParseIP(field) if ip == nil { - log.Infof("Can't parse ip %s, no City enrich", ip) + plog.Infof("Can't parse ip %s, no City enrich", ip) return nil, nil } record, err := ctx.(*geoip2.Reader).City(ip) if err != nil { - log.Debugf("Unable to enrich ip '%s'", ip) + plog.Debugf("Unable to enrich ip '%s'", ip) return nil, nil } if record.Country.IsoCode != "" { @@ -94,7 +94,7 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}) (map[string]string ret["Latitude"] = fmt.Sprintf("%f", record.Location.Latitude) ret["Longitude"] = fmt.Sprintf("%f", record.Location.Longitude) - log.Tracef("geoip City %s -> %s, %s", field, ret["IsoCode"], ret["IsInEU"]) + plog.Tracef("geoip City %s -> %s, %s", field, ret["IsoCode"], ret["IsInEU"]) return ret, nil } diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index 434b2e969..69e448b9f 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -157,7 +157,7 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er /*still way too hackish, but : inject all the results in enriched, and */ if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok { clog.Tracef("Found method '%s'", static.Method) - ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx) + ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx, n.Logger) if err != nil { clog.Errorf("method '%s' returned an error : %v", static.Method, err) } diff --git a/pkg/types/event.go b/pkg/types/event.go index 20adfadd5..ad47d194a 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -34,6 +34,7 @@ type Event struct { Overflow RuntimeAlert `yaml:"Overflow,omitempty" json:"Alert,omitempty"` Time time.Time `yaml:"Time,omitempty" json:"Time,omitempty"` //parsed time `json:"-"` `` StrTime string `yaml:"StrTime,omitempty" json:"StrTime,omitempty"` + StrTimeFormat string `yaml:"StrTimeFormat,omitempty" json:"StrTimeFormat,omitempty"` MarshaledTime string `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"` Process bool `yaml:"Process,omitempty" json:"Process,omitempty"` //can be set to false to avoid processing line /* Meta is the only part that will make it to the API - it should be normalized */