Generic dateparse approach (#1669)
* Allow any parser to suggest a format string for the date to be parsed. * allow the enricher functions to get the parser's logger so they can inherit the level
This commit is contained in:
parent
1fc29d094f
commit
866c200c31
|
@ -6,7 +6,7 @@ import (
|
|||
)
|
||||
|
||||
/* should be part of a packaged shared with enrich/geoip.go */
|
||||
type EnrichFunc func(string, *types.Event, interface{}) (map[string]string, error)
|
||||
type EnrichFunc func(string, *types.Event, interface{}, *log.Entry) (map[string]string, error)
|
||||
type InitFunc func(map[string]string) (interface{}, error)
|
||||
|
||||
type EnricherCtx struct {
|
||||
|
|
|
@ -7,6 +7,23 @@ import (
|
|||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func parseDateWithFormat(date, format string) (string, time.Time) {
|
||||
t, err := time.Parse(format, date)
|
||||
if err == nil && !t.IsZero() {
|
||||
//if the year isn't set, set it to current date :)
|
||||
if t.Year() == 0 {
|
||||
t = t.AddDate(time.Now().UTC().Year(), 0, 0)
|
||||
}
|
||||
retstr, err := t.MarshalText()
|
||||
if err != nil {
|
||||
log.Warningf("Failed marshaling '%v'", t)
|
||||
return "", time.Time{}
|
||||
}
|
||||
return string(retstr), t
|
||||
}
|
||||
return "", time.Time{}
|
||||
}
|
||||
|
||||
func GenDateParse(date string) (string, time.Time) {
|
||||
var (
|
||||
layouts = [...]string{
|
||||
|
@ -29,40 +46,44 @@ func GenDateParse(date string) (string, time.Time) {
|
|||
)
|
||||
|
||||
for _, dateFormat := range layouts {
|
||||
t, err := time.Parse(dateFormat, date)
|
||||
if err == nil && !t.IsZero() {
|
||||
//if the year isn't set, set it to current date :)
|
||||
if t.Year() == 0 {
|
||||
t = t.AddDate(time.Now().UTC().Year(), 0, 0)
|
||||
}
|
||||
retstr, err := t.MarshalText()
|
||||
if err != nil {
|
||||
log.Warningf("Failed marshaling '%v'", t)
|
||||
continue
|
||||
}
|
||||
return string(retstr), t
|
||||
retstr, parsedDate := parseDateWithFormat(date, dateFormat)
|
||||
if !parsedDate.IsZero() {
|
||||
return retstr, parsedDate
|
||||
}
|
||||
}
|
||||
return "", time.Time{}
|
||||
}
|
||||
|
||||
func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[string]string, error) {
|
||||
|
||||
var ret map[string]string = make(map[string]string)
|
||||
var strDate string
|
||||
var parsedDate time.Time
|
||||
|
||||
if p.StrTimeFormat != "" {
|
||||
strDate, parsedDate = parseDateWithFormat(in, p.StrTimeFormat)
|
||||
if !parsedDate.IsZero() {
|
||||
ret["MarshaledTime"] = strDate
|
||||
return ret, nil
|
||||
} else {
|
||||
plog.Debugf("unable to parse '%s' with layout '%s'", in, p.StrTimeFormat)
|
||||
}
|
||||
}
|
||||
strDate, parsedDate = GenDateParse(in)
|
||||
if !parsedDate.IsZero() {
|
||||
ret["MarshaledTime"] = strDate
|
||||
return ret, nil
|
||||
}
|
||||
plog.Debugf("no suitable date format found for '%s', falling back to now", in)
|
||||
now := time.Now().UTC()
|
||||
retstr, err := now.MarshalText()
|
||||
if err != nil {
|
||||
log.Warning("Failed marshaling current time")
|
||||
return "", time.Time{}
|
||||
plog.Warning("Failed marshaling current time")
|
||||
return ret, err
|
||||
}
|
||||
return string(retstr), now
|
||||
}
|
||||
ret["MarshaledTime"] = string(retstr)
|
||||
|
||||
func ParseDate(in string, p *types.Event, x interface{}) (map[string]string, error) {
|
||||
|
||||
var ret map[string]string = make(map[string]string)
|
||||
tstr, tbin := GenDateParse(in)
|
||||
if !tbin.IsZero() {
|
||||
ret["MarshaledTime"] = tstr
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func parseDateInit(cfg map[string]string) (interface{}, error) {
|
||||
|
|
65
pkg/parser/enrich_date_test.go
Normal file
65
pkg/parser/enrich_date_test.go
Normal file
|
@ -0,0 +1,65 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func TestDateParse(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
evt types.Event
|
||||
expected_err *error
|
||||
expected_strTime *string
|
||||
}{
|
||||
{
|
||||
name: "RFC3339",
|
||||
evt: types.Event{
|
||||
StrTime: "2019-10-12T07:20:50.52Z",
|
||||
},
|
||||
expected_err: nil,
|
||||
expected_strTime: types.StrPtr("2019-10-12T07:20:50.52Z"),
|
||||
},
|
||||
{
|
||||
name: "02/Jan/2006:15:04:05 -0700",
|
||||
evt: types.Event{
|
||||
StrTime: "02/Jan/2006:15:04:05 -0700",
|
||||
},
|
||||
expected_err: nil,
|
||||
expected_strTime: types.StrPtr("2006-01-02T15:04:05-07:00"),
|
||||
},
|
||||
{
|
||||
name: "Dec 17 08:17:43",
|
||||
evt: types.Event{
|
||||
StrTime: "2011 X 17 zz 08X17X43 oneone Dec",
|
||||
StrTimeFormat: "2006 X 2 zz 15X04X05 oneone Jan",
|
||||
},
|
||||
expected_err: nil,
|
||||
expected_strTime: types.StrPtr("2011-12-17T08:17:43Z"),
|
||||
},
|
||||
}
|
||||
|
||||
logger := log.WithFields(log.Fields{
|
||||
"test": "test",
|
||||
})
|
||||
for idx, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
strTime, err := ParseDate(tt.evt.StrTime, &tt.evt, nil, logger)
|
||||
if tt.expected_err != nil {
|
||||
if err != *tt.expected_err {
|
||||
t.Errorf("%s: expected error %v, got %v", tt.name, tt.expected_err, err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Errorf("%s: expected no error, got %v", tt.name, err)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if tt.expected_strTime != nil && strTime["MarshaledTime"] != *tt.expected_strTime {
|
||||
t.Errorf("%d: expected strTime %s, got %s", idx, *tt.expected_strTime, strTime["MarshaledTime"])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
|
@ -11,14 +11,14 @@ import (
|
|||
/* All plugins must export a list of function pointers for exported symbols */
|
||||
//var ExportedFuncs = []string{"reverse_dns"}
|
||||
|
||||
func reverse_dns(field string, p *types.Event, ctx interface{}) (map[string]string, error) {
|
||||
func reverse_dns(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
|
||||
ret := make(map[string]string)
|
||||
if field == "" {
|
||||
return nil, nil
|
||||
}
|
||||
rets, err := net.LookupAddr(field)
|
||||
if err != nil {
|
||||
log.Debugf("failed to resolve '%s'", field)
|
||||
plog.Debugf("failed to resolve '%s'", field)
|
||||
return nil, nil
|
||||
}
|
||||
//When using the host C library resolver, at most one result will be returned. To bypass the host resolver, use a custom Resolver.
|
||||
|
|
|
@ -12,7 +12,7 @@ import (
|
|||
"github.com/oschwald/maxminddb-golang"
|
||||
)
|
||||
|
||||
func IpToRange(field string, p *types.Event, ctx interface{}) (map[string]string, error) {
|
||||
func IpToRange(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
|
||||
var dummy interface{}
|
||||
ret := make(map[string]string)
|
||||
|
||||
|
@ -21,23 +21,23 @@ func IpToRange(field string, p *types.Event, ctx interface{}) (map[string]string
|
|||
}
|
||||
ip := net.ParseIP(field)
|
||||
if ip == nil {
|
||||
log.Infof("Can't parse ip %s, no range enrich", field)
|
||||
plog.Infof("Can't parse ip %s, no range enrich", field)
|
||||
return nil, nil
|
||||
}
|
||||
net, ok, err := ctx.(*maxminddb.Reader).LookupNetwork(ip, &dummy)
|
||||
if err != nil {
|
||||
log.Errorf("Failed to fetch network for %s : %v", ip.String(), err)
|
||||
plog.Errorf("Failed to fetch network for %s : %v", ip.String(), err)
|
||||
return nil, nil
|
||||
}
|
||||
if !ok {
|
||||
log.Debugf("Unable to find range of %s", ip.String())
|
||||
plog.Debugf("Unable to find range of %s", ip.String())
|
||||
return nil, nil
|
||||
}
|
||||
ret["SourceRange"] = net.String()
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func GeoIpASN(field string, p *types.Event, ctx interface{}) (map[string]string, error) {
|
||||
func GeoIpASN(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
|
||||
ret := make(map[string]string)
|
||||
if field == "" {
|
||||
return nil, nil
|
||||
|
@ -45,36 +45,36 @@ func GeoIpASN(field string, p *types.Event, ctx interface{}) (map[string]string,
|
|||
|
||||
ip := net.ParseIP(field)
|
||||
if ip == nil {
|
||||
log.Infof("Can't parse ip %s, no ASN enrich", ip)
|
||||
plog.Infof("Can't parse ip %s, no ASN enrich", ip)
|
||||
return nil, nil
|
||||
}
|
||||
record, err := ctx.(*geoip2.Reader).ASN(ip)
|
||||
if err != nil {
|
||||
log.Errorf("Unable to enrich ip '%s'", field)
|
||||
plog.Errorf("Unable to enrich ip '%s'", field)
|
||||
return nil, nil
|
||||
}
|
||||
ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
|
||||
ret["ASNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
|
||||
ret["ASNOrg"] = record.AutonomousSystemOrganization
|
||||
|
||||
log.Tracef("geoip ASN %s -> %s, %s", field, ret["ASNNumber"], ret["ASNOrg"])
|
||||
plog.Tracef("geoip ASN %s -> %s, %s", field, ret["ASNNumber"], ret["ASNOrg"])
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func GeoIpCity(field string, p *types.Event, ctx interface{}) (map[string]string, error) {
|
||||
func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
|
||||
ret := make(map[string]string)
|
||||
if field == "" {
|
||||
return nil, nil
|
||||
}
|
||||
ip := net.ParseIP(field)
|
||||
if ip == nil {
|
||||
log.Infof("Can't parse ip %s, no City enrich", ip)
|
||||
plog.Infof("Can't parse ip %s, no City enrich", ip)
|
||||
return nil, nil
|
||||
}
|
||||
record, err := ctx.(*geoip2.Reader).City(ip)
|
||||
if err != nil {
|
||||
log.Debugf("Unable to enrich ip '%s'", ip)
|
||||
plog.Debugf("Unable to enrich ip '%s'", ip)
|
||||
return nil, nil
|
||||
}
|
||||
if record.Country.IsoCode != "" {
|
||||
|
@ -94,7 +94,7 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}) (map[string]string
|
|||
ret["Latitude"] = fmt.Sprintf("%f", record.Location.Latitude)
|
||||
ret["Longitude"] = fmt.Sprintf("%f", record.Location.Longitude)
|
||||
|
||||
log.Tracef("geoip City %s -> %s, %s", field, ret["IsoCode"], ret["IsInEU"])
|
||||
plog.Tracef("geoip City %s -> %s, %s", field, ret["IsoCode"], ret["IsInEU"])
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
|
|
@ -157,7 +157,7 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er
|
|||
/*still way too hackish, but : inject all the results in enriched, and */
|
||||
if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok {
|
||||
clog.Tracef("Found method '%s'", static.Method)
|
||||
ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx)
|
||||
ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx, n.Logger)
|
||||
if err != nil {
|
||||
clog.Errorf("method '%s' returned an error : %v", static.Method, err)
|
||||
}
|
||||
|
|
|
@ -34,6 +34,7 @@ type Event struct {
|
|||
Overflow RuntimeAlert `yaml:"Overflow,omitempty" json:"Alert,omitempty"`
|
||||
Time time.Time `yaml:"Time,omitempty" json:"Time,omitempty"` //parsed time `json:"-"` ``
|
||||
StrTime string `yaml:"StrTime,omitempty" json:"StrTime,omitempty"`
|
||||
StrTimeFormat string `yaml:"StrTimeFormat,omitempty" json:"StrTimeFormat,omitempty"`
|
||||
MarshaledTime string `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"`
|
||||
Process bool `yaml:"Process,omitempty" json:"Process,omitempty"` //can be set to false to avoid processing line
|
||||
/* Meta is the only part that will make it to the API - it should be normalized */
|
||||
|
|
Loading…
Reference in a new issue