2020-05-15 09:39:16 +00:00
package parser
/ *
This file contains
- the runtime parsing routines
* /
import (
"errors"
"fmt"
"reflect"
"strings"
2023-03-16 10:01:25 +00:00
"sync"
2020-11-30 09:37:17 +00:00
"time"
2020-05-15 09:39:16 +00:00
"github.com/crowdsecurity/crowdsec/pkg/types"
"strconv"
2020-11-30 09:37:17 +00:00
"github.com/mohae/deepcopy"
2020-05-15 09:39:16 +00:00
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"github.com/antonmedv/expr"
)
/* ok, this is kinda experimental, I don't know how bad of an idea it is .. */
func SetTargetByName ( target string , value string , evt * types . Event ) bool {
if evt == nil {
return false
}
2020-05-20 09:00:25 +00:00
2020-05-15 09:39:16 +00:00
//it's a hack, we do it for the user
2020-05-20 09:00:25 +00:00
target = strings . TrimPrefix ( target , "evt." )
2020-05-15 09:39:16 +00:00
log . Debugf ( "setting target %s to %s" , target , value )
defer func ( ) {
if r := recover ( ) ; r != nil {
2022-05-19 14:28:25 +00:00
log . Errorf ( "Runtime error while trying to set '%s': %+v" , target , r )
2020-05-15 09:39:16 +00:00
return
}
} ( )
iter := reflect . ValueOf ( evt ) . Elem ( )
if ( iter == reflect . Value { } ) || iter . IsZero ( ) {
log . Tracef ( "event is nill" )
//event is nill
return false
}
for _ , f := range strings . Split ( target , "." ) {
/ *
* * According to current Event layout we only have to handle struct and map
* /
switch iter . Kind ( ) {
case reflect . Map :
tmp := iter . MapIndex ( reflect . ValueOf ( f ) )
/*if we're in a map and the field doesn't exist, the user wants to add it :) */
if ( tmp == reflect . Value { } ) || tmp . IsZero ( ) {
log . Debugf ( "map entry is zero in '%s'" , target )
}
iter . SetMapIndex ( reflect . ValueOf ( f ) , reflect . ValueOf ( value ) )
return true
case reflect . Struct :
tmp := iter . FieldByName ( f )
if ! tmp . IsValid ( ) {
2022-05-19 14:28:25 +00:00
log . Debugf ( "'%s' is not a valid target because '%s' is not valid" , target , f )
2020-05-15 09:39:16 +00:00
return false
}
2022-05-19 14:28:25 +00:00
if tmp . Kind ( ) == reflect . Ptr {
tmp = reflect . Indirect ( tmp )
}
2020-05-15 09:39:16 +00:00
iter = tmp
2022-06-16 12:41:54 +00:00
//nolint: gosimple
2020-05-15 09:39:16 +00:00
break
2022-05-19 14:28:25 +00:00
case reflect . Ptr :
tmp := iter . Elem ( )
iter = reflect . Indirect ( tmp . FieldByName ( f ) )
2020-05-15 09:39:16 +00:00
default :
log . Errorf ( "unexpected type %s in '%s'" , iter . Kind ( ) , target )
return false
}
}
//now we should have the final member :)
if ! iter . CanSet ( ) {
log . Errorf ( "'%s' can't be set" , target )
return false
}
if iter . Kind ( ) != reflect . String {
log . Errorf ( "Expected string, got %v when handling '%s'" , iter . Kind ( ) , target )
return false
}
iter . Set ( reflect . ValueOf ( value ) )
return true
}
2023-06-08 13:07:30 +00:00
func printStaticTarget ( static ExtraField ) string {
2020-05-15 09:39:16 +00:00
if static . Method != "" {
return static . Method
} else if static . Parsed != "" {
return fmt . Sprintf ( ".Parsed[%s]" , static . Parsed )
} else if static . Meta != "" {
return fmt . Sprintf ( ".Meta[%s]" , static . Meta )
} else if static . Enriched != "" {
return fmt . Sprintf ( ".Enriched[%s]" , static . Enriched )
} else if static . TargetByName != "" {
return static . TargetByName
} else {
return "?"
}
}
2023-06-08 13:07:30 +00:00
func ( n * Node ) ProcessStatics ( statics [ ] ExtraField , event * types . Event ) error {
2020-05-15 09:39:16 +00:00
//we have a few cases :
//(meta||key) + (static||reference||expr)
var value string
2020-12-14 13:12:22 +00:00
clog := n . Logger
2020-05-15 09:39:16 +00:00
for _ , static := range statics {
value = ""
if static . Value != "" {
value = static . Value
} else if static . RunTimeValue != nil {
2023-03-28 08:49:01 +00:00
output , err := expr . Run ( static . RunTimeValue , map [ string ] interface { } { "evt" : event } )
2020-05-15 09:39:16 +00:00
if err != nil {
clog . Warningf ( "failed to run RunTimeValue : %v" , err )
continue
}
2020-05-20 09:26:21 +00:00
switch out := output . ( type ) {
2020-05-15 09:39:16 +00:00
case string :
2020-05-20 09:26:21 +00:00
value = out
2020-05-15 09:39:16 +00:00
case int :
2020-05-20 09:26:21 +00:00
value = strconv . Itoa ( out )
2022-06-08 10:15:29 +00:00
case map [ string ] interface { } :
2023-05-16 07:10:38 +00:00
clog . Warnf ( "Expression '%s' returned a map, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string" , static . ExpValue )
2022-06-08 10:15:29 +00:00
case [ ] interface { } :
2023-05-16 07:10:38 +00:00
clog . Warnf ( "Expression '%s' returned an array, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string" , static . ExpValue )
2022-12-06 12:47:29 +00:00
case nil :
2023-05-16 07:10:38 +00:00
clog . Debugf ( "Expression '%s' returned nil, skipping" , static . ExpValue )
2020-05-15 09:39:16 +00:00
default :
2022-06-08 10:15:29 +00:00
clog . Errorf ( "unexpected return type for RunTimeValue : %T" , output )
2020-05-15 09:39:16 +00:00
return errors . New ( "unexpected return type for RunTimeValue" )
}
}
if value == "" {
2021-04-16 17:13:48 +00:00
//allow ParseDate to have empty input
if static . Method != "ParseDate" {
clog . Debugf ( "Empty value for %s, skip." , printStaticTarget ( static ) )
continue
}
2020-05-15 09:39:16 +00:00
}
if static . Method != "" {
processed := false
/*still way too hackish, but : inject all the results in enriched, and */
2021-09-09 14:27:30 +00:00
if enricherPlugin , ok := n . EnrichFunctions . Registered [ static . Method ] ; ok {
clog . Tracef ( "Found method '%s'" , static . Method )
2022-07-28 14:41:41 +00:00
ret , err := enricherPlugin . EnrichFunc ( value , event , enricherPlugin . Ctx , n . Logger )
2021-09-09 14:27:30 +00:00
if err != nil {
clog . Errorf ( "method '%s' returned an error : %v" , static . Method , err )
2020-05-15 09:39:16 +00:00
}
2021-09-09 14:27:30 +00:00
processed = true
clog . Debugf ( "+ Method %s('%s') returned %d entries to merge in .Enriched\n" , static . Method , value , len ( ret ) )
2022-12-06 12:47:29 +00:00
//Hackish check, but those methods do not return any data by design
2023-05-12 07:43:01 +00:00
if len ( ret ) == 0 && static . Method != "UnmarshalJSON" {
2021-09-09 14:27:30 +00:00
clog . Debugf ( "+ Method '%s' empty response on '%s'" , static . Method , value )
}
for k , v := range ret {
clog . Debugf ( "\t.Enriched[%s] = '%s'\n" , k , v )
event . Enriched [ k ] = v
}
} else {
clog . Debugf ( "method '%s' doesn't exist or plugin not initialized" , static . Method )
2020-05-15 09:39:16 +00:00
}
2020-05-20 08:49:17 +00:00
if ! processed {
2021-09-09 14:27:30 +00:00
clog . Debugf ( "method '%s' doesn't exist" , static . Method )
2020-05-15 09:39:16 +00:00
}
} else if static . Parsed != "" {
clog . Debugf ( ".Parsed[%s] = '%s'" , static . Parsed , value )
2020-11-30 09:37:17 +00:00
event . Parsed [ static . Parsed ] = value
2020-05-15 09:39:16 +00:00
} else if static . Meta != "" {
clog . Debugf ( ".Meta[%s] = '%s'" , static . Meta , value )
2020-11-30 09:37:17 +00:00
event . Meta [ static . Meta ] = value
2020-05-15 09:39:16 +00:00
} else if static . Enriched != "" {
clog . Debugf ( ".Enriched[%s] = '%s'" , static . Enriched , value )
2020-11-30 09:37:17 +00:00
event . Enriched [ static . Enriched ] = value
2020-05-15 09:39:16 +00:00
} else if static . TargetByName != "" {
2020-11-30 09:37:17 +00:00
if ! SetTargetByName ( static . TargetByName , value , event ) {
2020-05-15 09:39:16 +00:00
clog . Errorf ( "Unable to set value of '%s'" , static . TargetByName )
} else {
clog . Debugf ( "%s = '%s'" , static . TargetByName , value )
}
} else {
2022-06-06 13:24:48 +00:00
clog . Fatal ( "unable to process static : unknown target" )
2020-05-15 09:39:16 +00:00
}
}
return nil
}
var NodesHits = prometheus . NewCounterVec (
prometheus . CounterOpts {
2020-07-29 13:03:15 +00:00
Name : "cs_node_hits_total" ,
Help : "Total events entered node." ,
2020-05-15 09:39:16 +00:00
} ,
2021-06-11 07:53:53 +00:00
[ ] string { "source" , "type" , "name" } ,
2020-05-15 09:39:16 +00:00
)
var NodesHitsOk = prometheus . NewCounterVec (
prometheus . CounterOpts {
2020-07-29 13:03:15 +00:00
Name : "cs_node_hits_ok_total" ,
2021-12-06 16:29:23 +00:00
Help : "Total events successfully exited node." ,
2020-05-15 09:39:16 +00:00
} ,
2021-06-11 07:53:53 +00:00
[ ] string { "source" , "type" , "name" } ,
2020-05-15 09:39:16 +00:00
)
var NodesHitsKo = prometheus . NewCounterVec (
prometheus . CounterOpts {
2020-07-29 13:03:15 +00:00
Name : "cs_node_hits_ko_total" ,
2021-12-06 16:29:23 +00:00
Help : "Total events unsuccessfully exited node." ,
2020-05-15 09:39:16 +00:00
} ,
2021-06-11 07:53:53 +00:00
[ ] string { "source" , "type" , "name" } ,
2020-05-15 09:39:16 +00:00
)
func stageidx ( stage string , stages [ ] string ) int {
for i , v := range stages {
if stage == v {
return i
}
}
return - 1
}
2021-10-04 15:14:52 +00:00
type ParserResult struct {
Evt types . Event
Success bool
}
2020-06-10 10:14:27 +00:00
var ParseDump bool
2021-10-04 15:14:52 +00:00
var DumpFolder string
var StageParseCache map [ string ] map [ string ] [ ] ParserResult
2023-03-16 10:01:25 +00:00
var StageParseMutex sync . Mutex
2020-06-10 10:14:27 +00:00
2020-11-30 09:37:17 +00:00
func Parse ( ctx UnixParserCtx , xp types . Event , nodes [ ] Node ) ( types . Event , error ) {
2023-03-09 10:56:02 +00:00
var event = xp
2020-05-20 15:50:56 +00:00
2020-05-15 09:39:16 +00:00
/* the stage is undefined, probably line is freshly acquired, set to first stage !*/
if event . Stage == "" && len ( ctx . Stages ) > 0 {
event . Stage = ctx . Stages [ 0 ]
2020-11-30 09:37:17 +00:00
log . Tracef ( "no stage, set to : %s" , event . Stage )
2020-05-15 09:39:16 +00:00
}
event . Process = false
2020-11-30 09:37:17 +00:00
if event . Time . IsZero ( ) {
2022-01-19 13:56:05 +00:00
event . Time = time . Now ( ) . UTC ( )
2020-11-30 09:37:17 +00:00
}
2020-05-15 09:39:16 +00:00
if event . Parsed == nil {
event . Parsed = make ( map [ string ] string )
}
if event . Enriched == nil {
event . Enriched = make ( map [ string ] string )
}
if event . Meta == nil {
event . Meta = make ( map [ string ] string )
}
2023-05-16 07:10:38 +00:00
if event . Unmarshaled == nil {
event . Unmarshaled = make ( map [ string ] interface { } )
}
2020-05-15 09:39:16 +00:00
if event . Type == types . LOG {
log . Tracef ( "INPUT '%s'" , event . Line . Raw )
}
2020-06-10 10:14:27 +00:00
if ParseDump {
2021-10-04 15:14:52 +00:00
if StageParseCache == nil {
2023-03-16 10:01:25 +00:00
StageParseMutex . Lock ( )
2021-10-04 15:14:52 +00:00
StageParseCache = make ( map [ string ] map [ string ] [ ] ParserResult )
StageParseCache [ "success" ] = make ( map [ string ] [ ] ParserResult )
StageParseCache [ "success" ] [ "" ] = make ( [ ] ParserResult , 0 )
2023-03-16 10:01:25 +00:00
StageParseMutex . Unlock ( )
2021-10-04 15:14:52 +00:00
}
2020-06-10 10:14:27 +00:00
}
2020-05-15 09:39:16 +00:00
for _ , stage := range ctx . Stages {
2020-06-10 10:14:27 +00:00
if ParseDump {
2023-03-16 10:01:25 +00:00
StageParseMutex . Lock ( )
2021-10-04 15:14:52 +00:00
if _ , ok := StageParseCache [ stage ] ; ! ok {
StageParseCache [ stage ] = make ( map [ string ] [ ] ParserResult )
}
2023-03-16 10:01:25 +00:00
StageParseMutex . Unlock ( )
2020-06-10 10:14:27 +00:00
}
2022-03-16 08:40:00 +00:00
/* if the node is forward in stages, seek to this stage */
2020-05-15 09:39:16 +00:00
/* this is for example used by testing system to inject logs in post-syslog-parsing phase*/
if stageidx ( event . Stage , ctx . Stages ) > stageidx ( stage , ctx . Stages ) {
log . Tracef ( "skipping stage, we are already at [%s] expecting [%s]" , event . Stage , stage )
continue
}
log . Tracef ( "node stage : %s, current stage : %s" , event . Stage , stage )
/* if the stage is wrong, it means that the log didn't manage "pass" a stage with a onsuccess: next_stage tag */
if event . Stage != stage {
log . Debugf ( "Event not parsed, expected stage '%s' got '%s', abort" , stage , event . Stage )
2022-03-16 08:40:00 +00:00
event . Process = false
return event , nil
2020-05-15 09:39:16 +00:00
}
isStageOK := false
for idx , node := range nodes {
//Only process current stage's nodes
if event . Stage != node . Stage {
continue
}
2020-06-10 10:14:27 +00:00
clog := log . WithFields ( log . Fields {
"node-name" : node . rn ,
"stage" : event . Stage ,
} )
2020-05-15 09:39:16 +00:00
clog . Tracef ( "Processing node %d/%d -> %s" , idx , len ( nodes ) , node . rn )
2020-05-20 08:49:17 +00:00
if ctx . Profiling {
2020-05-15 09:39:16 +00:00
node . Profiling = true
}
2023-03-28 08:49:01 +00:00
ret , err := node . process ( & event , ctx , map [ string ] interface { } { "evt" : & event } )
2020-05-15 09:39:16 +00:00
if err != nil {
2022-06-08 10:15:29 +00:00
clog . Errorf ( "Error while processing node : %v" , err )
return event , err
2020-05-15 09:39:16 +00:00
}
clog . Tracef ( "node (%s) ret : %v" , node . rn , ret )
2021-10-04 15:14:52 +00:00
if ParseDump {
2023-05-26 14:23:50 +00:00
StageParseMutex . Lock ( )
2021-10-04 15:14:52 +00:00
if len ( StageParseCache [ stage ] [ node . Name ] ) == 0 {
StageParseCache [ stage ] [ node . Name ] = make ( [ ] ParserResult , 0 )
}
2023-05-26 14:23:50 +00:00
StageParseMutex . Unlock ( )
2021-10-04 15:14:52 +00:00
evtcopy := deepcopy . Copy ( event )
parserInfo := ParserResult { Evt : evtcopy . ( types . Event ) , Success : ret }
2023-03-16 10:01:25 +00:00
StageParseMutex . Lock ( )
2021-10-04 15:14:52 +00:00
StageParseCache [ stage ] [ node . Name ] = append ( StageParseCache [ stage ] [ node . Name ] , parserInfo )
2023-03-16 10:01:25 +00:00
StageParseMutex . Unlock ( )
2021-10-04 15:14:52 +00:00
}
2020-05-20 08:49:17 +00:00
if ret {
2020-05-15 09:39:16 +00:00
isStageOK = true
}
2020-05-20 08:49:17 +00:00
if ret && node . OnSuccess == "next_stage" {
2020-05-15 09:39:16 +00:00
clog . Debugf ( "node successful, stop end stage %s" , stage )
break
}
//the parsed object moved onto the next phase
if event . Stage != stage {
clog . Tracef ( "node moved stage, break and redo" )
break
}
}
2020-05-20 08:49:17 +00:00
if ! isStageOK {
2020-05-15 09:39:16 +00:00
log . Debugf ( "Log didn't finish stage %s" , event . Stage )
event . Process = false
return event , nil
}
}
event . Process = true
return event , nil
}