add waf_routines

This commit is contained in:
alteredCoder 2023-07-04 17:36:56 +02:00
parent 3fe6e3be14
commit 13512891e4
6 changed files with 301 additions and 117 deletions

View file

@ -8,10 +8,12 @@ import (
corazatypes "github.com/corazawaf/coraza/v3/types" corazatypes "github.com/corazawaf/coraza/v3/types"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/waf"
"github.com/pkg/errors" "github.com/pkg/errors"
log "github.com/sirupsen/logrus"
) )
func TxToEvents(r ParsedRequest, kind string) ([]types.Event, error) { func TxToEvents(r waf.ParsedRequest, kind string) ([]types.Event, error) {
evts := []types.Event{} evts := []types.Event{}
if r.Tx == nil { if r.Tx == nil {
return nil, fmt.Errorf("tx is nil") return nil, fmt.Errorf("tx is nil")
@ -32,7 +34,7 @@ func TxToEvents(r ParsedRequest, kind string) ([]types.Event, error) {
} }
// Transforms a coraza interruption to a crowdsec event // Transforms a coraza interruption to a crowdsec event
func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r ParsedRequest, kind string) (types.Event, error) { func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r waf.ParsedRequest, kind string) (types.Event, error) {
evt := types.Event{} evt := types.Event{}
//we might want to change this based on in-band vs out-of-band ? //we might want to change this based on in-band vs out-of-band ?
evt.Type = types.LOG evt.Type = types.LOG
@ -40,7 +42,7 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction,
//def needs fixing //def needs fixing
evt.Stage = "s00-raw" evt.Stage = "s00-raw"
evt.Process = true evt.Process = true
log.Infof("SOURCE IP: %+v", rule)
//we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later. //we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later.
//why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers //why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers
CorazaEvent := map[string]interface{}{ CorazaEvent := map[string]interface{}{

View file

@ -2,14 +2,15 @@ package wafacquisition
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"io/ioutil"
"net/http" "net/http"
"net/url"
"os" "os"
"strings" "strings"
"github.com/antonmedv/expr"
"github.com/corazawaf/coraza/v3" "github.com/corazawaf/coraza/v3"
"github.com/corazawaf/coraza/v3/experimental"
corazatypes "github.com/corazawaf/coraza/v3/types" corazatypes "github.com/corazawaf/coraza/v3/types"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
@ -29,11 +30,20 @@ const (
) )
type WafRunner struct { type WafRunner struct {
outChan chan types.Event outChan chan types.Event
inChan chan ParsedRequest inChan chan waf.ParsedRequest
inBandWaf coraza.WAF inBandWaf coraza.WAF
outOfBandWaf coraza.WAF outOfBandWaf coraza.WAF
UUID string UUID string
RulesCollections []*waf.WafRulesCollection
}
type WafSourceConfig struct {
ListenAddr string `yaml:"listen_addr"`
ListenPort int `yaml:"listen_port"`
Path string `yaml:"path"`
WafRoutines int `yaml:"waf_routines"`
configuration.DataSourceCommonCfg `yaml:",inline"`
} }
type WafSource struct { type WafSource struct {
@ -43,47 +53,15 @@ type WafSource struct {
server *http.Server server *http.Server
addr string addr string
outChan chan types.Event outChan chan types.Event
InChan chan ParsedRequest InChan chan waf.ParsedRequest
inBandWaf coraza.WAF inBandWaf coraza.WAF
outOfBandWaf coraza.WAF outOfBandWaf coraza.WAF
RulesCollections []*waf.WafRulesCollection
WafRunners []WafRunner WafRunners []WafRunner
} }
type ParsedRequest struct {
RemoteAddr string
Host string
ClientIP string
URI string
ClientHost string
Headers http.Header
URL *url.URL
Method string
Proto string
Body []byte
TransferEncoding []string
UUID string
Tx corazatypes.Transaction
ResponseChannel chan ResponseRequest
}
type ResponseRequest struct {
ResponseChannel chan ResponseRequest
UUID string
Tx corazatypes.Transaction
Interruption *corazatypes.Interruption
Err error
}
type WafSourceConfig struct {
ListenAddr string `yaml:"listen_addr"`
ListenPort int `yaml:"listen_port"`
Path string `yaml:"path"`
WafRoutines int `yaml:"waf_routines"`
configuration.DataSourceCommonCfg `yaml:",inline"`
}
func (w *WafSource) GetMetrics() []prometheus.Collector { func (w *WafSource) GetMetrics() []prometheus.Collector {
return nil return nil
} }
@ -156,6 +134,8 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error {
return fmt.Errorf("cannot load WAF rules: %w", err) return fmt.Errorf("cannot load WAF rules: %w", err)
} }
w.RulesCollections = rulesCollections
var inBandRules string var inBandRules string
var outOfBandRules string var outOfBandRules string
@ -179,7 +159,7 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error {
w.config.WafRoutines = 1 w.config.WafRoutines = 1
} }
w.InChan = make(chan ParsedRequest) w.InChan = make(chan waf.ParsedRequest)
w.WafRunners = make([]WafRunner, w.config.WafRoutines) w.WafRunners = make([]WafRunner, w.config.WafRoutines)
for nbRoutine := 0; nbRoutine < w.config.WafRoutines; nbRoutine++ { for nbRoutine := 0; nbRoutine < w.config.WafRoutines; nbRoutine++ {
w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n"))) w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n")))
@ -207,10 +187,11 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error {
} }
runner := WafRunner{ runner := WafRunner{
outOfBandWaf: outofbandwaf, outOfBandWaf: outofbandwaf,
inBandWaf: inbandwaf, inBandWaf: inbandwaf,
inChan: w.InChan, inChan: w.InChan,
UUID: uuid.New().String(), UUID: uuid.New().String(),
RulesCollections: rulesCollections,
} }
w.WafRunners[nbRoutine] = runner w.WafRunners[nbRoutine] = runner
} }
@ -284,51 +265,8 @@ func (w *WafSource) Dump() interface{} {
return w return w
} }
func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { func processReqWithEngine(tx experimental.FullTransaction, r waf.ParsedRequest, wafType string) (*corazatypes.Interruption, experimental.FullTransaction, error) {
var body []byte
var err error
if r.Body != nil {
body = make([]byte, 0)
body, err = ioutil.ReadAll(r.Body)
if err != nil {
return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err)
}
}
// the real source of the request is set in 'x-client-ip'
clientIP := r.Header.Get("X-Client-Ip")
// the real target Host of the request is set in 'x-client-host'
clientHost := r.Header.Get("X-Client-Host")
// the real URI of the request is set in 'x-client-uri'
clientURI := r.Header.Get("X-Client-Uri")
// delete those headers before coraza process the request
delete(r.Header, "x-client-ip")
delete(r.Header, "x-client-host")
delete(r.Header, "x-client-uri")
return ParsedRequest{
RemoteAddr: r.RemoteAddr,
UUID: uuid.New().String(),
ClientHost: clientHost,
ClientIP: clientIP,
URI: clientURI,
Host: r.Host,
Headers: r.Header,
URL: r.URL,
Method: r.Method,
Proto: r.Proto,
Body: body,
TransferEncoding: r.TransferEncoding,
ResponseChannel: make(chan ResponseRequest),
}, nil
}
func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType string) (*corazatypes.Interruption, corazatypes.Transaction, error) {
var in *corazatypes.Interruption var in *corazatypes.Interruption
tx := waf.NewTransactionWithID(uuid)
if tx.IsRuleEngineOff() { if tx.IsRuleEngineOff() {
log.Printf("engine is off") log.Printf("engine is off")
return nil, nil, nil return nil, nil, nil
@ -346,7 +284,6 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType
//txx := experimental.ToFullInterface(tx) //txx := experimental.ToFullInterface(tx)
//txx = tx.(experimental.FullTransaction) //txx = tx.(experimental.FullTransaction)
//txx.RemoveRuleByID(1) //txx.RemoveRuleByID(1)
tx.ProcessConnection(r.ClientIP, 0, "", 0) tx.ProcessConnection(r.ClientIP, 0, "", 0)
//tx.ProcessURI(r.URL.String(), r.Method, r.Proto) //FIXME: get it from the headers //tx.ProcessURI(r.URL.String(), r.Method, r.Proto) //FIXME: get it from the headers
@ -369,12 +306,15 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType
} }
in = tx.ProcessRequestHeaders() in = tx.ProcessRequestHeaders()
//spew.Dump(in) //spew.Dump(in)
//spew.Dump(tx.MatchedRules()) //spew.Dump(tx.MatchedRules())
/*for _, rule := range tx.MatchedRules() { for _, rule := range tx.MatchedRules() {
spew.Dump(rule.Rule()) if rule.Message() == "" {
}*/ continue
}
}
//if we're inband, we should stop here, but for outofband go to the end //if we're inband, we should stop here, but for outofband go to the end
if in != nil && wafType == InBand { if in != nil && wafType == InBand {
@ -422,17 +362,105 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
log.Infof("Waf Runner is dying") log.Infof("Waf Runner is dying")
return nil return nil
case request := <-r.inChan: case request := <-r.inChan:
in, tx, err := processReqWithEngine(r.inBandWaf, request, request.UUID, InBand) inBoundTx := r.inBandWaf.NewTransactionWithID(request.UUID)
response := ResponseRequest{ expTx := inBoundTx.(experimental.FullTransaction)
Tx: tx, // we use this internal transaction for the expr helpers
Interruption: in, tx := waf.NewTransaction(expTx)
Err: err,
UUID: request.UUID, //Run the pre_eval hooks
for _, rules := range r.RulesCollections {
if len(rules.CompiledPreEval) == 0 {
continue
}
for _, compiledHook := range rules.CompiledPreEval {
if compiledHook.Filter != nil {
res, err := expr.Run(compiledHook.Filter, map[string]interface{}{
"rules": rules,
"req": request,
})
if err != nil {
log.Errorf("unable to run PreEval filter: %s", err)
continue
}
switch t := res.(type) {
case bool:
if t == false {
log.Infof("filter didnt match")
continue
}
default:
log.Errorf("Filter must return a boolean, can't filter")
continue
}
}
// here means there is no filter or the filter matched
for _, applyExpr := range compiledHook.Apply {
_, err := expr.Run(applyExpr, map[string]interface{}{
"rules": rules,
"req": request,
"RemoveRuleByID": tx.RemoveRuleByIDWithError,
})
if err != nil {
log.Errorf("unable to apply filter: %s", err)
continue
}
}
}
} }
in, expTx, err := processReqWithEngine(expTx, request, InBand)
request.Tx = expTx
response := waf.NewResponseRequest(expTx, in, request.UUID, err)
// run the on_match hooks
for _, rules := range r.RulesCollections {
if len(rules.CompiledOnMatch) == 0 {
continue
}
for _, compiledHook := range rules.CompiledOnMatch {
if compiledHook.Filter != nil {
res, err := expr.Run(compiledHook.Filter, map[string]interface{}{
"rules": rules,
"req": request,
})
if err != nil {
log.Errorf("unable to run PreEval filter: %s", err)
continue
}
switch t := res.(type) {
case bool:
if t == false {
continue
}
default:
log.Errorf("Filter must return a boolean, can't filter")
continue
}
}
// here means there is no filter or the filter matched
for _, applyExpr := range compiledHook.Apply {
_, err := expr.Run(applyExpr, map[string]interface{}{
"rules": rules,
"req": request,
"RemoveRuleByID": tx.RemoveRuleByIDWithError,
"SetRemediation": response.SetRemediation,
"SetRemediationByID": response.SetRemediationByID,
"CancelEvent": response.CancelEvent,
})
if err != nil {
log.Errorf("unable to apply filter: %s", err)
continue
}
}
}
}
// send back the result to the HTTP handler for the InBand part // send back the result to the HTTP handler for the InBand part
request.ResponseChannel <- response request.ResponseChannel <- response
if in != nil { if in != nil && response.SendEvents {
request.Tx = tx
// Generate the events for InBand channel // Generate the events for InBand channel
events, err := TxToEvents(request, InBand) events, err := TxToEvents(request, InBand)
if err != nil { if err != nil {
@ -446,13 +474,15 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
} }
// Process outBand // Process outBand
in, tx, err = processReqWithEngine(r.outOfBandWaf, request, request.UUID, OutOfBand) outBandTx := r.outOfBandWaf.NewTransactionWithID(request.UUID)
expTx = outBandTx.(experimental.FullTransaction)
in, expTx, err = processReqWithEngine(expTx, request, OutOfBand)
if err != nil { //things went south if err != nil { //things went south
log.Errorf("Error while processing request : %s", err) log.Errorf("Error while processing request : %s", err)
continue continue
} }
request.Tx = tx request.Tx = expTx
if tx != nil && len(tx.MatchedRules()) > 0 { if expTx != nil && len(expTx.MatchedRules()) > 0 {
events, err := TxToEvents(request, OutOfBand) events, err := TxToEvents(request, OutOfBand)
log.Infof("Request triggered by WAF, %d events to send", len(events)) log.Infof("Request triggered by WAF, %d events to send", len(events))
for _, evt := range events { for _, evt := range events {
@ -467,9 +497,13 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
} }
} }
type BodyResponse struct {
Action string `json:"action"`
}
func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) {
// parse the request only once // parse the request only once
parsedRequest, err := NewParsedRequestFromRequest(r) parsedRequest, err := waf.NewParsedRequestFromRequest(r)
if err != nil { if err != nil {
log.Errorf("%s", err) log.Errorf("%s", err)
rw.WriteHeader(http.StatusForbidden) rw.WriteHeader(http.StatusForbidden)
@ -487,10 +521,22 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) {
if message.Interruption != nil { if message.Interruption != nil {
rw.WriteHeader(http.StatusForbidden) rw.WriteHeader(http.StatusForbidden)
body, err := json.Marshal(BodyResponse{Action: message.Interruption.Action})
if err != nil {
log.Errorf("unable to build response: %s", err)
} else {
rw.Write(body)
}
return return
} }
rw.WriteHeader(http.StatusOK) rw.WriteHeader(http.StatusOK)
body, err := json.Marshal(BodyResponse{Action: "allow"})
if err != nil {
log.Errorf("unable to build response: %s", err)
} else {
rw.Write(body)
}
return return
} }

32
pkg/waf/env.go Normal file
View file

@ -0,0 +1,32 @@
package waf
import "github.com/corazawaf/coraza/v3/experimental"
type Transaction struct {
Tx experimental.FullTransaction
}
func NewTransaction(tx experimental.FullTransaction) Transaction {
return Transaction{Tx: tx}
}
func (t *Transaction) RemoveRuleByIDWithError(id int) error {
t.Tx.RemoveRuleByID(id)
return nil
}
func GetEnv() map[string]interface{} {
ResponseRequest := ResponseRequest{}
ParsedRequest := ParsedRequest{}
Rules := &WafRulesCollection{}
Tx := Transaction{}
return map[string]interface{}{
"rules": Rules,
"req": ParsedRequest,
"SetRemediation": ResponseRequest.SetRemediation,
"SetRemediationByID": ResponseRequest.SetRemediationByID,
"CancelEvent": ResponseRequest.CancelEvent,
"RemoveRuleByID": Tx.RemoveRuleByIDWithError,
}
}

106
pkg/waf/request.go Normal file
View file

@ -0,0 +1,106 @@
package waf
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
"github.com/corazawaf/coraza/v3/experimental"
corazatypes "github.com/corazawaf/coraza/v3/types"
"github.com/google/uuid"
)
type ResponseRequest struct {
UUID string
Tx corazatypes.Transaction
Interruption *corazatypes.Interruption
Err error
SendEvents bool
}
func NewResponseRequest(Tx experimental.FullTransaction, in *corazatypes.Interruption, UUID string, err error) ResponseRequest {
return ResponseRequest{
UUID: UUID,
Tx: Tx,
Interruption: in,
Err: err,
SendEvents: true,
}
}
func (r *ResponseRequest) SetRemediation(remediation string) error {
r.Interruption.Action = remediation
return nil
}
func (r *ResponseRequest) SetRemediationByID(ID int, remediation string) error {
if r.Interruption.RuleID == ID {
r.Interruption.Action = remediation
}
return nil
}
func (r *ResponseRequest) CancelEvent() error {
// true by default
r.SendEvents = false
return nil
}
type ParsedRequest struct {
RemoteAddr string
Host string
ClientIP string
URI string
ClientHost string
Headers http.Header
URL *url.URL
Method string
Proto string
Body []byte
TransferEncoding []string
UUID string
Tx experimental.FullTransaction
ResponseChannel chan ResponseRequest
}
func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) {
var body []byte
var err error
if r.Body != nil {
body = make([]byte, 0)
body, err = ioutil.ReadAll(r.Body)
if err != nil {
return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err)
}
}
// the real source of the request is set in 'x-client-ip'
clientIP := r.Header.Get("X-Client-Ip")
// the real target Host of the request is set in 'x-client-host'
clientHost := r.Header.Get("X-Client-Host")
// the real URI of the request is set in 'x-client-uri'
clientURI := r.Header.Get("X-Client-Uri")
// delete those headers before coraza process the request
delete(r.Header, "x-client-ip")
delete(r.Header, "x-client-host")
delete(r.Header, "x-client-uri")
return ParsedRequest{
RemoteAddr: r.RemoteAddr,
UUID: uuid.New().String(),
ClientHost: clientHost,
ClientIP: clientIP,
URI: clientURI,
Host: r.Host,
Headers: r.Header,
URL: r.URL,
Method: r.Method,
Proto: r.Proto,
Body: body,
TransferEncoding: r.TransferEncoding,
ResponseChannel: make(chan ResponseRequest),
}, nil
}

View file

@ -67,9 +67,7 @@ func buildHook(hook Hook) (CompiledHook, error) {
compiledHook.Filter = program compiledHook.Filter = program
} }
for _, apply := range hook.Apply { for _, apply := range hook.Apply {
program, err := expr.Compile(apply, GetExprWAFOptions(map[string]interface{}{ program, err := expr.Compile(apply, GetExprWAFOptions(GetEnv())...)
"rules": &WafRulesCollection{},
})...)
if err != nil { if err != nil {
return CompiledHook{}, fmt.Errorf("unable to compile apply %s : %w", apply, err) return CompiledHook{}, fmt.Errorf("unable to compile apply %s : %w", apply, err)
} }
@ -173,9 +171,7 @@ func (w *WafRuleLoader) LoadWafRules() ([]*WafRulesCollection, error) {
//Ignore filter for on load ? //Ignore filter for on load ?
if onLoadHook.Apply != nil { if onLoadHook.Apply != nil {
for exprIdx, applyExpr := range onLoadHook.Apply { for exprIdx, applyExpr := range onLoadHook.Apply {
_, err := expr.Run(applyExpr, map[string]interface{}{ _, err := expr.Run(applyExpr, GetEnv())
"rules": collection,
})
if err != nil { if err != nil {
w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafConfig.OnLoad[hookIdx].Apply[exprIdx], err) w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafConfig.OnLoad[hookIdx].Apply[exprIdx], err)
continue continue

View file

@ -4,6 +4,8 @@ import "strings"
type WafRule struct { type WafRule struct {
RawRule string RawRule string
RuleID string
InBand bool
} }
// This is the "compiled" state of a WafConfig // This is the "compiled" state of a WafConfig