From 17cfc9909e0a457bd2abb0de014d9be8b0ba1d67 Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 4 Dec 2023 09:45:47 +0100 Subject: [PATCH] add request dumper with filters --- pkg/waf/request.go | 305 ++++++++++++++++++++++++++++++++-------- pkg/waf/request_test.go | 181 ++++++++++++++++++++++++ 2 files changed, 425 insertions(+), 61 deletions(-) create mode 100644 pkg/waf/request_test.go diff --git a/pkg/waf/request.go b/pkg/waf/request.go index c2dc3e1ac..ecd8e6472 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -1,11 +1,14 @@ package waf import ( + "encoding/json" "fmt" "io" "net" "net/http" "net/url" + "os" + "regexp" "github.com/google/uuid" log "github.com/sirupsen/logrus" @@ -19,68 +22,248 @@ const ( APIKeyHeaderName = "X-Crowdsec-Waap-Api-Key" ) -// type ResponseRequest struct { -// UUID string -// Tx corazatypes.Transaction -// Interruption *corazatypes.Interruption -// Err error -// SendEvents bool -// } - -// func NewResponseRequest(Tx experimental.FullTransaction, in *corazatypes.Interruption, UUID string, err error) ResponseRequest { -// return ResponseRequest{ -// UUID: UUID, -// Tx: Tx, -// Interruption: in, -// Err: err, -// SendEvents: true, -// } -// } - -// func (r *ResponseRequest) SetRemediation(remediation string) error { -// if r.Interruption == nil { -// return nil -// } -// r.Interruption.Action = remediation -// return nil -// } - -// func (r *ResponseRequest) SetRemediationByID(ID int, remediation string) error { -// if r.Interruption == nil { -// return nil -// } -// if r.Interruption.RuleID == ID { -// r.Interruption.Action = remediation -// } -// return nil -// } - -// func (r *ResponseRequest) CancelEvent() error { -// // true by default -// r.SendEvents = false -// return nil -// } - type ParsedRequest struct { - RemoteAddr string - Host string - ClientIP string - URI string - Args url.Values - ClientHost string - Headers http.Header - URL *url.URL - Method string - Proto string - Body []byte - TransferEncoding []string - UUID string - Tx ExtendedTransaction - ResponseChannel chan WaapTempResponse - IsInBand bool - IsOutBand bool - WaapEngine string - RemoteAddrNormalized string + RemoteAddr string `json:"remote_addr,omitempty"` + Host string `json:"host,omitempty"` + ClientIP string `json:"client_ip,omitempty"` + URI string `json:"uri,omitempty"` + Args url.Values `json:"args,omitempty"` + ClientHost string `json:"client_host,omitempty"` + Headers http.Header `json:"headers,omitempty"` + URL *url.URL `json:"url,omitempty"` + Method string `json:"method,omitempty"` + Proto string `json:"proto,omitempty"` + Body []byte `json:"body,omitempty"` + TransferEncoding []string `json:"transfer_encoding,omitempty"` + UUID string `json:"uuid,omitempty"` + Tx ExtendedTransaction `json:"transaction,omitempty"` + ResponseChannel chan WaapTempResponse `json:"-"` + IsInBand bool `json:"-"` + IsOutBand bool `json:"-"` + WaapEngine string `json:"waap_engine,omitempty"` + RemoteAddrNormalized string `json:"normalized_remote_addr,omitempty"` +} + +type ReqDumpFilter struct { + req *ParsedRequest + HeadersContentFilters []string + HeadersNameFilters []string + HeadersDrop bool + + BodyDrop bool + //BodyContentFilters []string TBD + + ArgsContentFilters []string + ArgsNameFilters []string + ArgsDrop bool +} + +func (r *ParsedRequest) DumpRequest(params ...any) *ReqDumpFilter { + filter := ReqDumpFilter{} + filter.BodyDrop = true + filter.HeadersNameFilters = []string{"cookie", "authorization"} + filter.req = r + return &filter +} + +// clear filters +func (r *ReqDumpFilter) NoFilters() *ReqDumpFilter { + r2 := ReqDumpFilter{} + r2.req = r.req + return &r2 +} + +func (r *ReqDumpFilter) WithEmptyHeadersFilters() *ReqDumpFilter { + r.HeadersContentFilters = []string{} + return r +} + +func (r *ReqDumpFilter) WithHeadersContentFilters(filter string) *ReqDumpFilter { + r.HeadersContentFilters = append(r.HeadersContentFilters, filter) + return r +} + +func (r *ReqDumpFilter) WithHeadersNameFilter(filter string) *ReqDumpFilter { + r.HeadersNameFilters = append(r.HeadersNameFilters, filter) + return r +} + +func (r *ReqDumpFilter) WithNoHeaders() *ReqDumpFilter { + r.HeadersDrop = true + return r +} + +func (r *ReqDumpFilter) WithHeaders() *ReqDumpFilter { + r.HeadersDrop = false + r.HeadersNameFilters = []string{} + return r +} + +func (r *ReqDumpFilter) WithBody() *ReqDumpFilter { + r.BodyDrop = false + return r +} + +func (r *ReqDumpFilter) WithNoBody() *ReqDumpFilter { + r.BodyDrop = true + return r +} + +func (r *ReqDumpFilter) WithEmptyArgsFilters() *ReqDumpFilter { + r.ArgsContentFilters = []string{} + return r +} + +func (r *ReqDumpFilter) WithArgsContentFilters(filter string) *ReqDumpFilter { + r.ArgsContentFilters = append(r.ArgsContentFilters, filter) + return r +} + +func (r *ReqDumpFilter) WithArgsNameFilter(filter string) *ReqDumpFilter { + r.ArgsNameFilters = append(r.ArgsNameFilters, filter) + return r +} + +func (r *ReqDumpFilter) FilterBody(out *ParsedRequest) error { + if r.BodyDrop { + return nil + } + out.Body = r.req.Body + return nil +} + +func (r *ReqDumpFilter) FilterArgs(out *ParsedRequest) error { + if r.ArgsDrop { + return nil + } + if len(r.ArgsContentFilters) == 0 && len(r.ArgsNameFilters) == 0 { + out.Args = r.req.Args + return nil + } + out.Args = make(url.Values) + for k, vals := range r.req.Args { + reject := false + //exclude by match on name + for _, filter := range r.ArgsNameFilters { + ok, err := regexp.MatchString("(?i)"+filter, k) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err) + continue + } + if ok { + reject = true + break + } + } + + for _, v := range vals { + //exclude by content + for _, filter := range r.ArgsContentFilters { + ok, err := regexp.MatchString("(?i)"+filter, v) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err) + continue + } + if ok { + reject = true + break + } + + } + } + //if it was not rejected, let's add it + if !reject { + out.Args[k] = vals + } + } + return nil +} + +func (r *ReqDumpFilter) FilterHeaders(out *ParsedRequest) error { + if r.HeadersDrop { + return nil + } + + if len(r.HeadersContentFilters) == 0 && len(r.HeadersNameFilters) == 0 { + out.Headers = r.req.Headers + return nil + } + + out.Headers = make(http.Header) + for k, vals := range r.req.Headers { + reject := false + //exclude by match on name + for _, filter := range r.HeadersNameFilters { + ok, err := regexp.MatchString("(?i)"+filter, k) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err) + continue + } + if ok { + reject = true + break + } + } + + for _, v := range vals { + //exclude by content + for _, filter := range r.HeadersContentFilters { + ok, err := regexp.MatchString("(?i)"+filter, v) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err) + continue + } + if ok { + reject = true + break + } + + } + } + //if it was not rejected, let's add it + if !reject { + out.Headers[k] = vals + } + } + return nil +} + +func (r *ReqDumpFilter) GetFilteredRequest() *ParsedRequest { + //if there are no filters, we return the original request + if len(r.HeadersContentFilters) == 0 && + len(r.HeadersNameFilters) == 0 && + len(r.ArgsContentFilters) == 0 && + len(r.ArgsNameFilters) == 0 && + !r.BodyDrop && !r.HeadersDrop && !r.ArgsDrop { + log.Warningf("no filters, returning original request") + return r.req + } + + r2 := ParsedRequest{} + r.FilterHeaders(&r2) + r.FilterBody(&r2) + r.FilterArgs(&r2) + return &r2 +} + +func (r *ReqDumpFilter) ToJSON() error { + fd, err := os.CreateTemp("/tmp/", "crowdsec_req_dump_*.json") + if err != nil { + return fmt.Errorf("while creating temp file: %w", err) + } + defer fd.Close() + enc := json.NewEncoder(fd) + enc.SetIndent("", " ") + + req := r.GetFilteredRequest() + + log.Warningf("dumping : %+v", req) + + if err := enc.Encode(req); err != nil { + return fmt.Errorf("while encoding request: %w", err) + } + log.Warningf("request dumped to %s", fd.Name()) + return nil } // Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the Waap Engine diff --git a/pkg/waf/request_test.go b/pkg/waf/request_test.go new file mode 100644 index 000000000..2625e11f5 --- /dev/null +++ b/pkg/waf/request_test.go @@ -0,0 +1,181 @@ +package waf + +import "testing" + +func TestBodyDumper(t *testing.T) { + + tests := []struct { + name string + req *ParsedRequest + expect *ParsedRequest + filter func(r *ReqDumpFilter) *ReqDumpFilter + }{ + { + name: "default filter (cookie+authorization stripped + no body)", + req: &ParsedRequest{ + Body: []byte("yo some body"), + Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"foo": {"bar", "baz"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r + }, + }, + { + name: "explicit empty filter", + req: &ParsedRequest{ + Body: []byte("yo some body"), + Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}}, + }, + expect: &ParsedRequest{ + Body: []byte("yo some body"), + Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.NoFilters() + }, + }, + { + name: "filter header", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}, "test2": {"tata"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithNoBody().WithHeadersNameFilter("test2") + }, + }, + { + name: "filter header content", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}, "test2": {"tata"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithHeadersContentFilters("tata") + }, + }, + { + name: "with headers", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"cookie1": {"lol"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"cookie1": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithHeaders() + }, + }, + { + name: "drop headers", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"toto": {"lol"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithNoHeaders() + }, + }, + { + name: "with body", + req: &ParsedRequest{ + Body: []byte("toto"), + Headers: map[string][]string{"toto": {"lol"}}, + }, + expect: &ParsedRequest{ + Body: []byte("toto"), + Headers: map[string][]string{"toto": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithBody() + }, + }, + { + name: "with empty args filter", + req: &ParsedRequest{ + Args: map[string][]string{"toto": {"lol"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"toto": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithEmptyArgsFilters() + }, + }, + { + name: "with args name filter", + req: &ParsedRequest{ + Args: map[string][]string{"toto": {"lol"}, "totolol": {"lol"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"totolol": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithArgsNameFilter("toto") + }, + }, + { + name: "WithEmptyHeadersFilters", + req: &ParsedRequest{ + Args: map[string][]string{"cookie": {"lol"}, "totolol": {"lol"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"cookie": {"lol"}, "totolol": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithEmptyHeadersFilters() + }, + }, + { + name: "WithArgsContentFilters", + req: &ParsedRequest{ + Args: map[string][]string{"test": {"lol"}, "test2": {"toto"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"test": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithArgsContentFilters("toto") + }, + }, + } + + for idx, test := range tests { + + t.Run(test.name, func(t *testing.T) { + orig_dr := test.req.DumpRequest() + result := test.filter(orig_dr).GetFilteredRequest() + + if len(result.Body) != len(test.expect.Body) { + t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(test.req.Body), len(test.expect.Body)) + } + if len(result.Headers) != len(test.expect.Headers) { + t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(test.req.Headers), len(test.expect.Headers)) + } + for k, v := range result.Headers { + if len(v) != len(test.expect.Headers[k]) { + t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(v), len(test.expect.Headers[k])) + } + } + }) + } + +}