63bd31b471
* fix our behavior to comply more with modsec, REQUEST_URI should be: path+query string * fix #2891 as well * add new transforms * add transform tests
382 lines
10 KiB
Go
382 lines
10 KiB
Go
package appsec
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"regexp"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/sirupsen/logrus"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
const (
|
|
URIHeaderName = "X-Crowdsec-Appsec-Uri"
|
|
VerbHeaderName = "X-Crowdsec-Appsec-Verb"
|
|
HostHeaderName = "X-Crowdsec-Appsec-Host"
|
|
IPHeaderName = "X-Crowdsec-Appsec-Ip"
|
|
APIKeyHeaderName = "X-Crowdsec-Appsec-Api-Key"
|
|
UserAgentHeaderName = "X-Crowdsec-Appsec-User-Agent"
|
|
)
|
|
|
|
type ParsedRequest struct {
|
|
RemoteAddr string `json:"remote_addr,omitempty"`
|
|
Host string `json:"host,omitempty"`
|
|
ClientIP string `json:"client_ip,omitempty"`
|
|
URI string `json:"uri,omitempty"`
|
|
Args url.Values `json:"args,omitempty"`
|
|
ClientHost string `json:"client_host,omitempty"`
|
|
Headers http.Header `json:"headers,omitempty"`
|
|
URL *url.URL `json:"url,omitempty"`
|
|
Method string `json:"method,omitempty"`
|
|
Proto string `json:"proto,omitempty"`
|
|
Body []byte `json:"body,omitempty"`
|
|
TransferEncoding []string `json:"transfer_encoding,omitempty"`
|
|
UUID string `json:"uuid,omitempty"`
|
|
Tx ExtendedTransaction `json:"-"`
|
|
ResponseChannel chan AppsecTempResponse `json:"-"`
|
|
IsInBand bool `json:"-"`
|
|
IsOutBand bool `json:"-"`
|
|
AppsecEngine string `json:"appsec_engine,omitempty"`
|
|
RemoteAddrNormalized string `json:"normalized_remote_addr,omitempty"`
|
|
HTTPRequest *http.Request `json:"-"`
|
|
}
|
|
|
|
type ReqDumpFilter struct {
|
|
req *ParsedRequest
|
|
HeadersContentFilters []string
|
|
HeadersNameFilters []string
|
|
HeadersDrop bool
|
|
|
|
BodyDrop bool
|
|
//BodyContentFilters []string TBD
|
|
|
|
ArgsContentFilters []string
|
|
ArgsNameFilters []string
|
|
ArgsDrop bool
|
|
}
|
|
|
|
func (r *ParsedRequest) DumpRequest(params ...any) *ReqDumpFilter {
|
|
filter := ReqDumpFilter{}
|
|
filter.BodyDrop = true
|
|
filter.HeadersNameFilters = []string{"cookie", "authorization"}
|
|
filter.req = r
|
|
return &filter
|
|
}
|
|
|
|
// clear filters
|
|
func (r *ReqDumpFilter) NoFilters() *ReqDumpFilter {
|
|
r2 := ReqDumpFilter{}
|
|
r2.req = r.req
|
|
return &r2
|
|
}
|
|
|
|
func (r *ReqDumpFilter) WithEmptyHeadersFilters() *ReqDumpFilter {
|
|
r.HeadersContentFilters = []string{}
|
|
return r
|
|
}
|
|
|
|
func (r *ReqDumpFilter) WithHeadersContentFilter(filter string) *ReqDumpFilter {
|
|
r.HeadersContentFilters = append(r.HeadersContentFilters, filter)
|
|
return r
|
|
}
|
|
|
|
func (r *ReqDumpFilter) WithHeadersNameFilter(filter string) *ReqDumpFilter {
|
|
r.HeadersNameFilters = append(r.HeadersNameFilters, filter)
|
|
return r
|
|
}
|
|
|
|
func (r *ReqDumpFilter) WithNoHeaders() *ReqDumpFilter {
|
|
r.HeadersDrop = true
|
|
return r
|
|
}
|
|
|
|
func (r *ReqDumpFilter) WithHeaders() *ReqDumpFilter {
|
|
r.HeadersDrop = false
|
|
r.HeadersNameFilters = []string{}
|
|
return r
|
|
}
|
|
|
|
func (r *ReqDumpFilter) WithBody() *ReqDumpFilter {
|
|
r.BodyDrop = false
|
|
return r
|
|
}
|
|
|
|
func (r *ReqDumpFilter) WithNoBody() *ReqDumpFilter {
|
|
r.BodyDrop = true
|
|
return r
|
|
}
|
|
|
|
func (r *ReqDumpFilter) WithEmptyArgsFilters() *ReqDumpFilter {
|
|
r.ArgsContentFilters = []string{}
|
|
return r
|
|
}
|
|
|
|
func (r *ReqDumpFilter) WithArgsContentFilter(filter string) *ReqDumpFilter {
|
|
r.ArgsContentFilters = append(r.ArgsContentFilters, filter)
|
|
return r
|
|
}
|
|
|
|
func (r *ReqDumpFilter) WithArgsNameFilter(filter string) *ReqDumpFilter {
|
|
r.ArgsNameFilters = append(r.ArgsNameFilters, filter)
|
|
return r
|
|
}
|
|
|
|
func (r *ReqDumpFilter) FilterBody(out *ParsedRequest) error {
|
|
if r.BodyDrop {
|
|
return nil
|
|
}
|
|
out.Body = r.req.Body
|
|
return nil
|
|
}
|
|
|
|
func (r *ReqDumpFilter) FilterArgs(out *ParsedRequest) error {
|
|
if r.ArgsDrop {
|
|
return nil
|
|
}
|
|
if len(r.ArgsContentFilters) == 0 && len(r.ArgsNameFilters) == 0 {
|
|
out.Args = r.req.Args
|
|
return nil
|
|
}
|
|
out.Args = make(url.Values)
|
|
for k, vals := range r.req.Args {
|
|
reject := false
|
|
//exclude by match on name
|
|
for _, filter := range r.ArgsNameFilters {
|
|
ok, err := regexp.MatchString("(?i)"+filter, k)
|
|
if err != nil {
|
|
log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err)
|
|
continue
|
|
}
|
|
if ok {
|
|
reject = true
|
|
break
|
|
}
|
|
}
|
|
|
|
for _, v := range vals {
|
|
//exclude by content
|
|
for _, filter := range r.ArgsContentFilters {
|
|
ok, err := regexp.MatchString("(?i)"+filter, v)
|
|
if err != nil {
|
|
log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err)
|
|
continue
|
|
}
|
|
if ok {
|
|
reject = true
|
|
break
|
|
}
|
|
|
|
}
|
|
}
|
|
//if it was not rejected, let's add it
|
|
if !reject {
|
|
out.Args[k] = vals
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *ReqDumpFilter) FilterHeaders(out *ParsedRequest) error {
|
|
if r.HeadersDrop {
|
|
return nil
|
|
}
|
|
|
|
if len(r.HeadersContentFilters) == 0 && len(r.HeadersNameFilters) == 0 {
|
|
out.Headers = r.req.Headers
|
|
return nil
|
|
}
|
|
|
|
out.Headers = make(http.Header)
|
|
for k, vals := range r.req.Headers {
|
|
reject := false
|
|
//exclude by match on name
|
|
for _, filter := range r.HeadersNameFilters {
|
|
ok, err := regexp.MatchString("(?i)"+filter, k)
|
|
if err != nil {
|
|
log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err)
|
|
continue
|
|
}
|
|
if ok {
|
|
reject = true
|
|
break
|
|
}
|
|
}
|
|
|
|
for _, v := range vals {
|
|
//exclude by content
|
|
for _, filter := range r.HeadersContentFilters {
|
|
ok, err := regexp.MatchString("(?i)"+filter, v)
|
|
if err != nil {
|
|
log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err)
|
|
continue
|
|
}
|
|
if ok {
|
|
reject = true
|
|
break
|
|
}
|
|
|
|
}
|
|
}
|
|
//if it was not rejected, let's add it
|
|
if !reject {
|
|
out.Headers[k] = vals
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *ReqDumpFilter) GetFilteredRequest() *ParsedRequest {
|
|
//if there are no filters, we return the original request
|
|
if len(r.HeadersContentFilters) == 0 &&
|
|
len(r.HeadersNameFilters) == 0 &&
|
|
len(r.ArgsContentFilters) == 0 &&
|
|
len(r.ArgsNameFilters) == 0 &&
|
|
!r.BodyDrop && !r.HeadersDrop && !r.ArgsDrop {
|
|
log.Warningf("no filters, returning original request")
|
|
return r.req
|
|
}
|
|
|
|
r2 := ParsedRequest{}
|
|
r.FilterHeaders(&r2)
|
|
r.FilterBody(&r2)
|
|
r.FilterArgs(&r2)
|
|
return &r2
|
|
}
|
|
|
|
func (r *ReqDumpFilter) ToJSON() error {
|
|
fd, err := os.CreateTemp("", "crowdsec_req_dump_*.json")
|
|
if err != nil {
|
|
return fmt.Errorf("while creating temp file: %w", err)
|
|
}
|
|
defer fd.Close()
|
|
enc := json.NewEncoder(fd)
|
|
enc.SetIndent("", " ")
|
|
|
|
req := r.GetFilteredRequest()
|
|
|
|
log.Tracef("dumping : %+v", req)
|
|
|
|
if err := enc.Encode(req); err != nil {
|
|
//Don't clobber the temp directory with empty files
|
|
err2 := os.Remove(fd.Name())
|
|
if err2 != nil {
|
|
log.Errorf("while removing temp file %s: %s", fd.Name(), err)
|
|
}
|
|
return fmt.Errorf("while encoding request: %w", err)
|
|
}
|
|
log.Infof("request dumped to %s", fd.Name())
|
|
return nil
|
|
}
|
|
|
|
// Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine
|
|
func NewParsedRequestFromRequest(r *http.Request, logger *logrus.Entry) (ParsedRequest, error) {
|
|
var err error
|
|
contentLength := r.ContentLength
|
|
if contentLength < 0 {
|
|
contentLength = 0
|
|
}
|
|
body := make([]byte, contentLength)
|
|
if r.Body != nil {
|
|
_, err = io.ReadFull(r.Body, body)
|
|
if err != nil {
|
|
return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err)
|
|
}
|
|
// reset the original body back as it's been read, i'm not sure its needed?
|
|
r.Body = io.NopCloser(bytes.NewBuffer(body))
|
|
|
|
}
|
|
clientIP := r.Header.Get(IPHeaderName)
|
|
if clientIP == "" {
|
|
return ParsedRequest{}, fmt.Errorf("missing '%s' header", IPHeaderName)
|
|
}
|
|
|
|
clientURI := r.Header.Get(URIHeaderName)
|
|
if clientURI == "" {
|
|
return ParsedRequest{}, fmt.Errorf("missing '%s' header", URIHeaderName)
|
|
}
|
|
|
|
clientMethod := r.Header.Get(VerbHeaderName)
|
|
if clientMethod == "" {
|
|
return ParsedRequest{}, fmt.Errorf("missing '%s' header", VerbHeaderName)
|
|
}
|
|
|
|
clientHost := r.Header.Get(HostHeaderName)
|
|
if clientHost == "" { //this might be empty
|
|
logger.Debugf("missing '%s' header", HostHeaderName)
|
|
}
|
|
|
|
userAgent := r.Header.Get(UserAgentHeaderName) //This one is optional
|
|
|
|
// delete those headers before coraza process the request
|
|
delete(r.Header, IPHeaderName)
|
|
delete(r.Header, HostHeaderName)
|
|
delete(r.Header, URIHeaderName)
|
|
delete(r.Header, VerbHeaderName)
|
|
delete(r.Header, UserAgentHeaderName)
|
|
delete(r.Header, APIKeyHeaderName)
|
|
|
|
originalHTTPRequest := r.Clone(r.Context())
|
|
originalHTTPRequest.Body = io.NopCloser(bytes.NewBuffer(body))
|
|
originalHTTPRequest.RemoteAddr = clientIP
|
|
originalHTTPRequest.RequestURI = clientURI
|
|
originalHTTPRequest.Method = clientMethod
|
|
originalHTTPRequest.Host = clientHost
|
|
if userAgent != "" {
|
|
originalHTTPRequest.Header.Set("User-Agent", userAgent)
|
|
r.Header.Set("User-Agent", userAgent) //Override the UA in the original request, as this is what will be used by the waf engine
|
|
} else {
|
|
//If we don't have a forwarded UA, delete the one that was set by the bouncer
|
|
originalHTTPRequest.Header.Del("User-Agent")
|
|
}
|
|
|
|
parsedURL, err := url.Parse(clientURI)
|
|
if err != nil {
|
|
return ParsedRequest{}, fmt.Errorf("unable to parse url '%s': %s", clientURI, err)
|
|
}
|
|
|
|
var remoteAddrNormalized string
|
|
if r.RemoteAddr == "@" {
|
|
r.RemoteAddr = "127.0.0.1:65535"
|
|
}
|
|
// TODO we need to implement forwrded headers
|
|
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
|
if err != nil {
|
|
log.Errorf("Invalid appsec remote IP source %v: %s", r.RemoteAddr, err.Error())
|
|
remoteAddrNormalized = r.RemoteAddr
|
|
} else {
|
|
ip := net.ParseIP(host)
|
|
if ip == nil {
|
|
log.Errorf("Invalid appsec remote IP address source %v", r.RemoteAddr)
|
|
remoteAddrNormalized = r.RemoteAddr
|
|
} else {
|
|
remoteAddrNormalized = ip.String()
|
|
}
|
|
}
|
|
|
|
return ParsedRequest{
|
|
RemoteAddr: r.RemoteAddr,
|
|
UUID: uuid.New().String(),
|
|
ClientHost: clientHost,
|
|
ClientIP: clientIP,
|
|
URI: clientURI,
|
|
Method: clientMethod,
|
|
Host: clientHost,
|
|
Headers: r.Header,
|
|
URL: parsedURL,
|
|
Proto: r.Proto,
|
|
Body: body,
|
|
Args: ParseQuery(parsedURL.RawQuery),
|
|
TransferEncoding: r.TransferEncoding,
|
|
ResponseChannel: make(chan AppsecTempResponse),
|
|
RemoteAddrNormalized: remoteAddrNormalized,
|
|
HTTPRequest: originalHTTPRequest,
|
|
}, nil
|
|
}
|