diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index bec3e05db..46b28ab36 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -284,6 +284,13 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e log.Warningln("Exprhelpers loaded without database client.") } + if cConfig.API.CTI != nil && *cConfig.API.CTI.Enabled { + log.Infof("Crowdsec CTI helper enabled") + if err := exprhelpers.InitCrowdsecCTI(cConfig.API.CTI.Key, cConfig.API.CTI.CacheTimeout, cConfig.API.CTI.CacheSize, cConfig.API.CTI.LogLevel); err != nil { + return errors.Wrap(err, "failed to init crowdsec cti") + } + } + if !cConfig.DisableAPI { if cConfig.API.Server.OnlineClient == nil || cConfig.API.Server.OnlineClient.Credentials == nil { log.Warningf("Communication with CrowdSec Central API disabled from configuration file") diff --git a/go.sum b/go.sum index 32bede118..ca7e0a6d3 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blackfireio/osinfo v1.0.3 h1:Yk2t2GTPjBcESv6nDSWZKO87bGMQgO+Hi9OoXPpxX8c= github.com/blackfireio/osinfo v1.0.3/go.mod h1:Pd987poVNmd5Wsx6PRPw4+w7kLlf9iJxoRKPtPAjOrA= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= diff --git a/pkg/apiserver/controllers/v1/alerts.go b/pkg/apiserver/controllers/v1/alerts.go index eebbe4d8b..cfef754c4 100644 --- a/pkg/apiserver/controllers/v1/alerts.go +++ b/pkg/apiserver/controllers/v1/alerts.go @@ -159,12 +159,13 @@ func (c *Controller) CreateAlert(gctx *gin.Context) { } alert.MachineID = machineID + //if coming from cscli, alert already has decisions if len(alert.Decisions) != 0 { for pIdx, profile := range c.Profiles { _, matched, err := profile.EvaluateProfile(alert) if err != nil { - gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) - return + profile.Logger.Warningf("error while evaluating profile %s : %v", profile.Cfg.Name, err) + continue } if !matched { continue @@ -183,9 +184,22 @@ func (c *Controller) CreateAlert(gctx *gin.Context) { for pIdx, profile := range c.Profiles { profileDecisions, matched, err := profile.EvaluateProfile(alert) + forceBreak := false if err != nil { - gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) - return + switch profile.Cfg.OnError { + case "apply": + profile.Logger.Warningf("applying profile %s despite error: %s", profile.Cfg.Name, err) + matched = true + case "continue": + profile.Logger.Warningf("skipping %s profile due to error: %s", profile.Cfg.Name, err) + case "break": + forceBreak = true + case "ignore": + profile.Logger.Warningf("ignoring error: %s", err) + default: + gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) + return + } } if !matched { @@ -197,7 +211,7 @@ func (c *Controller) CreateAlert(gctx *gin.Context) { } profileAlert := *alert c.sendAlertToPluginChannel(&profileAlert, uint(pIdx)) - if profile.Cfg.OnSuccess == "break" { + if profile.Cfg.OnSuccess == "break" || forceBreak { break } } diff --git a/pkg/csconfig/api.go b/pkg/csconfig/api.go index 5574d75aa..2066fe3c3 100644 --- a/pkg/csconfig/api.go +++ b/pkg/csconfig/api.go @@ -21,6 +21,7 @@ import ( type APICfg struct { Client *LocalApiClientCfg `yaml:"client"` Server *LocalApiServerCfg `yaml:"server"` + CTI *CTICfg `yaml:"cti"` } type ApiCredentialsCfg struct { @@ -45,6 +46,37 @@ type LocalApiClientCfg struct { InsecureSkipVerify *bool `yaml:"insecure_skip_verify"` // check if api certificate is bad or not } +type CTICfg struct { + Key *string `yaml:"key,omitempty"` + CacheTimeout *time.Duration `yaml:"cache_timeout,omitempty"` + CacheSize *int `yaml:"cache_size,omitempty"` + Enabled *bool `yaml:"enabled,omitempty"` + LogLevel *log.Level `yaml:"log_level,omitempty"` +} + +func (a *CTICfg) Load() error { + + if a.Key == nil { + *a.Enabled = false + } + if a.Key != nil && *a.Key == "" { + return fmt.Errorf("empty cti key") + } + if a.Enabled == nil { + a.Enabled = new(bool) + *a.Enabled = true + } + if a.CacheTimeout == nil { + a.CacheTimeout = new(time.Duration) + *a.CacheTimeout = 10 * time.Minute + } + if a.CacheSize == nil { + a.CacheSize = new(int) + *a.CacheSize = 100 + } + return nil +} + func (o *OnlineApiClientCfg) Load() error { o.Credentials = new(ApiCredentialsCfg) fcontent, err := os.ReadFile(o.CredentialsFilePath) @@ -92,7 +124,7 @@ func (l *LocalApiClientCfg) Load() error { apiclient.InsecureSkipVerify = *l.InsecureSkipVerify } - if l.Credentials.CACertPath != "" { + if l.Credentials.CACertPath != "" { caCert, err := os.ReadFile(l.Credentials.CACertPath) if err != nil { return errors.Wrapf(err, "failed to load cacert") @@ -230,6 +262,15 @@ func (c *Config) LoadAPIServer() error { return errors.Wrap(err, "loading online client credentials") } } + if c.API.Server.OnlineClient == nil || c.API.Server.OnlineClient.Credentials == nil { + log.Printf("push and pull to Central API disabled") + } + + if c.API.CTI != nil { + if err := c.API.CTI.Load(); err != nil { + return errors.Wrap(err, "loading CTI configuration") + } + } if err := c.LoadDBConfig(); err != nil { return err diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index c89f59fa0..2406a937a 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -110,6 +110,9 @@ func NewDefaultConfig() *Config { CredentialsFilePath: DefaultConfigPath("config", "online-api-secrets.yaml"), }, }, + CTI: &CTICfg{ + Enabled: types.BoolPtr(false), + }, } dbConfig := DatabaseCfg{ diff --git a/pkg/csconfig/profiles.go b/pkg/csconfig/profiles.go index b51ba33e3..16a4e454d 100644 --- a/pkg/csconfig/profiles.go +++ b/pkg/csconfig/profiles.go @@ -11,7 +11,13 @@ import ( "gopkg.in/yaml.v2" ) -//Profile structure(s) are used by the local API to "decide" what kind of decision should be applied when a scenario with an active remediation has been triggered +// var OnErrorDefault = OnErrorIgnore +// var OnErrorContinue = "continue" +// var OnErrorBreak = "break" +// var OnErrorApply = "apply" +// var OnErrorIgnore = "ignore" + +// Profile structure(s) are used by the local API to "decide" what kind of decision should be applied when a scenario with an active remediation has been triggered type ProfileCfg struct { Name string `yaml:"name,omitempty"` Debug *bool `yaml:"debug,omitempty"` @@ -20,6 +26,7 @@ type ProfileCfg struct { DurationExpr string `yaml:"duration_expr,omitempty"` OnSuccess string `yaml:"on_success,omitempty"` //continue or break OnFailure string `yaml:"on_failure,omitempty"` //continue or break + OnError string `yaml:"on_error,omitempty"` //continue, break, error, report, apply, ignore Notifications []string `yaml:"notifications,omitempty"` } diff --git a/pkg/csplugin/helpers.go b/pkg/csplugin/helpers.go index b1d66b145..857e9a53b 100644 --- a/pkg/csplugin/helpers.go +++ b/pkg/csplugin/helpers.go @@ -3,6 +3,7 @@ package csplugin import ( "text/template" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" ) @@ -18,6 +19,7 @@ var helpers = template.FuncMap{ } return metaValues }, + "CrowdsecCTI": exprhelpers.CrowdsecCTI, } func funcMap() template.FuncMap { diff --git a/pkg/csprofiles/csprofiles.go b/pkg/csprofiles/csprofiles.go index b2ca6026e..fea1f4b33 100644 --- a/pkg/csprofiles/csprofiles.go +++ b/pkg/csprofiles/csprofiles.go @@ -46,7 +46,12 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) { runtime.RuntimeFilters = make([]*vm.Program, len(profile.Filters)) runtime.DebugFilters = make([]*exprhelpers.ExprDebugger, len(profile.Filters)) runtime.Cfg = profile - + if runtime.Cfg.OnSuccess != "" && runtime.Cfg.OnSuccess != "continue" && runtime.Cfg.OnSuccess != "break" { + return []*Runtime{}, errors.Wrapf(err, "invalid 'on_success' for '%s' : %s", profile.Name, runtime.Cfg.OnSuccess) + } + if runtime.Cfg.OnFailure != "" && runtime.Cfg.OnFailure != "continue" && runtime.Cfg.OnFailure != "break" && runtime.Cfg.OnFailure != "apply" { + return []*Runtime{}, errors.Wrapf(err, "invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure) + } for fIdx, filter := range profile.Filters { if runtimeFilter, err = expr.Compile(filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"Alert": &models.Alert{}}))); err != nil { return []*Runtime{}, errors.Wrapf(err, "error compiling filter of '%s'", profile.Name) @@ -153,7 +158,7 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod return decisions, nil } -//EvaluateProfile is going to evaluate an Alert against a profile to generate Decisions +// EvaluateProfile is going to evaluate an Alert against a profile to generate Decisions func (Profile *Runtime) EvaluateProfile(Alert *models.Alert) ([]*models.Decision, bool, error) { var decisions []*models.Decision diff --git a/pkg/cticlient/client.go b/pkg/cticlient/client.go new file mode 100644 index 000000000..16876026a --- /dev/null +++ b/pkg/cticlient/client.go @@ -0,0 +1,157 @@ +package cticlient + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + log "github.com/sirupsen/logrus" +) + +const ( + CTIBaseUrl = "https://cti.api.crowdsec.net/v2" + smokeEndpoint = "/smoke" + fireEndpoint = "/fire" +) + +var ( + ErrUnauthorized = errors.New("unauthorized") + ErrLimit = errors.New("request quota exceeded, please reduce your request rate") + ErrNotFound = errors.New("ip not found") + ErrDisabled = errors.New("cti is disabled") + ErrUnknown = errors.New("unknown error") +) + +type CrowdsecCTIClient struct { + httpClient *http.Client + apiKey string + Logger *log.Entry +} + +func (c *CrowdsecCTIClient) doRequest(method string, endpoint string, params map[string]string) ([]byte, error) { + url := CTIBaseUrl + endpoint + if len(params) > 0 { + url += "?" + for k, v := range params { + url += fmt.Sprintf("%s=%s&", k, v) + } + } + req, err := http.NewRequest(method, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("x-api-key", c.apiKey) + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusForbidden { + return nil, ErrUnauthorized + } + if resp.StatusCode == http.StatusTooManyRequests { + return nil, ErrLimit + } + if resp.StatusCode == http.StatusNotFound { + return nil, ErrNotFound + } + return nil, fmt.Errorf("unexpected http code : %s", resp.Status) + } + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return respBody, nil +} + +func (c *CrowdsecCTIClient) GetIPInfo(ip string) (*SmokeItem, error) { + body, err := c.doRequest(http.MethodGet, smokeEndpoint+"/"+ip, nil) + if err != nil { + if err == ErrNotFound { + return &SmokeItem{}, nil + } + return nil, err + } + item := SmokeItem{} + err = json.Unmarshal(body, &item) + if err != nil { + return nil, err + } + return &item, nil +} + +func (c *CrowdsecCTIClient) SearchIPs(ips []string) (*SearchIPResponse, error) { + params := make(map[string]string) + params["ips"] = strings.Join(ips, ",") + body, err := c.doRequest(http.MethodGet, smokeEndpoint, params) + if err != nil { + return nil, err + } + searchIPResponse := SearchIPResponse{} + err = json.Unmarshal(body, &searchIPResponse) + if err != nil { + return nil, err + } + return &searchIPResponse, nil +} + +func (c *CrowdsecCTIClient) Fire(params FireParams) (*FireResponse, error) { + paramsMap := make(map[string]string) + if params.Page != nil { + paramsMap["page"] = fmt.Sprintf("%d", *params.Page) + } + if params.Since != nil { + paramsMap["since"] = *params.Since + } + if params.Limit != nil { + paramsMap["limit"] = fmt.Sprintf("%d", *params.Limit) + } + + body, err := c.doRequest(http.MethodGet, fireEndpoint, paramsMap) + if err != nil { + return nil, err + } + fireResponse := FireResponse{} + err = json.Unmarshal(body, &fireResponse) + if err != nil { + return nil, err + } + return &fireResponse, nil +} + +func NewCrowdsecCTIClient(options ...func(*CrowdsecCTIClient)) *CrowdsecCTIClient { + client := &CrowdsecCTIClient{} + for _, option := range options { + option(client) + } + if client.httpClient == nil { + client.httpClient = &http.Client{} + } + // we cannot return with a ni logger, so we set a default one + if client.Logger == nil { + client.Logger = log.NewEntry(log.New()) + } + return client +} + +func WithLogger(logger *log.Entry) func(*CrowdsecCTIClient) { + return func(c *CrowdsecCTIClient) { + c.Logger = logger + } +} + +func WithHTTPClient(httpClient *http.Client) func(*CrowdsecCTIClient) { + return func(c *CrowdsecCTIClient) { + c.httpClient = httpClient + } +} + +func WithAPIKey(apiKey string) func(*CrowdsecCTIClient) { + return func(c *CrowdsecCTIClient) { + c.apiKey = apiKey + } +} diff --git a/pkg/cticlient/client_test.go b/pkg/cticlient/client_test.go new file mode 100644 index 000000000..a487f1098 --- /dev/null +++ b/pkg/cticlient/client_test.go @@ -0,0 +1,294 @@ +package cticlient + +import ( + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "testing" + + log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/stretchr/testify/assert" +) + +const validApiKey = "my-api-key" + +// Copy pasted from actual API response +var smokeResponses = map[string]string{ + "1.1.1.1": `{"ip_range_score": 0, "ip": "1.1.1.1", "ip_range": "1.1.1.0/24", "as_name": "CLOUDFLARENET", "as_num": 13335, "location": {"country": null, "city": null, "latitude": null, "longitude": null}, "reverse_dns": "one.one.one.one", "behaviors": [{"name": "ssh:bruteforce", "label": "SSH Bruteforce", "description": "IP has been reported for performing brute force on ssh services."}, {"name": "tcp:scan", "label": "TCP Scan", "description": "IP has been reported for performing TCP port scanning."}, {"name": "http:scan", "label": "HTTP Scan", "description": "IP has been reported for performing actions related to HTTP vulnerability scanning and discovery."}], "history": {"first_seen": "2021-04-18T18:00:00+00:00", "last_seen": "2022-11-23T13:00:00+00:00", "full_age": 583, "days_age": 583}, "classifications": {"false_positives": [], "classifications": [{"name": "profile:insecure_services", "label": "Dangerous Services Exposed", "description": "IP exposes dangerous services (vnc, telnet, rdp), possibly due to a misconfiguration or because it's a honeypot."}, {"name": "profile:many_services", "label": "Many Services Exposed", "description": "IP exposes many open port, possibly due to a misconfiguration or because it's a honeypot."}]}, "attack_details": [{"name": "crowdsecurity/ssh-bf", "label": "SSH Bruteforce", "description": "Detect ssh brute force", "references": []}, {"name": "crowdsecurity/iptables-scan-multi_ports", "label": "Port Scanner", "description": "Detect tcp port scan", "references": []}, {"name": "crowdsecurity/ssh-slow-bf", "label": "Slow SSH Bruteforce", "description": "Detect slow ssh brute force", "references": []}, {"name": "crowdsecurity/http-probing", "label": "HTTP Scanner", "description": "Detect site scanning/probing from a single ip", "references": []}, {"name": "crowdsecurity/http-path-traversal-probing", "label": "Path Traversal Scanner", "description": "Detect path traversal attempt", "references": []}, {"name": "crowdsecurity/http-bad-user-agent", "label": "Known Bad User-Agent", "description": "Detect bad user-agents", "references": []}], "target_countries": {"DE": 33, "FR": 25, "US": 12, "CA": 8, "JP": 8, "AT": 4, "GB": 4, "AE": 4}, "background_noise_score": 4, "scores": {"overall": {"aggressiveness": 2, "threat": 2, "trust": 1, "anomaly": 2, "total": 2}, "last_day": {"aggressiveness": 0, "threat": 0, "trust": 0, "anomaly": 2, "total": 0}, "last_week": {"aggressiveness": 1, "threat": 2, "trust": 0, "anomaly": 2, "total": 1}, "last_month": {"aggressiveness": 3, "threat": 2, "trust": 0, "anomaly": 2, "total": 2}}, "references": []}`, +} + +var fireResponses []string + +// RoundTripFunc . +type RoundTripFunc func(req *http.Request) *http.Response + +// RoundTrip . +func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +// wip +func fireHandler(req *http.Request) *http.Response { + var err error + apiKey := req.Header.Get("x-api-key") + if apiKey != validApiKey { + log.Warningf("invalid api key: %s", apiKey) + return &http.Response{ + StatusCode: http.StatusForbidden, + Body: nil, + Header: make(http.Header), + } + } + //unmarshal data + if fireResponses == nil { + page1, err := os.ReadFile("tests/fire-page1.json") + if err != nil { + panic("can't read file") + } + page2, err := os.ReadFile("tests/fire-page2.json") + if err != nil { + panic("can't read file") + } + fireResponses = []string{string(page1), string(page2)} + } + //let's assume we have two valid pages. + page := 1 + if req.URL.Query().Get("page") != "" { + page, err = strconv.Atoi(req.URL.Query().Get("page")) + if err != nil { + log.Warningf("no page ?!") + return &http.Response{StatusCode: http.StatusInternalServerError} + } + } + + //how to react if you give a page number that is too big ? + if page > len(fireResponses) { + log.Warningf(" page too big %d vs %d", page, len(fireResponses)) + emptyResponse := `{ + "_links": { + "first": { + "href": "https://cti.api.crowdsec.net/v1/fire/" + }, + "self": { + "href": "https://cti.api.crowdsec.net/v1/fire/?page=3&limit=3" + } + }, + "items": [] + } + ` + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(emptyResponse))} + } + reader := io.NopCloser(strings.NewReader(fireResponses[page-1])) + //we should care about limit too + return &http.Response{ + StatusCode: http.StatusOK, + // Send response to be tested + Body: reader, + Header: make(http.Header), + ContentLength: 0, + } +} + +func smokeHandler(req *http.Request) *http.Response { + apiKey := req.Header.Get("x-api-key") + if apiKey != validApiKey { + return &http.Response{ + StatusCode: http.StatusForbidden, + Body: nil, + Header: make(http.Header), + } + } + + requestedIP := strings.Split(req.URL.Path, "/")[3] + response, ok := smokeResponses[requestedIP] + if !ok { + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(strings.NewReader(`{"message": "IP address information not found"}`)), + Header: make(http.Header), + } + } + + reader := io.NopCloser(strings.NewReader(response)) + + return &http.Response{ + StatusCode: http.StatusOK, + // Send response to be tested + Body: reader, + Header: make(http.Header), + ContentLength: 0, + } +} + +func rateLimitedHandler(req *http.Request) *http.Response { + apiKey := req.Header.Get("x-api-key") + if apiKey != validApiKey { + return &http.Response{ + StatusCode: http.StatusForbidden, + Body: nil, + Header: make(http.Header), + } + } + return &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: nil, + Header: make(http.Header), + } +} + +func searchHandler(req *http.Request) *http.Response { + apiKey := req.Header.Get("x-api-key") + if apiKey != validApiKey { + return &http.Response{ + StatusCode: http.StatusForbidden, + Body: nil, + Header: make(http.Header), + } + } + url, _ := url.Parse(req.URL.String()) + ipsParam := url.Query().Get("ips") + if ipsParam == "" { + return &http.Response{ + StatusCode: http.StatusBadRequest, + Body: nil, + Header: make(http.Header), + } + } + + totalIps := 0 + notFound := 0 + ips := strings.Split(ipsParam, ",") + for _, ip := range ips { + _, ok := smokeResponses[ip] + if ok { + totalIps++ + } else { + notFound++ + } + } + response := fmt.Sprintf(`{"total": %d, "not_found": %d, "items": [`, totalIps, notFound) + for _, ip := range ips { + response += smokeResponses[ip] + } + response += "]}" + reader := io.NopCloser(strings.NewReader(response)) + return &http.Response{ + StatusCode: http.StatusOK, + Body: reader, + Header: make(http.Header), + } +} + +func TestBadFireAuth(t *testing.T) { + ctiClient := NewCrowdsecCTIClient(WithAPIKey("asdasd"), WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(fireHandler), + })) + _, err := ctiClient.Fire(FireParams{}) + assert.EqualError(t, err, ErrUnauthorized.Error()) +} + +func TestFireOk(t *testing.T) { + cticlient := NewCrowdsecCTIClient(WithAPIKey(validApiKey), WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(fireHandler), + })) + data, err := cticlient.Fire(FireParams{}) + assert.Equal(t, err, nil) + assert.Equal(t, len(data.Items), 3) + assert.Equal(t, data.Items[0].Ip, "1.2.3.4") + //page 1 is the default + data, err = cticlient.Fire(FireParams{Page: types.IntPtr(1)}) + assert.Equal(t, err, nil) + assert.Equal(t, len(data.Items), 3) + assert.Equal(t, data.Items[0].Ip, "1.2.3.4") + //page 2 + data, err = cticlient.Fire(FireParams{Page: types.IntPtr(2)}) + assert.Equal(t, err, nil) + assert.Equal(t, len(data.Items), 3) + assert.Equal(t, data.Items[0].Ip, "4.2.3.4") +} + +func TestFirePaginator(t *testing.T) { + cticlient := NewCrowdsecCTIClient(WithAPIKey(validApiKey), WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(fireHandler), + })) + paginator := NewFirePaginator(cticlient, FireParams{}) + items, err := paginator.Next() + assert.Equal(t, err, nil) + assert.Equal(t, len(items), 3) + assert.Equal(t, items[0].Ip, "1.2.3.4") + items, err = paginator.Next() + assert.Equal(t, err, nil) + assert.Equal(t, len(items), 3) + assert.Equal(t, items[0].Ip, "4.2.3.4") + items, err = paginator.Next() + assert.Equal(t, err, nil) + assert.Equal(t, len(items), 0) + +} + +func TestBadSmokeAuth(t *testing.T) { + ctiClient := NewCrowdsecCTIClient(WithAPIKey("asdasd"), WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(smokeHandler), + })) + _, err := ctiClient.GetIPInfo("1.1.1.1") + assert.EqualError(t, err, ErrUnauthorized.Error()) +} + +func TestSmokeInfoValidIP(t *testing.T) { + ctiClient := NewCrowdsecCTIClient(WithAPIKey(validApiKey), WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(smokeHandler), + })) + resp, err := ctiClient.GetIPInfo("1.1.1.1") + if err != nil { + t.Fatalf("failed to get ip info: %s", err) + } + + assert.Equal(t, "1.1.1.1", resp.Ip) + assert.Equal(t, types.StrPtr("1.1.1.0/24"), resp.IpRange) +} + +func TestSmokeUnknownIP(t *testing.T) { + ctiClient := NewCrowdsecCTIClient(WithAPIKey(validApiKey), WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(smokeHandler), + })) + resp, err := ctiClient.GetIPInfo("42.42.42.42") + if err != nil { + t.Fatalf("failed to get ip info: %s", err) + } + + assert.Equal(t, "", resp.Ip) +} + +func TestRateLimit(t *testing.T) { + ctiClient := NewCrowdsecCTIClient(WithAPIKey(validApiKey), WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(rateLimitedHandler), + })) + _, err := ctiClient.GetIPInfo("1.1.1.1") + assert.EqualError(t, err, ErrLimit.Error()) +} + +func TestSearchIPs(t *testing.T) { + ctiClient := NewCrowdsecCTIClient(WithAPIKey(validApiKey), WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(searchHandler), + })) + resp, err := ctiClient.SearchIPs([]string{"1.1.1.1", "42.42.42.42"}) + if err != nil { + t.Fatalf("failed to search ips: %s", err) + } + assert.Equal(t, 1, resp.Total) + assert.Equal(t, 1, resp.NotFound) + assert.Equal(t, 1, len(resp.Items)) + assert.Equal(t, "1.1.1.1", resp.Items[0].Ip) +} + +//TODO: fire tests + pagination + +func TestFireInit(t *testing.T) { + +} diff --git a/pkg/cticlient/cti_test.go b/pkg/cticlient/cti_test.go new file mode 100644 index 000000000..1eb0a94de --- /dev/null +++ b/pkg/cticlient/cti_test.go @@ -0,0 +1,303 @@ +package cticlient + +// import ( +// "encoding/json" +// "net/http" +// "net/http/httptest" +// "net/url" +// "strings" +// "testing" +// "time" + +// "github.com/stretchr/testify/assert" +// ) + +// var sampledata = map[string]CTIResponse{ +// //1.2.3.4 is a known false positive +// "1.2.3.4": { +// Ip: "1.2.3.4", +// Classifications: CTIClassifications{ +// FalsePositives: []CTIClassification{ +// { +// Name: "example_false_positive", +// Label: "Example False Positive", +// }, +// }, +// }, +// }, +// //1.2.3.5 is a known bad-guy, and part of FIRE +// "1.2.3.5": { +// Ip: "1.2.3.5", +// Classifications: CTIClassifications{ +// Classifications: []CTIClassification{ +// { +// Name: "community-blocklist", +// Label: "CrowdSec Community Blocklist", +// Description: "IP belong to the CrowdSec Community Blocklist", +// }, +// }, +// }, +// }, +// //1.2.3.6 is a bad guy (high bg noise), but not in FIRE +// "1.2.3.6": { +// Ip: "1.2.3.6", +// BackgroundNoiseScore: new(int), +// Behaviors: []*CTIBehavior{ +// {Name: "ssh:bruteforce", Label: "SSH Bruteforce", Description: "SSH Bruteforce"}, +// }, +// AttackDetails: []*CTIAttackDetails{ +// {Name: "crowdsecurity/ssh-bf", Label: "Example Attack"}, +// {Name: "crowdsecurity/ssh-slow-bf", Label: "Example Attack"}, +// }, +// }, +// //1.2.3.7 is a ok guy, but part of a bad range +// "1.2.3.7": CTIResponse{}, +// } + +// func EmptyCTIResponse(ip string) CTIResponse { +// return CTIResponse{ +// IpRangeScore: 0, +// Ip: ip, +// Location: CTILocationInfo{}, +// } +// } + +// /* +// TBD : Simulate correctly quotas exhaustion +// */ +// func setup() (Router *http.ServeMux, serverURL string, teardown func()) { + +// //set static values +// *sampledata["1.2.3.6"].BackgroundNoiseScore = 10 + +// // mux is the HTTP request multiplexer used with the test server. +// Router = http.NewServeMux() +// baseURLPath := "/v2" + +// apiHandler := http.NewServeMux() +// apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, Router)) + +// // server is a test HTTP server used to provide mock API responses. +// server := httptest.NewServer(apiHandler) + +// // let's mock the API endpoints +// Router.HandleFunc("/smoke/", func(w http.ResponseWriter, r *http.Request) { +// //testMethod(t, r, "GET") +// if r.Header.Get("X-Api-Key") != "EXAMPLE_API_KEY" { +// w.WriteHeader(http.StatusForbidden) +// w.Write([]byte(`{"message":"Forbidden"}`)) +// return +// } + +// frags := strings.Split(r.RequestURI, "/") +// //[empty] [smoke] [v2] [actual_ip] +// if len(frags) != 4 { +// w.WriteHeader(http.StatusBadRequest) +// w.Write([]byte(`{"message":"Bad Request"}`)) +// return +// } +// ip := frags[3] + +// if ip == "" { +// //to be fixed to stick w/ real behavior +// panic("empty ip") + +// } +// // vars := mux.Vars(r) +// if v, ok := sampledata[ip]; ok { +// data, err := json.Marshal(v) +// if err != nil { +// panic("unable to marshal") +// } +// w.WriteHeader(http.StatusOK) +// w.Write(data) +// return +// } +// w.WriteHeader(http.StatusOK) +// data, err := json.Marshal(EmptyCTIResponse(ip)) +// if err != nil { +// panic("unable to marshal") +// } +// w.Write(data) +// return +// }) +// return Router, server.URL, server.Close +// } + +// func TestCTIAuthKO(t *testing.T) { + +// _, urlx, teardown := setup() +// apiURL, err := url.Parse(urlx + "/") +// if err != nil { +// t.Fatalf("parsing api url: %s", apiURL) +// } + +// defer teardown() +// defer ShutdownCTI() +// CTIUrl = urlx +// key := "BAD_KEY" +// if err := InitCTI(&key, nil, nil); err != nil { +// t.Fatalf("InitCTI failed: %s", err) +// } + +// ret := IpCTI("1.2.3.4") +// assert.Equal(t, false, ret.Ok(), "should be ko") +// assert.Equal(t, CTIResponse{}, ret, "auth failed, empty answer") +// assert.Equal(t, CTIApiEnabled, false, "auth failed, api disabled") +// //auth is disabled, we should always receive empty object +// ret = IpCTI("1.2.3.4") +// assert.Equal(t, false, ret.Ok(), "should be ko") +// assert.Equal(t, CTIResponse{}, ret, "auth failed, empty answer") +// } + +// func TestCTINoKey(t *testing.T) { + +// _, urlx, teardown := setup() +// apiURL, err := url.Parse(urlx + "/") +// if err != nil { +// t.Fatalf("parsing api url: %s", apiURL) +// } + +// defer teardown() +// defer ShutdownCTI() +// CTIUrl = urlx +// //key := "" +// err = InitCTI(nil, nil, nil) +// assert.NotEqual(t, err, nil, "InitCTI should fail") +// ret := IpCTI("1.2.3.4") +// assert.Equal(t, false, ret.Ok(), "should be ko") +// assert.Equal(t, CTIResponse{}, ret, "auth failed, empty answer") +// assert.Equal(t, CTIApiEnabled, false, "auth failed, api disabled") +// } + +// func TestCTIAuthOK(t *testing.T) { + +// _, urlx, teardown := setup() +// apiURL, err := url.Parse(urlx + "/") +// if err != nil { +// t.Fatalf("parsing api url: %s", apiURL) +// } + +// defer teardown() +// defer ShutdownCTI() + +// CTIUrl = urlx +// key := "EXAMPLE_API_KEY" +// if err := InitCTI(&key, nil, nil); err != nil { +// t.Fatalf("InitCTI failed: %s", err) +// } + +// ret := IpCTI("1.2.3.4") +// assert.Equal(t, true, ret.Ok(), "should be ok") +// assert.Equal(t, "1.2.3.4", ret.Ip, "auth failed, empty answer") +// assert.Equal(t, CTIApiEnabled, true, "auth failed, api disabled") +// } +// func TestCTIKnownFP(t *testing.T) { +// _, urlx, teardown := setup() +// apiURL, err := url.Parse(urlx + "/") +// if err != nil { +// t.Fatalf("parsing api url: %s", apiURL) +// } + +// defer teardown() +// defer ShutdownCTI() + +// CTIUrl = urlx +// key := "EXAMPLE_API_KEY" +// if err := InitCTI(&key, nil, nil); err != nil { +// t.Fatalf("InitCTI failed: %s", err) +// } + +// ret := IpCTI("1.2.3.4") +// assert.Equal(t, true, ret.Ok(), "should be ok") +// assert.Equal(t, "1.2.3.4", ret.Ip, "auth failed, empty answer") +// assert.Equal(t, ret.IsFalsePositive(), true, "1.2.3.4 is a known false positive") +// } + +// func TestCTIBelongsToFire(t *testing.T) { +// _, urlx, teardown := setup() +// apiURL, err := url.Parse(urlx + "/") +// if err != nil { +// t.Fatalf("parsing api url: %s", apiURL) +// } + +// defer teardown() +// defer ShutdownCTI() + +// CTIUrl = urlx +// key := "EXAMPLE_API_KEY" +// if err := InitCTI(&key, nil, nil); err != nil { +// t.Fatalf("InitCTI failed: %s", err) +// } + +// ret := IpCTI("1.2.3.5") +// assert.Equal(t, true, ret.Ok(), "should be ok") +// assert.Equal(t, "1.2.3.5", ret.Ip, "auth failed, empty answer") +// assert.Equal(t, ret.IsPartOfCommunityBlocklist(), true, "1.2.3.5 is a known false positive") +// } + +// func TestCTIBehaviors(t *testing.T) { +// _, urlx, teardown := setup() +// apiURL, err := url.Parse(urlx + "/") +// if err != nil { +// t.Fatalf("parsing api url: %s", apiURL) +// } + +// defer teardown() +// defer ShutdownCTI() + +// CTIUrl = urlx +// key := "EXAMPLE_API_KEY" +// if err := InitCTI(&key, nil, nil); err != nil { +// t.Fatalf("InitCTI failed: %s", err) +// } + +// ret := IpCTI("1.2.3.6") +// assert.Equal(t, true, ret.Ok(), "should be ok") +// assert.Equal(t, ret.Ip, "1.2.3.6", "auth failed, empty answer") +// //ssh:bruteforce +// assert.Equal(t, []string{"ssh:bruteforce"}, ret.GetBehaviors(), "error matching behaviors") +// assert.Equal(t, []string{"crowdsecurity/ssh-bf", "crowdsecurity/ssh-slow-bf"}, ret.GetAttackDetails(), "error matching behaviors") +// assert.Equal(t, 10, ret.GetBackgroundNoiseScore(), "error matching bg noise") +// } + +// func TestCacheFetch(t *testing.T) { +// _, urlx, teardown := setup() +// apiURL, err := url.Parse(urlx + "/") +// if err != nil { +// t.Fatalf("parsing api url: %s", apiURL) +// } + +// defer teardown() +// defer ShutdownCTI() + +// CTIUrl = urlx +// key := "EXAMPLE_API_KEY" +// ttl := 1 * time.Second +// if err := InitCTI(&key, &ttl, nil); err != nil { +// t.Fatalf("InitCTI failed: %s", err) +// } + +// ret := IpCTI("1.2.3.6") +// assert.Equal(t, true, ret.Ok(), "should be ok") +// assert.Equal(t, "1.2.3.6", ret.Ip, "initial fetch : bad item") +// assert.Equal(t, 1, CTICache.Len(true), "initial fetch : bad cache size") +// assert.Equal(t, "1.2.3.6", CTICache.Keys(true)[0].(string), "initial fetch : bad cache keys") +// //get it a second time before it expires +// ret = IpCTI("1.2.3.6") +// assert.Equal(t, true, ret.Ok(), "should be ok") +// assert.Equal(t, "1.2.3.6", ret.Ip, "initial fetch : bad item") +// assert.Equal(t, 1, CTICache.Len(true), "initial fetch : bad cache size") +// assert.Equal(t, "1.2.3.6", CTICache.Keys(true)[0].(string), "initial fetch : bad cache keys") +// //let data expire +// time.Sleep(1 * time.Second) +// assert.Equal(t, 0, CTICache.Len(true), "after ttl : bad cache size") +// //fetch again +// ret = IpCTI("1.2.3.6") +// assert.Equal(t, true, ret.Ok(), "should be ok") +// assert.Equal(t, "1.2.3.6", ret.Ip, "second fetch : bad item") +// assert.Equal(t, 1, CTICache.Len(true), "second fetch : bad cache size") +// assert.Equal(t, "1.2.3.6", CTICache.Keys(true)[0].(string), "initial fetch : bad cache keys") +// } + +// //GetMaliciousnessScore diff --git a/pkg/cticlient/example/fire.go b/pkg/cticlient/example/fire.go new file mode 100644 index 000000000..7bcf814a0 --- /dev/null +++ b/pkg/cticlient/example/fire.go @@ -0,0 +1,59 @@ +package main + +import ( + "encoding/csv" + "fmt" + "os" + "time" + + "github.com/crowdsecurity/crowdsec/pkg/cticlient" +) + +func intPtr(i int) *int { + return &i +} + +func main() { + client := cticlient.NewCrowdsecCTIClient(cticlient.WithAPIKey(os.Getenv("CTI_API_KEY"))) + paginator := cticlient.NewFirePaginator(client, cticlient.FireParams{ + Limit: intPtr(1000), + }) + + csvHeader := []string{ + "value", + "reason", + "type", + "scope", + "duration", + } + csvFile, err := os.Create("fire.csv") + if err != nil { + panic(err) + } + defer csvFile.Close() + csvWriter := csv.NewWriter(csvFile) + allItems := make([][]string, 0) + + for { + items, err := paginator.Next() + if err != nil { + panic(err) + } + if items == nil { + break + } + + for _, item := range items { + banDuration := time.Until(item.Expiration.Time) + allItems = append(allItems, []string{ + item.Ip, + "fire-import", + "ban", + "ip", + fmt.Sprintf("%ds", int(banDuration.Seconds())), + }) + } + } + csvWriter.Write(csvHeader) + csvWriter.WriteAll(allItems) +} diff --git a/pkg/cticlient/pagination.go b/pkg/cticlient/pagination.go new file mode 100644 index 000000000..d980aef8b --- /dev/null +++ b/pkg/cticlient/pagination.go @@ -0,0 +1,36 @@ +package cticlient + +type FirePaginator struct { + client *CrowdsecCTIClient + params FireParams + currentPage int + done bool +} + +func (p *FirePaginator) Next() ([]FireItem, error) { + if p.done { + return nil, nil + } + p.params.Page = &p.currentPage + resp, err := p.client.Fire(p.params) + if err != nil { + return nil, err + } + p.currentPage++ + if resp.Links.Next == nil { + p.done = true + } + return resp.Items, nil +} + +func NewFirePaginator(client *CrowdsecCTIClient, params FireParams) *FirePaginator { + startPage := 1 + if params.Page != nil { + startPage = *params.Page + } + return &FirePaginator{ + client: client, + params: params, + currentPage: startPage, + } +} diff --git a/pkg/cticlient/tests/fire-page1.json b/pkg/cticlient/tests/fire-page1.json new file mode 100644 index 000000000..ced0c7ba8 --- /dev/null +++ b/pkg/cticlient/tests/fire-page1.json @@ -0,0 +1,320 @@ +{ + "_links": { + "first": { + "href": "https://cti.api.crowdsec.net/v2/fire" + }, + "self": { + "href": "https://cti.api.crowdsec.net/v2/fire?page=1&limit=3" + }, + "next": { + "href": "https://cti.api.crowdsec.net/v2/fire?page=2&limit=3" + } + }, + "items": [ + { + "ip_range_score": 5, + "ip": "1.2.3.4", + "ip_range": "1.2.3.0/24", + "as_name": "AFFINITY-FTL", + "as_num": 3064, + "location": { + "country": "US", + "city": null, + "latitude": 37.751, + "longitude": -97.822 + }, + "reverse_dns": "lsxx.com", + "behaviors": [ + { + "name": "http:bruteforce", + "label": "HTTP Bruteforce", + "description": "IP has been reported for performing a HTTP brute force attack (either generic http probing or applicative related brute force)." + }, + { + "name": "http:scan", + "label": "HTTP Scan", + "description": "IP has been reported for performing actions related to HTTP vulnerability scanning and discovery." + } + ], + "history": { + "first_seen": "2022-09-18T14:00:00+00:00", + "last_seen": "2022-11-26T12:00:00+00:00", + "full_age": 77, + "days_age": 69 + }, + "classifications": { + "false_positives": [], + "classifications": [] + }, + "attack_details": [ + { + "name": "crowdsecurity/http-wordpress_user-enum", + "label": "WordPress Bruteforce", + "description": "Detect wordpress brute force", + "references": [] + }, + { + "name": "crowdsecurity/http-probing", + "label": "HTTP Scanner", + "description": "Detect site scanning/probing from a single ip", + "references": [] + }, + { + "name": "crowdsecurity/http-bf-wordpress_bf_xmlrpc", + "label": "WordPress XMLRPC Bruteforce", + "description": "Detect wordpress brute force on xmlrpc", + "references": [] + }, + { + "name": "crowdsecurity/http-bad-user-agent", + "label": "Known Bad User-Agent", + "description": "Detect bad user-agents", + "references": [] + } + ], + "state": "validated", + "expiration": "2022-12-11T14:15:47.553000", + "target_countries": { + "US": 43, + "DE": 20, + "NL": 8, + "GB": 7, + "FR": 6, + "PL": 3, + "SG": 2, + "CA": 2, + "DK": 2, + "ZA": 1 + }, + "background_noise_score": 5, + "scores": { + "overall": { + "aggressiveness": 5, + "threat": 0, + "trust": 5, + "anomaly": 0, + "total": 3 + }, + "last_day": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 0, + "total": 0 + }, + "last_week": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 0, + "total": 0 + }, + "last_month": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 0, + "total": 0 + } + }, + "references": [] + }, + { + "ip_range_score": 5, + "ip": "2.3.4.5", + "ip_range": "2.3.0./16", + "as_name": "Linode, LLC", + "as_num": 63949, + "location": { + "country": "DE", + "city": "Frankfurt am Main", + "latitude": 50.1188, + "longitude": 8.6843 + }, + "reverse_dns": "172xxent.com", + "behaviors": [ + { + "name": "http:exploit", + "label": "HTTP Exploit", + "description": "IP has been reported for attempting to exploit a vulnerability in a web application." + }, + { + "name": "http:scan", + "label": "HTTP Scan", + "description": "IP has been reported for performing actions related to HTTP vulnerability scanning and discovery." + }, + { + "name": "http:crawl", + "label": "HTTP Crawl", + "description": "IP has been reported for performing aggressive crawling of web applications." + } + ], + "history": { + "first_seen": "2022-10-15T16:00:00+00:00", + "last_seen": "2022-11-18T18:15:00+00:00", + "full_age": 50, + "days_age": 35 + }, + "classifications": { + "false_positives": [], + "classifications": [] + }, + "attack_details": [ + { + "name": "crowdsecurity/jira_cve-2021-26086", + "label": "Atlassian Jira CVE-2021-26086", + "description": "Detect Atlassian Jira CVE-2021-26086 exploitation attemps", + "references": [] + }, + { + "name": "crowdsecurity/http-probing", + "label": "HTTP Scanner", + "description": "Detect site scanning/probing from a single ip", + "references": [] + }, + { + "name": "crowdsecurity/CVE-2022-40684", + "label": "CVE-2022-40684", + "description": "Detect CVE-2022-40684 exploitation attempts (fortinet)", + "references": [ + "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-40684" + ] + }, + { + "name": "crowdsecurity/http-crawl-non_statics", + "label": "HTTP Crawler", + "description": "Detect aggressive crawl from single ip", + "references": [] + } + ], + "state": "validated", + "expiration": "2022-12-14T16:16:46.507000", + "target_countries": { + "US": 36, + "DE": 19, + "FR": 17, + "RU": 8, + "NL": 5, + "GB": 4, + "CA": 2, + "RO": 2, + "IT": 1, + "BR": 1 + }, + "background_noise_score": 9, + "scores": { + "overall": { + "aggressiveness": 5, + "threat": 2, + "trust": 5, + "anomaly": 0, + "total": 4 + }, + "last_day": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 0, + "total": 0 + }, + "last_week": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 0, + "total": 0 + }, + "last_month": { + "aggressiveness": 2, + "threat": 2, + "trust": 0, + "anomaly": 0, + "total": 1 + } + }, + "references": [] + }, + { + "ip_range_score": 0, + "ip": "3.2.3.4", + "ip_range": "3.2.3.0/24", + "as_name": "TOTxxited", + "as_num": 23969, + "location": { + "country": "TH", + "city": "Bangkok", + "latitude": 13.7366, + "longitude": 100.4995 + }, + "reverse_dns": "nxxxt.net", + "behaviors": [ + { + "name": "smb:bruteforce", + "label": "SMB Bruteforce", + "description": "IP has been reported for performing brute force on samba services." + } + ], + "history": { + "first_seen": "2022-11-26T05:15:00+00:00", + "last_seen": "2022-11-26T12:00:00+00:00", + "full_age": 9, + "days_age": 1 + }, + "classifications": { + "false_positives": [], + "classifications": [ + { + "name": "profile:insecure_services", + "label": "Dangerous Services Exposed", + "description": "IP exposes dangerous services (vnc, telnet, rdp), possibly due to a misconfiguration or because it's a honeypot." + } + ] + }, + "attack_details": [ + { + "name": "crowdsecurity/smb-bf", + "label": "Samba Bruteforce", + "description": "Detect smb brute force", + "references": [] + } + ], + "state": "validated", + "expiration": "2022-12-14T16:18:00.671000", + "target_countries": { + "GB": 100 + }, + "background_noise_score": 5, + "scores": { + "overall": { + "aggressiveness": 2, + "threat": 4, + "trust": 5, + "anomaly": 1, + "total": 4 + }, + "last_day": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 1, + "total": 0 + }, + "last_week": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 1, + "total": 0 + }, + "last_month": { + "aggressiveness": 2, + "threat": 4, + "trust": 5, + "anomaly": 1, + "total": 4 + } + }, + "references": [] + } + ] +} diff --git a/pkg/cticlient/tests/fire-page2.json b/pkg/cticlient/tests/fire-page2.json new file mode 100644 index 000000000..f7866b280 --- /dev/null +++ b/pkg/cticlient/tests/fire-page2.json @@ -0,0 +1,315 @@ +{ + "_links": { + "first": { + "href": "https://cti.api.crowdsec.net/v2/fire" + }, + "self": { + "href": "https://cti.api.crowdsec.net/v2/fire?page=2&limit=3" + }, + "prev": { + "href": "https://cti.api.crowdsec.net/v2/fire?page=1&limit=3" + }, + "next": { + "href": "https://cti.api.crowdsec.net/v2/fire?page=3&limit=3" + } + }, + "items": [ + { + "ip_range_score": 0, + "ip": "4.2.3.4", + "ip_range": "4.2.0.0/16", + "as_name": "Chxxoup", + "as_num": 4812, + "location": { + "country": "CN", + "city": null, + "latitude": 34.7732, + "longitude": 113.722 + }, + "reverse_dns": "xxxweqwwe.com.cn", + "behaviors": [ + { + "name": "smb:bruteforce", + "label": "SMB Bruteforce", + "description": "IP has been reported for performing brute force on samba services." + }, + { + "name": "windows:bruteforce", + "label": "SMB/RDP bruteforce", + "description": "IP has been reported for performing brute force on Windows (samba, remote desktop) services." + } + ], + "history": { + "first_seen": "2022-11-25T04:15:00+00:00", + "last_seen": "2022-11-25T13:30:00+00:00", + "full_age": 9, + "days_age": 1 + }, + "classifications": { + "false_positives": [], + "classifications": [ + { + "name": "proxy:vpn", + "label": "VPN", + "description": "IP exposes a VPN service or is being flagged as one." + } + ] + }, + "attack_details": [ + { + "name": "crowdsecurity/smb-bf", + "label": "Samba Bruteforce", + "description": "Detect smb brute force", + "references": [] + }, + { + "name": "crowdsecurity/windows-bf", + "label": "SMB/RDP brute force", + "description": "Detect samba/remote-desktop user brute force", + "references": [] + } + ], + "state": "validated", + "expiration": "2022-12-14T16:17:24.865000", + "target_countries": { + "FR": 100 + }, + "background_noise_score": 6, + "scores": { + "overall": { + "aggressiveness": 2, + "threat": 4, + "trust": 5, + "anomaly": 1, + "total": 4 + }, + "last_day": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 1, + "total": 0 + }, + "last_week": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 1, + "total": 0 + }, + "last_month": { + "aggressiveness": 2, + "threat": 4, + "trust": 5, + "anomaly": 1, + "total": 4 + } + }, + "references": [] + }, + { + "ip_range_score": 2, + "ip": "5.2.3.4", + "ip_range": "5.2.3.0/24", + "as_name": "Turxxri A.s.", + "as_num": 16135, + "location": { + "country": "TR", + "city": "Istanbul", + "latitude": 41.0551, + "longitude": 28.9347 + }, + "reverse_dns": null, + "behaviors": [ + { + "name": "ssh:bruteforce", + "label": "SSH Bruteforce", + "description": "IP has been reported for performing brute force on ssh services." + }, + { + "name": "tcp:scan", + "label": "TCP Scan", + "description": "IP has been reported for performing TCP port scanning." + } + ], + "history": { + "first_seen": "2022-08-26T02:00:00+00:00", + "last_seen": "2022-11-18T09:45:00+00:00", + "full_age": 100, + "days_age": 85 + }, + "classifications": { + "false_positives": [], + "classifications": [ + { + "name": "profile:insecure_services", + "label": "Dangerous Services Exposed", + "description": "IP exposes dangerous services (vnc, telnet, rdp), possibly due to a misconfiguration or because it's a honeypot." + }, + { + "name": "profile:many_services", + "label": "Many Services Exposed", + "description": "IP exposes many open port, possibly due to a misconfiguration or because it's a honeypot." + } + ] + }, + "attack_details": [ + { + "name": "crowdsecurity/ssh-slow-bf", + "label": "Slow SSH Bruteforce", + "description": "Detect slow ssh brute force", + "references": [] + }, + { + "name": "crowdsecurity/ssh-bf", + "label": "SSH Bruteforce", + "description": "Detect ssh brute force", + "references": [] + }, + { + "name": "crowdsecurity/iptables-scan-multi_ports", + "label": "Port Scanner", + "description": "Detect tcp port scan", + "references": [] + } + ], + "state": "validated", + "expiration": "2022-12-12T15:16:33.246000", + "target_countries": { + "FR": 21, + "HK": 19, + "US": 19, + "DE": 11, + "AU": 7, + "GB": 4, + "RU": 4, + "BR": 4, + "CA": 4, + "VE": 2 + }, + "background_noise_score": 4, + "scores": { + "overall": { + "aggressiveness": 2, + "threat": 3, + "trust": 2, + "anomaly": 3, + "total": 3 + }, + "last_day": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 3, + "total": 0 + }, + "last_week": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 3, + "total": 0 + }, + "last_month": { + "aggressiveness": 1, + "threat": 3, + "trust": 1, + "anomaly": 3, + "total": 2 + } + }, + "references": [] + }, + { + "ip_range_score": 5, + "ip": "6.2.3.4", + "ip_range": "6.2.0.0/17", + "as_name": "SMILESERV", + "as_num": 38700, + "location": { + "country": "KR", + "city": null, + "latitude": 37.5112, + "longitude": 126.9741 + }, + "reverse_dns": null, + "behaviors": [ + { + "name": "ssh:bruteforce", + "label": "SSH Bruteforce", + "description": "IP has been reported for performing brute force on ssh services." + } + ], + "history": { + "first_seen": "2022-09-20T15:30:00+00:00", + "last_seen": "2022-11-25T11:30:00+00:00", + "full_age": 74, + "days_age": 66 + }, + "classifications": { + "false_positives": [], + "classifications": [] + }, + "attack_details": [ + { + "name": "crowdsecurity/ssh-slow-bf", + "label": "Slow SSH Bruteforce", + "description": "Detect slow ssh brute force", + "references": [] + }, + { + "name": "crowdsecurity/ssh-bf", + "label": "SSH Bruteforce", + "description": "Detect ssh brute force", + "references": [] + } + ], + "state": "validated", + "expiration": "2022-12-14T16:19:30.654000", + "target_countries": { + "FR": 32, + "US": 21, + "DE": 17, + "NL": 5, + "FI": 5, + "RU": 3, + "GB": 3, + "SI": 2, + "RO": 2, + "HK": 2 + }, + "background_noise_score": 4, + "scores": { + "overall": { + "aggressiveness": 4, + "threat": 4, + "trust": 5, + "anomaly": 1, + "total": 4 + }, + "last_day": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 1, + "total": 0 + }, + "last_week": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 1, + "total": 0 + }, + "last_month": { + "aggressiveness": 3, + "threat": 4, + "trust": 1, + "anomaly": 1, + "total": 3 + } + }, + "references": [] + } + ] +} diff --git a/pkg/cticlient/types.go b/pkg/cticlient/types.go new file mode 100644 index 000000000..c26da39fb --- /dev/null +++ b/pkg/cticlient/types.go @@ -0,0 +1,295 @@ +package cticlient + +import ( + "time" +) + +type CTIScores struct { + Overall CTIScore `json:"overall"` + LastDay CTIScore `json:"last_day"` + LastWeek CTIScore `json:"last_week"` + LastMonth CTIScore `json:"last_month"` +} + +type CTIScore struct { + Aggressiveness int `json:"aggressiveness"` + Threat int `json:"threat"` + Trust int `json:"trust"` + Anomaly int `json:"anomaly"` + Total int `json:"total"` +} + +type CTIAttackDetails struct { + Name string `json:"name"` + Label string `json:"label"` + Description string `json:"description"` + References []string `json:"references"` +} + +type CTIClassifications struct { + FalsePositives []CTIClassification `json:"false_positives"` + Classifications []CTIClassification `json:"classifications"` +} + +type CTIClassification struct { + Name string `json:"name"` + Label string `json:"label"` + Description string `json:"description"` +} +type CTIHistory struct { + FirstSeen *string `json:"first_seen"` + LastSeen *string `json:"last_seen"` + FullAge int `json:"full_age"` + DaysAge int `json:"days_age"` +} + +type CTIBehavior struct { + Name string `json:"name"` + Label string `json:"label"` + Description string `json:"description"` +} +type CTILocationInfo struct { + Country *string `json:"country"` + City *string `json:"city"` + Latitude *float64 `json:"latitude"` + Longitude *float64 `json:"longitude"` +} + +type CTIReferences struct { + Name string `json:"name"` + Label string `json:"label"` + Description string `json:"description"` +} + +type SmokeItem struct { + IpRangeScore int `json:"ip_range_score"` + Ip string `json:"ip"` + IpRange *string `json:"ip_range"` + AsName *string `json:"as_name"` + AsNum *int `json:"as_num"` + Location CTILocationInfo `json:"location"` + ReverseDNS *string `json:"reverse_dns"` + Behaviors []*CTIBehavior `json:"behaviors"` + History CTIHistory `json:"history"` + Classifications CTIClassifications `json:"classifications"` + AttackDetails []*CTIAttackDetails `json:"attack_details"` + TargetCountries map[string]int `json:"target_countries"` + BackgroundNoiseScore *int `json:"background_noise_score"` + Scores CTIScores `json:"scores"` + References []CTIReferences `json:"references"` + IsOk bool `json:"-"` +} + +type SearchIPResponse struct { + Total int `json:"total"` + NotFound int `json:"not_found"` + Items []SmokeItem `json:"items"` +} + +type CustomTime struct { + time.Time +} + +func (ct *CustomTime) UnmarshalJSON(b []byte) error { + if string(b) == "null" { + return nil + } + + t, err := time.Parse(`"2006-01-02T15:04:05.999999999"`, string(b)) + if err != nil { + return err + } + + ct.Time = t + return nil +} + +type FireItem struct { + IpRangeScore int `json:"ip_range_score"` + Ip string `json:"ip"` + IpRange *string `json:"ip_range"` + AsName *string `json:"as_name"` + AsNum *int `json:"as_num"` + Location CTILocationInfo `json:"location"` + ReverseDNS *string `json:"reverse_dns"` + Behaviors []*CTIBehavior `json:"behaviors"` + History CTIHistory `json:"history"` + Classifications CTIClassifications `json:"classifications"` + AttackDetails []*CTIAttackDetails `json:"attack_details"` + TargetCountries map[string]int `json:"target_countries"` + BackgroundNoiseScore *int `json:"background_noise_score"` + Scores CTIScores `json:"scores"` + References []CTIReferences `json:"references"` + Status string `json:"status"` + Expiration CustomTime `json:"expiration"` +} + +type FireParams struct { + Since *string `json:"since"` + Page *int `json:"page"` + Limit *int `json:"limit"` +} + +type Href struct { + Href string `json:"href"` +} + +type Links struct { + First *Href `json:"first"` + Self *Href `json:"self"` + Prev *Href `json:"prev"` + Next *Href `json:"next"` +} + +type FireResponse struct { + Links Links `json:"_links"` + Items []FireItem `json:"items"` +} + +func (c *SmokeItem) GetAttackDetails() []string { + var ret []string = make([]string, 0) + + if c.AttackDetails != nil { + for _, b := range c.AttackDetails { + ret = append(ret, b.Name) + } + } + return ret +} + +func (c *SmokeItem) GetBehaviors() []string { + var ret []string = make([]string, 0) + + if c.Behaviors != nil { + for _, b := range c.Behaviors { + ret = append(ret, b.Name) + } + } + return ret +} + +// Provide the likelihood of the IP being bad +func (c *SmokeItem) GetMaliciousnessScore() float32 { + if c.IsPartOfCommunityBlocklist() { + return 1.0 + } + if c.Scores.LastDay.Total > 0 { + return float32(c.Scores.LastDay.Total) / 10.0 + } + return 0.0 +} + +func (c *SmokeItem) IsPartOfCommunityBlocklist() bool { + if c.Classifications.Classifications != nil { + for _, v := range c.Classifications.Classifications { + if v.Name == "community-blocklist" { + return true + } + } + } + + return false +} + +func (c *SmokeItem) GetBackgroundNoiseScore() int { + if c.BackgroundNoiseScore != nil { + return *c.BackgroundNoiseScore + } + return 0 +} + +func (c *SmokeItem) GetFalsePositives() []string { + var ret []string = make([]string, 0) + + if c.Classifications.FalsePositives != nil { + for _, b := range c.Classifications.FalsePositives { + ret = append(ret, b.Name) + } + } + return ret +} + +func (c *SmokeItem) IsFalsePositive() bool { + + if c.Classifications.FalsePositives != nil { + if len(c.Classifications.FalsePositives) > 0 { + return true + } + } + + return false +} + +func (c *FireItem) GetAttackDetails() []string { + var ret []string = make([]string, 0) + + if c.AttackDetails != nil { + for _, b := range c.AttackDetails { + ret = append(ret, b.Name) + } + } + return ret +} + +func (c *FireItem) GetBehaviors() []string { + var ret []string = make([]string, 0) + + if c.Behaviors != nil { + for _, b := range c.Behaviors { + ret = append(ret, b.Name) + } + } + return ret +} + +// Provide the likelihood of the IP being bad +func (c *FireItem) GetMaliciousnessScore() float32 { + if c.IsPartOfCommunityBlocklist() { + return 1.0 + } + if c.Scores.LastDay.Total > 0 { + return float32(c.Scores.LastDay.Total) / 10.0 + } + return 0.0 +} + +func (c *FireItem) IsPartOfCommunityBlocklist() bool { + if c.Classifications.Classifications != nil { + for _, v := range c.Classifications.Classifications { + if v.Name == "community-blocklist" { + return true + } + } + } + + return false +} + +func (c *FireItem) GetBackgroundNoiseScore() int { + if c.BackgroundNoiseScore != nil { + return *c.BackgroundNoiseScore + } + return 0 +} + +func (c *FireItem) GetFalsePositives() []string { + var ret []string = make([]string, 0) + + if c.Classifications.FalsePositives != nil { + for _, b := range c.Classifications.FalsePositives { + ret = append(ret, b.Name) + } + } + return ret +} + +func (c *FireItem) IsFalsePositive() bool { + + if c.Classifications.FalsePositives != nil { + if len(c.Classifications.FalsePositives) > 0 { + return true + } + } + + return false +} diff --git a/pkg/cticlient/types_test.go b/pkg/cticlient/types_test.go new file mode 100644 index 000000000..046d28692 --- /dev/null +++ b/pkg/cticlient/types_test.go @@ -0,0 +1,114 @@ +package cticlient + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/crowdsecurity/crowdsec/pkg/types" +) + +//func (c *SmokeItem) GetAttackDetails() []string { + +func getSampleSmokeItem() SmokeItem { + lat := 48.8566 + long := 2.3522 + emptyItem := SmokeItem{ + IpRangeScore: 2.0, + Ip: "1.2.3.4", + IpRange: types.StrPtr("1.2.3.0/24"), + AsName: types.StrPtr("AS1234"), + AsNum: types.IntPtr(1234), + Location: CTILocationInfo{ + Country: types.StrPtr("FR"), + City: types.StrPtr("Paris"), + Latitude: &lat, + Longitude: &long, + }, + ReverseDNS: types.StrPtr("foo.bar.com"), + Behaviors: []*CTIBehavior{ + { + Name: "ssh:bruteforce", + Label: "SSH Bruteforce", + Description: "IP has been reported for performing brute force on ssh services.", + }, + }, + History: CTIHistory{ + FirstSeen: types.StrPtr("2022-12-05T17:45:00+00:00"), + LastSeen: types.StrPtr("2022-12-06T19:15:00+00:00"), + FullAge: 3, + DaysAge: 1, + }, + Classifications: CTIClassifications{ + FalsePositives: []CTIClassification{}, + Classifications: []CTIClassification{}, + }, + AttackDetails: []*CTIAttackDetails{ + { + Name: "ssh:bruteforce", + Label: "SSH Bruteforce", + Description: "Detect ssh brute force", + References: []string{}, + }, + }, + TargetCountries: map[string]int{ + "HK": 71, + "GB": 14, + "US": 14, + }, + BackgroundNoiseScore: types.IntPtr(3), + Scores: CTIScores{ + Overall: CTIScore{ + Aggressiveness: 2, + Threat: 1, + Trust: 1, + Anomaly: 0, + Total: 1, + }, + LastDay: CTIScore{ + Aggressiveness: 2, + Threat: 1, + Trust: 1, + Anomaly: 0, + Total: 1, + }, + LastWeek: CTIScore{ + Aggressiveness: 2, + Threat: 1, + Trust: 1, + Anomaly: 0, + Total: 1, + }, + LastMonth: CTIScore{ + Aggressiveness: 2, + Threat: 1, + Trust: 1, + Anomaly: 0, + Total: 1, + }, + }, + } + return emptyItem +} + +func TestBasicSmokeItem(t *testing.T) { + item := getSampleSmokeItem() + assert.Equal(t, item.GetAttackDetails(), []string{"ssh:bruteforce"}) + assert.Equal(t, item.GetBehaviors(), []string{"ssh:bruteforce"}) + assert.Equal(t, item.GetMaliciousnessScore(), float32(0.1)) + assert.Equal(t, item.IsPartOfCommunityBlocklist(), false) + assert.Equal(t, item.GetBackgroundNoiseScore(), int(3)) + assert.Equal(t, item.GetFalsePositives(), []string{}) + assert.Equal(t, item.IsFalsePositive(), false) +} + +func TestEmptySmokeItem(t *testing.T) { + item := SmokeItem{} + assert.Equal(t, item.GetAttackDetails(), []string{}) + assert.Equal(t, item.GetBehaviors(), []string{}) + assert.Equal(t, item.GetMaliciousnessScore(), float32(0.0)) + assert.Equal(t, item.IsPartOfCommunityBlocklist(), false) + assert.Equal(t, item.GetBackgroundNoiseScore(), int(0)) + assert.Equal(t, item.GetFalsePositives(), []string{}) + assert.Equal(t, item.IsFalsePositive(), false) +} diff --git a/pkg/exprhelpers/crowdsec_cti.go b/pkg/exprhelpers/crowdsec_cti.go new file mode 100644 index 000000000..5596f8278 --- /dev/null +++ b/pkg/exprhelpers/crowdsec_cti.go @@ -0,0 +1,135 @@ +package exprhelpers + +import ( + "fmt" + "time" + + "github.com/bluele/gcache" + "github.com/crowdsecurity/crowdsec/pkg/cticlient" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +var CTIUrl = "https://cti.api.crowdsec.net" +var CTIUrlSuffix = "/v2/smoke/" +var CTIApiKey = "" + +// this is set for non-recoverable errors, such as 403 when querying API or empty API key +var CTIApiEnabled = true + +// when hitting quotas or auth errors, we temporarily disable the API +var CTIBackOffUntil time.Time +var CTIBackOffDuration time.Duration = 5 * time.Minute + +var ctiClient *cticlient.CrowdsecCTIClient + +func InitCrowdsecCTI(Key *string, TTL *time.Duration, Size *int, LogLevel *log.Level) error { + if Key != nil { + CTIApiKey = *Key + } else { + CTIApiEnabled = false + return fmt.Errorf("CTI API key not set, CTI will not be available") + } + if Size == nil { + Size = new(int) + *Size = 1000 + } + if TTL == nil { + TTL = new(time.Duration) + *TTL = 5 * time.Minute + } + //dedicated logger + clog := log.New() + if err := types.ConfigureLogger(clog); err != nil { + return errors.Wrap(err, "while configuring datasource logger") + } + if LogLevel != nil { + clog.SetLevel(*LogLevel) + } + customLog := log.Fields{ + "type": "crowdsec-cti", + } + subLogger := clog.WithFields(customLog) + CrowdsecCTIInitCache(*Size, *TTL) + ctiClient = cticlient.NewCrowdsecCTIClient(cticlient.WithAPIKey(CTIApiKey), cticlient.WithLogger(subLogger)) + return nil +} + +func ShutdownCrowdsecCTI() { + if CTICache != nil { + CTICache.Purge() + } + CTIApiKey = "" + CTIApiEnabled = true +} + +// Cache for responses +var CTICache gcache.Cache +var CacheExpiration time.Duration + +func CrowdsecCTIInitCache(size int, ttl time.Duration) { + CTICache = gcache.New(size).LRU().Build() + CacheExpiration = ttl +} + +func CrowdsecCTI(ip string) (*cticlient.SmokeItem, error) { + if !CTIApiEnabled { + ctiClient.Logger.Warningf("Crowdsec CTI API is disabled, please check your configuration") + return &cticlient.SmokeItem{}, cticlient.ErrDisabled + } + + if CTIApiKey == "" { + ctiClient.Logger.Warningf("CrowdsecCTI : no key provided, skipping") + return &cticlient.SmokeItem{}, cticlient.ErrDisabled + } + + if ctiClient == nil { + ctiClient.Logger.Warningf("CrowdsecCTI: no client, skipping") + return &cticlient.SmokeItem{}, cticlient.ErrDisabled + } + + if val, err := CTICache.Get(ip); err == nil && val != nil { + ctiClient.Logger.Debugf("cti cache fetch for %s", ip) + ret, ok := val.(*cticlient.SmokeItem) + if !ok { + ctiClient.Logger.Warningf("CrowdsecCTI: invalid type in cache, removing") + CTICache.Remove(ip) + } else { + return ret, nil + } + } + + if !CTIBackOffUntil.IsZero() && time.Now().Before(CTIBackOffUntil) { + //ctiClient.Logger.Warningf("Crowdsec CTI client is in backoff mode, ending in %s", time.Until(CTIBackOffUntil)) + return &cticlient.SmokeItem{}, cticlient.ErrLimit + } + + ctiClient.Logger.Infof("cti call for %s", ip) + before := time.Now() + ctiResp, err := ctiClient.GetIPInfo(ip) + ctiClient.Logger.Debugf("request for %s took %v", ip, time.Since(before)) + if err != nil { + if err == cticlient.ErrUnauthorized { + CTIApiEnabled = false + ctiClient.Logger.Errorf("Invalid API key provided, disabling CTI API") + return &cticlient.SmokeItem{}, cticlient.ErrUnauthorized + } else if err == cticlient.ErrLimit { + CTIBackOffUntil = time.Now().Add(CTIBackOffDuration) + ctiClient.Logger.Errorf("CTI API is throttled, will try again in %s", CTIBackOffDuration) + return &cticlient.SmokeItem{}, cticlient.ErrLimit + } else { + ctiClient.Logger.Warnf("CTI API error : %s", err) + return &cticlient.SmokeItem{}, fmt.Errorf("unexpected error : %v", err) + } + } + + if err := CTICache.SetWithExpire(ip, ctiResp, CacheExpiration); err != nil { + ctiClient.Logger.Warningf("IpCTI : error while caching CTI : %s", err) + return &cticlient.SmokeItem{}, cticlient.ErrUnknown + } + + ctiClient.Logger.Tracef("CTI response : %v", *ctiResp) + + return ctiResp, nil +} diff --git a/pkg/exprhelpers/crowdsec_cti_test.go b/pkg/exprhelpers/crowdsec_cti_test.go new file mode 100644 index 000000000..90beee0f8 --- /dev/null +++ b/pkg/exprhelpers/crowdsec_cti_test.go @@ -0,0 +1,181 @@ +package exprhelpers + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/crowdsecurity/crowdsec/pkg/cticlient" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/stretchr/testify/assert" +) + +var sampledata = map[string]cticlient.SmokeItem{ + //1.2.3.4 is a known false positive + "1.2.3.4": { + Ip: "1.2.3.4", + Classifications: cticlient.CTIClassifications{ + FalsePositives: []cticlient.CTIClassification{ + { + Name: "example_false_positive", + Label: "Example False Positive", + }, + }, + }, + }, + //1.2.3.5 is a known bad-guy, and part of FIRE + "1.2.3.5": { + Ip: "1.2.3.5", + Classifications: cticlient.CTIClassifications{ + Classifications: []cticlient.CTIClassification{ + { + Name: "community-blocklist", + Label: "CrowdSec Community Blocklist", + Description: "IP belong to the CrowdSec Community Blocklist", + }, + }, + }, + }, + //1.2.3.6 is a bad guy (high bg noise), but not in FIRE + "1.2.3.6": { + Ip: "1.2.3.6", + BackgroundNoiseScore: new(int), + Behaviors: []*cticlient.CTIBehavior{ + {Name: "ssh:bruteforce", Label: "SSH Bruteforce", Description: "SSH Bruteforce"}, + }, + AttackDetails: []*cticlient.CTIAttackDetails{ + {Name: "crowdsecurity/ssh-bf", Label: "Example Attack"}, + {Name: "crowdsecurity/ssh-slow-bf", Label: "Example Attack"}, + }, + }, + //1.2.3.7 is a ok guy, but part of a bad range + "1.2.3.7": cticlient.SmokeItem{}, +} + +const validApiKey = "my-api-key" + +type RoundTripFunc func(req *http.Request) *http.Response + +func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func smokeHandler(req *http.Request) *http.Response { + apiKey := req.Header.Get("x-api-key") + if apiKey != validApiKey { + return &http.Response{ + StatusCode: http.StatusForbidden, + Body: nil, + Header: make(http.Header), + } + } + + requestedIP := strings.Split(req.URL.Path, "/")[3] + sample, ok := sampledata[requestedIP] + if !ok { + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: nil, + Header: make(http.Header), + } + } + + body, err := json.Marshal(sample) + if err != nil { + return &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: nil, + Header: make(http.Header), + } + } + + reader := io.NopCloser(bytes.NewReader(body)) + + return &http.Response{ + StatusCode: http.StatusOK, + // Send response to be tested + Body: reader, + Header: make(http.Header), + ContentLength: 0, + } +} + +func TestInvalidAuth(t *testing.T) { + defer ShutdownCrowdsecCTI() + if err := InitCrowdsecCTI(types.StrPtr("asdasd"), nil, nil, nil); err != nil { + t.Fatalf("failed to init CTI : %s", err) + } + //Replace the client created by InitCrowdsecCTI with one that uses a custom transport + ctiClient = cticlient.NewCrowdsecCTIClient(cticlient.WithAPIKey("asdasd"), cticlient.WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(smokeHandler), + })) + + item, err := CrowdsecCTI("1.2.3.4") + assert.Equal(t, item, &cticlient.SmokeItem{}) + assert.Equal(t, CTIApiEnabled, false) + assert.Equal(t, err, cticlient.ErrUnauthorized) + + //CTI is now disabled, all requests should return empty + ctiClient = cticlient.NewCrowdsecCTIClient(cticlient.WithAPIKey(validApiKey), cticlient.WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(smokeHandler), + })) + + item, err = CrowdsecCTI("1.2.3.4") + assert.Equal(t, item, &cticlient.SmokeItem{}) + assert.Equal(t, CTIApiEnabled, false) + assert.Equal(t, err, cticlient.ErrDisabled) +} + +func TestNoKey(t *testing.T) { + defer ShutdownCrowdsecCTI() + err := InitCrowdsecCTI(nil, nil, nil, nil) + assert.ErrorContains(t, err, "CTI API key not set") + //Replace the client created by InitCrowdsecCTI with one that uses a custom transport + ctiClient = cticlient.NewCrowdsecCTIClient(cticlient.WithAPIKey("asdasd"), cticlient.WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(smokeHandler), + })) + + item, err := CrowdsecCTI("1.2.3.4") + assert.Equal(t, item, &cticlient.SmokeItem{}) + assert.Equal(t, CTIApiEnabled, false) + assert.Equal(t, err, cticlient.ErrDisabled) +} + +func TestCache(t *testing.T) { + defer ShutdownCrowdsecCTI() + cacheDuration := 1 * time.Second + if err := InitCrowdsecCTI(types.StrPtr(validApiKey), &cacheDuration, nil, nil); err != nil { + t.Fatalf("failed to init CTI : %s", err) + } + //Replace the client created by InitCrowdsecCTI with one that uses a custom transport + ctiClient = cticlient.NewCrowdsecCTIClient(cticlient.WithAPIKey(validApiKey), cticlient.WithHTTPClient(&http.Client{ + Transport: RoundTripFunc(smokeHandler), + })) + + item, err := CrowdsecCTI("1.2.3.4") + assert.Equal(t, "1.2.3.4", item.Ip) + assert.Equal(t, CTIApiEnabled, true) + assert.Equal(t, CTICache.Len(true), 1) + assert.Equal(t, err, nil) + + item, err = CrowdsecCTI("1.2.3.4") + assert.Equal(t, "1.2.3.4", item.Ip) + assert.Equal(t, CTIApiEnabled, true) + assert.Equal(t, CTICache.Len(true), 1) + assert.Equal(t, err, nil) + + time.Sleep(2 * time.Second) + + assert.Equal(t, CTICache.Len(true), 0) + + item, err = CrowdsecCTI("1.2.3.4") + assert.Equal(t, "1.2.3.4", item.Ip) + assert.Equal(t, CTIApiEnabled, true) + assert.Equal(t, CTICache.Len(true), 1) + assert.Equal(t, err, nil) + +} diff --git a/pkg/exprhelpers/exprlib.go b/pkg/exprhelpers/exprlib.go index c911a0871..88971c291 100644 --- a/pkg/exprhelpers/exprlib.go +++ b/pkg/exprhelpers/exprlib.go @@ -69,6 +69,7 @@ func GetExprEnv(ctx map[string]interface{}) map[string]interface{} { "GetDecisionsCount": GetDecisionsCount, "GetDecisionsSinceCount": GetDecisionsSinceCount, "Sprintf": fmt.Sprintf, + "CrowdsecCTI": CrowdsecCTI, "ParseUnix": ParseUnix, "GetFromStash": cache.GetKey, "SetInStash": cache.SetKey, @@ -258,6 +259,7 @@ func GetDecisionsCount(value string) int { if dbClient == nil { log.Error("No database config to call GetDecisionsCount()") return 0 + } count, err := dbClient.CountDecisionsByValue(value) if err != nil {