From 0a39066f9de916ce1f3b27a3d2cb802f2bfb272c Mon Sep 17 00:00:00 2001 From: AlteredCoder <64792091+AlteredCoder@users.noreply.github.com> Date: Wed, 22 Jun 2022 10:29:02 +0200 Subject: [PATCH] Fix #1552 (#1569) --- cmd/crowdsec-cli/decisions.go | 3 +- pkg/apiserver/alerts_test.go | 3 + pkg/apiserver/controllers/v1/decisions.go | 29 +- pkg/apiserver/decisions_test.go | 1226 +++++++++++++++--- pkg/apiserver/tests/alert_duplicate.json | 266 ++++ pkg/database/decisions.go | 148 ++- tests/bats/99_lapi-stream-mode-scenario.bats | 233 ++++ tests/lib/db/instance-mysql | 8 +- tests/lib/setup_file.sh | 4 + 9 files changed, 1680 insertions(+), 240 deletions(-) create mode 100644 pkg/apiserver/tests/alert_duplicate.json create mode 100644 tests/bats/99_lapi-stream-mode-scenario.bats diff --git a/cmd/crowdsec-cli/decisions.go b/cmd/crowdsec-cli/decisions.go index 7eccff15d..019e0e621 100644 --- a/cmd/crowdsec-cli/decisions.go +++ b/cmd/crowdsec-cli/decisions.go @@ -284,7 +284,7 @@ cscli decisions list -t ban cmdDecisionsList.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)") cmdDecisionsList.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)") cmdDecisionsList.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)") - cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", "restrict to this origin (ie. lists,CAPI,cscli)") + cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", "restrict to this origin (ie. lists,CAPI,cscli,cscli-import,crowdsec)") cmdDecisionsList.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)") cmdDecisionsList.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)") cmdDecisionsList.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value )") @@ -419,7 +419,6 @@ cscli decisions add --scope username --value foobar Aliases: []string{"remove"}, Example: `cscli decisions delete -r 1.2.3.0/24 cscli decisions delete -i 1.2.3.4 -cscli decisions delete -s crowdsecurity/ssh-bf cscli decisions delete --id 42 cscli decisions delete --type captcha `, diff --git a/pkg/apiserver/alerts_test.go b/pkg/apiserver/alerts_test.go index 6c98a2fbf..b7c254e81 100644 --- a/pkg/apiserver/alerts_test.go +++ b/pkg/apiserver/alerts_test.go @@ -23,6 +23,7 @@ type LAPI struct { loginResp models.WatcherAuthResponse bouncerKey string t *testing.T + DBConfig *csconfig.DatabaseCfg } func SetupLAPITest(t *testing.T) LAPI { @@ -36,10 +37,12 @@ func SetupLAPITest(t *testing.T) LAPI { if err != nil { t.Fatalf("%s", err.Error()) } + return LAPI{ router: router, loginResp: loginResp, bouncerKey: APIKey, + DBConfig: config.API.Server.DbConfig, } } diff --git a/pkg/apiserver/controllers/v1/decisions.go b/pkg/apiserver/controllers/v1/decisions.go index 1c4c7528b..847e514a8 100644 --- a/pkg/apiserver/controllers/v1/decisions.go +++ b/pkg/apiserver/controllers/v1/decisions.go @@ -11,9 +11,20 @@ import ( log "github.com/sirupsen/logrus" ) -func FormatDecisions(decisions []*ent.Decision) ([]*models.Decision, error) { +//Format decisions for the bouncers, and deduplicate them by keeping only the longest one +func FormatDecisions(decisions []*ent.Decision, dedup bool) ([]*models.Decision, error) { var results []*models.Decision + + seen := make(map[string]struct{}, 0) + for _, dbDecision := range decisions { + if dedup { + key := dbDecision.Value + dbDecision.Scope + dbDecision.Type + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + } duration := dbDecision.Until.Sub(time.Now().UTC()).String() decision := models.Decision{ ID: int64(dbDecision.ID), @@ -46,7 +57,7 @@ func (c *Controller) GetDecision(gctx *gin.Context) { return } - results, err = FormatDecisions(data) + results, err = FormatDecisions(data, false) if err != nil { gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) return @@ -82,14 +93,14 @@ func (c *Controller) DeleteDecisionById(gctx *gin.Context) { gctx.JSON(http.StatusBadRequest, gin.H{"message": "decision_id must be valid integer"}) return } - err = c.DBClient.SoftDeleteDecisionByID(decisionID) + nbDeleted, err := c.DBClient.SoftDeleteDecisionByID(decisionID) if err != nil { c.HandleDBErrors(gctx, err) return } deleteDecisionResp := models.DeleteDecisionResponse{ - NbDeleted: "1", + NbDeleted: strconv.Itoa(nbDeleted), } gctx.JSON(http.StatusOK, deleteDecisionResp) @@ -138,7 +149,8 @@ func (c *Controller) StreamDecision(gctx *gin.Context) { gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) return } - ret["new"], err = FormatDecisions(data) + //data = KeepLongestDecision(data) + ret["new"], err = FormatDecisions(data, true) if err != nil { log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) @@ -152,7 +164,7 @@ func (c *Controller) StreamDecision(gctx *gin.Context) { gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) return } - ret["deleted"], err = FormatDecisions(data) + ret["deleted"], err = FormatDecisions(data, true) if err != nil { log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) @@ -180,7 +192,8 @@ func (c *Controller) StreamDecision(gctx *gin.Context) { gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) return } - ret["new"], err = FormatDecisions(data) + //data = KeepLongestDecision(data) + ret["new"], err = FormatDecisions(data, true) if err != nil { log.Errorf("unable to format new decision for '%s' : %v", bouncerInfo.Name, err) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) @@ -194,7 +207,7 @@ func (c *Controller) StreamDecision(gctx *gin.Context) { gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) return } - ret["deleted"], err = FormatDecisions(data) + ret["deleted"], err = FormatDecisions(data, true) if err != nil { log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) diff --git a/pkg/apiserver/decisions_test.go b/pkg/apiserver/decisions_test.go index b830f136e..c22bd3460 100644 --- a/pkg/apiserver/decisions_test.go +++ b/pkg/apiserver/decisions_test.go @@ -1,12 +1,18 @@ package apiserver import ( + "fmt" + "os" "testing" - "time" "github.com/stretchr/testify/assert" ) +const ( + APIKEY = "apikey" + PASSWORD = "password" +) + func TestDeleteDecisionRange(t *testing.T) { lapi := SetupLAPITest(t) @@ -14,20 +20,20 @@ func TestDeleteDecisionRange(t *testing.T) { lapi.InsertAlertFromFile("./tests/alert_minibulk.json") // delete by ip wrong - w := lapi.RecordResponse("DELETE", "/v1/decisions?range=1.2.3.0/24", emptyBody, "password") + w := lapi.RecordResponse("DELETE", "/v1/decisions?range=1.2.3.0/24", emptyBody, PASSWORD) assert.Equal(t, 200, w.Code) assert.Equal(t, `{"nbDeleted":"0"}`, w.Body.String()) // delete by range - w = lapi.RecordResponse("DELETE", "/v1/decisions?range=91.121.79.0/24&contains=false", emptyBody, "password") + w = lapi.RecordResponse("DELETE", "/v1/decisions?range=91.121.79.0/24&contains=false", emptyBody, PASSWORD) assert.Equal(t, 200, w.Code) assert.Equal(t, `{"nbDeleted":"2"}`, w.Body.String()) // delete by range : ensure it was already deleted - w = lapi.RecordResponse("DELETE", "/v1/decisions?range=91.121.79.0/24", emptyBody, "password") + w = lapi.RecordResponse("DELETE", "/v1/decisions?range=91.121.79.0/24", emptyBody, PASSWORD) assert.Equal(t, 200, w.Code) assert.Equal(t, `{"nbDeleted":"0"}`, w.Body.String()) } @@ -40,19 +46,19 @@ func TestDeleteDecisionFilter(t *testing.T) { // delete by ip wrong - w := lapi.RecordResponse("DELETE", "/v1/decisions?ip=1.2.3.4", emptyBody, "password") + w := lapi.RecordResponse("DELETE", "/v1/decisions?ip=1.2.3.4", emptyBody, PASSWORD) assert.Equal(t, 200, w.Code) assert.Equal(t, `{"nbDeleted":"0"}`, w.Body.String()) // delete by ip good - w = lapi.RecordResponse("DELETE", "/v1/decisions?ip=91.121.79.179", emptyBody, "password") + w = lapi.RecordResponse("DELETE", "/v1/decisions?ip=91.121.79.179", emptyBody, PASSWORD) assert.Equal(t, 200, w.Code) assert.Equal(t, `{"nbDeleted":"1"}`, w.Body.String()) // delete by scope/value - w = lapi.RecordResponse("DELETE", "/v1/decisions?scopes=Ip&value=91.121.79.178", emptyBody, "password") + w = lapi.RecordResponse("DELETE", "/v1/decisions?scopes=Ip&value=91.121.79.178", emptyBody, PASSWORD) assert.Equal(t, 200, w.Code) assert.Equal(t, `{"nbDeleted":"1"}`, w.Body.String()) } @@ -65,7 +71,7 @@ func TestGetDecisionFilters(t *testing.T) { // Get Decision - w := lapi.RecordResponse("GET", "/v1/decisions", emptyBody, "apikey") + w := lapi.RecordResponse("GET", "/v1/decisions", emptyBody, APIKEY) assert.Equal(t, 200, w.Code) decisions, code, err := readDecisionsGetResp(w) assert.Nil(t, err) @@ -80,7 +86,7 @@ func TestGetDecisionFilters(t *testing.T) { // Get Decision : type filter - w = lapi.RecordResponse("GET", "/v1/decisions?type=ban", emptyBody, "apikey") + w = lapi.RecordResponse("GET", "/v1/decisions?type=ban", emptyBody, APIKEY) assert.Equal(t, 200, w.Code) decisions, code, err = readDecisionsGetResp(w) assert.Nil(t, err) @@ -98,7 +104,7 @@ func TestGetDecisionFilters(t *testing.T) { // Get Decision : scope/value - w = lapi.RecordResponse("GET", "/v1/decisions?scopes=Ip&value=91.121.79.179", emptyBody, "apikey") + w = lapi.RecordResponse("GET", "/v1/decisions?scopes=Ip&value=91.121.79.179", emptyBody, APIKEY) assert.Equal(t, 200, w.Code) decisions, code, err = readDecisionsGetResp(w) assert.Nil(t, err) @@ -113,7 +119,7 @@ func TestGetDecisionFilters(t *testing.T) { // Get Decision : ip filter - w = lapi.RecordResponse("GET", "/v1/decisions?ip=91.121.79.179", emptyBody, "apikey") + w = lapi.RecordResponse("GET", "/v1/decisions?ip=91.121.79.179", emptyBody, APIKEY) assert.Equal(t, 200, w.Code) decisions, code, err = readDecisionsGetResp(w) assert.Nil(t, err) @@ -127,7 +133,7 @@ func TestGetDecisionFilters(t *testing.T) { // assert.NotContains(t, w.Body.String(), `"id":2,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.178"`) // Get decision : by range - w = lapi.RecordResponse("GET", "/v1/decisions?range=91.121.79.0/24&contains=false", emptyBody, "apikey") + w = lapi.RecordResponse("GET", "/v1/decisions?range=91.121.79.0/24&contains=false", emptyBody, APIKEY) assert.Equal(t, 200, w.Code) decisions, code, err = readDecisionsGetResp(w) assert.Nil(t, err) @@ -145,7 +151,7 @@ func TestGetDecision(t *testing.T) { lapi.InsertAlertFromFile("./tests/alert_sample.json") // Get Decision - w := lapi.RecordResponse("GET", "/v1/decisions", emptyBody, "apikey") + w := lapi.RecordResponse("GET", "/v1/decisions", emptyBody, APIKEY) assert.Equal(t, 200, w.Code) decisions, code, err := readDecisionsGetResp(w) assert.Nil(t, err) @@ -165,7 +171,7 @@ func TestGetDecision(t *testing.T) { assert.Equal(t, int64(3), decisions[2].ID) // Get Decision with invalid filter. It should ignore this filter - w = lapi.RecordResponse("GET", "/v1/decisions?test=test", emptyBody, "apikey") + w = lapi.RecordResponse("GET", "/v1/decisions?test=test", emptyBody, APIKEY) assert.Equal(t, 200, w.Code) assert.Equal(t, 3, len(decisions)) } @@ -177,49 +183,49 @@ func TestDeleteDecisionByID(t *testing.T) { lapi.InsertAlertFromFile("./tests/alert_sample.json") //Have one alerts - w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey") + w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY) decisions, code, err := readDecisionsStreamResp(w) assert.Equal(t, err, nil) - assert.Equal(t, code, 200) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 1) + assert.Equal(t, 200, code) + assert.Equal(t, 0, len(decisions["deleted"])) + assert.Equal(t, 1, len(decisions["new"])) // Delete alert with Invalid ID - w = lapi.RecordResponse("DELETE", "/v1/decisions/test", emptyBody, "password") + w = lapi.RecordResponse("DELETE", "/v1/decisions/test", emptyBody, PASSWORD) assert.Equal(t, 400, w.Code) err_resp, _, err := readDecisionsErrorResp(w) assert.NoError(t, err) - assert.Equal(t, err_resp["message"], "decision_id must be valid integer") + assert.Equal(t, "decision_id must be valid integer", err_resp["message"]) // Delete alert with ID that not exist - w = lapi.RecordResponse("DELETE", "/v1/decisions/100", emptyBody, "password") + w = lapi.RecordResponse("DELETE", "/v1/decisions/100", emptyBody, PASSWORD) assert.Equal(t, 500, w.Code) err_resp, _, err = readDecisionsErrorResp(w) assert.NoError(t, err) - assert.Equal(t, err_resp["message"], "decision with id '100' doesn't exist: unable to delete") + assert.Equal(t, "decision with id '100' doesn't exist: unable to delete", err_resp["message"]) //Have one alerts - w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey") + w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY) decisions, code, err = readDecisionsStreamResp(w) assert.Equal(t, err, nil) - assert.Equal(t, code, 200) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 1) + assert.Equal(t, 200, code) + assert.Equal(t, 0, len(decisions["deleted"])) + assert.Equal(t, 1, len(decisions["new"])) // Delete alert with valid ID - w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody, "password") + w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody, PASSWORD) assert.Equal(t, 200, w.Code) resp, _, err := readDecisionsDeleteResp(w) assert.NoError(t, err) assert.Equal(t, resp.NbDeleted, "1") //Have one alert (because we delete an alert that has dup targets) - w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey") + w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY) decisions, code, err = readDecisionsStreamResp(w) assert.Equal(t, err, nil) - assert.Equal(t, code, 200) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 1) + assert.Equal(t, 200, code) + assert.Equal(t, 0, len(decisions["deleted"])) + assert.Equal(t, 1, len(decisions["new"])) } func TestDeleteDecision(t *testing.T) { @@ -229,14 +235,14 @@ func TestDeleteDecision(t *testing.T) { lapi.InsertAlertFromFile("./tests/alert_sample.json") // Delete alert with Invalid filter - w := lapi.RecordResponse("DELETE", "/v1/decisions?test=test", emptyBody, "password") + w := lapi.RecordResponse("DELETE", "/v1/decisions?test=test", emptyBody, PASSWORD) assert.Equal(t, 500, w.Code) err_resp, _, err := readDecisionsErrorResp(w) assert.NoError(t, err) assert.Equal(t, err_resp["message"], "'test' doesn't exist: invalid filter") // Delete all alert - w = lapi.RecordResponse("DELETE", "/v1/decisions", emptyBody, "password") + w = lapi.RecordResponse("DELETE", "/v1/decisions", emptyBody, PASSWORD) assert.Equal(t, 200, w.Code) resp, _, err := readDecisionsDeleteResp(w) assert.NoError(t, err) @@ -251,181 +257,1061 @@ func TestStreamStartDecisionDedup(t *testing.T) { lapi.InsertAlertFromFile("./tests/alert_sample.json") // Get Stream, we only get one decision (the longest one) - w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey") + w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY) decisions, code, err := readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, code, 200) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 1) - assert.Equal(t, decisions["new"][0].ID, int64(3)) - assert.Equal(t, *decisions["new"][0].Origin, "test") - assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1") + assert.Equal(t, nil, err) + assert.Equal(t, 200, code) + assert.Equal(t, 0, len(decisions["deleted"])) + assert.Equal(t, 1, len(decisions["new"])) + assert.Equal(t, int64(3), decisions["new"][0].ID) + assert.Equal(t, "test", *decisions["new"][0].Origin) + assert.Equal(t, "127.0.0.1", *decisions["new"][0].Value) // id=3 decision is deleted, this won't affect `deleted`, because there are decisions on the same ip - w = lapi.RecordResponse("DELETE", "/v1/decisions/3", emptyBody, "password") + w = lapi.RecordResponse("DELETE", "/v1/decisions/3", emptyBody, PASSWORD) assert.Equal(t, 200, w.Code) // Get Stream, we only get one decision (the longest one, id=2) - w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey") + w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY) decisions, code, err = readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, code, 200) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 1) - assert.Equal(t, decisions["new"][0].ID, int64(2)) - assert.Equal(t, *decisions["new"][0].Origin, "test") - assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1") + assert.Equal(t, nil, err) + assert.Equal(t, 200, code) + assert.Equal(t, 0, len(decisions["deleted"])) + assert.Equal(t, 1, len(decisions["new"])) + assert.Equal(t, int64(2), decisions["new"][0].ID) + assert.Equal(t, "test", *decisions["new"][0].Origin) + assert.Equal(t, "127.0.0.1", *decisions["new"][0].Value) // We delete another decision, yet don't receive it in stream, since there's another decision on same IP - w = lapi.RecordResponse("DELETE", "/v1/decisions/2", emptyBody, "password") + w = lapi.RecordResponse("DELETE", "/v1/decisions/2", emptyBody, PASSWORD) assert.Equal(t, 200, w.Code) // And get the remaining decision (1) - w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey") + w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY) decisions, code, err = readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, code, 200) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 1) - assert.Equal(t, decisions["new"][0].ID, int64(1)) - assert.Equal(t, *decisions["new"][0].Origin, "test") - assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1") + assert.Equal(t, nil, err) + assert.Equal(t, 200, code) + assert.Equal(t, 0, len(decisions["deleted"])) + assert.Equal(t, 1, len(decisions["new"])) + assert.Equal(t, int64(1), decisions["new"][0].ID) + assert.Equal(t, "test", *decisions["new"][0].Origin) + assert.Equal(t, "127.0.0.1", *decisions["new"][0].Value) // We delete the last decision, we receive the delete order - w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody, "password") + w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody, PASSWORD) assert.Equal(t, 200, w.Code) //and now we only get a deleted decision - w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey") + w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY) decisions, code, err = readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, code, 200) - assert.Equal(t, len(decisions["deleted"]), 1) - assert.Equal(t, decisions["deleted"][0].ID, int64(1)) - assert.Equal(t, *decisions["deleted"][0].Origin, "test") - assert.Equal(t, *decisions["deleted"][0].Value, "127.0.0.1") - assert.Equal(t, len(decisions["new"]), 0) + assert.Equal(t, nil, err) + assert.Equal(t, 200, code) + assert.Equal(t, 1, len(decisions["deleted"])) + assert.Equal(t, int64(1), decisions["deleted"][0].ID) + assert.Equal(t, "test", *decisions["deleted"][0].Origin) + assert.Equal(t, "127.0.0.1", *decisions["deleted"][0].Value) + assert.Equal(t, 0, len(decisions["new"])) } -func TestStreamDecisionDedup(t *testing.T) { - //Ensure that at stream startup we only get the longest decision +type DecisionCheck struct { + ID int64 + Origin string + Scenario string + Value string + Duration string + Type string +} + +type DecisionTest struct { + TestName string + Method string + Route string + CheckCodeOnly bool + Code int + LenNew int + LenDeleted int + NewChecks []DecisionCheck + DelChecks []DecisionCheck + AuthType string +} + +func TestStreamDecisionStart(t *testing.T) { lapi := SetupLAPITest(t) - // Create Valid Alert : 3 decisions for 127.0.0.1, longest has id=3 - lapi.InsertAlertFromFile("./tests/alert_sample.json") + /* + Create multiple alerts: + - 3 alerts for 127.0.0.1 with ID 1/2/3 : Different duration / scenario / origin + - 3 alerts for 127.0.0.2 with ID 4/5/6/7 : Different duration / scenario / origin + */ + lapi.InsertAlertFromFile("./tests/alert_duplicate.json") - time.Sleep(2 * time.Second) + tests := []DecisionTest{ + { + TestName: "test startup", + Method: "GET", + Route: "/v1/decisions/stream?startup=true", + CheckCodeOnly: false, + Code: 200, + LenNew: 3, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{ + { + ID: int64(3), + Origin: "test", + Scenario: "crowdsecurity/longest", + Value: "127.0.0.1", + Duration: "4h59", + Type: "ban", + }, + { + ID: int64(4), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(8), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "captcha", + }, + }, + }, + { + TestName: "test startup with scenarios containing", + Method: "GET", + Route: "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf", + CheckCodeOnly: false, + Code: 200, + LenNew: 2, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{ + { + ID: int64(2), + Origin: "another_origin", + Scenario: "crowdsecurity/ssh_bf", + Value: "127.0.0.1", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(5), + Origin: "test", + Scenario: "crowdsecurity/ssh_bf", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + }, + }, + { + TestName: "test startup with multiple scenarios containing", + Method: "GET", + Route: "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf,test", + CheckCodeOnly: false, + Code: 200, + LenNew: 3, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, - // Get Stream, we only get one decision (the longest one) - w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey") - decisions, code, err := readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, code, 200) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 1) - assert.Equal(t, decisions["new"][0].ID, int64(3)) - assert.Equal(t, *decisions["new"][0].Origin, "test") - assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1") + NewChecks: []DecisionCheck{ + { + ID: int64(2), + Origin: "another_origin", + Scenario: "crowdsecurity/ssh_bf", + Value: "127.0.0.1", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(4), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(8), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "captcha", + }, + }, + }, + { + TestName: "test startup with unknown scenarios containing", + Method: "GET", + Route: "/v1/decisions/stream?startup=true&scenarios_containing=unknown", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, - // id=3 decision is deleted, this won't affect `deleted`, because there are decisions on the same ip - w = lapi.RecordResponse("DELETE", "/v1/decisions/3", emptyBody, "password") - assert.Equal(t, 200, w.Code) + NewChecks: []DecisionCheck{}, + }, + { + TestName: "test startup with scenarios containing and not containing", + Method: "GET", + Route: "/v1/decisions/stream?startup=true&scenarios_containing=test&scenarios_not_containing=ssh_bf", + CheckCodeOnly: false, + Code: 200, + LenNew: 3, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{ + { + ID: int64(1), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.1", + Duration: "59m", + Type: "ban", + }, + { + ID: int64(4), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(8), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "captcha", + }, + }, + }, + { + TestName: "test startup with scenarios containing and not containing 2", + Method: "GET", + Route: "/v1/decisions/stream?startup=true&scenarios_containing=longest&scenarios_not_containing=ssh_bf,test", + CheckCodeOnly: false, + Code: 200, + LenNew: 1, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{ + { + ID: int64(3), + Origin: "test", + Scenario: "crowdsecurity/longest", + Value: "127.0.0.1", + Duration: "4h59", + Type: "ban", + }, + }, + }, + { + TestName: "test startup with scenarios not containing", + Method: "GET", + Route: "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf", + CheckCodeOnly: false, + Code: 200, + LenNew: 3, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, - w = lapi.RecordResponse("GET", "/v1/decisions/stream", emptyBody, "apikey") - assert.Equal(t, err, nil) - decisions, code, err = readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, code, 200) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 0) - // We delete another decision, yet don't receive it in stream, since there's another decision on same IP - w = lapi.RecordResponse("DELETE", "/v1/decisions/2", emptyBody, "password") - assert.Equal(t, 200, w.Code) + NewChecks: []DecisionCheck{ + { + ID: int64(3), + Origin: "test", + Scenario: "crowdsecurity/longest", + Value: "127.0.0.1", + Duration: "4h59", + Type: "ban", + }, + { + ID: int64(4), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(8), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "captcha", + }, + }, + }, + { + TestName: "test startup with multiple scenarios not containing", + Method: "GET", + Route: "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf,test", + CheckCodeOnly: false, + Code: 200, + LenNew: 1, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, - w = lapi.RecordResponse("GET", "/v1/decisions/stream", emptyBody, "apikey") - decisions, code, err = readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, code, 200) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 0) + NewChecks: []DecisionCheck{ + { + ID: int64(3), + Origin: "test", + Scenario: "crowdsecurity/longest", + Value: "127.0.0.1", + Duration: "4h59", + Type: "ban", + }, + }, + }, + { + TestName: "test startup with origins parameter", + Method: "GET", + Route: "/v1/decisions/stream?startup=true&origins=another_origin", + CheckCodeOnly: false, + Code: 200, + LenNew: 2, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, - // We delete the last decision, we receive the delete order - w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody, "password") - assert.Equal(t, 200, w.Code) + NewChecks: []DecisionCheck{ + { + ID: int64(2), + Origin: "another_origin", + Scenario: "crowdsecurity/ssh_bf", + Value: "127.0.0.1", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(7), + Origin: "another_origin", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "1h59", + Type: "ban", + }, + }, + }, + { + TestName: "test startup with multiple origins parameter", + Method: "GET", + Route: "/v1/decisions/stream?startup=true&origins=another_origin,test", + CheckCodeOnly: false, + Code: 200, + LenNew: 3, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, - w = lapi.RecordResponse("GET", "/v1/decisions/stream", emptyBody, "apikey") - decisions, code, err = readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, code, 200) - assert.Equal(t, len(decisions["deleted"]), 1) - assert.Equal(t, decisions["deleted"][0].ID, int64(1)) - assert.Equal(t, *decisions["deleted"][0].Origin, "test") - assert.Equal(t, *decisions["deleted"][0].Value, "127.0.0.1") - assert.Equal(t, len(decisions["new"]), 0) + NewChecks: []DecisionCheck{ + { + ID: int64(3), + Origin: "test", + Scenario: "crowdsecurity/longest", + Value: "127.0.0.1", + Duration: "4h59", + Type: "ban", + }, + { + ID: int64(4), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(8), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "captcha", + }, + }, + }, + { + TestName: "test startup with unknown origins", + Method: "GET", + Route: "/v1/decisions/stream?startup=true&origins=unknown", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "delete decisions 3 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/3", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "check that 127.0.0.1 is not in deleted IP", + Method: "GET", + Route: "/v1/decisions/stream?startup=true", + CheckCodeOnly: false, + Code: 200, + LenNew: 3, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{ + { + ID: int64(2), + Origin: "another_origin", + Scenario: "crowdsecurity/ssh_bf", + Value: "127.0.0.1", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(4), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(8), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "captcha", + }, + }, + }, + { + TestName: "delete decisions 2 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/2", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "check that 127.0.0.1 is not in deleted IP", + Method: "GET", + Route: "/v1/decisions/stream?startup=true", + CheckCodeOnly: false, + Code: 200, + LenNew: 3, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{ + { + ID: int64(1), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.1", + Duration: "59", + Type: "ban", + }, + { + ID: int64(4), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(8), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "captcha", + }, + }, + }, + { + TestName: "delete decisions 1 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/1", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "127.0.0.1 should be in deleted now", + Method: "GET", + Route: "/v1/decisions/stream?startup=true", + CheckCodeOnly: false, + Code: 200, + LenNew: 2, + LenDeleted: 1, + AuthType: APIKEY, + DelChecks: []DecisionCheck{ + { + ID: int64(1), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.1", + Duration: "-", // we check that the time is negative + Type: "ban", + }, + }, + NewChecks: []DecisionCheck{ + { + ID: int64(4), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(8), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "captcha", + }, + }, + }, + } + + for _, test := range tests { + runTest(lapi, test, t) + } } -func TestStreamDecisionFilters(t *testing.T) { +func TestStreamDecision(t *testing.T) { - lapi := SetupLAPITest(t) + /* + Create multiple alerts: + - 3 alerts for 127.0.0.1 with ID 1/2/3 : Different duration / scenario / origin + - 3 alerts for 127.0.0.2 with ID 4/5/6/7 : Different duration / scenario / origin + */ - // Create Valid Alert - lapi.InsertAlertFromFile("./tests/alert_stream_fixture.json") + // this test just init the stream with startup=true + preTests := []DecisionTest{ + { + TestName: "test startup", + Method: "GET", + Route: "/v1/decisions/stream?startup=true", + CheckCodeOnly: false, + Code: 200, + AuthType: APIKEY, + LenNew: 0, + LenDeleted: 0, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + } - w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey") - decisions, code, err := readDecisionsStreamResp(w) + tests := map[string][]DecisionTest{ + "Test without parameter": { + { + TestName: "get stream", + Method: "GET", + Route: "/v1/decisions/stream", + CheckCodeOnly: false, + Code: 200, + LenNew: 3, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{ + { + ID: int64(3), + Origin: "test", + Scenario: "crowdsecurity/longest", + Value: "127.0.0.1", + Duration: "4h59", + Type: "ban", + }, + { + ID: int64(4), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(8), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "captcha", + }, + }, + }, + { + TestName: "delete decisions 3 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/3", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "check that 127.0.0.1 is not in deleted IP", + Method: "GET", + Route: "/v1/decisions/stream", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "delete decisions 2 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/2", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "check that 127.0.0.1 is not in deleted IP", + Method: "GET", + Route: "/v1/decisions/stream", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "delete decisions 1 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/1", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "127.0.0.1 should be in deleted now", + Method: "GET", + Route: "/v1/decisions/stream", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 1, + AuthType: APIKEY, + DelChecks: []DecisionCheck{ + { + ID: int64(1), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.1", + Duration: "-", - assert.Equal(t, 200, code) - assert.Equal(t, err, nil) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 3) - assert.Equal(t, decisions["new"][0].ID, int64(1)) - assert.Equal(t, *decisions["new"][0].Origin, "test1") - assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1") - assert.Equal(t, *decisions["new"][0].Scenario, "crowdsecurity/http_bf") - assert.Equal(t, decisions["new"][1].ID, int64(2)) - assert.Equal(t, *decisions["new"][1].Origin, "test2") - assert.Equal(t, *decisions["new"][1].Value, "127.0.0.1") - assert.Equal(t, *decisions["new"][1].Scenario, "crowdsecurity/ssh_bf") - assert.Equal(t, decisions["new"][2].ID, int64(3)) - assert.Equal(t, *decisions["new"][2].Origin, "test3") - assert.Equal(t, *decisions["new"][2].Value, "127.0.0.1") - assert.Equal(t, *decisions["new"][2].Scenario, "crowdsecurity/ddos") + Type: "ban", + }, + }, + NewChecks: []DecisionCheck{}, + }, + }, + "test with scenarios containing": { + { + TestName: "get stream", + Method: "GET", + Route: "/v1/decisions/stream?scenarios_containing=ssh_bf", + CheckCodeOnly: false, + Code: 200, + LenNew: 2, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{ + { + ID: int64(2), + Origin: "another_origin", + Scenario: "crowdsecurity/ssh_bf", + Value: "127.0.0.1", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(5), + Origin: "test", + Scenario: "crowdsecurity/ssh_bf", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + }, + }, + { + TestName: "delete decisions 3 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/3", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "check that 127.0.0.1 is not in deleted IP", + Method: "GET", + Route: "/v1/decisions/stream?scenarios_containing=ssh_bf", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "delete decisions 2 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/2", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "check that 127.0.0.1 is deleted (decision for ssh_bf was with ID 2)", + Method: "GET", + Route: "/v1/decisions/stream?scenarios_containing=ssh_bf", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 1, + AuthType: APIKEY, + DelChecks: []DecisionCheck{ + { + ID: int64(2), + Origin: "another_origin", + Scenario: "crowdsecurity/ssh_bf", + Value: "127.0.0.1", + Duration: "-", - // test filter scenarios_not_containing - w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&scenarios_not_containing=http", emptyBody, "apikey") - decisions, code, err = readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, 200, code) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 2) - assert.Equal(t, decisions["new"][0].ID, int64(2)) - assert.Equal(t, decisions["new"][1].ID, int64(3)) + Type: "ban", + }, + }, + NewChecks: []DecisionCheck{}, + }, + }, + "test with scenarios not containing": { + { + TestName: "get stream", + Method: "GET", + Route: "/v1/decisions/stream?scenarios_not_containing=ssh_bf", + CheckCodeOnly: false, + Code: 200, + LenNew: 3, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{ + { + ID: int64(3), + Origin: "test", + Scenario: "crowdsecurity/longest", + Value: "127.0.0.1", + Duration: "4h59", + Type: "ban", + }, + { + ID: int64(4), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(8), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "2h59", + Type: "captcha", + }, + }, + }, + { + TestName: "delete decisions 3 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/3", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "check that 127.0.0.1 is not in deleted IP", + Method: "GET", + Route: "/v1/decisions/stream?scenarios_not_containing=ssh_bf", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "delete decisions 2 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/2", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "check that 127.0.0.1 is not deleted", + Method: "GET", + Route: "/v1/decisions/stream?scenarios_not_containing=ssh_bf", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "delete decisions 1 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/1", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "check that 127.0.0.1 is deleted", + Method: "GET", + Route: "/v1/decisions/stream?scenarios_not_containing=ssh_bf", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 1, + AuthType: APIKEY, + DelChecks: []DecisionCheck{ + { + ID: int64(1), + Origin: "test", + Scenario: "crowdsecurity/test", + Value: "127.0.0.1", + Duration: "-", - // test filter scenarios_containing - w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&scenarios_containing=http", emptyBody, "apikey") - decisions, code, err = readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, 200, code) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 1) - assert.Equal(t, decisions["new"][0].ID, int64(1)) + Type: "ban", + }, + }, + NewChecks: []DecisionCheck{}, + }, + }, + "test with origins": { + { + TestName: "get stream", + Method: "GET", + Route: "/v1/decisions/stream?origins=another_origin", + CheckCodeOnly: false, + Code: 200, + LenNew: 2, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{ + { + ID: int64(2), + Origin: "another_origin", + Scenario: "crowdsecurity/ssh_bf", + Value: "127.0.0.1", + Duration: "2h59", + Type: "ban", + }, + { + ID: int64(7), + Origin: "another_origin", + Scenario: "crowdsecurity/test", + Value: "127.0.0.2", + Duration: "1h59", + Type: "ban", + }, + }, + }, + { + TestName: "delete decisions 3 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/3", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "check that 127.0.0.1 is not in deleted IP", + Method: "GET", + Route: "/v1/decisions/stream?origins=another_origin", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: APIKEY, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "delete decisions 2 (127.0.0.1)", + Method: "DELETE", + Route: "/v1/decisions/2", + CheckCodeOnly: true, + Code: 200, + LenNew: 0, + LenDeleted: 0, + AuthType: PASSWORD, + DelChecks: []DecisionCheck{}, + NewChecks: []DecisionCheck{}, + }, + { + TestName: "check that 127.0.0.1 is deleted", + Method: "GET", + Route: "/v1/decisions/stream?origins=another_origin", + CheckCodeOnly: false, + Code: 200, + LenNew: 0, + LenDeleted: 1, + AuthType: APIKEY, + DelChecks: []DecisionCheck{ + { + ID: int64(2), + Origin: "another_origin", + Scenario: "crowdsecurity/ssh_bf", + Value: "127.0.0.1", + Duration: "-", - // test filters both by scenarios_not_containing and scenarios_containing - w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh&scenarios_containing=ddos", emptyBody, "apikey") - decisions, code, err = readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, 200, code) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 1) - assert.Equal(t, decisions["new"][0].ID, int64(3)) + Type: "ban", + }, + }, + NewChecks: []DecisionCheck{}, + }, + }, + } - // test filter by origin - w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&origins=test1,test2", emptyBody, "apikey") - decisions, code, err = readDecisionsStreamResp(w) - assert.Equal(t, err, nil) - assert.Equal(t, 200, code) - assert.Equal(t, len(decisions["deleted"]), 0) - assert.Equal(t, len(decisions["new"]), 2) - assert.Equal(t, decisions["new"][0].ID, int64(1)) - assert.Equal(t, decisions["new"][1].ID, int64(2)) + // run tests for the stream + for testName, test := range tests { + + // init a new LAPI + lapi := SetupLAPITest(t) + + // run pre-test, mostly to init the stream + for _, test := range preTests { + runTest(lapi, test, t) + } + // insert decisions now that the stream is initiated + lapi.InsertAlertFromFile("./tests/alert_duplicate.json") + + for _, oneTest := range test { + oneTest.TestName = fmt.Sprintf("%s (%s)", oneTest.TestName, testName) + runTest(lapi, oneTest, t) + } + + // clean the db after each test + os.Remove(lapi.DBConfig.DbPath) + } +} + +func runTest(lapi LAPI, test DecisionTest, t *testing.T) { + w := lapi.RecordResponse(test.Method, test.Route, emptyBody, test.AuthType) + assert.Equal(t, test.Code, w.Code) + if test.CheckCodeOnly { + return + } + decisions, _, err := readDecisionsStreamResp(w) + assert.Equal(t, nil, err) + assert.Equal(t, test.LenDeleted, len(decisions["deleted"]), fmt.Sprintf("'%s': len(deleted)", test.TestName)) + assert.Equal(t, test.LenNew, len(decisions["new"]), fmt.Sprintf("'%s': len(new)", test.TestName)) + + for i, check := range test.NewChecks { + assert.Equal(t, check.ID, decisions["new"][i].ID, fmt.Sprintf("'%s' (idx: %d): field: ID", test.TestName, i)) + assert.Equal(t, check.Origin, *decisions["new"][i].Origin, fmt.Sprintf("'%s' (idx: %d): field: Origin", test.TestName, i)) + assert.Equal(t, check.Scenario, *decisions["new"][i].Scenario, fmt.Sprintf("'%s' (idx: %d): field: Scenario", test.TestName, i)) + assert.Equal(t, check.Value, *decisions["new"][i].Value, fmt.Sprintf("'%s' (idx: %d): field: Value", test.TestName, i)) + assert.Equal(t, check.Type, *decisions["new"][i].Type, fmt.Sprintf("'%s' (idx: %d): field: Type", test.TestName, i)) + assert.Contains(t, *decisions["new"][i].Duration, check.Duration, fmt.Sprintf("'%s' (idx: %d): field: Duration", test.TestName, i)) + } + + for i, check := range test.DelChecks { + assert.Equal(t, check.ID, decisions["deleted"][i].ID, fmt.Sprintf("'%s' (idx: %d): field: ID", test.TestName, i)) + assert.Equal(t, check.Origin, *decisions["deleted"][i].Origin, fmt.Sprintf("'%s' (idx: %d): field: Origin", test.TestName, i)) + assert.Equal(t, check.Scenario, *decisions["deleted"][i].Scenario, fmt.Sprintf("'%s' (idx: %d): field: Scenario", test.TestName, i)) + assert.Equal(t, check.Value, *decisions["deleted"][i].Value, fmt.Sprintf("'%s' (idx: %d): field: Value", test.TestName, i)) + assert.Equal(t, check.Type, *decisions["deleted"][i].Type, fmt.Sprintf("'%s' (idx: %d): field: Type", test.TestName, i)) + assert.Contains(t, *decisions["deleted"][i].Duration, check.Duration, fmt.Sprintf("'%s' (idx: %d): field: Duration", test.TestName, i)) + } } diff --git a/pkg/apiserver/tests/alert_duplicate.json b/pkg/apiserver/tests/alert_duplicate.json new file mode 100644 index 000000000..8377895f4 --- /dev/null +++ b/pkg/apiserver/tests/alert_duplicate.json @@ -0,0 +1,266 @@ +[ + { + "id": 42, + "machine_id": "test", + "capacity": 1, + "created_at": "2020-10-09T10:00:10Z", + "decisions": [ + { + "duration": "1h", + "origin": "test", + "scenario": "crowdsecurity/test", + "scope": "Ip", + "value": "127.0.0.1", + "type": "ban" + } + ], + "source": { + "ip": "127.0.0.1", + "range": "127.0.0.1/32", + "scope": "ip", + "value": "127.0.0.1" + }, + "Events": [ + ], + "events_count": 1, + "leakspeed": "0.5s", + "message": "test", + "scenario_hash": "hashtest", + "scenario_version": "v1", + "simulated": false, + "scenario": "crowdsecurity/test", + "start_at": "2020-10-09T10:00:01Z", + "stop_at": "2020-10-09T10:00:05Z" + }, + { + "id": 44, + "machine_id": "test", + "created_at": "2020-10-09T10:00:10Z", + "decisions": [ + { + "duration": "3h", + "origin": "another_origin", + "scenario": "crowdsecurity/ssh_bf", + "scope": "Ip", + "value": "127.0.0.1", + "type": "ban" + } + ], + "source": { + "ip": "127.0.0.1", + "range": "127.0.0.1/32", + "scope": "ip", + "value": "127.0.0.1" + }, + "Events": [ + ], + "events_count": 1, + "leakspeed": "0.5s", + "message": "test", + "scenario_hash": "hashtest", + "scenario_version": "v1", + "simulated": false, + "capacity": 1, + "scenario": "crowdsecurity/ssh_bf", + "start_at": "2020-10-09T10:00:01Z", + "stop_at": "2020-10-09T10:00:05Z" + }, + { + "id": 45, + "machine_id": "test", + "created_at": "2020-10-09T10:00:10Z", + "decisions": [ + { + "duration": "5h", + "origin": "test", + "scenario": "crowdsecurity/longest", + "scope": "Ip", + "value": "127.0.0.1", + "type": "ban" + } + ], + "source": { + "ip": "127.0.0.1", + "range": "127.0.0.1/32", + "scope": "ip", + "value": "127.0.0.1" + }, + "Events": [ + ], + "events_count": 1, + "leakspeed": "0.5s", + "message": "test", + "scenario_hash": "hashtest", + "scenario_version": "v1", + "simulated": false, + "capacity": 1, + "scenario": "crowdsecurity/longest", + "start_at": "2020-10-09T10:00:01Z", + "stop_at": "2020-10-09T10:00:05Z" + }, + { + "id": 46, + "machine_id": "test", + "created_at": "2020-10-09T10:00:10Z", + "decisions": [ + { + "duration": "3h", + "origin": "test", + "scenario": "crowdsecurity/test", + "scope": "Ip", + "value": "127.0.0.2", + "type": "ban" + } + ], + "source": { + "ip": "127.0.0.2", + "range": "127.0.0.2/32", + "scope": "ip", + "value": "127.0.0.2" + }, + "Events": [ + ], + "events_count": 1, + "leakspeed": "0.5s", + "message": "test", + "scenario_hash": "hashtest", + "scenario_version": "v1", + "simulated": false, + "capacity": 1, + "scenario": "crowdsecurity/test", + "start_at": "2020-10-09T10:00:01Z", + "stop_at": "2020-10-09T10:00:05Z" + }, + { + "id": 47, + "machine_id": "test", + "created_at": "2020-10-09T10:00:10Z", + "decisions": [ + { + "duration": "3h", + "origin": "test", + "scenario": "crowdsecurity/ssh_bf", + "scope": "Ip", + "value": "127.0.0.2", + "type": "ban" + } + ], + "source": { + "ip": "127.0.0.2", + "range": "127.0.0.2/32", + "scope": "ip", + "value": "127.0.0.2" + }, + "Events": [ + ], + "events_count": 1, + "leakspeed": "0.5s", + "message": "test", + "scenario_hash": "hashtest", + "scenario_version": "v1", + "simulated": false, + "capacity": 1, + "scenario": "crowdsecurity/ssh_bf", + "start_at": "2020-10-09T10:00:01Z", + "stop_at": "2020-10-09T10:00:05Z" + }, + { + "id": 48, + "machine_id": "test", + "created_at": "2020-10-09T10:00:10Z", + "decisions": [ + { + "duration": "1h", + "origin": "test", + "scenario": "crowdsecurity/ssh_bf", + "scope": "Ip", + "value": "127.0.0.2", + "type": "ban" + } + ], + "source": { + "ip": "127.0.0.2", + "range": "127.0.0.2/32", + "scope": "ip", + "value": "127.0.0.2" + }, + "Events": [ + ], + "events_count": 1, + "leakspeed": "0.5s", + "message": "test", + "scenario_hash": "hashtest", + "scenario_version": "v1", + "simulated": false, + "capacity": 1, + "scenario": "crowdsecurity/ssh_bf", + "start_at": "2020-10-09T10:00:01Z", + "stop_at": "2020-10-09T10:00:05Z" + }, + { + "id": 49, + "machine_id": "test", + "created_at": "2020-10-09T10:00:10Z", + "decisions": [ + { + "duration": "2h", + "origin": "another_origin", + "scenario": "crowdsecurity/test", + "scope": "Ip", + "value": "127.0.0.2", + "type": "ban" + } + ], + "source": { + "ip": "127.0.0.2", + "range": "127.0.0.2/32", + "scope": "ip", + "value": "127.0.0.2" + }, + "Events": [ + ], + "events_count": 1, + "leakspeed": "0.5s", + "message": "test", + "scenario_hash": "hashtest", + "scenario_version": "v1", + "simulated": false, + "capacity": 1, + "scenario": "crowdsecurity/test", + "start_at": "2020-10-09T10:00:01Z", + "stop_at": "2020-10-09T10:00:05Z" + }, + { + "id": 50, + "machine_id": "test", + "created_at": "2020-10-09T10:00:10Z", + "decisions": [ + { + "duration": "3h", + "origin": "test", + "scenario": "crowdsecurity/test", + "scope": "Ip", + "value": "127.0.0.2", + "type": "captcha" + } + ], + "source": { + "ip": "127.0.0.2", + "range": "127.0.0.2/32", + "scope": "ip", + "value": "127.0.0.2" + }, + "Events": [ + ], + "events_count": 1, + "leakspeed": "0.5s", + "message": "test", + "scenario_hash": "hashtest", + "scenario_version": "v1", + "simulated": false, + "capacity": 1, + "scenario": "crowdsecurity/test", + "start_at": "2020-10-09T10:00:01Z", + "stop_at": "2020-10-09T10:00:05Z" + } +] \ No newline at end of file diff --git a/pkg/database/decisions.go b/pkg/database/decisions.go index ea86f8c8c..4fddb46a2 100644 --- a/pkg/database/decisions.go +++ b/pkg/database/decisions.go @@ -15,16 +15,19 @@ import ( "github.com/pkg/errors" ) -func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string][]string) (*ent.DecisionQuery, error) { +func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string][]string) (*ent.DecisionQuery, []*sql.Predicate, error) { var err error var start_ip, start_sfx, end_ip, end_sfx int64 var ip_sz int var contains bool = true - /*if contains is true, return bans that *contains* the given value (value is the inner) - else, return bans that are *contained* by the given value (value is the outer)*/ - /*the simulated filter is a bit different : if it's not present *or* set to false, specifically exclude records with simulated to true */ + // contains == true -> return bans that *contain* the given value (value is the inner) + // contains == false or missing -> return bans *contained* in the given value (value is the outer) + + // simulated == true -> include simulated rows + // simulated == false or missing -> exclude simulated rows + if v, ok := filter["simulated"]; ok { if v[0] == "false" { query = query.Where(decision.SimulatedEQ(false)) @@ -33,13 +36,14 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string] } else { query = query.Where(decision.SimulatedEQ(false)) } - + t := sql.Table(decision.Table) + joinPredicate := make([]*sql.Predicate, 0) for param, value := range filter { switch param { case "contains": contains, err = strconv.ParseBool(value[0]) if err != nil { - return nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err) + return nil, nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err) } case "scopes": scopes := strings.Split(value[0], ",") @@ -64,9 +68,24 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string] query = query.Where( decision.OriginIn(strings.Split(value[0], ",")...), ) + origins := strings.Split(value[0], ",") + originsContainsPredicate := make([]*sql.Predicate, 0) + for _, origin := range origins { + pred := sql.EqualFold(t.C(decision.FieldOrigin), origin) + originsContainsPredicate = append(originsContainsPredicate, pred) + } + joinPredicate = append(joinPredicate, sql.Or(originsContainsPredicate...)) case "scenarios_containing": predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold) query = query.Where(decision.Or(predicates...)) + + scenarios := strings.Split(value[0], ",") + scenariosContainsPredicate := make([]*sql.Predicate, 0) + for _, scenario := range scenarios { + pred := sql.ContainsFold(t.C(decision.FieldScenario), scenario) + scenariosContainsPredicate = append(scenariosContainsPredicate, pred) + } + joinPredicate = append(joinPredicate, sql.Or(scenariosContainsPredicate...)) case "scenarios_not_containing": predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold) query = query.Where(decision.Not( @@ -74,10 +93,17 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string] predicates..., ), )) + scenarios := strings.Split(value[0], ",") + scenariosContainsPredicate := make([]*sql.Predicate, 0) + for _, scenario := range scenarios { + pred := sql.ContainsFold(t.C(decision.FieldScenario), scenario) + scenariosContainsPredicate = append(scenariosContainsPredicate, sql.Not(pred)) + } + joinPredicate = append(joinPredicate, sql.Or(scenariosContainsPredicate...)) case "ip", "range": ip_sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(value[0]) if err != nil { - return nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err) + return nil, nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err) } } } @@ -149,9 +175,9 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string] )) } } else if ip_sz != 0 { - return nil, errors.Wrapf(InvalidFilter, "Unknown ip size %d", ip_sz) + return nil, nil, errors.Wrapf(InvalidFilter, "Unknown ip size %d", ip_sz) } - return query, nil + return query, joinPredicate, nil } func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Decision, error) { @@ -161,7 +187,7 @@ func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Dec decisions := c.Ent.Decision.Query(). Where(decision.UntilGTE(time.Now().UTC())) - decisions, err = BuildDecisionRequestWithFilter(decisions, filter) + decisions, _, err = BuildDecisionRequestWithFilter(decisions, filter) if err != nil { return []*ent.Decision{}, err } @@ -185,86 +211,89 @@ func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Dec return data, nil } -// ent translation of https://stackoverflow.com/a/28090544 -func longestDecisionForScopeTypeValue(s *sql.Selector) { - t := sql.Table(decision.Table) - s.LeftJoin(t).OnP(sql.And( - sql.ColumnsEQ( - t.C(decision.FieldValue), - s.C(decision.FieldValue), - ), - sql.ColumnsEQ( - t.C(decision.FieldType), - s.C(decision.FieldType), - ), - sql.ColumnsEQ( - t.C(decision.FieldScope), - s.C(decision.FieldScope), - ), - sql.ColumnsGT( - t.C(decision.FieldUntil), - s.C(decision.FieldUntil), - ), - )) - s.Where( - sql.IsNull( - t.C(decision.FieldUntil), - ), - ) -} - func (c *Client) QueryAllDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) { query := c.Ent.Decision.Query().Where( decision.UntilGT(time.Now().UTC()), - longestDecisionForScopeTypeValue, ) - query, err := BuildDecisionRequestWithFilter(query, filters) - + query, _, err := BuildDecisionRequestWithFilter(query, filters) if err != nil { c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err) return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters") } - data, err := query.All(c.CTX) + //Order is *very* important, the dedup assumes that decisions are sorted per IP and per time left + data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX) if err != nil { c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err) return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters") } + return data, nil } func (c *Client) QueryExpiredDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) { + now := time.Now().UTC() query := c.Ent.Decision.Query().Where( decision.UntilLT(time.Now().UTC()), - longestDecisionForScopeTypeValue, ) - query, err := BuildDecisionRequestWithFilter(query, filters) - + query, predicates, err := BuildDecisionRequestWithFilter(query, filters) if err != nil { c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err) return []*ent.Decision{}, errors.Wrap(QueryFail, "get expired decisions with filters") } - data, err := query.All(c.CTX) + query = query.Where(func(s *sql.Selector) { + t := sql.Table(decision.Table) + + subQuery := sql.Select(t.C(decision.FieldValue)).From(t).Where(sql.GT(t.C(decision.FieldUntil), now)) + for _, predicate := range predicates { + subQuery.Where(predicate) + } + s.Where( + sql.NotIn( + s.C(decision.FieldValue), + subQuery, + ), + ) + }) + + data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX) if err != nil { c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err) return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions") } + return data, nil } func (c *Client) QueryExpiredDecisionsSinceWithFilters(since time.Time, filters map[string][]string) ([]*ent.Decision, error) { + now := time.Now().UTC() + query := c.Ent.Decision.Query().Where( - decision.UntilLT(time.Now().UTC()), + decision.UntilLT(now), decision.UntilGT(since), - longestDecisionForScopeTypeValue, ) - query, err := BuildDecisionRequestWithFilter(query, filters) + query, predicates, err := BuildDecisionRequestWithFilter(query, filters) if err != nil { c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err) return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters") } - data, err := query.All(c.CTX) + query = query.Where(func(s *sql.Selector) { + t := sql.Table(decision.Table) + + subQuery := sql.Select(t.C(decision.FieldValue)).From(t).Where(sql.GT(t.C(decision.FieldUntil), now)) + for _, predicate := range predicates { + subQuery.Where(predicate) + } + s.Where( + sql.NotIn( + s.C(decision.FieldValue), + subQuery, + ), + ) + }) + + data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX) if err != nil { c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err) return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters") @@ -277,14 +306,15 @@ func (c *Client) QueryNewDecisionsSinceWithFilters(since time.Time, filters map[ query := c.Ent.Decision.Query().Where( decision.CreatedAtGT(since), decision.UntilGT(time.Now().UTC()), - longestDecisionForScopeTypeValue, ) - query, err := BuildDecisionRequestWithFilter(query, filters) + query, _, err := BuildDecisionRequestWithFilter(query, filters) if err != nil { - c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err) - return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String()) + c.Log.Warningf("BuildDecisionRequestWithFilter : %s", err) + return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters") } - data, err := query.All(c.CTX) + + //Order is *very* important, the dedup assumes that decisions are sorted per IP and per time left + data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX) if err != nil { c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err) return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String()) @@ -521,17 +551,17 @@ func (c *Client) SoftDeleteDecisionsWithFilter(filter map[string][]string) (stri } //SoftDeleteDecisionByID set the expiration of a decision to now() -func (c *Client) SoftDeleteDecisionByID(decisionID int) error { +func (c *Client) SoftDeleteDecisionByID(decisionID int) (int, error) { nbUpdated, err := c.Ent.Decision.Update().Where(decision.IDEQ(decisionID)).SetUntil(time.Now().UTC()).Save(c.CTX) if err != nil || nbUpdated == 0 { c.Log.Warningf("SoftDeleteDecisionByID : %v (nb soft deleted: %d)", err, nbUpdated) - return errors.Wrapf(DeleteFail, "decision with id '%d' doesn't exist", decisionID) + return 0, errors.Wrapf(DeleteFail, "decision with id '%d' doesn't exist", decisionID) } if nbUpdated == 0 { - return ItemNotFound + return 0, ItemNotFound } - return nil + return nbUpdated, nil } func decisionPredicatesFromStr(s string, predicateFunc func(string) predicate.Decision) []predicate.Decision { diff --git a/tests/bats/99_lapi-stream-mode-scenario.bats b/tests/bats/99_lapi-stream-mode-scenario.bats new file mode 100644 index 000000000..22981da53 --- /dev/null +++ b/tests/bats/99_lapi-stream-mode-scenario.bats @@ -0,0 +1,233 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + ./instance-crowdsec start + API_KEY=$(cscli bouncers add testbouncer -o raw) + export API_KEY + CROWDSEC_API_URL="http://localhost:8080" + export CROWDSEC_API_URL +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + skip +} + +#---------- + +api() { + URI="$1" + curl -s -H "X-Api-Key:${API_KEY}" "${CROWDSEC_API_URL}${URI}" +} + +output_new_decisions() { + jq -c '.new | map(select(.origin!="CAPI")) | .[] | del(.id) | (.. | .duration?) |= capture("(?[[:digit:]]+h[[:digit:]]+m)").d' <(output) | sort +} + + +@test "${FILE} adding decisions with different duration, scenario, origin" { + # origin: test + run -0 cscli decisions add -i 127.0.0.1 -d 1h -R crowdsecurity/test + ./instance-crowdsec stop + run -0 ./instance-db exec_sql "update decisions set origin='test' where origin='cscli'" + ./instance-crowdsec start + + run -0 cscli decisions add -i 127.0.0.1 -d 3h -R crowdsecurity/ssh_bf + ./instance-crowdsec stop + run -0 ./instance-db exec_sql "update decisions set origin='another_origin' where origin='cscli'" + ./instance-crowdsec start + + run -0 cscli decisions add -i 127.0.0.1 -d 5h -R crowdsecurity/longest + run -0 cscli decisions add -i 127.0.0.2 -d 3h -R crowdsecurity/test + run -0 cscli decisions add -i 127.0.0.2 -d 3h -R crowdsecurity/ssh_bf + run -0 cscli decisions add -i 127.0.0.2 -d 1h -R crowdsecurity/ssh_bf + ./instance-crowdsec stop + run -0 ./instance-db exec_sql "update decisions set origin='test' where origin='cscli'" + ./instance-crowdsec start + + # origin: another_origin + run -0 cscli decisions add -i 127.0.0.2 -d 2h -R crowdsecurity/test + ./instance-crowdsec stop + run -0 ./instance-db exec_sql "update decisions set origin='another_origin' where origin='cscli'" + ./instance-crowdsec start +} + +@test "${FILE} test startup" { + run -0 api "/v1/decisions/stream?startup=true" + run -0 output_new_decisions + assert_output - <<-EOT + {"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"} + {"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"} + EOT +} + +@test "${FILE} test startup with scenarios containing" { + run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf" + run -0 output_new_decisions + assert_output - <<-EOT + {"duration":"2h59m","origin":"another_origin","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.1"} + {"duration":"2h59m","origin":"test","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.2"} + EOT +} + +@test "${FILE} test startup with multiple scenarios containing" { + run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf,test" + run -0 output_new_decisions + assert_output - <<-EOT + {"duration":"2h59m","origin":"another_origin","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.1"} + {"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"} + EOT +} + +@test "${FILE} test startup with unknown scenarios containing" { + run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=unknown" + assert_output '{"deleted":null,"new":null}' +} + +@test "${FILE} test startup with scenarios containing and not containing" { + run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=test&scenarios_not_containing=ssh_bf" + run -0 output_new_decisions + assert_output - <<-EOT + {"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"} + {"origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.1"} + EOT +} + +@test "${FILE} test startup with scenarios containing and not containing 2" { + run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=longest&scenarios_not_containing=ssh_bf,test" + run -0 output_new_decisions + assert_output - <<-EOT + {"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"} + EOT +} + +@test "${FILE} test startup with scenarios not containing" { + run -0 api "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf" + run -0 output_new_decisions + assert_output - <<-EOT + {"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"} + {"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"} + EOT +} + +@test "${FILE} test startup with multiple scenarios not containing" { + run -0 api "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf,test" + run -0 output_new_decisions + assert_output - <<-EOT + {"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"} + EOT +} + +@test "${FILE} test startup with origins parameter" { + run -0 api "/v1/decisions/stream?startup=true&origins=another_origin" + run -0 output_new_decisions + assert_output - <<-EOT + {"duration":"1h59m","origin":"another_origin","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"} + {"duration":"2h59m","origin":"another_origin","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.1"} + EOT +} + +@test "${FILE} test startup with multiple origins parameter" { + run -0 api "/v1/decisions/stream?startup=true&origins=another_origin,test" + run -0 output_new_decisions + assert_output - <<-EOT + {"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"} + {"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"} + EOT +} + +@test "${FILE} test startup with unknown origins" { + run -0 api "/v1/decisions/stream?startup=true&origins=unknown" + assert_output '{"deleted":null,"new":null}' +} + +#@test "${FILE} delete decision 3 (127.0.0.1)" { +# +# { +# TestName: "delete decisions 3 (127.0.0.1)", +# Method: "DELETE", +# Route: "/v1/decisions/3", +# CheckCodeOnly: true, +# Code: 200, +# LenNew: 0, +# LenDeleted: 0, +# AuthType: PASSWORD, +# DelChecks: []DecisionCheck{}, +# NewChecks: []DecisionCheck{}, +# TestName: "check that 127.0.0.1 is not in deleted IP", +# Method: "GET", +# Route: "/v1/decisions/stream?startup=true", +# CheckCodeOnly: false, +# Code: 200, +# LenNew: 2, +# LenDeleted: 0, +# AuthType: APIKEY, +# DelChecks: []DecisionCheck{}, +# NewChecks: []DecisionCheck{}, +# }, +# { +# TestName: "delete decisions 2 (127.0.0.1)", +# Method: "DELETE", +# Route: "/v1/decisions/2", +# CheckCodeOnly: true, +# Code: 200, +# LenNew: 0, +# LenDeleted: 0, +# AuthType: PASSWORD, +# DelChecks: []DecisionCheck{}, +# NewChecks: []DecisionCheck{}, +# }, +# { +# TestName: "check that 127.0.0.1 is not in deleted IP", +# Method: "GET", +# Route: "/v1/decisions/stream?startup=true", +# CheckCodeOnly: false, +# Code: 200, +# LenNew: 2, +# LenDeleted: 0, +# AuthType: APIKEY, +# DelChecks: []DecisionCheck{}, +# NewChecks: []DecisionCheck{}, +# }, +# { +# TestName: "delete decisions 1 (127.0.0.1)", +# Method: "DELETE", +# Route: "/v1/decisions/1", +# CheckCodeOnly: true, +# Code: 200, +# LenNew: 0, +# LenDeleted: 0, +# AuthType: PASSWORD, +# DelChecks: []DecisionCheck{}, +# NewChecks: []DecisionCheck{}, +# }, +# TestName: "127.0.0.1 should be in deleted now", +# Method: "GET", +# Route: "/v1/decisions/stream?startup=true", +# CheckCodeOnly: false, +# Code: 200, +# LenNew: 1, +# LenDeleted: 1, +# AuthType: APIKEY, +# DelChecks: []DecisionCheck{ +# { +# ID: int64(1), +# Origin: "test", +# Scenario: "crowdsecurity/test", +# Value: "127.0.0.1", +# Duration: "-", // we check that the time is negative +# }, +# }, +# NewChecks: []DecisionCheck{}, +# }, +#} + diff --git a/tests/lib/db/instance-mysql b/tests/lib/db/instance-mysql index bbf7b5e1b..e908b6e45 100755 --- a/tests/lib/db/instance-mysql +++ b/tests/lib/db/instance-mysql @@ -115,7 +115,13 @@ case "$1" in ;; exec_sql) shift - exec_sql "$@" + # + # This command is meant to run a query against the the crowdsec database. + # The exec_sql() function is more generic and is also used for database setup and backups. + # + # For this reason, we select the database here. + # + exec_sql "use crowdsec_test; $@" ;; *) about diff --git a/tests/lib/setup_file.sh b/tests/lib/setup_file.sh index be1106a54..e41fdafc6 100755 --- a/tests/lib/setup_file.sh +++ b/tests/lib/setup_file.sh @@ -4,6 +4,10 @@ # https://github.com/bats-core/bats-core/blob/master/docs/source/warnings/BW02.rst bats_require_minimum_version 1.5.0 +# this should have effect globally, for all tests +# https://github.com/bats-core/bats-core/blob/master/docs/source/warnings/BW02.rst +bats_require_minimum_version 1.5.0 + debug() { echo 'exec 1<&-; exec 2<&-; exec 1>&3; exec 2>&1' }