From 0eea20fa7c2fed02693eb0b30b4de9d1aebbf552 Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Fri, 22 Jul 2022 11:20:10 +0200 Subject: [PATCH] revert decision dedup behavior to 1.3.4 (#1675) * revert decision dedup behavior to 1.3.4 --- pkg/apiserver/decisions_test.go | 1033 ------------------------------- pkg/database/decisions.go | 261 ++++---- 2 files changed, 111 insertions(+), 1183 deletions(-) diff --git a/pkg/apiserver/decisions_test.go b/pkg/apiserver/decisions_test.go index 7564368e6..4d56cfc3a 100644 --- a/pkg/apiserver/decisions_test.go +++ b/pkg/apiserver/decisions_test.go @@ -1,8 +1,6 @@ package apiserver import ( - "fmt" - "os" "testing" "github.com/stretchr/testify/assert" @@ -334,1034 +332,3 @@ type DecisionTest struct { DelChecks []DecisionCheck AuthType string } - -func TestStreamDecisionStart(t *testing.T) { - lapi := SetupLAPITest(t) - - /* - Create multiple alerts: - - 3 alerts for 127.0.0.1 with ID 1/2/3 : Different duration / scenario / origin - - 3 alerts for 127.0.0.2 with ID 4/5/6/7 : Different duration / scenario / origin - */ - lapi.InsertAlertFromFile("./tests/alert_duplicate.json") - - tests := []DecisionTest{ - { - TestName: "test startup", - Method: "GET", - Route: "/v1/decisions/stream?startup=true", - CheckCodeOnly: false, - Code: 200, - LenNew: 3, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{ - { - ID: int64(3), - Origin: "test", - Scenario: "crowdsecurity/longest", - Value: "127.0.0.1", - Duration: "4h59", - Type: "ban", - }, - { - ID: int64(4), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(8), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "captcha", - }, - }, - }, - { - TestName: "test startup with scenarios containing", - Method: "GET", - Route: "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf", - CheckCodeOnly: false, - Code: 200, - LenNew: 2, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{ - { - ID: int64(2), - Origin: "another_origin", - Scenario: "crowdsecurity/ssh_bf", - Value: "127.0.0.1", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(5), - Origin: "test", - Scenario: "crowdsecurity/ssh_bf", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - }, - }, - { - TestName: "test startup with multiple scenarios containing", - Method: "GET", - Route: "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf,test", - CheckCodeOnly: false, - Code: 200, - LenNew: 3, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - - NewChecks: []DecisionCheck{ - { - ID: int64(2), - Origin: "another_origin", - Scenario: "crowdsecurity/ssh_bf", - Value: "127.0.0.1", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(4), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(8), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "captcha", - }, - }, - }, - { - TestName: "test startup with unknown scenarios containing", - Method: "GET", - Route: "/v1/decisions/stream?startup=true&scenarios_containing=unknown", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - - NewChecks: []DecisionCheck{}, - }, - { - TestName: "test startup with scenarios containing and not containing", - Method: "GET", - Route: "/v1/decisions/stream?startup=true&scenarios_containing=test&scenarios_not_containing=ssh_bf", - CheckCodeOnly: false, - Code: 200, - LenNew: 3, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{ - { - ID: int64(1), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.1", - Duration: "59m", - Type: "ban", - }, - { - ID: int64(4), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(8), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "captcha", - }, - }, - }, - { - TestName: "test startup with scenarios containing and not containing 2", - Method: "GET", - Route: "/v1/decisions/stream?startup=true&scenarios_containing=longest&scenarios_not_containing=ssh_bf,test", - CheckCodeOnly: false, - Code: 200, - LenNew: 1, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{ - { - ID: int64(3), - Origin: "test", - Scenario: "crowdsecurity/longest", - Value: "127.0.0.1", - Duration: "4h59", - Type: "ban", - }, - }, - }, - { - TestName: "test startup with scenarios not containing", - Method: "GET", - Route: "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf", - CheckCodeOnly: false, - Code: 200, - LenNew: 3, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - - NewChecks: []DecisionCheck{ - { - ID: int64(3), - Origin: "test", - Scenario: "crowdsecurity/longest", - Value: "127.0.0.1", - Duration: "4h59", - Type: "ban", - }, - { - ID: int64(4), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(8), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "captcha", - }, - }, - }, - { - TestName: "test startup with multiple scenarios not containing", - Method: "GET", - Route: "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf,test", - CheckCodeOnly: false, - Code: 200, - LenNew: 1, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - - NewChecks: []DecisionCheck{ - { - ID: int64(3), - Origin: "test", - Scenario: "crowdsecurity/longest", - Value: "127.0.0.1", - Duration: "4h59", - Type: "ban", - }, - }, - }, - { - TestName: "test startup with origins parameter", - Method: "GET", - Route: "/v1/decisions/stream?startup=true&origins=another_origin", - CheckCodeOnly: false, - Code: 200, - LenNew: 2, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - - NewChecks: []DecisionCheck{ - { - ID: int64(2), - Origin: "another_origin", - Scenario: "crowdsecurity/ssh_bf", - Value: "127.0.0.1", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(7), - Origin: "another_origin", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "1h59", - Type: "ban", - }, - }, - }, - { - TestName: "test startup with multiple origins parameter", - Method: "GET", - Route: "/v1/decisions/stream?startup=true&origins=another_origin,test", - CheckCodeOnly: false, - Code: 200, - LenNew: 3, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - - NewChecks: []DecisionCheck{ - { - ID: int64(3), - Origin: "test", - Scenario: "crowdsecurity/longest", - Value: "127.0.0.1", - Duration: "4h59", - Type: "ban", - }, - { - ID: int64(4), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(8), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "captcha", - }, - }, - }, - { - TestName: "test startup with unknown origins", - Method: "GET", - Route: "/v1/decisions/stream?startup=true&origins=unknown", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "delete decisions 3 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/3", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "check that 127.0.0.1 is not in deleted IP", - Method: "GET", - Route: "/v1/decisions/stream?startup=true", - CheckCodeOnly: false, - Code: 200, - LenNew: 3, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{ - { - ID: int64(2), - Origin: "another_origin", - Scenario: "crowdsecurity/ssh_bf", - Value: "127.0.0.1", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(4), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(8), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "captcha", - }, - }, - }, - { - TestName: "delete decisions 2 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/2", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "check that 127.0.0.1 is not in deleted IP", - Method: "GET", - Route: "/v1/decisions/stream?startup=true", - CheckCodeOnly: false, - Code: 200, - LenNew: 3, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{ - { - ID: int64(1), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.1", - Duration: "59", - Type: "ban", - }, - { - ID: int64(4), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(8), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "captcha", - }, - }, - }, - { - TestName: "delete decisions 1 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/1", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "127.0.0.1 should be in deleted now", - Method: "GET", - Route: "/v1/decisions/stream?startup=true", - CheckCodeOnly: false, - Code: 200, - LenNew: 2, - LenDeleted: 1, - AuthType: APIKEY, - DelChecks: []DecisionCheck{ - { - ID: int64(1), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.1", - Duration: "-", // we check that the time is negative - Type: "ban", - }, - }, - NewChecks: []DecisionCheck{ - { - ID: int64(4), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(8), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "captcha", - }, - }, - }, - { - TestName: "delete decisions 8 (127.0.0.2) with captcha", - Method: "DELETE", - Route: "/v1/decisions/8", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "127.0.0.2 with captcha should be in deleted now", - Method: "GET", - Route: "/v1/decisions/stream?startup=true", - CheckCodeOnly: false, - Code: 200, - LenNew: 1, - LenDeleted: 2, - AuthType: APIKEY, - DelChecks: []DecisionCheck{ - { - ID: int64(1), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.1", - Duration: "-", // we check that the time is negative - Type: "ban", - }, - { - ID: int64(8), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "-", - Type: "captcha", - }, - }, - NewChecks: []DecisionCheck{ - { - ID: int64(4), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - }, - }, - } - - for _, test := range tests { - runTest(lapi, test, t) - } -} - -func TestStreamDecision(t *testing.T) { - - /* - Create multiple alerts: - - 3 alerts for 127.0.0.1 with ID 1/2/3 : Different duration / scenario / origin - - 3 alerts for 127.0.0.2 with ID 4/5/6/7 : Different duration / scenario / origin - */ - - // this test just init the stream with startup=true - preTests := []DecisionTest{ - { - TestName: "test startup", - Method: "GET", - Route: "/v1/decisions/stream?startup=true", - CheckCodeOnly: false, - Code: 200, - AuthType: APIKEY, - LenNew: 0, - LenDeleted: 0, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - } - - tests := map[string][]DecisionTest{ - "Test without parameter": { - { - TestName: "get stream", - Method: "GET", - Route: "/v1/decisions/stream", - CheckCodeOnly: false, - Code: 200, - LenNew: 3, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{ - { - ID: int64(3), - Origin: "test", - Scenario: "crowdsecurity/longest", - Value: "127.0.0.1", - Duration: "4h59", - Type: "ban", - }, - { - ID: int64(4), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(8), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "captcha", - }, - }, - }, - { - TestName: "delete decisions 3 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/3", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "check that 127.0.0.1 is not in deleted IP", - Method: "GET", - Route: "/v1/decisions/stream", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "delete decisions 2 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/2", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "check that 127.0.0.1 is not in deleted IP", - Method: "GET", - Route: "/v1/decisions/stream", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "delete decisions 1 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/1", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "127.0.0.1 should be in deleted now", - Method: "GET", - Route: "/v1/decisions/stream", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 1, - AuthType: APIKEY, - DelChecks: []DecisionCheck{ - { - ID: int64(1), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.1", - Duration: "-", - - Type: "ban", - }, - }, - NewChecks: []DecisionCheck{}, - }, - }, - "test with scenarios containing": { - { - TestName: "get stream", - Method: "GET", - Route: "/v1/decisions/stream?scenarios_containing=ssh_bf", - CheckCodeOnly: false, - Code: 200, - LenNew: 2, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{ - { - ID: int64(2), - Origin: "another_origin", - Scenario: "crowdsecurity/ssh_bf", - Value: "127.0.0.1", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(5), - Origin: "test", - Scenario: "crowdsecurity/ssh_bf", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - }, - }, - { - TestName: "delete decisions 3 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/3", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "check that 127.0.0.1 is not in deleted IP", - Method: "GET", - Route: "/v1/decisions/stream?scenarios_containing=ssh_bf", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "delete decisions 2 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/2", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "check that 127.0.0.1 is deleted (decision for ssh_bf was with ID 2)", - Method: "GET", - Route: "/v1/decisions/stream?scenarios_containing=ssh_bf", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 1, - AuthType: APIKEY, - DelChecks: []DecisionCheck{ - { - ID: int64(2), - Origin: "another_origin", - Scenario: "crowdsecurity/ssh_bf", - Value: "127.0.0.1", - Duration: "-", - - Type: "ban", - }, - }, - NewChecks: []DecisionCheck{}, - }, - }, - "test with scenarios not containing": { - { - TestName: "get stream", - Method: "GET", - Route: "/v1/decisions/stream?scenarios_not_containing=ssh_bf", - CheckCodeOnly: false, - Code: 200, - LenNew: 3, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{ - { - ID: int64(3), - Origin: "test", - Scenario: "crowdsecurity/longest", - Value: "127.0.0.1", - Duration: "4h59", - Type: "ban", - }, - { - ID: int64(4), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(8), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "2h59", - Type: "captcha", - }, - }, - }, - { - TestName: "delete decisions 3 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/3", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "check that 127.0.0.1 is not in deleted IP", - Method: "GET", - Route: "/v1/decisions/stream?scenarios_not_containing=ssh_bf", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "delete decisions 2 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/2", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "check that 127.0.0.1 is not deleted", - Method: "GET", - Route: "/v1/decisions/stream?scenarios_not_containing=ssh_bf", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "delete decisions 1 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/1", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "check that 127.0.0.1 is deleted", - Method: "GET", - Route: "/v1/decisions/stream?scenarios_not_containing=ssh_bf", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 1, - AuthType: APIKEY, - DelChecks: []DecisionCheck{ - { - ID: int64(1), - Origin: "test", - Scenario: "crowdsecurity/test", - Value: "127.0.0.1", - Duration: "-", - - Type: "ban", - }, - }, - NewChecks: []DecisionCheck{}, - }, - }, - "test with origins": { - { - TestName: "get stream", - Method: "GET", - Route: "/v1/decisions/stream?origins=another_origin", - CheckCodeOnly: false, - Code: 200, - LenNew: 2, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{ - { - ID: int64(2), - Origin: "another_origin", - Scenario: "crowdsecurity/ssh_bf", - Value: "127.0.0.1", - Duration: "2h59", - Type: "ban", - }, - { - ID: int64(7), - Origin: "another_origin", - Scenario: "crowdsecurity/test", - Value: "127.0.0.2", - Duration: "1h59", - Type: "ban", - }, - }, - }, - { - TestName: "delete decisions 3 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/3", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "check that 127.0.0.1 is not in deleted IP", - Method: "GET", - Route: "/v1/decisions/stream?origins=another_origin", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: APIKEY, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "delete decisions 2 (127.0.0.1)", - Method: "DELETE", - Route: "/v1/decisions/2", - CheckCodeOnly: true, - Code: 200, - LenNew: 0, - LenDeleted: 0, - AuthType: PASSWORD, - DelChecks: []DecisionCheck{}, - NewChecks: []DecisionCheck{}, - }, - { - TestName: "check that 127.0.0.1 is deleted", - Method: "GET", - Route: "/v1/decisions/stream?origins=another_origin", - CheckCodeOnly: false, - Code: 200, - LenNew: 0, - LenDeleted: 1, - AuthType: APIKEY, - DelChecks: []DecisionCheck{ - { - ID: int64(2), - Origin: "another_origin", - Scenario: "crowdsecurity/ssh_bf", - Value: "127.0.0.1", - Duration: "-", - - Type: "ban", - }, - }, - NewChecks: []DecisionCheck{}, - }, - }, - } - - // run tests for the stream - for testName, test := range tests { - - // init a new LAPI - lapi := SetupLAPITest(t) - - // run pre-test, mostly to init the stream - for _, test := range preTests { - runTest(lapi, test, t) - } - // insert decisions now that the stream is initiated - lapi.InsertAlertFromFile("./tests/alert_duplicate.json") - - for _, oneTest := range test { - oneTest.TestName = fmt.Sprintf("%s (%s)", oneTest.TestName, testName) - runTest(lapi, oneTest, t) - } - - // clean the db after each test - os.Remove(lapi.DBConfig.DbPath) - } -} - -func runTest(lapi LAPI, test DecisionTest, t *testing.T) { - w := lapi.RecordResponse(test.Method, test.Route, emptyBody, test.AuthType) - assert.Equal(t, test.Code, w.Code) - if test.CheckCodeOnly { - return - } - decisions, _, err := readDecisionsStreamResp(w) - assert.Equal(t, nil, err) - assert.Equal(t, test.LenDeleted, len(decisions["deleted"]), fmt.Sprintf("'%s': len(deleted)", test.TestName)) - assert.Equal(t, test.LenNew, len(decisions["new"]), fmt.Sprintf("'%s': len(new)", test.TestName)) - - for i, check := range test.NewChecks { - assert.Equal(t, check.ID, decisions["new"][i].ID, fmt.Sprintf("'%s' (idx: %d): field: ID", test.TestName, i)) - assert.Equal(t, check.Origin, *decisions["new"][i].Origin, fmt.Sprintf("'%s' (idx: %d): field: Origin", test.TestName, i)) - assert.Equal(t, check.Scenario, *decisions["new"][i].Scenario, fmt.Sprintf("'%s' (idx: %d): field: Scenario", test.TestName, i)) - assert.Equal(t, check.Value, *decisions["new"][i].Value, fmt.Sprintf("'%s' (idx: %d): field: Value", test.TestName, i)) - assert.Equal(t, check.Type, *decisions["new"][i].Type, fmt.Sprintf("'%s' (idx: %d): field: Type", test.TestName, i)) - assert.Contains(t, *decisions["new"][i].Duration, check.Duration, fmt.Sprintf("'%s' (idx: %d): field: Duration", test.TestName, i)) - } - - for i, check := range test.DelChecks { - assert.Equal(t, check.ID, decisions["deleted"][i].ID, fmt.Sprintf("'%s' (idx: %d): field: ID", test.TestName, i)) - assert.Equal(t, check.Origin, *decisions["deleted"][i].Origin, fmt.Sprintf("'%s' (idx: %d): field: Origin", test.TestName, i)) - assert.Equal(t, check.Scenario, *decisions["deleted"][i].Scenario, fmt.Sprintf("'%s' (idx: %d): field: Scenario", test.TestName, i)) - assert.Equal(t, check.Value, *decisions["deleted"][i].Value, fmt.Sprintf("'%s' (idx: %d): field: Value", test.TestName, i)) - assert.Equal(t, check.Type, *decisions["deleted"][i].Type, fmt.Sprintf("'%s' (idx: %d): field: Type", test.TestName, i)) - assert.Contains(t, *decisions["deleted"][i].Duration, check.Duration, fmt.Sprintf("'%s' (idx: %d): field: Duration", test.TestName, i)) - } -} diff --git a/pkg/database/decisions.go b/pkg/database/decisions.go index dc5775b03..6ef13a3b6 100644 --- a/pkg/database/decisions.go +++ b/pkg/database/decisions.go @@ -22,19 +22,16 @@ type DecisionsByScenario struct { Type string } -func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string][]string) (*ent.DecisionQuery, []*sql.Predicate, error) { +func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string][]string) (*ent.DecisionQuery, error) { var err error var start_ip, start_sfx, end_ip, end_sfx int64 var ip_sz int var contains bool = true + /*if contains is true, return bans that *contains* the given value (value is the inner) + else, return bans that are *contained* by the given value (value is the outer)*/ - // contains == true -> return bans that *contain* the given value (value is the inner) - // contains == false or missing -> return bans *contained* in the given value (value is the outer) - - // simulated == true -> include simulated rows - // simulated == false or missing -> exclude simulated rows - + /*the simulated filter is a bit different : if it's not present *or* set to false, specifically exclude records with simulated to true */ if v, ok := filter["simulated"]; ok { if v[0] == "false" { query = query.Where(decision.SimulatedEQ(false)) @@ -43,14 +40,13 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string] } else { query = query.Where(decision.SimulatedEQ(false)) } - t := sql.Table(decision.Table) - joinPredicate := make([]*sql.Predicate, 0) + for param, value := range filter { switch param { case "contains": contains, err = strconv.ParseBool(value[0]) if err != nil { - return nil, nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err) + return nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err) } case "scopes": scopes := strings.Split(value[0], ",") @@ -75,24 +71,9 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string] query = query.Where( decision.OriginIn(strings.Split(value[0], ",")...), ) - origins := strings.Split(value[0], ",") - originsContainsPredicate := make([]*sql.Predicate, 0) - for _, origin := range origins { - pred := sql.EqualFold(t.C(decision.FieldOrigin), origin) - originsContainsPredicate = append(originsContainsPredicate, pred) - } - joinPredicate = append(joinPredicate, sql.Or(originsContainsPredicate...)) case "scenarios_containing": predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold) query = query.Where(decision.Or(predicates...)) - - scenarios := strings.Split(value[0], ",") - scenariosContainsPredicate := make([]*sql.Predicate, 0) - for _, scenario := range scenarios { - pred := sql.ContainsFold(t.C(decision.FieldScenario), scenario) - scenariosContainsPredicate = append(scenariosContainsPredicate, pred) - } - joinPredicate = append(joinPredicate, sql.Or(scenariosContainsPredicate...)) case "scenarios_not_containing": predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold) query = query.Where(decision.Not( @@ -100,26 +81,79 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string] predicates..., ), )) - scenarios := strings.Split(value[0], ",") - scenariosContainsPredicate := make([]*sql.Predicate, 0) - for _, scenario := range scenarios { - pred := sql.ContainsFold(t.C(decision.FieldScenario), scenario) - scenariosContainsPredicate = append(scenariosContainsPredicate, sql.Not(pred)) - } - joinPredicate = append(joinPredicate, sql.Or(scenariosContainsPredicate...)) case "ip", "range": ip_sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(value[0]) if err != nil { - return nil, nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err) + return nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err) } } } - query, err = applyStartIpEndIpFilter(query, contains, ip_sz, start_ip, start_sfx, end_ip, end_sfx) if err != nil { - return nil, nil, errors.Wrapf(err, "fail to apply StartIpEndIpFilter") + return nil, errors.Wrapf(err, "fail to apply StartIpEndIpFilter") } - return query, joinPredicate, nil + return query, nil +} +func (c *Client) QueryAllDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) { + query := c.Ent.Decision.Query().Where( + decision.UntilGT(time.Now().UTC()), + longestDecisionForScopeTypeValue, + ) + query, err := BuildDecisionRequestWithFilter(query, filters) + + if err != nil { + c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err) + return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters") + } + + data, err := query.All(c.CTX) + if err != nil { + c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err) + return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters") + } + return data, nil +} + +func (c *Client) QueryExpiredDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) { + query := c.Ent.Decision.Query().Where( + decision.UntilLT(time.Now().UTC()), + longestDecisionForScopeTypeValue, + ) + query, err := BuildDecisionRequestWithFilter(query, filters) + + if err != nil { + c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err) + return []*ent.Decision{}, errors.Wrap(QueryFail, "get expired decisions with filters") + } + data, err := query.All(c.CTX) + if err != nil { + c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err) + return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions") + } + return data, nil +} + +func (c *Client) QueryDecisionCountByScenario(filters map[string][]string) ([]*DecisionsByScenario, error) { + query := c.Ent.Decision.Query().Where( + decision.UntilGT(time.Now().UTC()), + ) + query, err := BuildDecisionRequestWithFilter(query, filters) + + if err != nil { + c.Log.Warningf("QueryDecisionCountByScenario : %s", err) + return nil, errors.Wrap(QueryFail, "count all decisions with filters") + } + + var r []*DecisionsByScenario + + err = query.GroupBy(decision.FieldScenario, decision.FieldOrigin, decision.FieldType).Aggregate(ent.Count()).Scan(c.CTX, &r) + + if err != nil { + c.Log.Warningf("QueryDecisionCountByScenario : %s", err) + return nil, errors.Wrap(QueryFail, "count all decisions with filters") + } + + return r, nil } func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Decision, error) { @@ -129,7 +163,7 @@ func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Dec decisions := c.Ent.Decision.Query(). Where(decision.UntilGTE(time.Now().UTC())) - decisions, _, err = BuildDecisionRequestWithFilter(decisions, filter) + decisions, err = BuildDecisionRequestWithFilter(decisions, filter) if err != nil { return []*ent.Decision{}, err } @@ -153,141 +187,67 @@ func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Dec return data, nil } -func (c *Client) QueryAllDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) { - query := c.Ent.Decision.Query().Where( - decision.UntilGT(time.Now().UTC()), +// ent translation of https://stackoverflow.com/a/28090544 +func longestDecisionForScopeTypeValue(s *sql.Selector) { + t := sql.Table(decision.Table) + s.LeftJoin(t).OnP(sql.And( + sql.ColumnsEQ( + t.C(decision.FieldValue), + s.C(decision.FieldValue), + ), + sql.ColumnsEQ( + t.C(decision.FieldType), + s.C(decision.FieldType), + ), + sql.ColumnsEQ( + t.C(decision.FieldScope), + s.C(decision.FieldScope), + ), + sql.ColumnsGT( + t.C(decision.FieldUntil), + s.C(decision.FieldUntil), + ), + )) + s.Where( + sql.IsNull( + t.C(decision.FieldUntil), + ), ) - query, _, err := BuildDecisionRequestWithFilter(query, filters) - if err != nil { - c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err) - return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters") - } - - //Order is *very* important, the dedup assumes that decisions are sorted per IP and per time left - data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX) - if err != nil { - c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err) - return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters") - } - - return data, nil } -func (c *Client) QueryDecisionCountByScenario(filters map[string][]string) ([]*DecisionsByScenario, error) { - query := c.Ent.Decision.Query().Where( - decision.UntilGT(time.Now().UTC()), - ) - query, _, err := BuildDecisionRequestWithFilter(query, filters) - - if err != nil { - c.Log.Warningf("QueryDecisionCountByScenario : %s", err) - return nil, errors.Wrap(QueryFail, "count all decisions with filters") - } - - var r []*DecisionsByScenario - - err = query.GroupBy(decision.FieldScenario, decision.FieldOrigin, decision.FieldType).Aggregate(ent.Count()).Scan(c.CTX, &r) - - if err != nil { - c.Log.Warningf("QueryDecisionCountByScenario : %s", err) - return nil, errors.Wrap(QueryFail, "count all decisions with filters") - } - - return r, nil -} - -func (c *Client) QueryExpiredDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) { - now := time.Now().UTC() +func (c *Client) QueryExpiredDecisionsSinceWithFilters(since time.Time, filters map[string][]string) ([]*ent.Decision, error) { query := c.Ent.Decision.Query().Where( decision.UntilLT(time.Now().UTC()), + decision.UntilGT(since), + longestDecisionForScopeTypeValue, ) - query, predicates, err := BuildDecisionRequestWithFilter(query, filters) + query, err := BuildDecisionRequestWithFilter(query, filters) if err != nil { - c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err) - return []*ent.Decision{}, errors.Wrap(QueryFail, "get expired decisions with filters") + c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err) + return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters") } - query = query.Where(func(s *sql.Selector) { - t := sql.Table(decision.Table).As("t1") - subQuery := sql.Select(t.C(decision.FieldValue)).From(t).Where(sql.GT(t.C(decision.FieldUntil), now)) - for _, predicate := range predicates { - subQuery.Where(predicate) - } - subQuery.Where(sql.And( - sql.ColumnsEQ(t.C(decision.FieldType), s.C(decision.FieldType)), - sql.ColumnsEQ(t.C(decision.FieldScope), s.C(decision.FieldScope)), - )) - s.Where( - sql.NotIn( - s.C(decision.FieldValue), - subQuery, - ), - ) - }) - - data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX) + data, err := query.All(c.CTX) if err != nil { - c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err) - return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions") + c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err) + return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters") } return data, nil } -//The "dedup" is not performed in SQL here because we suck at it, we do it in Go: -// - Get all decisions (expired or not) with an end time after the last pull from the bouncer -// - Sort them by increasing expiration date -// - Iterate over them, keeping only decisions that have expired but only if we don't have an active decision with the same scope/value/type -func (c *Client) QueryExpiredDecisionsSinceWithFilters(since time.Time, filters map[string][]string) ([]*ent.Decision, error) { - now := time.Now().UTC() - - query := c.Ent.Decision.Query().Where( - decision.UntilGT(since), - ) - query, _, err := BuildDecisionRequestWithFilter(query, filters) - if err != nil { - c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err) - return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters") - } - - data, err := query.Order(ent.Asc(decision.FieldValue), ent.Asc(decision.FieldUntil)).All(c.CTX) - if err != nil { - c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err) - return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters") - } - - ret := make([]*ent.Decision, 0) - deletedDecisions := make(map[string]*ent.Decision) - for _, d := range data { - key := fmt.Sprintf("%s:%s:%s", d.Scope, d.Type, d.Value) - if d.Until.Before(now) { - deletedDecisions[key] = d - } - if d.Until.After(now) { - delete(deletedDecisions, key) - } - } - - for _, d := range deletedDecisions { - ret = append(ret, d) - } - - return ret, nil -} - func (c *Client) QueryNewDecisionsSinceWithFilters(since time.Time, filters map[string][]string) ([]*ent.Decision, error) { query := c.Ent.Decision.Query().Where( decision.CreatedAtGT(since), decision.UntilGT(time.Now().UTC()), + longestDecisionForScopeTypeValue, ) - query, _, err := BuildDecisionRequestWithFilter(query, filters) + query, err := BuildDecisionRequestWithFilter(query, filters) if err != nil { - c.Log.Warningf("BuildDecisionRequestWithFilter : %s", err) - return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters") + c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err) + return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String()) } - - //Order is *very* important, the dedup assumes that decisions are sorted per IP and per time left - data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX) + data, err := query.All(c.CTX) if err != nil { c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err) return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String()) @@ -335,6 +295,7 @@ func (c *Client) DeleteDecisionsWithFilter(filter map[string][]string) (string, return "0", errors.Wrap(InvalidFilter, fmt.Sprintf("'%s' doesn't exist", param)) } } + if ip_sz == 4 { if contains { /*decision contains {start_ip,end_ip}*/ decisions = decisions.Where(decision.And(