beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Revamp unit tests (#1368)

Browse source 
* Revamp unit tests
* Increase coverage
* Use go-acc to get cross packages coverage

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>
This commit is contained in:
Thibault "bui" Koechlin 2022-03-29 14:20:26 +02:00 committed by GitHub
parent 3f24bcdbcf
commit d8dc01cd94
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 8536 additions and 989 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.github/workflows/ci_go-test.yml vendored
View file

@ -62,11 +62,9 @@ jobs:
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Build
run: make build && go get -u github.com/jandelgado/gcov2lcov
- name: Build package
run: make package
run: make build && go get -u github.com/jandelgado/gcov2lcov && go get -u github.com/ory/go-acc
- name: All tests
run: go test -coverprofile=coverage.out -covermode=atomic ./...
run: go run github.com/ory/go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models
- name: gcov2lcov
uses: jandelgado/gcov2lcov-action@v1.0.2
with:

18
cmd/crowdsec-cli/explain.go
View file

@ -1,13 +1,13 @@
package main
import (
"bufio"
"fmt"
"os"
"os/exec"
"path/filepath"
"github.com/crowdsecurity/crowdsec/pkg/cstest"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
@ -63,7 +63,7 @@ cscli explain --dsn "file://myfile.log" --type nginx
log.Fatalf("unable to get absolue path of '%s', exiting", logFile)
}
dsn = fmt.Sprintf("file://%s", absolutePath)
lineCount := getLineCountForFile(absolutePath)
lineCount := types.GetLineCountForFile(absolutePath)
if lineCount > 100 {
log.Warnf("log file contains %d lines. This may take lot of resources.", lineCount)
}
@ -112,17 +112,3 @@ cscli explain --dsn "file://myfile.log" --type nginx
return cmdExplain
}
func getLineCountForFile(filepath string) int {
f, err := os.Open(filepath)
if err != nil {
log.Fatalf("unable to open log file %s", filepath)
}
defer f.Close()
lc := 0
fs := bufio.NewScanner(f)
for fs.Scan() {
lc++
}
return lc
}

3
go.mod
View file

@ -38,6 +38,7 @@ require (
github.com/hashicorp/go-version v1.2.1
github.com/influxdata/go-syslog/v3 v3.0.0
github.com/jackc/pgx/v4 v4.14.1
github.com/jarcoal/httpmock v1.1.0
github.com/jszwec/csvutil v1.5.1
github.com/lib/pq v1.10.4
github.com/mattn/go-sqlite3 v1.14.10
@ -148,7 +149,7 @@ require (
go.mongodb.org/mongo-driver v1.4.4 // indirect
golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220317022123-2c4bbad7e934 // indirect
golang.org/x/sys v0.0.0-20220318055525-2edf467146b5 // indirect
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/appengine v1.6.7 // indirect

6
go.sum
View file

@ -610,6 +610,8 @@ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0f
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jarcoal/httpmock v1.1.0 h1:F47ChZj1Y2zFsCXxNkBPwNNKnAyOATcdQibk0qEdVCE=
github.com/jarcoal/httpmock v1.1.0/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
@ -1253,8 +1255,8 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220317022123-2c4bbad7e934 h1:GwUTNnIS5asZGjc34dMBLO/LLp4kEvyZr/8wlQs1Bt8=
golang.org/x/sys v0.0.0-20220317022123-2c4bbad7e934/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220318055525-2edf467146b5 h1:saXMvIOKvRFwbOMicHXr0B1uwoxq9dGmLe5ExMES6c4=
golang.org/x/sys v0.0.0-20220318055525-2edf467146b5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=

5
pkg/apiclient/client.go
View file

@ -63,8 +63,11 @@ func NewClient(config *Config) (*ApiClient, error) {
func NewDefaultClient(URL *url.URL, prefix string, userAgent string, client *http.Client) (*ApiClient, error) {
if client == nil {
client = &http.Client{}
if ht, ok := http.DefaultTransport.(*http.Transport); ok {
ht.TLSClientConfig = &tls.Config{InsecureSkipVerify: InsecureSkipVerify}
client.Transport = ht
}
}
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: InsecureSkipVerify}
c := &ApiClient{client: client, BaseURL: URL, UserAgent: userAgent, URLPrefix: prefix}
c.common.client = c
c.Decisions = (*DecisionsService)(&c.common)

441
pkg/apiserver/alerts_test.go
View file

@ -3,13 +3,11 @@ package apiserver
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/csplugin"
"github.com/crowdsecurity/crowdsec/pkg/models"
@ -19,6 +17,48 @@ import (
"github.com/stretchr/testify/assert"
)
type LAPI struct {
router *gin.Engine
loginResp models.WatcherAuthResponse
bouncerKey string
t *testing.T
}
func SetupLAPITest(t *testing.T) LAPI {
t.Helper()
router, loginResp, err := InitMachineTest()
if err != nil {
t.Fatal(err.Error())
}
APIKey, err := CreateTestBouncer()
if err != nil {
t.Fatalf("%s", err.Error())
}
return LAPI{
router: router,
loginResp: loginResp,
bouncerKey: APIKey,
}
}
func (l *LAPI) InsertAlertFromFile(path string) *httptest.ResponseRecorder {
alertReader := GetAlertReaderFromFile(path)
return l.RecordResponse("POST", "/v1/alerts", alertReader)
}
func (l *LAPI) RecordResponse(verb string, url string, body *strings.Reader) *httptest.ResponseRecorder {
w := httptest.NewRecorder()
req, err := http.NewRequest(verb, url, body)
if err != nil {
l.t.Fatal(err)
}
req.Header.Add("X-Api-Key", l.bouncerKey)
AddAuthHeaders(req, l.loginResp)
l.router.ServeHTTP(w, req)
return w
}
func InitMachineTest() (*gin.Engine, models.WatcherAuthResponse, error) {
router, err := NewAPITest()
if err != nil {
@ -61,82 +101,40 @@ func AddAuthHeaders(request *http.Request, authResponse models.WatcherAuthRespon
}
func TestSimulatedAlert(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
alertContentBytes, err := ioutil.ReadFile("./tests/alert_minibulk+simul.json")
if err != nil {
log.Fatal(err)
}
alertContent := string(alertContentBytes)
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(alertContent))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
lapi := SetupLAPITest(t)
lapi.InsertAlertFromFile("./tests/alert_minibulk+simul.json")
alertContent := GetAlertReaderFromFile("./tests/alert_minibulk+simul.json")
//exclude decision in simulation mode
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?simulated=false", strings.NewReader(alertContent))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w := lapi.RecordResponse("GET", "/v1/alerts?simulated=false", alertContent)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), `"message":"Ip 91.121.79.178 performed crowdsecurity/ssh-bf (6 events over `)
assert.NotContains(t, w.Body.String(), `"message":"Ip 91.121.79.179 performed crowdsecurity/ssh-bf (6 events over `)
//include decision in simulation mode
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?simulated=true", strings.NewReader(alertContent))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?simulated=true", alertContent)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), `"message":"Ip 91.121.79.178 performed crowdsecurity/ssh-bf (6 events over `)
assert.Contains(t, w.Body.String(), `"message":"Ip 91.121.79.179 performed crowdsecurity/ssh-bf (6 events over `)
}
func TestCreateAlert(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
lapi := SetupLAPITest(t)
// Create Alert with invalid format
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader("test"))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w := lapi.RecordResponse("POST", "/v1/alerts", strings.NewReader("test"))
assert.Equal(t, 400, w.Code)
assert.Equal(t, "{\"message\":\"invalid character 'e' in literal true (expecting 'r')\"}", w.Body.String())
// Create Alert with invalid input
alertContentBytes, err := ioutil.ReadFile("./tests/invalidAlert_sample.json")
if err != nil {
log.Fatal(err)
}
alertContent := string(alertContentBytes)
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/alerts", strings.NewReader(alertContent))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
alertContent := GetAlertReaderFromFile("./tests/invalidAlert_sample.json")
w = lapi.RecordResponse("POST", "/v1/alerts", alertContent)
assert.Equal(t, 500, w.Code)
assert.Equal(t, "{\"message\":\"validation failure list:\\n0.scenario in body is required\\n0.scenario_hash in body is required\\n0.scenario_version in body is required\\n0.simulated in body is required\\n0.source in body is required\"}", w.Body.String())
// Create Valid Alert
alertContentBytes, err = ioutil.ReadFile("./tests/alert_sample.json")
if err != nil {
log.Fatal(err)
}
alertContent = string(alertContentBytes)
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/alerts", strings.NewReader(alertContent))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.InsertAlertFromFile("./tests/alert_sample.json")
assert.Equal(t, 201, w.Code)
assert.Equal(t, "[\"1\"]", w.Body.String())
}
@ -154,12 +152,7 @@ func TestCreateAlertChannels(t *testing.T) {
if err != nil {
log.Fatalln(err.Error())
}
alertContentBytes, err := ioutil.ReadFile("./tests/alert_ssh-bf.json")
if err != nil {
log.Fatal(err)
}
alertContent := string(alertContentBytes)
lapi := LAPI{router: apiServer.router, loginResp: loginResp}
var pd csplugin.ProfileAlert
var wg sync.WaitGroup
@ -170,389 +163,248 @@ func TestCreateAlertChannels(t *testing.T) {
wg.Done()
}()
go func() {
for {
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(alertContent))
AddAuthHeaders(req, loginResp)
apiServer.controller.Router.ServeHTTP(w, req)
break
}
}()
go lapi.InsertAlertFromFile("./tests/alert_ssh-bf.json")
wg.Wait()
assert.Equal(t, len(pd.Alert.Decisions), 1)
apiServer.Close()
}
func TestAlertListFilters(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
alertContentBytes, err := ioutil.ReadFile("./tests/alert_ssh-bf.json")
if err != nil {
log.Fatal(err)
}
alerts := make([]*models.Alert, 0)
if err := json.Unmarshal(alertContentBytes, &alerts); err != nil {
log.Fatal(err)
}
for _, alert := range alerts {
*alert.StartAt = time.Now().UTC().Format(time.RFC3339)
*alert.StopAt = time.Now().UTC().Format(time.RFC3339)
}
alertContent, err := json.Marshal(alerts)
if err != nil {
log.Fatal(err)
}
//create one alert
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(string(alertContent)))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
lapi := SetupLAPITest(t)
lapi.InsertAlertFromFile("./tests/alert_ssh-bf.json")
alertContent := GetAlertReaderFromFile("./tests/alert_ssh-bf.json")
//bad filter
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?test=test", strings.NewReader(string(alertContent)))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w := lapi.RecordResponse("GET", "/v1/alerts?test=test", alertContent)
assert.Equal(t, 500, w.Code)
assert.Equal(t, "{\"message\":\"Filter parameter 'test' is unknown (=test): invalid filter\"}", w.Body.String())
//get without filters
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts", emptyBody)
assert.Equal(t, 200, w.Code)
//check alert and decision
assert.Contains(t, w.Body.String(), "Ip 91.121.79.195 performed 'crowdsecurity/ssh-bf' (6 events over ")
assert.Contains(t, w.Body.String(), `scope":"Ip","simulated":false,"type":"ban","value":"91.121.79.195"`)
//test decision_type filter (ok)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?decision_type=ban", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?decision_type=ban", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "Ip 91.121.79.195 performed 'crowdsecurity/ssh-bf' (6 events over ")
assert.Contains(t, w.Body.String(), `scope":"Ip","simulated":false,"type":"ban","value":"91.121.79.195"`)
//test decision_type filter (bad value)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?decision_type=ratata", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?decision_type=ratata", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "null", w.Body.String())
//test scope (ok)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?scope=Ip", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?scope=Ip", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "Ip 91.121.79.195 performed 'crowdsecurity/ssh-bf' (6 events over ")
assert.Contains(t, w.Body.String(), `scope":"Ip","simulated":false,"type":"ban","value":"91.121.79.195"`)
//test scope (bad value)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?scope=rarara", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?scope=rarara", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "null", w.Body.String())
//test scenario (ok)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?scenario=crowdsecurity/ssh-bf", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?scenario=crowdsecurity/ssh-bf", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "Ip 91.121.79.195 performed 'crowdsecurity/ssh-bf' (6 events over ")
assert.Contains(t, w.Body.String(), `scope":"Ip","simulated":false,"type":"ban","value":"91.121.79.195"`)
//test scenario (bad value)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?scenario=crowdsecurity/nope", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?scenario=crowdsecurity/nope", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "null", w.Body.String())
//test ip (ok)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?ip=91.121.79.195", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?ip=91.121.79.195", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "Ip 91.121.79.195 performed 'crowdsecurity/ssh-bf' (6 events over ")
assert.Contains(t, w.Body.String(), `scope":"Ip","simulated":false,"type":"ban","value":"91.121.79.195"`)
//test ip (bad value)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?ip=99.122.77.195", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?ip=99.122.77.195", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "null", w.Body.String())
//test ip (invalid value)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?ip=gruueq", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?ip=gruueq", emptyBody)
assert.Equal(t, 500, w.Code)
assert.Equal(t, `{"message":"unable to convert 'gruueq' to int: invalid address: invalid ip address / range"}`, w.Body.String())
//test range (ok)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?range=91.121.79.0/24&contains=false", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?range=91.121.79.0/24&contains=false", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "Ip 91.121.79.195 performed 'crowdsecurity/ssh-bf' (6 events over ")
assert.Contains(t, w.Body.String(), `scope":"Ip","simulated":false,"type":"ban","value":"91.121.79.195"`)
//test range
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?range=99.122.77.0/24&contains=false", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?range=99.122.77.0/24&contains=false", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "null", w.Body.String())
//test range (invalid value)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?range=ratata", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?range=ratata", emptyBody)
assert.Equal(t, 500, w.Code)
assert.Equal(t, `{"message":"unable to convert 'ratata' to int: invalid address: invalid ip address / range"}`, w.Body.String())
//test since (ok)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?since=1h", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?since=1h", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "Ip 91.121.79.195 performed 'crowdsecurity/ssh-bf' (6 events over ")
assert.Contains(t, w.Body.String(), `scope":"Ip","simulated":false,"type":"ban","value":"91.121.79.195"`)
//test since (ok but yelds no results)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?since=1ns", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?since=1ns", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "null", w.Body.String())
//test since (invalid value)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?since=1zuzu", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?since=1zuzu", emptyBody)
assert.Equal(t, 500, w.Code)
assert.Contains(t, w.Body.String(), `{"message":"while parsing duration: time: unknown unit`)
//test until (ok)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?until=1ns", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?until=1ns", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "Ip 91.121.79.195 performed 'crowdsecurity/ssh-bf' (6 events over ")
assert.Contains(t, w.Body.String(), `scope":"Ip","simulated":false,"type":"ban","value":"91.121.79.195"`)
//test until (ok but no return)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?until=1m", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?until=1m", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "null", w.Body.String())
//test until (invalid value)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?until=1zuzu", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?until=1zuzu", emptyBody)
assert.Equal(t, 500, w.Code)
assert.Contains(t, w.Body.String(), `{"message":"while parsing duration: time: unknown unit`)
//test simulated (ok)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?simulated=true", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?simulated=true", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "Ip 91.121.79.195 performed 'crowdsecurity/ssh-bf' (6 events over ")
assert.Contains(t, w.Body.String(), `scope":"Ip","simulated":false,"type":"ban","value":"91.121.79.195"`)
//test simulated (ok)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?simulated=false", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?simulated=false", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "Ip 91.121.79.195 performed 'crowdsecurity/ssh-bf' (6 events over ")
assert.Contains(t, w.Body.String(), `scope":"Ip","simulated":false,"type":"ban","value":"91.121.79.195"`)
//test has active decision
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?has_active_decision=true", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?has_active_decision=true", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "Ip 91.121.79.195 performed 'crowdsecurity/ssh-bf' (6 events over ")
assert.Contains(t, w.Body.String(), `scope":"Ip","simulated":false,"type":"ban","value":"91.121.79.195"`)
//test has active decision
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?has_active_decision=false", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?has_active_decision=false", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "null", w.Body.String())
//test has active decision (invalid value)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?has_active_decision=ratatqata", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts?has_active_decision=ratatqata", emptyBody)
assert.Equal(t, 500, w.Code)
assert.Equal(t, `{"message":"'ratatqata' is not a boolean: strconv.ParseBool: parsing \"ratatqata\": invalid syntax: unable to parse type"}`, w.Body.String())
}
func TestAlertBulkInsert(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
lapi := SetupLAPITest(t)
//insert a bulk of 20 alerts to trigger bulk insert
alertContentBytes, err := ioutil.ReadFile("./tests/alert_bulk.json")
if err != nil {
log.Fatal(err)
}
alertContent := string(alertContentBytes)
lapi.InsertAlertFromFile("./tests/alert_bulk.json")
alertContent := GetAlertReaderFromFile("./tests/alert_bulk.json")
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(alertContent))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts", strings.NewReader(alertContent))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w := lapi.RecordResponse("GET", "/v1/alerts", alertContent)
assert.Equal(t, 200, w.Code)
}
func TestListAlert(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
alertContentBytes, err := ioutil.ReadFile("./tests/alert_sample.json")
if err != nil {
log.Fatal(err)
}
alertContent := string(alertContentBytes)
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(alertContent))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
lapi := SetupLAPITest(t)
lapi.InsertAlertFromFile("./tests/alert_sample.json")
// List Alert with invalid filter
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts?test=test", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w := lapi.RecordResponse("GET", "/v1/alerts?test=test", emptyBody)
assert.Equal(t, 500, w.Code)
assert.Equal(t, "{\"message\":\"Filter parameter 'test' is unknown (=test): invalid filter\"}", w.Body.String())
// List Alert
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/alerts", nil)
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/alerts", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "crowdsecurity/test")
}
func TestCreateAlertErrors(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
alertContentBytes, err := ioutil.ReadFile("./tests/alert_sample.json")
if err != nil {
log.Fatal(err)
}
alertContent := string(alertContentBytes)
lapi := SetupLAPITest(t)
alertContent := GetAlertReaderFromFile("./tests/alert_sample.json")
//test invalid bearer
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(alertContent))
req, _ := http.NewRequest("POST", "/v1/alerts", alertContent)
req.Header.Add("User-Agent", UserAgent)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", "ratata"))
router.ServeHTTP(w, req)
lapi.router.ServeHTTP(w, req)
assert.Equal(t, 401, w.Code)
//test invalid bearer
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/alerts", strings.NewReader(alertContent))
req, _ = http.NewRequest("POST", "/v1/alerts", alertContent)
req.Header.Add("User-Agent", UserAgent)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", loginResp.Token+"s"))
router.ServeHTTP(w, req)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", lapi.loginResp.Token+"s"))
lapi.router.ServeHTTP(w, req)
assert.Equal(t, 401, w.Code)
}
func TestDeleteAlert(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
alertContentBytes, err := ioutil.ReadFile("./tests/alert_sample.json")
if err != nil {
log.Fatal(err)
}
alertContent := string(alertContentBytes)
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(alertContent))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
lapi := SetupLAPITest(t)
lapi.InsertAlertFromFile("./tests/alert_sample.json")
// Fail Delete Alert
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/alerts", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
w := httptest.NewRecorder()
req, _ := http.NewRequest("DELETE", "/v1/alerts", strings.NewReader(""))
AddAuthHeaders(req, lapi.loginResp)
req.RemoteAddr = "127.0.0.2:4242"
router.ServeHTTP(w, req)
lapi.router.ServeHTTP(w, req)
assert.Equal(t, 403, w.Code)
assert.Equal(t, `{"message":"access forbidden from this IP (127.0.0.2)"}`, w.Body.String())
// Delete Alert
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/alerts", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
AddAuthHeaders(req, lapi.loginResp)
req.RemoteAddr = "127.0.0.1:4242"
router.ServeHTTP(w, req)
lapi.router.ServeHTTP(w, req)
assert.Equal(t, 200, w.Code)
assert.Equal(t, `{"nbDeleted":"1"}`, w.Body.String())
}
@ -579,17 +431,10 @@ func TestDeleteAlertTrustedIPS(t *testing.T) {
if err != nil {
log.Fatal(err.Error())
}
insertAlert := func() {
alertContentBytes, err := ioutil.ReadFile("./tests/alert_sample.json")
if err != nil {
log.Fatal(err)
}
alertContent := string(alertContentBytes)
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(alertContent))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
lapi := LAPI{
router: router,
loginResp: loginResp,
t: t,
}
assertAlertDeleteFailedFromIP := func(ip string) {
@ -598,6 +443,7 @@ func TestDeleteAlertTrustedIPS(t *testing.T) {
AddAuthHeaders(req, loginResp)
req.RemoteAddr = ip + ":1234"
router.ServeHTTP(w, req)
assert.Equal(t, 403, w.Code)
assert.Contains(t, w.Body.String(), fmt.Sprintf(`{"message":"access forbidden from this IP (%s)"}`, ip))
@ -608,23 +454,24 @@ func TestDeleteAlertTrustedIPS(t *testing.T) {
req, _ := http.NewRequest("DELETE", "/v1/alerts", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
req.RemoteAddr = ip + ":1234"
router.ServeHTTP(w, req)
assert.Equal(t, 200, w.Code)
assert.Equal(t, `{"nbDeleted":"1"}`, w.Body.String())
}
insertAlert()
lapi.InsertAlertFromFile("./tests/alert_sample.json")
assertAlertDeleteFailedFromIP("4.3.2.1")
assertAlertDeletedFromIP("1.2.3.4")
insertAlert()
lapi.InsertAlertFromFile("./tests/alert_sample.json")
assertAlertDeletedFromIP("1.2.4.0")
insertAlert()
lapi.InsertAlertFromFile("./tests/alert_sample.json")
assertAlertDeletedFromIP("1.2.4.1")
insertAlert()
lapi.InsertAlertFromFile("./tests/alert_sample.json")
assertAlertDeletedFromIP("1.2.4.255")
insertAlert()
lapi.InsertAlertFromFile("./tests/alert_sample.json")
assertAlertDeletedFromIP("127.0.0.1")
}

375
pkg/apiserver/apic.go
View file

@ -24,12 +24,16 @@ import (
"gopkg.in/tomb.v2"
)
const (
PullInterval = "2h"
PushInterval = "30s"
MetricsInterval = "30m"
var (
PullInterval = time.Hour * 2
PushInterval = time.Second * 30
MetricsInterval = time.Minute * 30
)
var SCOPE_CAPI string = "CAPI"
var SCOPE_CAPI_ALIAS string = "crowdsecurity/community-blocklist" //we don't use "CAPI" directly, to make it less confusing for the user
var SCOPE_LISTS string = "lists"
type apic struct {
pullInterval time.Duration
pushInterval time.Duration
@ -47,15 +51,6 @@ type apic struct {
consoleConfig *csconfig.ConsoleConfig
}
func IsInSlice(a string, b []string) bool {
for _, v := range b {
if a == v {
return true
}
}
return false
}
func (a *apic) FetchScenariosListFromDB() ([]string, error) {
scenarios := make([]string, 0)
machines, err := a.dbClient.ListMachines()
@ -67,7 +62,7 @@ func (a *apic) FetchScenariosListFromDB() ([]string, error) {
machineScenarios := strings.Split(v.Scenarios, ",")
log.Debugf("%d scenarios for machine %d", len(machineScenarios), v.ID)
for _, sv := range machineScenarios {
if !IsInSlice(sv, scenarios) && sv != "" {
if !types.InSlice(sv, scenarios) && sv != "" {
scenarios = append(scenarios, sv)
}
}
@ -76,7 +71,7 @@ func (a *apic) FetchScenariosListFromDB() ([]string, error) {
return scenarios, nil
}
func AlertToSignal(alert *models.Alert, scenarioTrust string) *models.AddSignalsRequestItem {
func alertToSignal(alert *models.Alert, scenarioTrust string) *models.AddSignalsRequestItem {
return &models.AddSignalsRequestItem{
Message: alert.Message,
Scenario: alert.Scenario,
@ -94,29 +89,19 @@ func AlertToSignal(alert *models.Alert, scenarioTrust string) *models.AddSignals
func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, consoleConfig *csconfig.ConsoleConfig) (*apic, error) {
var err error
ret := &apic{
alertToPush: make(chan []*models.Alert),
dbClient: dbClient,
mu: sync.Mutex{},
startup: true,
credentials: config.Credentials,
pullTomb: tomb.Tomb{},
pushTomb: tomb.Tomb{},
metricsTomb: tomb.Tomb{},
scenarioList: make([]string, 0),
consoleConfig: consoleConfig,
}
ret.pullInterval, err = time.ParseDuration(PullInterval)
if err != nil {
return ret, err
}
ret.pushInterval, err = time.ParseDuration(PushInterval)
if err != nil {
return ret, err
}
ret.metricsInterval, err = time.ParseDuration(MetricsInterval)
if err != nil {
return ret, err
alertToPush: make(chan []*models.Alert),
dbClient: dbClient,
mu: sync.Mutex{},
startup: true,
credentials: config.Credentials,
pullTomb: tomb.Tomb{},
pushTomb: tomb.Tomb{},
metricsTomb: tomb.Tomb{},
scenarioList: make([]string, 0),
consoleConfig: consoleConfig,
pullInterval: PullInterval,
pushInterval: PushInterval,
metricsInterval: MetricsInterval,
}
password := strfmt.Password(config.Credentials.Password)
@ -140,6 +125,7 @@ func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, con
return ret, err
}
// keep track of all alerts in cache and push it to CAPI every PushInterval.
func (a *apic) Push() error {
defer types.CatchPanic("lapi/pushToAPIC")
@ -170,39 +156,9 @@ func (a *apic) Push() error {
case alerts := <-a.alertToPush:
var signals []*models.AddSignalsRequestItem
for _, alert := range alerts {
if *alert.Simulated {
log.Debugf("simulation enabled for alert (id:%d), will not be sent to CAPI", alert.ID)
continue
if ok := shouldShareAlert(alert, a.consoleConfig); ok {
signals = append(signals, alertToSignal(alert, getScenarioTrustOfAlert(alert)))
}
scenarioTrust := "certified"
if alert.ScenarioHash == nil || *alert.ScenarioHash == "" {
scenarioTrust = "custom"
} else if alert.ScenarioVersion == nil || *alert.ScenarioVersion == "" || *alert.ScenarioVersion == "?" {
scenarioTrust = "tainted"
}
if len(alert.Decisions) > 0 {
if *alert.Decisions[0].Origin == "cscli" {
scenarioTrust = "manual"
}
}
switch scenarioTrust {
case "manual":
if !*a.consoleConfig.ShareManualDecisions {
log.Debugf("manual decision generated an alert, doesn't send it to CAPI because options is disabled")
continue
}
case "tainted":
if !*a.consoleConfig.ShareTaintedScenarios {
log.Debugf("tainted scenario generated an alert, doesn't send it to CAPI because options is disabled")
continue
}
case "custom":
if !*a.consoleConfig.ShareCustomScenarios {
log.Debugf("custom scenario generated an alert, doesn't send it to CAPI because options is disabled")
continue
}
}
signals = append(signals, AlertToSignal(alert, scenarioTrust))
}
a.mu.Lock()
cache = append(cache, signals...)
@ -211,6 +167,46 @@ func (a *apic) Push() error {
}
}
func getScenarioTrustOfAlert(alert *models.Alert) string {
scenarioTrust := "certified"
if alert.ScenarioHash == nil || *alert.ScenarioHash == "" {
scenarioTrust = "custom"
} else if alert.ScenarioVersion == nil || *alert.ScenarioVersion == "" || *alert.ScenarioVersion == "?" {
scenarioTrust = "tainted"
}
if len(alert.Decisions) > 0 {
if *alert.Decisions[0].Origin == "cscli" {
scenarioTrust = "manual"
}
}
return scenarioTrust
}
func shouldShareAlert(alert *models.Alert, consoleConfig *csconfig.ConsoleConfig) bool {
if *alert.Simulated {
log.Debugf("simulation enabled for alert (id:%d), will not be sent to CAPI", alert.ID)
return false
}
switch scenarioTrust := getScenarioTrustOfAlert(alert); scenarioTrust {
case "manual":
if !*consoleConfig.ShareManualDecisions {
log.Debugf("manual decision generated an alert, doesn't send it to CAPI because options is disabled")
return false
}
case "tainted":
if !*consoleConfig.ShareTaintedScenarios {
log.Debugf("tainted scenario generated an alert, doesn't send it to CAPI because options is disabled")
return false
}
case "custom":
if !*consoleConfig.ShareCustomScenarios {
log.Debugf("custom scenario generated an alert, doesn't send it to CAPI because options is disabled")
return false
}
}
return true
}
func (a *apic) Send(cacheOrig *models.AddSignalsRequest) {
/*we do have a problem with this :
The apic.Push background routine reads from alertToPush chan.
@ -256,54 +252,26 @@ func (a *apic) Send(cacheOrig *models.AddSignalsRequest) {
}
}
var SCOPE_CAPI string = "CAPI"
var SCOPE_CAPI_ALIAS string = "crowdsecurity/community-blocklist" //we don't use "CAPI" directly, to make it less confusing for the user
var SCOPE_LISTS string = "lists"
func (a *apic) PullTop() error {
var err error
func (a *apic) CAPIPullIsOld() (bool, error) {
/*only pull community blocklist if it's older than 1h30 */
alerts := a.dbClient.Ent.Alert.Query()
alerts = alerts.Where(alert.HasDecisionsWith(decision.OriginEQ(database.CapiMachineID)))
alerts = alerts.Where(alert.CreatedAtGTE(time.Now().UTC().Add(-time.Duration(1*time.Hour + 30*time.Minute))))
count, err := alerts.Count(a.dbClient.CTX)
if err != nil {
return errors.Wrap(err, "while looking for CAPI alert")
return false, errors.Wrap(err, "while looking for CAPI alert")
}
if count > 0 {
log.Printf("last CAPI pull is newer than 1h30, skip.")
return nil
return false, nil
}
data, _, err := a.apiClient.Decisions.GetStream(context.Background(), apiclient.DecisionsStreamOpts{Startup: a.startup})
if err != nil {
return errors.Wrap(err, "get stream")
}
if a.startup {
a.startup = false
}
/*to count additions/deletions accross lists*/
var add_counters map[string]map[string]int
var delete_counters map[string]map[string]int
return true, nil
}
add_counters = make(map[string]map[string]int)
add_counters[SCOPE_CAPI] = make(map[string]int)
add_counters[SCOPE_LISTS] = make(map[string]int)
delete_counters = make(map[string]map[string]int)
delete_counters[SCOPE_CAPI] = make(map[string]int)
delete_counters[SCOPE_LISTS] = make(map[string]int)
func (a *apic) HandleDeletedDecisions(deletedDecisions []*models.Decision, delete_counters map[string]map[string]int) (int, error) {
var filter map[string][]string
var nbDeleted int
// process deleted decisions
for _, decision := range data.Deleted {
//count individual deletions
if *decision.Origin == SCOPE_CAPI {
delete_counters[SCOPE_CAPI][*decision.Scenario]++
} else if *decision.Origin == SCOPE_LISTS {
delete_counters[SCOPE_LISTS][*decision.Scenario]++
} else {
log.Warningf("Unknown origin %s", *decision.Origin)
}
for _, decision := range deletedDecisions {
if strings.ToLower(*decision.Scope) == "ip" {
filter = make(map[string][]string, 1)
filter["value"] = []string{*decision.Value}
@ -311,36 +279,30 @@ func (a *apic) PullTop() error {
filter = make(map[string][]string, 3)
filter["value"] = []string{*decision.Value}
filter["type"] = []string{*decision.Type}
filter["value"] = []string{*decision.Scope}
filter["scopes"] = []string{*decision.Scope}
}
filter["origin"] = []string{*decision.Origin}
dbCliRet, err := a.dbClient.SoftDeleteDecisionsWithFilter(filter)
if err != nil {
return errors.Wrap(err, "deleting decisions error")
return 0, errors.Wrap(err, "deleting decisions error")
}
dbCliDel, err := strconv.Atoi(dbCliRet)
if err != nil {
return errors.Wrapf(err, "converting db ret %d", dbCliDel)
return 0, errors.Wrapf(err, "converting db ret %d", dbCliDel)
}
updateCounterForDecision(delete_counters, decision, dbCliDel)
nbDeleted += dbCliDel
}
log.Printf("capi/community-blocklist : %d explicit deletions", nbDeleted)
return nbDeleted, nil
if len(data.New) == 0 {
log.Warnf("capi/community-blocklist : received 0 new entries, CAPI failure ?")
return nil
}
}
//we receive only one list of decisions, that we need to break-up :
// one alert for "community blocklist"
// one alert per list we're subscribed to
var alertsFromCapi []*models.Alert
alertsFromCapi = make([]*models.Alert, 0)
//iterate over all new decisions, and simply create corresponding alerts
for _, decision := range data.New {
func createAlertsForDecisions(decisions []*models.Decision) []*models.Alert {
newAlerts := make([]*models.Alert, 0)
for _, decision := range decisions {
found := false
for _, sub := range alertsFromCapi {
for _, sub := range newAlerts {
if sub.Source.Scope == nil {
log.Warningf("nil scope in %+v", sub)
continue
@ -366,42 +328,44 @@ func (a *apic) PullTop() error {
}
if !found {
log.Debugf("Create entry for origin:%s scenario:%s", *decision.Origin, *decision.Scenario)
newAlert := models.Alert{}
newAlert.Message = types.StrPtr("")
newAlert.Source = &models.Source{}
if *decision.Origin == SCOPE_CAPI { //to make things more user friendly, we replace CAPI with community-blocklist
newAlert.Source.Scope = types.StrPtr(SCOPE_CAPI)
newAlert.Scenario = types.StrPtr(SCOPE_CAPI)
} else if *decision.Origin == SCOPE_LISTS {
newAlert.Source.Scope = types.StrPtr(SCOPE_LISTS)
newAlert.Scenario = types.StrPtr(*decision.Scenario)
} else {
log.Warningf("unknown origin %s", *decision.Origin)
}
newAlert.Source.Value = types.StrPtr("")
newAlert.StartAt = types.StrPtr(time.Now().UTC().Format(time.RFC3339))
newAlert.StopAt = types.StrPtr(time.Now().UTC().Format(time.RFC3339))
newAlert.Capacity = types.Int32Ptr(0)
newAlert.Simulated = types.BoolPtr(false)
newAlert.EventsCount = types.Int32Ptr(int32(len(data.New)))
newAlert.Leakspeed = types.StrPtr("")
newAlert.ScenarioHash = types.StrPtr("")
newAlert.ScenarioVersion = types.StrPtr("")
newAlert.MachineID = database.CapiMachineID
alertsFromCapi = append(alertsFromCapi, &newAlert)
newAlerts = append(newAlerts, createAlertForDecision(decision))
}
}
return newAlerts
}
//iterate a second time and fill the alerts with the new decisions
for _, decision := range data.New {
func createAlertForDecision(decision *models.Decision) *models.Alert {
newAlert := &models.Alert{}
newAlert.Source = &models.Source{}
newAlert.Source.Scope = types.StrPtr("")
if *decision.Origin == SCOPE_CAPI { //to make things more user friendly, we replace CAPI with community-blocklist
newAlert.Scenario = types.StrPtr(SCOPE_CAPI)
newAlert.Source.Scope = types.StrPtr(SCOPE_CAPI)
} else if *decision.Origin == SCOPE_LISTS {
newAlert.Scenario = types.StrPtr(*decision.Scenario)
newAlert.Source.Scope = types.StrPtr(SCOPE_LISTS)
} else {
log.Warningf("unknown origin %s", *decision.Origin)
}
newAlert.Message = types.StrPtr("")
newAlert.Source.Value = types.StrPtr("")
newAlert.StartAt = types.StrPtr(time.Now().UTC().Format(time.RFC3339))
newAlert.StopAt = types.StrPtr(time.Now().UTC().Format(time.RFC3339))
newAlert.Capacity = types.Int32Ptr(0)
newAlert.Simulated = types.BoolPtr(false)
newAlert.EventsCount = types.Int32Ptr(0)
newAlert.Leakspeed = types.StrPtr("")
newAlert.ScenarioHash = types.StrPtr("")
newAlert.ScenarioVersion = types.StrPtr("")
newAlert.MachineID = database.CapiMachineID
return newAlert
}
// This function takes in list of parent alerts and decisions and then pairs them up.
func fillAlertsWithDecisions(alerts []*models.Alert, decisions []*models.Decision, add_counters map[string]map[string]int) []*models.Alert {
for _, decision := range decisions {
//count and create separate alerts for each list
if *decision.Origin == SCOPE_CAPI {
add_counters[SCOPE_CAPI]["all"]++
} else if *decision.Origin == SCOPE_LISTS {
add_counters[SCOPE_LISTS][*decision.Scenario]++
} else {
log.Warningf("Unknown origin %s", *decision.Origin)
}
updateCounterForDecision(add_counters, decision, 1)
/*CAPI might send lower case scopes, unify it.*/
switch strings.ToLower(*decision.Scope) {
@ -412,16 +376,16 @@ func (a *apic) PullTop() error {
}
found := false
//add the individual decisions to the right list
for idx, alert := range alertsFromCapi {
for idx, alert := range alerts {
if *decision.Origin == SCOPE_CAPI {
if *alert.Source.Scope == SCOPE_CAPI {
alertsFromCapi[idx].Decisions = append(alertsFromCapi[idx].Decisions, decision)
alerts[idx].Decisions = append(alerts[idx].Decisions, decision)
found = true
break
}
} else if *decision.Origin == SCOPE_LISTS {
if *alert.Source.Scope == SCOPE_LISTS && *alert.Scenario == *decision.Scenario {
alertsFromCapi[idx].Decisions = append(alertsFromCapi[idx].Decisions, decision)
alerts[idx].Decisions = append(alerts[idx].Decisions, decision)
found = true
break
}
@ -433,18 +397,49 @@ func (a *apic) PullTop() error {
log.Warningf("Orphaned decision for %s - %s", *decision.Origin, *decision.Scenario)
}
}
return alerts
}
//we receive only one list of decisions, that we need to break-up :
// one alert for "community blocklist"
// one alert per list we're subscribed to
func (a *apic) PullTop() error {
var err error
if lastPullIsOld, err := a.CAPIPullIsOld(); err != nil {
return err
} else if !lastPullIsOld {
return nil
}
data, _, err := a.apiClient.Decisions.GetStream(context.Background(), apiclient.DecisionsStreamOpts{Startup: a.startup})
if err != nil {
return errors.Wrap(err, "get stream")
}
a.startup = false
/*to count additions/deletions accross lists*/
add_counters, delete_counters := makeAddAndDeleteCounters()
// process deleted decisions
if nbDeleted, err := a.HandleDeletedDecisions(data.Deleted, delete_counters); err != nil {
return err
} else {
log.Printf("capi/community-blocklist : %d explicit deletions", nbDeleted)
}
if len(data.New) == 0 {
log.Warnf("capi/community-blocklist : received 0 new entries, CAPI failure ?")
return nil
}
//we receive only one list of decisions, that we need to break-up :
// one alert for "community blocklist"
// one alert per list we're subscribed to
alertsFromCapi := createAlertsForDecisions(data.New)
alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, data.New, add_counters)
for idx, alert := range alertsFromCapi {
formatted_update := ""
if *alertsFromCapi[idx].Source.Scope == SCOPE_CAPI {
*alertsFromCapi[idx].Source.Scope = SCOPE_CAPI_ALIAS
formatted_update = fmt.Sprintf("update : +%d/-%d IPs", add_counters[SCOPE_CAPI]["all"], delete_counters[SCOPE_CAPI]["all"])
} else if *alertsFromCapi[idx].Source.Scope == SCOPE_LISTS {
*alertsFromCapi[idx].Source.Scope = fmt.Sprintf("%s:%s", SCOPE_LISTS, *alertsFromCapi[idx].Scenario)
formatted_update = fmt.Sprintf("update : +%d/-%d IPs", add_counters[SCOPE_LISTS][*alert.Scenario], delete_counters[SCOPE_LISTS][*alert.Scenario])
}
alertsFromCapi[idx].Scenario = types.StrPtr(formatted_update)
alertsFromCapi[idx] = setAlertScenario(add_counters, delete_counters, alert)
log.Debugf("%s has %d decisions", *alertsFromCapi[idx].Source.Scope, len(alertsFromCapi[idx].Decisions))
alertID, inserted, deleted, err := a.dbClient.UpdateCommunityBlocklist(alertsFromCapi[idx])
if err != nil {
@ -455,14 +450,27 @@ func (a *apic) PullTop() error {
return nil
}
func setAlertScenario(add_counters map[string]map[string]int, delete_counters map[string]map[string]int, alert *models.Alert) *models.Alert {
if *alert.Source.Scope == SCOPE_CAPI {
*alert.Source.Scope = SCOPE_CAPI_ALIAS
alert.Scenario = types.StrPtr(fmt.Sprintf("update : +%d/-%d IPs", add_counters[SCOPE_CAPI]["all"], delete_counters[SCOPE_CAPI]["all"]))
} else if *alert.Source.Scope == SCOPE_LISTS {
*alert.Source.Scope = fmt.Sprintf("%s:%s", SCOPE_LISTS, *alert.Scenario)
alert.Scenario = types.StrPtr(fmt.Sprintf("update : +%d/-%d IPs", add_counters[SCOPE_LISTS][*alert.Scenario], delete_counters[SCOPE_LISTS][*alert.Scenario]))
}
return alert
}
func (a *apic) Pull() error {
defer types.CatchPanic("lapi/pullFromAPIC")
log.Infof("start crowdsec api pull (interval: %s)", PullInterval)
var err error
scenario := a.scenarioList
toldOnce := false
for {
scenario, err := a.FetchScenariosListFromDB()
if err != nil {
log.Errorf("unable to fetch scenarios from db: %s", err)
}
if len(scenario) > 0 {
break
}
@ -471,10 +479,6 @@ func (a *apic) Pull() error {
toldOnce = true
}
time.Sleep(1 * time.Second)
scenario, err = a.FetchScenariosListFromDB()
if err != nil {
log.Errorf("unable to fetch scenarios from db: %s", err)
}
}
if err := a.PullTop(); err != nil {
log.Errorf("capi pull top: %s", err)
@ -496,9 +500,8 @@ func (a *apic) Pull() error {
}
func (a *apic) GetMetrics() (*models.Metrics, error) {
version := cwversion.VersionStr()
metric := &models.Metrics{
ApilVersion: &version,
ApilVersion: types.StrPtr(cwversion.VersionStr()),
Machines: make([]*models.MetricsAgentInfo, 0),
Bouncers: make([]*models.MetricsBouncerInfo, 0),
}
@ -578,3 +581,25 @@ func (a *apic) Shutdown() {
a.pullTomb.Kill(nil)
a.metricsTomb.Kill(nil)
}
func makeAddAndDeleteCounters() (map[string]map[string]int, map[string]map[string]int) {
add_counters := make(map[string]map[string]int)
add_counters[SCOPE_CAPI] = make(map[string]int)
add_counters[SCOPE_LISTS] = make(map[string]int)
delete_counters := make(map[string]map[string]int)
delete_counters[SCOPE_CAPI] = make(map[string]int)
delete_counters[SCOPE_LISTS] = make(map[string]int)
return add_counters, delete_counters
}
func updateCounterForDecision(counter map[string]map[string]int, decision *models.Decision, totalDecisions int) {
if *decision.Origin == SCOPE_CAPI {
counter[*decision.Origin]["all"] += totalDecisions
return
} else if *decision.Origin == SCOPE_LISTS {
counter[*decision.Origin][*decision.Scenario] += totalDecisions
}
log.Warningf("Unknown origin %s", *decision.Origin)
}

956
pkg/apiserver/apic_test.go Normal file
View file

@ -0,0 +1,956 @@
package apiserver
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/url"
"os"
"reflect"
"sort"
"sync"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/database/ent/decision"
"github.com/crowdsecurity/crowdsec/pkg/database/ent/machine"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/jarcoal/httpmock"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"gopkg.in/tomb.v2"
)
func getDBClient(t *testing.T) *database.Client {
t.Helper()
dbPath, err := os.CreateTemp("", "*sqlite")
if err != nil {
t.Fatal(err)
}
dbClient, err := database.NewClient(&csconfig.DatabaseCfg{
Type: "sqlite",
DbName: "crowdsec",
DbPath: dbPath.Name(),
})
if err != nil {
t.Fatal(err)
}
return dbClient
}
func getAPIC(t *testing.T) *apic {
t.Helper()
dbClient := getDBClient(t)
return &apic{
alertToPush: make(chan []*models.Alert),
dbClient: dbClient,
mu: sync.Mutex{},
startup: true,
pullTomb: tomb.Tomb{},
pushTomb: tomb.Tomb{},
metricsTomb: tomb.Tomb{},
scenarioList: make([]string, 0),
consoleConfig: &csconfig.ConsoleConfig{
ShareManualDecisions: types.BoolPtr(false),
ShareTaintedScenarios: types.BoolPtr(false),
ShareCustomScenarios: types.BoolPtr(false),
},
}
}
func absDiff(a int, b int) (c int) {
if c = a - b; c < 0 {
return -1 * c
}
return c
}
func assertTotalDecisionCount(t *testing.T, dbClient *database.Client, count int) {
d := dbClient.Ent.Decision.Query().AllX(context.Background())
assert.Len(t, d, count)
}
func assertTotalValidDecisionCount(t *testing.T, dbClient *database.Client, count int) {
d := dbClient.Ent.Decision.Query().Where(
decision.UntilGT(time.Now()),
).AllX(context.Background())
assert.Len(t, d, count)
}
func jsonMarshalX(v interface{}) []byte {
data, err := json.Marshal(v)
if err != nil {
panic(err)
}
return data
}
func assertTotalAlertCount(t *testing.T, dbClient *database.Client, count int) {
d := dbClient.Ent.Alert.Query().AllX(context.Background())
assert.Len(t, d, count)
}
func TestAPICCAPIPullIsOld(t *testing.T) {
api := getAPIC(t)
isOld, err := api.CAPIPullIsOld()
if err != nil {
t.Fatal(err)
}
assert.True(t, isOld)
decision := api.dbClient.Ent.Decision.Create().
SetUntil(time.Now().Add(time.Hour)).
SetScenario("crowdsec/test").
SetType("IP").
SetScope("Country").
SetValue("Blah").
SetOrigin(SCOPE_CAPI).
SaveX(context.Background())
api.dbClient.Ent.Alert.Create().
SetCreatedAt(time.Now()).
SetScenario("crowdsec/test").
AddDecisions(
decision,
).
SaveX(context.Background())
isOld, err = api.CAPIPullIsOld()
if err != nil {
t.Fatal(err)
}
assert.False(t, isOld)
}
func TestAPICFetchScenariosListFromDB(t *testing.T) {
api := getAPIC(t)
testCases := []struct {
name string
machineIDsWithScenarios map[string]string
expectedScenarios []string
}{
{
name: "Simple one machine with two scenarios",
machineIDsWithScenarios: map[string]string{
"a": "crowdsecurity/http-bf,crowdsecurity/ssh-bf",
},
expectedScenarios: []string{"crowdsecurity/ssh-bf", "crowdsecurity/http-bf"},
},
{
name: "Multi machine with custom+hub scenarios",
machineIDsWithScenarios: map[string]string{
"a": "crowdsecurity/http-bf,crowdsecurity/ssh-bf,my_scenario",
"b": "crowdsecurity/http-bf,crowdsecurity/ssh-bf,foo_scenario",
},
expectedScenarios: []string{"crowdsecurity/ssh-bf", "crowdsecurity/http-bf", "my_scenario", "foo_scenario"},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
for machineID, scenarios := range tc.machineIDsWithScenarios {
api.dbClient.Ent.Machine.Create().
SetMachineId(machineID).
SetPassword(testPassword.String()).
SetIpAddress("1.2.3.4").
SetScenarios(scenarios).
ExecX(context.Background())
}
scenarios, err := api.FetchScenariosListFromDB()
for machineID := range tc.machineIDsWithScenarios {
api.dbClient.Ent.Machine.Delete().Where(machine.MachineIdEQ(machineID)).ExecX(context.Background())
}
if err != nil {
t.Fatal(err)
} else {
sort.Strings(scenarios)
sort.Strings(tc.expectedScenarios)
assert.Equal(t, scenarios, tc.expectedScenarios)
}
})
}
}
func TestNewAPIC(t *testing.T) {
var testConfig *csconfig.OnlineApiClientCfg
setConfig := func() {
testConfig = &csconfig.OnlineApiClientCfg{
Credentials: &csconfig.ApiCredentialsCfg{
URL: "foobar",
Login: "foo",
Password: "bar",
},
}
}
type args struct {
dbClient *database.Client
consoleConfig *csconfig.ConsoleConfig
}
tests := []struct {
name string
args args
wantErr bool
errorContains string
action func()
}{
{
name: "simple",
action: func() {},
args: args{
dbClient: getDBClient(t),
consoleConfig: LoadTestConfig().API.Server.ConsoleConfig,
},
},
{
name: "error in parsing URL",
action: func() { testConfig.Credentials.URL = "foobar http://" },
args: args{
dbClient: getDBClient(t),
consoleConfig: LoadTestConfig().API.Server.ConsoleConfig,
},
wantErr: true,
errorContains: "first path segment in URL cannot contain colon",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
setConfig()
tt.action()
_, err := NewAPIC(testConfig, tt.args.dbClient, tt.args.consoleConfig)
if tt.wantErr {
assert.ErrorContains(t, err, tt.errorContains)
} else {
assert.NoError(t, err)
}
})
}
}
func TestAPICHandleDeletedDecisions(t *testing.T) {
api := getAPIC(t)
_, deleteCounters := makeAddAndDeleteCounters()
decision1 := api.dbClient.Ent.Decision.Create().
SetUntil(time.Now().Add(time.Hour)).
SetScenario("crowdsec/test").
SetType("ban").
SetScope("IP").
SetValue("1.2.3.4").
SetOrigin(SCOPE_CAPI).
SaveX(context.Background())
api.dbClient.Ent.Decision.Create().
SetUntil(time.Now().Add(time.Hour)).
SetScenario("crowdsec/test").
SetType("ban").
SetScope("IP").
SetValue("1.2.3.4").
SetOrigin(SCOPE_CAPI).
SaveX(context.Background())
assertTotalDecisionCount(t, api.dbClient, 2)
nbDeleted, err := api.HandleDeletedDecisions([]*models.Decision{{
Value: types.StrPtr("1.2.3.4"),
Origin: &SCOPE_CAPI,
Type: &decision1.Type,
Scenario: types.StrPtr("crowdsec/test"),
Scope: types.StrPtr("IP"),
}}, deleteCounters)
assert.NoError(t, err)
assert.Equal(t, nbDeleted, 2)
assert.Equal(t, deleteCounters[SCOPE_CAPI]["all"], 2)
}
func TestAPICGetMetrics(t *testing.T) {
api := getAPIC(t)
cleanUp := func() {
api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background())
api.dbClient.Ent.Machine.Delete().ExecX(context.Background())
}
testCases := []struct {
name string
machineIDs []string
bouncers []string
expectedMetric *models.Metrics
}{
{
name: "simple",
machineIDs: []string{"a", "b", "c"},
bouncers: []string{"1", "2", "3"},
expectedMetric: &models.Metrics{
ApilVersion: types.StrPtr(cwversion.VersionStr()),
Bouncers: []*models.MetricsBouncerInfo{
{
CustomName: "1",
LastPull: time.Time{}.String(),
}, {
CustomName: "2",
LastPull: time.Time{}.String(),
}, {
CustomName: "3",
LastPull: time.Time{}.String(),
},
},
Machines: []*models.MetricsAgentInfo{
{
Name: "a",
LastPush: time.Time{}.String(),
LastUpdate: time.Time{}.String(),
},
{
Name: "b",
LastPush: time.Time{}.String(),
LastUpdate: time.Time{}.String(),
},
{
Name: "c",
LastPush: time.Time{}.String(),
LastUpdate: time.Time{}.String(),
},
},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
cleanUp()
for i, machineID := range testCase.machineIDs {
api.dbClient.Ent.Machine.Create().
SetMachineId(machineID).
SetPassword(testPassword.String()).
SetIpAddress(fmt.Sprintf("1.2.3.%d", i)).
SetScenarios("crowdsecurity/test").
SetLastPush(time.Time{}).
SetUpdatedAt(time.Time{}).
ExecX(context.Background())
}
for i, bouncerName := range testCase.bouncers {
api.dbClient.Ent.Bouncer.Create().
SetIPAddress(fmt.Sprintf("1.2.3.%d", i)).
SetName(bouncerName).
SetAPIKey("foobar").
SetRevoked(false).
SetLastPull(time.Time{}).
ExecX(context.Background())
}
if foundMetrics, err := api.GetMetrics(); err != nil {
t.Fatal(err)
} else {
assert.Equal(t, foundMetrics.Bouncers, testCase.expectedMetric.Bouncers)
assert.Equal(t, foundMetrics.Machines, testCase.expectedMetric.Machines)
}
})
}
}
func TestCreateAlertsForDecision(t *testing.T) {
httpBfDecisionList := &models.Decision{
Origin: &SCOPE_LISTS,
Scenario: types.StrPtr("crowdsecurity/http-bf"),
}
sshBfDecisionList := &models.Decision{
Origin: &SCOPE_LISTS,
Scenario: types.StrPtr("crowdsecurity/ssh-bf"),
}
httpBfDecisionCommunity := &models.Decision{
Origin: &SCOPE_CAPI,
Scenario: types.StrPtr("crowdsecurity/http-bf"),
}
sshBfDecisionCommunity := &models.Decision{
Origin: &SCOPE_CAPI,
Scenario: types.StrPtr("crowdsecurity/ssh-bf"),
}
type args struct {
decisions []*models.Decision
}
tests := []struct {
name string
args args
want []*models.Alert
}{
{
name: "2 decisions CAPI List Decisions should create 2 alerts",
args: args{
decisions: []*models.Decision{
httpBfDecisionList,
sshBfDecisionList,
},
},
want: []*models.Alert{
createAlertForDecision(httpBfDecisionList),
createAlertForDecision(sshBfDecisionList),
},
},
{
name: "2 decisions CAPI List same scenario decisions should create 1 alert",
args: args{
decisions: []*models.Decision{
httpBfDecisionList,
httpBfDecisionList,
},
},
want: []*models.Alert{
createAlertForDecision(httpBfDecisionList),
},
},
{
name: "5 decisions from community list should create 1 alert",
args: args{
decisions: []*models.Decision{
httpBfDecisionCommunity,
httpBfDecisionCommunity,
sshBfDecisionCommunity,
sshBfDecisionCommunity,
sshBfDecisionCommunity,
},
},
want: []*models.Alert{
createAlertForDecision(sshBfDecisionCommunity),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := createAlertsForDecisions(tt.args.decisions); !reflect.DeepEqual(got, tt.want) {
t.Errorf("createAlertsForDecisions() = %v, want %v", got, tt.want)
}
})
}
}
func TestFillAlertsWithDecisions(t *testing.T) {
httpBfDecisionCommunity := &models.Decision{
Origin: &SCOPE_CAPI,
Scenario: types.StrPtr("crowdsecurity/http-bf"),
Scope: types.StrPtr("ip"),
}
sshBfDecisionCommunity := &models.Decision{
Origin: &SCOPE_CAPI,
Scenario: types.StrPtr("crowdsecurity/ssh-bf"),
Scope: types.StrPtr("ip"),
}
httpBfDecisionList := &models.Decision{
Origin: &SCOPE_LISTS,
Scenario: types.StrPtr("crowdsecurity/http-bf"),
Scope: types.StrPtr("ip"),
}
sshBfDecisionList := &models.Decision{
Origin: &SCOPE_LISTS,
Scenario: types.StrPtr("crowdsecurity/ssh-bf"),
Scope: types.StrPtr("ip"),
}
type args struct {
alerts []*models.Alert
decisions []*models.Decision
}
tests := []struct {
name string
args args
want []*models.Alert
}{
{
name: "1 CAPI alert should pair up with n CAPI decisions",
args: args{
alerts: []*models.Alert{createAlertForDecision(httpBfDecisionCommunity)},
decisions: []*models.Decision{httpBfDecisionCommunity, sshBfDecisionCommunity, sshBfDecisionCommunity, httpBfDecisionCommunity},
},
want: []*models.Alert{
func() *models.Alert {
a := createAlertForDecision(httpBfDecisionCommunity)
a.Decisions = []*models.Decision{httpBfDecisionCommunity, sshBfDecisionCommunity, sshBfDecisionCommunity, httpBfDecisionCommunity}
return a
}(),
},
},
{
name: "List alert should pair up only with decisions having same scenario",
args: args{
alerts: []*models.Alert{createAlertForDecision(httpBfDecisionList), createAlertForDecision(sshBfDecisionList)},
decisions: []*models.Decision{httpBfDecisionList, httpBfDecisionList, sshBfDecisionList, sshBfDecisionList},
},
want: []*models.Alert{
func() *models.Alert {
a := createAlertForDecision(httpBfDecisionList)
a.Decisions = []*models.Decision{httpBfDecisionList, httpBfDecisionList}
return a
}(),
func() *models.Alert {
a := createAlertForDecision(sshBfDecisionList)
a.Decisions = []*models.Decision{sshBfDecisionList, sshBfDecisionList}
return a
}(),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
add_counters, _ := makeAddAndDeleteCounters()
if got := fillAlertsWithDecisions(tt.args.alerts, tt.args.decisions, add_counters); !reflect.DeepEqual(got, tt.want) {
t.Errorf("fillAlertsWithDecisions() = %v, want %v", got, tt.want)
}
})
}
}
func TestAPICPullTop(t *testing.T) {
api := getAPIC(t)
api.dbClient.Ent.Decision.Create().
SetOrigin(SCOPE_LISTS).
SetType("ban").
SetValue("9.9.9.9").
SetScope("Ip").
SetScenario("crowdsecurity/ssh-bf").
SetUntil(time.Now().Add(time.Hour)).
ExecX(context.Background())
assertTotalDecisionCount(t, api.dbClient, 1)
assertTotalValidDecisionCount(t, api.dbClient, 1)
httpmock.Activate()
defer httpmock.DeactivateAndReset()
httpmock.RegisterResponder("GET", "http://api.crowdsec.net/api/decisions/stream", httpmock.NewBytesResponder(
200, jsonMarshalX(
models.DecisionsStreamResponse{
Deleted: models.GetDecisionsResponse{
&models.Decision{
Origin: &SCOPE_LISTS,
Scenario: types.StrPtr("crowdsecurity/ssh-bf"),
Value: types.StrPtr("9.9.9.9"),
Scope: types.StrPtr("Ip"),
Duration: types.StrPtr("24h"),
Type: types.StrPtr("ban"),
}, // Thie is already present in DB
&models.Decision{
Origin: &SCOPE_LISTS,
Scenario: types.StrPtr("crowdsecurity/ssh-bf"),
Value: types.StrPtr("9.1.9.9"),
Scope: types.StrPtr("Ip"),
Duration: types.StrPtr("24h"),
Type: types.StrPtr("ban"),
}, // This not present in DB.
},
New: models.GetDecisionsResponse{
&models.Decision{
Origin: &SCOPE_CAPI,
Scenario: types.StrPtr("crowdsecurity/test1"),
Value: types.StrPtr("1.2.3.4"),
Scope: types.StrPtr("Ip"),
Duration: types.StrPtr("24h"),
Type: types.StrPtr("ban"),
},
&models.Decision{
Origin: &SCOPE_CAPI,
Scenario: types.StrPtr("crowdsecurity/test2"),
Value: types.StrPtr("1.2.3.5"),
Scope: types.StrPtr("Ip"),
Duration: types.StrPtr("24h"),
Type: types.StrPtr("ban"),
}, // These two are from community list.
&models.Decision{
Origin: &SCOPE_LISTS,
Scenario: types.StrPtr("crowdsecurity/http-bf"),
Value: types.StrPtr("1.2.3.6"),
Scope: types.StrPtr("Ip"),
Duration: types.StrPtr("24h"),
Type: types.StrPtr("ban"),
},
&models.Decision{
Origin: &SCOPE_LISTS,
Scenario: types.StrPtr("crowdsecurity/ssh-bf"),
Value: types.StrPtr("1.2.3.7"),
Scope: types.StrPtr("Ip"),
Duration: types.StrPtr("24h"),
Type: types.StrPtr("ban"),
}, // These two are from list subscription.
},
},
),
))
url, err := url.ParseRequestURI("http://api.crowdsec.net/")
if err != nil {
t.Fatal(err)
}
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
nil,
)
if err != nil {
t.Fatal(err)
}
api.apiClient = apic
err = api.PullTop()
if err != nil {
t.Fatal(err)
}
assertTotalDecisionCount(t, api.dbClient, 5)
assertTotalValidDecisionCount(t, api.dbClient, 4)
assertTotalAlertCount(t, api.dbClient, 3) // 2 for list sub , 1 for community list.
alerts := api.dbClient.Ent.Alert.Query().AllX(context.Background())
validDecisions := api.dbClient.Ent.Decision.Query().Where(
decision.UntilGT(time.Now())).
AllX(context.Background())
decisionScenarioFreq := make(map[string]int)
alertScenario := make(map[string]int)
for _, alert := range alerts {
alertScenario[alert.SourceScope]++
}
assert.Equal(t, len(alertScenario), 3)
assert.Equal(t, alertScenario[SCOPE_CAPI_ALIAS], 1)
assert.Equal(t, alertScenario["lists:crowdsecurity/ssh-bf"], 1)
assert.Equal(t, alertScenario["lists:crowdsecurity/http-bf"], 1)
for _, decisions := range validDecisions {
decisionScenarioFreq[decisions.Scenario]++
}
assert.Equal(t, decisionScenarioFreq["crowdsecurity/http-bf"], 1)
assert.Equal(t, decisionScenarioFreq["crowdsecurity/ssh-bf"], 1)
assert.Equal(t, decisionScenarioFreq["crowdsecurity/test1"], 1)
assert.Equal(t, decisionScenarioFreq["crowdsecurity/test2"], 1)
}
func TestAPICPush(t *testing.T) {
testCases := []struct {
name string
alerts []*models.Alert
expectedCalls int
}{
{
name: "simple single alert",
alerts: []*models.Alert{
{
Scenario: types.StrPtr("crowdsec/test"),
ScenarioHash: types.StrPtr("certified"),
ScenarioVersion: types.StrPtr("v1.0"),
Simulated: types.BoolPtr(false),
},
},
expectedCalls: 1,
},
{
name: "simulated alert is not pushed",
alerts: []*models.Alert{
{
Scenario: types.StrPtr("crowdsec/test"),
ScenarioHash: types.StrPtr("certified"),
ScenarioVersion: types.StrPtr("v1.0"),
Simulated: types.BoolPtr(true),
},
},
expectedCalls: 0,
},
{
name: "1 request per 50 alerts",
expectedCalls: 2,
alerts: func() []*models.Alert {
alerts := make([]*models.Alert, 100)
for i := 0; i < 100; i++ {
alerts[i] = &models.Alert{
Scenario: types.StrPtr("crowdsec/test"),
ScenarioHash: types.StrPtr("certified"),
ScenarioVersion: types.StrPtr("v1.0"),
Simulated: types.BoolPtr(false),
}
}
return alerts
}(),
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
api := getAPIC(t)
api.pushInterval = time.Millisecond
url, err := url.ParseRequestURI("http://api.crowdsec.net/")
if err != nil {
t.Fatal(err)
}
httpmock.Activate()
defer httpmock.DeactivateAndReset()
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
nil,
)
if err != nil {
t.Fatal(err)
}
api.apiClient = apic
httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/signals", httpmock.NewBytesResponder(200, []byte{}))
go func() {
api.alertToPush <- testCase.alerts
time.Sleep(time.Second)
api.Shutdown()
}()
if err := api.Push(); err != nil {
t.Fatal(err)
}
assert.Equal(t, httpmock.GetTotalCallCount(), testCase.expectedCalls)
})
}
}
func TestAPICSendMetrics(t *testing.T) {
api := getAPIC(t)
testCases := []struct {
name string
duration time.Duration
expectedCalls int
setUp func()
metricsInterval time.Duration
}{
{
name: "basic",
duration: time.Millisecond * 5,
metricsInterval: time.Millisecond,
expectedCalls: 5,
setUp: func() {},
},
{
name: "with some metrics",
duration: time.Millisecond * 5,
metricsInterval: time.Millisecond,
expectedCalls: 5,
setUp: func() {
api.dbClient.Ent.Machine.Create().
SetMachineId("1234").
SetPassword(testPassword.String()).
SetIpAddress("1.2.3.4").
SetScenarios("crowdsecurity/test").
SetLastPush(time.Time{}).
SetUpdatedAt(time.Time{}).
ExecX(context.Background())
api.dbClient.Ent.Bouncer.Create().
SetIPAddress("1.2.3.6").
SetName("someBouncer").
SetAPIKey("foobar").
SetRevoked(false).
SetLastPull(time.Time{}).
ExecX(context.Background())
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
api = getAPIC(t)
api.pushInterval = time.Millisecond
url, err := url.ParseRequestURI("http://api.crowdsec.net/")
if err != nil {
t.Fatal(err)
}
httpmock.Activate()
defer httpmock.DeactivateAndReset()
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
nil,
)
if err != nil {
t.Fatal(err)
}
api.apiClient = apic
api.metricsInterval = testCase.metricsInterval
httpmock.RegisterNoResponder(httpmock.NewBytesResponder(200, []byte{}))
testCase.setUp()
go func() {
if err := api.SendMetrics(); err != nil {
panic(err)
}
}()
time.Sleep(testCase.duration)
assert.LessOrEqual(t, absDiff(testCase.expectedCalls, httpmock.GetTotalCallCount()), 2)
})
}
}
func TestAPICPull(t *testing.T) {
api := getAPIC(t)
testCases := []struct {
name string
setUp func()
expectedDecisionCount int
logContains string
}{
{
name: "test pull if no scenarios are present",
setUp: func() {},
logContains: "scenario list is empty, will not pull yet",
},
{
name: "test pull",
setUp: func() {
api.dbClient.Ent.Machine.Create().
SetMachineId("1.2.3.4").
SetPassword(testPassword.String()).
SetIpAddress("1.2.3.4").
SetScenarios("crowdsecurity/ssh-bf").
ExecX(context.Background())
},
expectedDecisionCount: 1,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
api = getAPIC(t)
api.pullInterval = time.Millisecond
url, err := url.ParseRequestURI("http://api.crowdsec.net/")
if err != nil {
t.Fatal(err)
}
httpmock.Activate()
defer httpmock.DeactivateAndReset()
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
nil,
)
if err != nil {
t.Fatal(err)
}
api.apiClient = apic
httpmock.RegisterNoResponder(httpmock.NewBytesResponder(200, jsonMarshalX(
models.DecisionsStreamResponse{
New: models.GetDecisionsResponse{
&models.Decision{
Origin: &SCOPE_CAPI,
Scenario: types.StrPtr("crowdsecurity/test2"),
Value: types.StrPtr("1.2.3.5"),
Scope: types.StrPtr("Ip"),
Duration: types.StrPtr("24h"),
Type: types.StrPtr("ban"),
},
},
},
)))
testCase.setUp()
var buf bytes.Buffer
go func() {
logrus.SetOutput(&buf)
if err := api.Pull(); err != nil {
panic(err)
}
}()
time.Sleep(time.Millisecond * 10)
logrus.SetOutput(os.Stderr)
assert.Contains(t, buf.String(), testCase.logContains)
assertTotalDecisionCount(t, api.dbClient, testCase.expectedDecisionCount)
})
}
}
func TestShouldShareAlert(t *testing.T) {
testCases := []struct {
name string
consoleConfig *csconfig.ConsoleConfig
alert *models.Alert
expectedRet bool
expectedTrust string
}{
{
name: "custom alert should be shared if config enables it",
consoleConfig: &csconfig.ConsoleConfig{
ShareCustomScenarios: types.BoolPtr(true),
},
alert: &models.Alert{Simulated: types.BoolPtr(false)},
expectedRet: true,
expectedTrust: "custom",
},
{
name: "custom alert should not be shared if config disables it",
consoleConfig: &csconfig.ConsoleConfig{
ShareCustomScenarios: types.BoolPtr(false),
},
alert: &models.Alert{Simulated: types.BoolPtr(false)},
expectedRet: false,
expectedTrust: "custom",
},
{
name: "manual alert should be shared if config enables it",
consoleConfig: &csconfig.ConsoleConfig{
ShareManualDecisions: types.BoolPtr(true),
},
alert: &models.Alert{
Simulated: types.BoolPtr(false),
Decisions: []*models.Decision{{Origin: types.StrPtr("cscli")}},
},
expectedRet: true,
expectedTrust: "manual",
},
{
name: "manaul alert should not be shared if config disables it",
consoleConfig: &csconfig.ConsoleConfig{
ShareManualDecisions: types.BoolPtr(false),
},
alert: &models.Alert{
Simulated: types.BoolPtr(false),
Decisions: []*models.Decision{{Origin: types.StrPtr("cscli")}},
},
expectedRet: false,
expectedTrust: "manual",
},
{
name: "manual alert should be shared if config enables it",
consoleConfig: &csconfig.ConsoleConfig{
ShareTaintedScenarios: types.BoolPtr(true),
},
alert: &models.Alert{
Simulated: types.BoolPtr(false),
ScenarioHash: types.StrPtr("whateverHash"),
},
expectedRet: true,
expectedTrust: "tainted",
},
{
name: "manaul alert should not be shared if config disables it",
consoleConfig: &csconfig.ConsoleConfig{
ShareTaintedScenarios: types.BoolPtr(false),
},
alert: &models.Alert{
Simulated: types.BoolPtr(false),
ScenarioHash: types.StrPtr("whateverHash"),
},
expectedRet: false,
expectedTrust: "tainted",
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
ret := shouldShareAlert(testCase.alert, testCase.consoleConfig)
assert.Equal(t, ret, testCase.expectedRet)
})
}
}

81
pkg/apiserver/apiserver_test.go
View file

@ -3,7 +3,6 @@ package apiserver
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
@ -16,6 +15,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/database"
@ -33,6 +33,7 @@ var MachineTest = models.WatcherAuthRequest{
}
var UserAgent = fmt.Sprintf("crowdsec-test/%s", cwversion.Version)
var emptyBody = strings.NewReader("")
func LoadTestConfig() csconfig.Config {
config := csconfig.Config{}
@ -177,6 +178,79 @@ func GetMachineIP(machineID string) (string, error) {
return "", nil
}
func GetAlertReaderFromFile(path string) *strings.Reader {
alertContentBytes, err := os.ReadFile(path)
if err != nil {
log.Fatal(err)
}
alerts := make([]*models.Alert, 0)
if err := json.Unmarshal(alertContentBytes, &alerts); err != nil {
log.Fatal(err)
}
for _, alert := range alerts {
*alert.StartAt = time.Now().UTC().Format(time.RFC3339)
*alert.StopAt = time.Now().UTC().Format(time.RFC3339)
}
alertContent, err := json.Marshal(alerts)
if err != nil {
log.Fatal(err)
}
return strings.NewReader(string(alertContent))
}
func readDecisionsGetResp(resp *httptest.ResponseRecorder) ([]*models.Decision, int, error) {
var response []*models.Decision
if resp == nil {
return nil, 0, errors.New("response is nil")
}
err := json.Unmarshal(resp.Body.Bytes(), &response)
if err != nil {
return nil, resp.Code, err
}
return response, resp.Code, nil
}
func readDecisionsErrorResp(resp *httptest.ResponseRecorder) (map[string]string, int, error) {
var response map[string]string
if resp == nil {
return nil, 0, errors.New("response is nil")
}
err := json.Unmarshal(resp.Body.Bytes(), &response)
if err != nil {
return nil, resp.Code, err
}
return response, resp.Code, nil
}
func readDecisionsDeleteResp(resp *httptest.ResponseRecorder) (*models.DeleteDecisionResponse, int, error) {
var response models.DeleteDecisionResponse
if resp == nil {
return nil, 0, errors.New("response is nil")
}
err := json.Unmarshal(resp.Body.Bytes(), &response)
if err != nil {
return nil, resp.Code, err
}
return &response, resp.Code, nil
}
func readDecisionsStreamResp(resp *httptest.ResponseRecorder) (map[string][]*models.Decision, int, error) {
response := make(map[string][]*models.Decision)
if resp == nil {
return nil, 0, errors.New("response is nil")
}
err := json.Unmarshal(resp.Body.Bytes(), &response)
if err != nil {
return nil, resp.Code, err
}
return response, resp.Code, nil
}
func CreateTestMachine(router *gin.Engine) (string, error) {
b, err := json.Marshal(MachineTest)
if err != nil {
@ -306,7 +380,7 @@ func TestLoggingDebugToFileConfig(t *testing.T) {
time.Sleep(500 * time.Millisecond)
//check file content
data, err := ioutil.ReadFile(expectedFile)
data, err := os.ReadFile(expectedFile)
if err != nil {
t.Fatalf("failed to read file : %s", err)
}
@ -368,12 +442,11 @@ func TestLoggingErrorToFileConfig(t *testing.T) {
time.Sleep(500 * time.Millisecond)
//check file content
x, err := ioutil.ReadFile(expectedFile)
x, err := os.ReadFile(expectedFile)
if err == nil && len(x) > 0 {
t.Fatalf("file should be empty, got '%s'", x)
}
os.Remove("./crowdsec.log")
os.Remove(expectedFile)
}

752
pkg/apiserver/decisions_test.go
View file

@ -1,571 +1,429 @@
package apiserver
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/models"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func TestDeleteDecisionRange(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
lapi := SetupLAPITest(t)
// Create Valid Alert
alertContentBytes, err := ioutil.ReadFile("./tests/alert_minibulk.json")
if err != nil {
log.Fatal(err)
}
alerts := make([]*models.Alert, 0)
if err := json.Unmarshal(alertContentBytes, &alerts); err != nil {
log.Fatal(err)
}
for _, alert := range alerts {
*alert.StartAt = time.Now().UTC().Format(time.RFC3339)
*alert.StopAt = time.Now().UTC().Format(time.RFC3339)
}
alertContent, err := json.Marshal(alerts)
if err != nil {
log.Fatal(err)
}
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(string(alertContent)))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
lapi.InsertAlertFromFile("./tests/alert_minibulk.json")
// delete by ip wrong
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/decisions?range=1.2.3.0/24", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w := lapi.RecordResponse("DELETE", "/v1/decisions?range=1.2.3.0/24", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, `{"nbDeleted":"0"}`, w.Body.String())
// delete by range
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/decisions?range=91.121.79.0/24&contains=false", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("DELETE", "/v1/decisions?range=91.121.79.0/24&contains=false", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, `{"nbDeleted":"2"}`, w.Body.String())
// delete by range : ensure it was already deleted
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/decisions?range=91.121.79.0/24", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("DELETE", "/v1/decisions?range=91.121.79.0/24", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, `{"nbDeleted":"0"}`, w.Body.String())
}
func TestDeleteDecisionFilter(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
lapi := SetupLAPITest(t)
// Create Valid Alert
alertContentBytes, err := ioutil.ReadFile("./tests/alert_minibulk.json")
if err != nil {
log.Fatal(err)
}
alerts := make([]*models.Alert, 0)
if err := json.Unmarshal(alertContentBytes, &alerts); err != nil {
log.Fatal(err)
}
for _, alert := range alerts {
*alert.StartAt = time.Now().UTC().Format(time.RFC3339)
*alert.StopAt = time.Now().UTC().Format(time.RFC3339)
}
alertContent, err := json.Marshal(alerts)
if err != nil {
log.Fatal(err)
}
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(string(alertContent)))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
lapi.InsertAlertFromFile("./tests/alert_minibulk.json")
// delete by ip wrong
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/decisions?ip=1.2.3.4", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w := lapi.RecordResponse("DELETE", "/v1/decisions?ip=1.2.3.4", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, `{"nbDeleted":"0"}`, w.Body.String())
// delete by ip good
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/decisions?ip=91.121.79.179", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("DELETE", "/v1/decisions?ip=91.121.79.179", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, `{"nbDeleted":"1"}`, w.Body.String())
// delete by scope/value
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/decisions?scopes=Ip&value=91.121.79.178", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("DELETE", "/v1/decisions?scopes=Ip&value=91.121.79.178", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, `{"nbDeleted":"1"}`, w.Body.String())
}
func TestGetDecisionFilters(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
lapi := SetupLAPITest(t)
// Create Valid Alert
alertContentBytes, err := ioutil.ReadFile("./tests/alert_minibulk.json")
if err != nil {
log.Fatal(err)
}
alerts := make([]*models.Alert, 0)
if err := json.Unmarshal(alertContentBytes, &alerts); err != nil {
log.Fatal(err)
}
for _, alert := range alerts {
*alert.StartAt = time.Now().UTC().Format(time.RFC3339)
*alert.StopAt = time.Now().UTC().Format(time.RFC3339)
}
alertContent, err := json.Marshal(alerts)
if err != nil {
log.Fatal(err)
}
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(string(alertContent)))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
APIKey, err := CreateTestBouncer()
if err != nil {
log.Fatalf("%s", err.Error())
}
lapi.InsertAlertFromFile("./tests/alert_minibulk.json")
// Get Decision
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions", strings.NewReader(""))
req.Header.Add("User-Agent", UserAgent)
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
w := lapi.RecordResponse("GET", "/v1/decisions", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), `"id":1,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.179"`)
assert.Contains(t, w.Body.String(), `"id":2,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.178"`)
decisions, code, err := readDecisionsGetResp(w)
assert.Nil(t, err)
assert.Equal(t, 200, code)
assert.Equal(t, 2, len(decisions))
assert.Equal(t, "crowdsecurity/ssh-bf", *decisions[0].Scenario)
assert.Equal(t, "91.121.79.179", *decisions[0].Value)
assert.Equal(t, int64(1), decisions[0].ID)
assert.Equal(t, "crowdsecurity/ssh-bf", *decisions[1].Scenario)
assert.Equal(t, "91.121.79.178", *decisions[1].Value)
assert.Equal(t, int64(2), decisions[1].ID)
// Get Decision : type filter
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions?type=ban", strings.NewReader(""))
req.Header.Add("User-Agent", UserAgent)
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/decisions?type=ban", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), `"id":1,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.179"`)
assert.Contains(t, w.Body.String(), `"id":2,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.178"`)
decisions, code, err = readDecisionsGetResp(w)
assert.Nil(t, err)
assert.Equal(t, 200, code)
assert.Equal(t, 2, len(decisions))
assert.Equal(t, "crowdsecurity/ssh-bf", *decisions[0].Scenario)
assert.Equal(t, "91.121.79.179", *decisions[0].Value)
assert.Equal(t, int64(1), decisions[0].ID)
assert.Equal(t, "crowdsecurity/ssh-bf", *decisions[1].Scenario)
assert.Equal(t, "91.121.79.178", *decisions[1].Value)
assert.Equal(t, int64(2), decisions[1].ID)
// assert.Contains(t, w.Body.String(), `"id":1,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.179"`)
// assert.Contains(t, w.Body.String(), `"id":2,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.178"`)
// Get Decision : scope/value
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions?scopes=Ip&value=91.121.79.179", strings.NewReader(""))
req.Header.Add("User-Agent", UserAgent)
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/decisions?scopes=Ip&value=91.121.79.179", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), `"id":1,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.179"`)
assert.NotContains(t, w.Body.String(), `"id":2,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.178"`)
decisions, code, err = readDecisionsGetResp(w)
assert.Nil(t, err)
assert.Equal(t, 200, code)
assert.Equal(t, 1, len(decisions))
assert.Equal(t, "crowdsecurity/ssh-bf", *decisions[0].Scenario)
assert.Equal(t, "91.121.79.179", *decisions[0].Value)
assert.Equal(t, int64(1), decisions[0].ID)
// assert.Contains(t, w.Body.String(), `"id":1,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.179"`)
// assert.NotContains(t, w.Body.String(), `"id":2,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.178"`)
// Get Decision : ip filter
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions?ip=91.121.79.179", strings.NewReader(""))
req.Header.Add("User-Agent", UserAgent)
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/decisions?ip=91.121.79.179", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), `"id":1,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.179"`)
assert.NotContains(t, w.Body.String(), `"id":2,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.178"`)
decisions, code, err = readDecisionsGetResp(w)
assert.Nil(t, err)
assert.Equal(t, 200, code)
assert.Equal(t, 1, len(decisions))
assert.Equal(t, "crowdsecurity/ssh-bf", *decisions[0].Scenario)
assert.Equal(t, "91.121.79.179", *decisions[0].Value)
assert.Equal(t, int64(1), decisions[0].ID)
// assert.Contains(t, w.Body.String(), `"id":1,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.179"`)
// assert.NotContains(t, w.Body.String(), `"id":2,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.178"`)
// Get decision : by range
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions?range=91.121.79.0/24&contains=false", strings.NewReader(""))
req.Header.Add("User-Agent", UserAgent)
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/decisions?range=91.121.79.0/24&contains=false", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), `"id":1,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.179"`)
assert.Contains(t, w.Body.String(), `"id":2,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.178"`)
decisions, code, err = readDecisionsGetResp(w)
assert.Nil(t, err)
assert.Equal(t, 200, code)
assert.Equal(t, 2, len(decisions))
assert.Contains(t, []string{*decisions[0].Value, *decisions[1].Value}, "91.121.79.179")
assert.Contains(t, []string{*decisions[0].Value, *decisions[1].Value}, "91.121.79.178")
}
func TestGetDecision(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
lapi := SetupLAPITest(t)
// Create Valid Alert
alertContentBytes, err := ioutil.ReadFile("./tests/alert_sample.json")
if err != nil {
log.Fatal(err)
}
alerts := make([]*models.Alert, 0)
if err := json.Unmarshal(alertContentBytes, &alerts); err != nil {
log.Fatal(err)
}
for _, alert := range alerts {
*alert.StartAt = time.Now().UTC().Format(time.RFC3339)
*alert.StopAt = time.Now().UTC().Format(time.RFC3339)
}
alertContent, err := json.Marshal(alerts)
if err != nil {
log.Fatal(err)
}
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(string(alertContent)))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
APIKey, err := CreateTestBouncer()
if err != nil {
log.Fatalf("%s", err.Error())
}
lapi.InsertAlertFromFile("./tests/alert_sample.json")
// Get Decision
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions", strings.NewReader(""))
req.Header.Add("User-Agent", UserAgent)
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
w := lapi.RecordResponse("GET", "/v1/decisions", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "\"id\":3,\"origin\":\"test\",\"scenario\":\"crowdsecurity/test\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"}]")
decisions, code, err := readDecisionsGetResp(w)
assert.Nil(t, err)
assert.Equal(t, 200, code)
assert.Equal(t, 3, len(decisions))
/*decisions get doesn't perform deduplication*/
assert.Equal(t, "crowdsecurity/test", *decisions[0].Scenario)
assert.Equal(t, "127.0.0.1", *decisions[0].Value)
assert.Equal(t, int64(1), decisions[0].ID)
assert.Equal(t, "crowdsecurity/test", *decisions[1].Scenario)
assert.Equal(t, "127.0.0.1", *decisions[1].Value)
assert.Equal(t, int64(2), decisions[1].ID)
assert.Equal(t, "crowdsecurity/test", *decisions[2].Scenario)
assert.Equal(t, "127.0.0.1", *decisions[2].Value)
assert.Equal(t, int64(3), decisions[2].ID)
// Get Decision with invalid filter. It should ignore this filter
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions?test=test", strings.NewReader(""))
req.Header.Add("User-Agent", UserAgent)
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("GET", "/v1/decisions?test=test", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "\"id\":3,\"origin\":\"test\",\"scenario\":\"crowdsecurity/test\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"}]")
assert.Equal(t, 3, len(decisions))
}
func TestDeleteDecisionByID(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
lapi := SetupLAPITest(t)
// Create Valid Alert
alertContentBytes, err := ioutil.ReadFile("./tests/alert_sample.json")
if err != nil {
log.Fatal(err)
}
alerts := make([]*models.Alert, 0)
if err := json.Unmarshal(alertContentBytes, &alerts); err != nil {
log.Fatal(err)
}
lapi.InsertAlertFromFile("./tests/alert_sample.json")
for _, alert := range alerts {
*alert.StartAt = time.Now().UTC().Format(time.RFC3339)
*alert.StopAt = time.Now().UTC().Format(time.RFC3339)
}
alertContent, err := json.Marshal(alerts)
if err != nil {
log.Fatal(err)
}
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(string(alertContent)))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
//Have one alerts
w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody)
decisions, code, err := readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, code, 200)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 1)
// Delete alert with Invalid ID
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/decisions/test", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("DELETE", "/v1/decisions/test", emptyBody)
assert.Equal(t, 400, w.Code)
assert.Equal(t, "{\"message\":\"decision_id must be valid integer\"}", w.Body.String())
err_resp, _, err := readDecisionsErrorResp(w)
assert.NoError(t, err)
assert.Equal(t, err_resp["message"], "decision_id must be valid integer")
// Delete alert with ID that not exist
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/decisions/100", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("DELETE", "/v1/decisions/100", emptyBody)
assert.Equal(t, 500, w.Code)
assert.Equal(t, "{\"message\":\"decision with id '100' doesn't exist: unable to delete\"}", w.Body.String())
err_resp, _, err = readDecisionsErrorResp(w)
assert.NoError(t, err)
assert.Equal(t, err_resp["message"], "decision with id '100' doesn't exist: unable to delete")
//Have one alerts
w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, code, 200)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 1)
// Delete alert with valid ID
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/decisions/1", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "{\"nbDeleted\":\"1\"}", w.Body.String())
resp, _, err := readDecisionsDeleteResp(w)
assert.NoError(t, err)
assert.Equal(t, resp.NbDeleted, "1")
//Have one alert (because we delete an alert that has dup targets)
w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, code, 200)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 1)
}
func TestDeleteDecision(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
lapi := SetupLAPITest(t)
// Create Valid Alert
alertContentBytes, err := ioutil.ReadFile("./tests/alert_sample.json")
if err != nil {
log.Fatal(err)
}
alerts := make([]*models.Alert, 0)
if err := json.Unmarshal(alertContentBytes, &alerts); err != nil {
log.Fatal(err)
}
for _, alert := range alerts {
*alert.StartAt = time.Now().UTC().Format(time.RFC3339)
*alert.StopAt = time.Now().UTC().Format(time.RFC3339)
}
alertContent, err := json.Marshal(alerts)
if err != nil {
log.Fatal(err)
}
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(string(alertContent)))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
lapi.InsertAlertFromFile("./tests/alert_sample.json")
// Delete alert with Invalid filter
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/decisions?test=test", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w := lapi.RecordResponse("DELETE", "/v1/decisions?test=test", emptyBody)
assert.Equal(t, 500, w.Code)
assert.Equal(t, "{\"message\":\"'test' doesn't exist: invalid filter\"}", w.Body.String())
// Delete alert
w = httptest.NewRecorder()
req, _ = http.NewRequest("DELETE", "/v1/decisions", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
err_resp, _, err := readDecisionsErrorResp(w)
assert.NoError(t, err)
assert.Equal(t, err_resp["message"], "'test' doesn't exist: invalid filter")
// Delete all alert
w = lapi.RecordResponse("DELETE", "/v1/decisions", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "{\"nbDeleted\":\"3\"}", w.Body.String())
resp, _, err := readDecisionsDeleteResp(w)
assert.NoError(t, err)
assert.Equal(t, resp.NbDeleted, "3")
}
func TestStreamDecision(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
func TestStreamStartDecisionDedup(t *testing.T) {
//Ensure that at stream startup we only get the longest decision
lapi := SetupLAPITest(t)
// Create Valid Alert
alertContentBytes, err := ioutil.ReadFile("./tests/alert_sample.json")
if err != nil {
log.Fatal(err)
}
alerts := make([]*models.Alert, 0)
if err := json.Unmarshal(alertContentBytes, &alerts); err != nil {
log.Fatal(err)
}
// Create Valid Alert : 3 decisions for 127.0.0.1, longest has id=3
lapi.InsertAlertFromFile("./tests/alert_sample.json")
for _, alert := range alerts {
*alert.StartAt = time.Now().UTC().Format(time.RFC3339)
*alert.StopAt = time.Now().UTC().Format(time.RFC3339)
}
// Get Stream, we only get one decision (the longest one)
w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody)
decisions, code, err := readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, code, 200)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 1)
assert.Equal(t, decisions["new"][0].ID, int64(3))
assert.Equal(t, *decisions["new"][0].Origin, "test")
assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1")
alertContent, err := json.Marshal(alerts)
if err != nil {
log.Fatal(err)
}
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/alerts", strings.NewReader(string(alertContent)))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", loginResp.Token))
router.ServeHTTP(w, req)
APIKey, err := CreateTestBouncer()
if err != nil {
log.Fatalf("%s", err.Error())
}
// Get Stream
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions/stream", strings.NewReader(""))
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "{\"deleted\":null,\"new\":null}", w.Body.String())
// Get Stream just startup
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions/stream?startup=true", strings.NewReader(""))
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
// the decision with id=3 is only returned because it's the longest decision
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "\"id\":3,\"origin\":\"test\",\"scenario\":\"crowdsecurity/test\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"}]}")
assert.NotContains(t, w.Body.String(), "\"id\":2")
assert.NotContains(t, w.Body.String(), "\"id\":1")
assert.Contains(t, w.Body.String(), "2h")
// id=3 decision is deleted, this won't affect `deleted`, because there are decisions
// targetting same IP
req, _ = http.NewRequest("DELETE", "/v1/decisions/3", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
// id=3 decision is deleted, this won't affect `deleted`, because there are decisions on the same ip
w = lapi.RecordResponse("DELETE", "/v1/decisions/3", emptyBody)
assert.Equal(t, 200, w.Code)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions/stream?startup=true", strings.NewReader(""))
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
assert.Equal(t, 200, w.Code)
// the decision with id=2 is only returned because it's the longest decision
assert.Contains(t, w.Body.String(), "\"id\":2,\"origin\":\"test\",\"scenario\":\"crowdsecurity/test\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"}]}")
assert.NotContains(t, w.Body.String(), "\"id\":3")
assert.NotContains(t, w.Body.String(), "\"id\":1")
assert.Contains(t, w.Body.String(), "1h")
assert.Contains(t, w.Body.String(), "\"deleted\":null")
// Get Stream, we only get one decision (the longest one, id=2)
w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, code, 200)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 1)
assert.Equal(t, decisions["new"][0].ID, int64(2))
assert.Equal(t, *decisions["new"][0].Origin, "test")
assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1")
// We delete another decision, yet don't receive it in stream, since there's another decision on same IP
req, _ = http.NewRequest("DELETE", "/v1/decisions/2", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions/stream", strings.NewReader(""))
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
w = lapi.RecordResponse("DELETE", "/v1/decisions/2", emptyBody)
assert.Equal(t, 200, w.Code)
assert.Equal(t, "{\"deleted\":null,\"new\":null}", w.Body.String())
// Now all decisions for this IP are deleted, we should receive it in stream
req, _ = http.NewRequest("DELETE", "/v1/decisions/1", strings.NewReader(""))
AddAuthHeaders(req, loginResp)
router.ServeHTTP(w, req)
// And get the remaining decision (1)
w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, code, 200)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 1)
assert.Equal(t, decisions["new"][0].ID, int64(1))
assert.Equal(t, *decisions["new"][0].Origin, "test")
assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1")
// We delete the last decision, we receive the delete order
w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody)
assert.Equal(t, 200, w.Code)
//and now we only get a deleted decision
w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, code, 200)
assert.Equal(t, len(decisions["deleted"]), 1)
assert.Equal(t, decisions["deleted"][0].ID, int64(1))
assert.Equal(t, *decisions["deleted"][0].Origin, "test")
assert.Equal(t, *decisions["deleted"][0].Value, "127.0.0.1")
assert.Equal(t, len(decisions["new"]), 0)
}
func TestStreamDecisionDedup(t *testing.T) {
//Ensure that at stream startup we only get the longest decision
lapi := SetupLAPITest(t)
// Create Valid Alert : 3 decisions for 127.0.0.1, longest has id=3
lapi.InsertAlertFromFile("./tests/alert_sample.json")
// Get Stream, we only get one decision (the longest one)
w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody)
decisions, code, err := readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, code, 200)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 1)
assert.Equal(t, decisions["new"][0].ID, int64(3))
assert.Equal(t, *decisions["new"][0].Origin, "test")
assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1")
// id=3 decision is deleted, this won't affect `deleted`, because there are decisions on the same ip
w = lapi.RecordResponse("DELETE", "/v1/decisions/3", emptyBody)
assert.Equal(t, 200, w.Code)
w = lapi.RecordResponse("GET", "/v1/decisions/stream", emptyBody)
assert.Equal(t, err, nil)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, code, 200)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 0)
// We delete another decision, yet don't receive it in stream, since there's another decision on same IP
w = lapi.RecordResponse("DELETE", "/v1/decisions/2", emptyBody)
assert.Equal(t, 200, w.Code)
w = lapi.RecordResponse("GET", "/v1/decisions/stream", emptyBody)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, code, 200)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 0)
// We delete the last decision, we receive the delete order
w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody)
assert.Equal(t, 200, w.Code)
w = lapi.RecordResponse("GET", "/v1/decisions/stream", emptyBody)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, code, 200)
assert.Equal(t, len(decisions["deleted"]), 1)
assert.Equal(t, decisions["deleted"][0].ID, int64(1))
assert.Equal(t, *decisions["deleted"][0].Origin, "test")
assert.Equal(t, *decisions["deleted"][0].Value, "127.0.0.1")
assert.Equal(t, len(decisions["new"]), 0)
}
func TestStreamDecisionFilters(t *testing.T) {
router, loginResp, err := InitMachineTest()
if err != nil {
log.Fatalln(err.Error())
}
lapi := SetupLAPITest(t)
// Create Valid Alert
alertContentBytes, err := ioutil.ReadFile("./tests/alert_stream_fixture.json")
if err != nil {
log.Fatal(err)
}
alerts := make([]*models.Alert, 0)
if err := json.Unmarshal(alertContentBytes, &alerts); err != nil {
log.Fatal(err)
}
lapi.InsertAlertFromFile("./tests/alert_stream_fixture.json")
for _, alert := range alerts {
*alert.StartAt = time.Now().UTC().Format(time.RFC3339)
*alert.StopAt = time.Now().UTC().Format(time.RFC3339)
}
w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody)
decisions, code, err := readDecisionsStreamResp(w)
alertContent, err := json.Marshal(alerts)
if err != nil {
log.Fatal(err)
}
w := httptest.NewRecorder()
req, err := http.NewRequest("POST", "/v1/alerts", strings.NewReader(string(alertContent)))
if err != nil {
log.Fatalf("%s", err.Error())
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", loginResp.Token))
router.ServeHTTP(w, req)
APIKey, err := CreateTestBouncer()
if err != nil {
log.Fatalf("%s", err.Error())
}
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions/stream?startup=true", strings.NewReader(""))
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "\"id\":1,\"origin\":\"test1\",\"scenario\":\"crowdsecurity/http_bf\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
assert.Contains(t, w.Body.String(), "\"id\":2,\"origin\":\"test2\",\"scenario\":\"crowdsecurity/ssh_bf\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
assert.Contains(t, w.Body.String(), "\"id\":3,\"origin\":\"test3\",\"scenario\":\"crowdsecurity/ddos\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
assert.Equal(t, 200, code)
assert.Equal(t, err, nil)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 3)
assert.Equal(t, decisions["new"][0].ID, int64(1))
assert.Equal(t, *decisions["new"][0].Origin, "test1")
assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1")
assert.Equal(t, *decisions["new"][0].Scenario, "crowdsecurity/http_bf")
assert.Equal(t, decisions["new"][1].ID, int64(2))
assert.Equal(t, *decisions["new"][1].Origin, "test2")
assert.Equal(t, *decisions["new"][1].Value, "127.0.0.1")
assert.Equal(t, *decisions["new"][1].Scenario, "crowdsecurity/ssh_bf")
assert.Equal(t, decisions["new"][2].ID, int64(3))
assert.Equal(t, *decisions["new"][2].Origin, "test3")
assert.Equal(t, *decisions["new"][2].Value, "127.0.0.1")
assert.Equal(t, *decisions["new"][2].Scenario, "crowdsecurity/ddos")
// test filter scenarios_not_containing
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions/stream?startup=true&scenarios_not_containing=http", strings.NewReader(""))
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
assert.Equal(t, 200, w.Code)
assert.NotContains(t, w.Body.String(), "\"id\":1,\"origin\":\"test1\",\"scenario\":\"crowdsecurity/http_bf\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
assert.Contains(t, w.Body.String(), "\"id\":2,\"origin\":\"test2\",\"scenario\":\"crowdsecurity/ssh_bf\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
assert.Contains(t, w.Body.String(), "\"id\":3,\"origin\":\"test3\",\"scenario\":\"crowdsecurity/ddos\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&scenarios_not_containing=http", emptyBody)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, 200, code)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 2)
assert.Equal(t, decisions["new"][0].ID, int64(2))
assert.Equal(t, decisions["new"][1].ID, int64(3))
// test filter scenarios_containing
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions/stream?startup=true&scenarios_containing=http", strings.NewReader(""))
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "\"id\":1,\"origin\":\"test1\",\"scenario\":\"crowdsecurity/http_bf\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
assert.NotContains(t, w.Body.String(), "\"id\":2,\"origin\":\"test2\",\"scenario\":\"crowdsecurity/ssh_bf\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
assert.NotContains(t, w.Body.String(), "\"id\":3,\"origin\":\"test3\",\"scenario\":\"crowdsecurity/ddos\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&scenarios_containing=http", emptyBody)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, 200, code)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 1)
assert.Equal(t, decisions["new"][0].ID, int64(1))
// test filters both by scenarios_not_containing and scenarios_containing
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh&scenarios_containing=ddos", strings.NewReader(""))
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
assert.Equal(t, 200, w.Code)
assert.NotContains(t, w.Body.String(), "\"id\":1,\"origin\":\"test1\",\"scenario\":\"crowdsecurity/http_bf\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
assert.NotContains(t, w.Body.String(), "\"id\":2,\"origin\":\"test2\",\"scenario\":\"crowdsecurity/ssh_bf\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
assert.Contains(t, w.Body.String(), "\"id\":3,\"origin\":\"test3\",\"scenario\":\"crowdsecurity/ddos\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh&scenarios_containing=ddos", emptyBody)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, 200, code)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 1)
assert.Equal(t, decisions["new"][0].ID, int64(3))
// test filter by origin
w = httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/v1/decisions/stream?startup=true&origins=test1,test2", strings.NewReader(""))
req.Header.Add("X-Api-Key", APIKey)
router.ServeHTTP(w, req)
assert.Equal(t, 200, w.Code)
assert.Contains(t, w.Body.String(), "\"id\":1,\"origin\":\"test1\",\"scenario\":\"crowdsecurity/http_bf\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
assert.Contains(t, w.Body.String(), "\"id\":2,\"origin\":\"test2\",\"scenario\":\"crowdsecurity/ssh_bf\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
assert.NotContains(t, w.Body.String(), "\"id\":3,\"origin\":\"test3\",\"scenario\":\"crowdsecurity/ddos\",\"scope\":\"Ip\",\"type\":\"ban\",\"value\":\"127.0.0.1\"")
w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&origins=test1,test2", emptyBody)
decisions, code, err = readDecisionsStreamResp(w)
assert.Equal(t, err, nil)
assert.Equal(t, 200, code)
assert.Equal(t, len(decisions["deleted"]), 0)
assert.Equal(t, len(decisions["new"]), 2)
assert.Equal(t, decisions["new"][0].ID, int64(1))
assert.Equal(t, decisions["new"][1].ID, int64(2))
}

5382
pkg/apiserver/tests/alert_bulk.json
View file

File diff suppressed because it is too large Load diff

550
pkg/apiserver/tests/alert_minibulk+simul.json
View file

@ -1,4 +1,548 @@
[
{"capacity":5,"decisions":null,"events":[{"meta":[{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"source_ip","value":"91.121.79.179"},{"key":"IsoCode","value":"FR"},{"key":"ASNNumber","value":"16276"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"source_ip","value":"91.121.79.179"},{"key":"IsoCode","value":"FR"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"service","value":"ssh"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"target_user","value":"root"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.179"},{"key":"IsoCode","value":"FR"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"service","value":"ssh"},{"key":"source_ip","value":"91.121.79.179"},{"key":"IsoCode","value":"FR"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.179"},{"key":"IsoCode","value":"FR"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.179"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"service","value":"ssh"},{"key":"IsoCode","value":"FR"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"}],"events_count":6,"labels":null,"leakspeed":"10s","message":"Ip 91.121.79.179 performed crowdsecurity/ssh-bf (6 events over 46.375699ms) at 2020-10-26 12:52:58.200237122 +0100 CET m=+8.191478202","remediation":true,"scenario":"crowdsecurity/ssh-bf","scenario_hash":"4441dcff07020f6690d998b7101e642359ba405c2abb83565bbbdcee36de280f","scenario_version":"0.1","simulated":true,"source":{"as_name":"OVH SAS","cn":"FR","ip":"91.121.79.179","latitude":50.646,"longitude":3.0758,"range":"91.121.72.0/21","scope":"Ip","value":"91.121.79.179"},"start_at":"2020-10-26T12:52:58.153861334+01:00","stop_at":"2020-10-26T12:52:58.200236582+01:00"},
{"capacity":5,"decisions":null,"events":[{"meta":[{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"source_ip","value":"91.121.79.178"},{"key":"IsoCode","value":"FR"},{"key":"ASNNumber","value":"16276"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"source_ip","value":"91.121.79.178"},{"key":"IsoCode","value":"FR"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"service","value":"ssh"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"target_user","value":"root"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.178"},{"key":"IsoCode","value":"FR"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"service","value":"ssh"},{"key":"source_ip","value":"91.121.79.178"},{"key":"IsoCode","value":"FR"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.178"},{"key":"IsoCode","value":"FR"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.178"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"service","value":"ssh"},{"key":"IsoCode","value":"FR"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"}],"events_count":6,"labels":null,"leakspeed":"10s","message":"Ip 91.121.79.178 performed crowdsecurity/ssh-bf (6 events over 46.375699ms) at 2020-10-26 12:52:58.200237122 +0100 CET m=+8.191478202","remediation":true,"scenario":"crowdsecurity/ssh-bf","scenario_hash":"4441dcff07020f6690d998b7101e642359ba405c2abb83565bbbdcee36de280f","scenario_version":"0.1","simulated":false,"source":{"as_name":"OVH SAS","cn":"FR","ip":"91.121.79.178","latitude":50.646,"longitude":3.0758,"range":"91.121.72.0/21","scope":"Ip","value":"91.121.79.178"},"start_at":"2020-10-26T12:52:58.153861334+01:00","stop_at":"2020-10-26T12:52:58.200236582+01:00"}
]
{
"capacity": 5,
"decisions": null,
"events": [
{
"meta": [
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "target_user",
"value": "root"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
}
],
"events_count": 6,
"labels": null,
"leakspeed": "10s",
"message": "Ip 91.121.79.179 performed crowdsecurity/ssh-bf (6 events over 46.375699ms) at 2020-10-26 12:52:58.200237122 +0100 CET m=+8.191478202",
"remediation": true,
"scenario": "crowdsecurity/ssh-bf",
"scenario_hash": "4441dcff07020f6690d998b7101e642359ba405c2abb83565bbbdcee36de280f",
"scenario_version": "0.1",
"simulated": true,
"source": {
"as_name": "OVH SAS",
"cn": "FR",
"ip": "91.121.79.179",
"latitude": 50.646,
"longitude": 3.0758,
"range": "91.121.72.0/21",
"scope": "Ip",
"value": "91.121.79.179"
},
"start_at": "2020-10-26T12:52:58.153861334+01:00",
"stop_at": "2020-10-26T12:52:58.200236582+01:00"
},
{
"capacity": 5,
"decisions": null,
"events": [
{
"meta": [
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "target_user",
"value": "root"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
}
],
"events_count": 6,
"labels": null,
"leakspeed": "10s",
"message": "Ip 91.121.79.178 performed crowdsecurity/ssh-bf (6 events over 46.375699ms) at 2020-10-26 12:52:58.200237122 +0100 CET m=+8.191478202",
"remediation": true,
"scenario": "crowdsecurity/ssh-bf",
"scenario_hash": "4441dcff07020f6690d998b7101e642359ba405c2abb83565bbbdcee36de280f",
"scenario_version": "0.1",
"simulated": false,
"source": {
"as_name": "OVH SAS",
"cn": "FR",
"ip": "91.121.79.178",
"latitude": 50.646,
"longitude": 3.0758,
"range": "91.121.72.0/21",
"scope": "Ip",
"value": "91.121.79.178"
},
"start_at": "2020-10-26T12:52:58.153861334+01:00",
"stop_at": "2020-10-26T12:52:58.200236582+01:00"
}
]

550
pkg/apiserver/tests/alert_minibulk.json
View file

@ -1,4 +1,548 @@
[
{"capacity":5,"decisions":null,"events":[{"meta":[{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"source_ip","value":"91.121.79.179"},{"key":"IsoCode","value":"FR"},{"key":"ASNNumber","value":"16276"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"source_ip","value":"91.121.79.179"},{"key":"IsoCode","value":"FR"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"service","value":"ssh"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"target_user","value":"root"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.179"},{"key":"IsoCode","value":"FR"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"service","value":"ssh"},{"key":"source_ip","value":"91.121.79.179"},{"key":"IsoCode","value":"FR"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.179"},{"key":"IsoCode","value":"FR"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.179"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"service","value":"ssh"},{"key":"IsoCode","value":"FR"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"}],"events_count":6,"labels":null,"leakspeed":"10s","message":"Ip 91.121.79.179 performed crowdsecurity/ssh-bf (6 events over 46.375699ms) at 2020-10-26 12:52:58.200237122 +0100 CET m=+8.191478202","remediation":true,"scenario":"crowdsecurity/ssh-bf","scenario_hash":"4441dcff07020f6690d998b7101e642359ba405c2abb83565bbbdcee36de280f","scenario_version":"0.1","simulated":false,"source":{"as_name":"OVH SAS","cn":"FR","ip":"91.121.79.179","latitude":50.646,"longitude":3.0758,"range":"91.121.72.0/21","scope":"Ip","value":"91.121.79.179"},"start_at":"2020-10-26T12:52:58.153861334+01:00","stop_at":"2020-10-26T12:52:58.200236582+01:00"},
{"capacity":5,"decisions":null,"events":[{"meta":[{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"source_ip","value":"91.121.79.178"},{"key":"IsoCode","value":"FR"},{"key":"ASNNumber","value":"16276"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"source_ip","value":"91.121.79.178"},{"key":"IsoCode","value":"FR"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"service","value":"ssh"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"target_user","value":"root"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.178"},{"key":"IsoCode","value":"FR"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"service","value":"ssh"},{"key":"source_ip","value":"91.121.79.178"},{"key":"IsoCode","value":"FR"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"service","value":"ssh"},{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.178"},{"key":"IsoCode","value":"FR"},{"key":"IsInEU","value":"true"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"}],"timestamp":"2020-10-02T17:09:08Z"},{"meta":[{"key":"log_type","value":"ssh_failed-auth"},{"key":"source_ip","value":"91.121.79.178"},{"key":"ASNNumber","value":"16276"},{"key":"ASNOrg","value":"OVH SAS"},{"key":"SourceRange","value":"91.121.72.0/21"},{"key":"target_user","value":"root"},{"key":"service","value":"ssh"},{"key":"IsoCode","value":"FR"},{"key":"IsInEU","value":"true"}],"timestamp":"2020-10-02T17:09:08Z"}],"events_count":6,"labels":null,"leakspeed":"10s","message":"Ip 91.121.79.178 performed crowdsecurity/ssh-bf (6 events over 46.375699ms) at 2020-10-26 12:52:58.200237122 +0100 CET m=+8.191478202","remediation":true,"scenario":"crowdsecurity/ssh-bf","scenario_hash":"4441dcff07020f6690d998b7101e642359ba405c2abb83565bbbdcee36de280f","scenario_version":"0.1","simulated":false,"source":{"as_name":"OVH SAS","cn":"FR","ip":"91.121.79.178","latitude":50.646,"longitude":3.0758,"range":"91.121.72.0/21","scope":"Ip","value":"91.121.79.178"},"start_at":"2020-10-26T12:52:58.153861334+01:00","stop_at":"2020-10-26T12:52:58.200236582+01:00"}
]
{
"capacity": 5,
"decisions": null,
"events": [
{
"meta": [
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "target_user",
"value": "root"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.179"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
}
],
"events_count": 6,
"labels": null,
"leakspeed": "10s",
"message": "Ip 91.121.79.179 performed crowdsecurity/ssh-bf (6 events over 46.375699ms) at 2020-10-26 12:52:58.200237122 +0100 CET m=+8.191478202",
"remediation": true,
"scenario": "crowdsecurity/ssh-bf",
"scenario_hash": "4441dcff07020f6690d998b7101e642359ba405c2abb83565bbbdcee36de280f",
"scenario_version": "0.1",
"simulated": false,
"source": {
"as_name": "OVH SAS",
"cn": "FR",
"ip": "91.121.79.179",
"latitude": 50.646,
"longitude": 3.0758,
"range": "91.121.72.0/21",
"scope": "Ip",
"value": "91.121.79.179"
},
"start_at": "2020-10-26T12:52:58.153861334+01:00",
"stop_at": "2020-10-26T12:52:58.200236582+01:00"
},
{
"capacity": 5,
"decisions": null,
"events": [
{
"meta": [
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "target_user",
"value": "root"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "IsInEU",
"value": "true"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
}
],
"timestamp": "2020-10-02T17:09:08Z"
},
{
"meta": [
{
"key": "log_type",
"value": "ssh_failed-auth"
},
{
"key": "source_ip",
"value": "91.121.79.178"
},
{
"key": "ASNNumber",
"value": "16276"
},
{
"key": "ASNOrg",
"value": "OVH SAS"
},
{
"key": "SourceRange",
"value": "91.121.72.0/21"
},
{
"key": "target_user",
"value": "root"
},
{
"key": "service",
"value": "ssh"
},
{
"key": "IsoCode",
"value": "FR"
},
{
"key": "IsInEU",
"value": "true"
}
],
"timestamp": "2020-10-02T17:09:08Z"
}
],
"events_count": 6,
"labels": null,
"leakspeed": "10s",
"message": "Ip 91.121.79.178 performed crowdsecurity/ssh-bf (6 events over 46.375699ms) at 2020-10-26 12:52:58.200237122 +0100 CET m=+8.191478202",
"remediation": true,
"scenario": "crowdsecurity/ssh-bf",
"scenario_hash": "4441dcff07020f6690d998b7101e642359ba405c2abb83565bbbdcee36de280f",
"scenario_version": "0.1",
"simulated": false,
"source": {
"as_name": "OVH SAS",
"cn": "FR",
"ip": "91.121.79.178",
"latitude": 50.646,
"longitude": 3.0758,
"range": "91.121.72.0/21",
"scope": "Ip",
"value": "91.121.79.178"
},
"start_at": "2020-10-26T12:52:58.153861334+01:00",
"stop_at": "2020-10-26T12:52:58.200236582+01:00"
}
]

2
pkg/apiserver/tests/alert_sample.json
View file

@ -74,4 +74,4 @@
"start_at": "2020-10-09T10:00:01Z",
"stop_at": "2020-10-09T10:00:05Z"
}
]
]

2
pkg/apiserver/tests/alert_ssh-bf.json
View file

@ -272,4 +272,4 @@
"start_at": "2020-10-26T09:50:32.025353849+01:00",
"stop_at": "2020-10-26T09:50:32.055534398+01:00"
}
]
]

8
pkg/csplugin/broker.go
View file

@ -30,7 +30,6 @@ import (
"gopkg.in/yaml.v2"
)
var testMode bool = false
var pluginMutex sync.Mutex
const (
@ -255,6 +254,9 @@ func (pb *PluginBroker) loadNotificationPlugin(name string, binaryPath string) (
}
cmd := exec.Command(binaryPath)
if pb.pluginProcConfig.User != "" || pb.pluginProcConfig.Group != "" {
if !(pb.pluginProcConfig.User != "" && pb.pluginProcConfig.Group != "") {
return nil, errors.New("while getting process attributes: both plugin user and group must be set")
}
cmd.SysProcAttr, err = getProcessAttr(pb.pluginProcConfig.User, pb.pluginProcConfig.Group)
if err != nil {
return nil, errors.Wrap(err, "while getting process attributes")
@ -360,9 +362,6 @@ func setRequiredFields(pluginCfg *PluginConfig) {
}
func pluginIsValid(path string) error {
if testMode {
return nil
}
var details fs.FileInfo
var err error
@ -387,7 +386,6 @@ func pluginIsValid(path string) error {
mode := details.Mode()
perm := uint32(mode)
if (perm & 00002) != 0 {
return fmt.Errorf("plugin at %s is world writable, world writable plugins are invalid", path)
}

196
pkg/csplugin/broker_test.go
View file

@ -1,17 +1,36 @@
package csplugin
import (
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"reflect"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"gopkg.in/tomb.v2"
)
var testPath string
func Test_getPluginNameAndTypeFromPath(t *testing.T) {
func setPluginPermTo744() {
setPluginPermTo("744")
}
func setPluginPermTo722() {
setPluginPermTo("722")
}
func setPluginPermTo724() {
setPluginPermTo("724")
}
func TestGetPluginNameAndTypeFromPath(t *testing.T) {
setUp()
defer tearDown()
type args struct {
@ -69,7 +88,7 @@ func Test_getPluginNameAndTypeFromPath(t *testing.T) {
}
}
func Test_listFilesAtPath(t *testing.T) {
func TestListFilesAtPath(t *testing.T) {
setUp()
defer tearDown()
type args struct {
@ -113,9 +132,176 @@ func Test_listFilesAtPath(t *testing.T) {
}
}
func TestBrokerInit(t *testing.T) {
tests := []struct {
name string
action func()
errContains string
wantErr bool
procCfg csconfig.PluginCfg
}{
{
name: "valid config",
action: setPluginPermTo744,
wantErr: false,
},
{
name: "group writable binary",
wantErr: true,
errContains: "notification-dummy is world writable",
action: setPluginPermTo722,
},
{
name: "group writable binary",
wantErr: true,
errContains: "notification-dummy is group writable",
action: setPluginPermTo724,
},
{
name: "no plugin dir",
wantErr: true,
errContains: "no such file or directory",
action: tearDown,
},
{
name: "no plugin binary",
wantErr: true,
errContains: "binary for plugin dummy_default not found",
action: func() {
err := os.Remove(path.Join(testPath, "notification-dummy"))
if err != nil {
t.Fatal(err)
}
},
},
{
name: "only specify user",
wantErr: true,
errContains: "both plugin user and group must be set",
procCfg: csconfig.PluginCfg{
User: "123445555551122toto",
},
action: setPluginPermTo744,
},
{
name: "only specify group",
wantErr: true,
errContains: "both plugin user and group must be set",
procCfg: csconfig.PluginCfg{
Group: "123445555551122toto",
},
action: setPluginPermTo744,
},
{
name: "Fails to run as root",
wantErr: true,
errContains: "operation not permitted",
procCfg: csconfig.PluginCfg{
User: "root",
Group: "root",
},
action: setPluginPermTo744,
},
{
name: "Invalid user and group",
wantErr: true,
errContains: "unknown user toto1234",
procCfg: csconfig.PluginCfg{
User: "toto1234",
Group: "toto1234",
},
action: setPluginPermTo744,
},
{
name: "Valid user and invalid group",
wantErr: true,
errContains: "unknown group toto1234",
procCfg: csconfig.PluginCfg{
User: "nobody",
Group: "toto1234",
},
action: setPluginPermTo744,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
defer tearDown()
buildDummyPlugin()
if test.action != nil {
test.action()
}
pb := PluginBroker{}
profiles := csconfig.NewDefaultConfig().API.Server.Profiles
profiles = append(profiles, &csconfig.ProfileCfg{
Notifications: []string{"dummy_default"},
})
err := pb.Init(&test.procCfg, profiles, &csconfig.ConfigurationPaths{
PluginDir: testPath,
NotificationDir: "./tests/notifications",
})
defer pb.Kill()
if test.wantErr {
assert.ErrorContains(t, err, test.errContains)
} else {
assert.NoError(t, err)
}
})
}
}
func TestBrokerRun(t *testing.T) {
buildDummyPlugin()
setPluginPermTo744()
defer tearDown()
procCfg := csconfig.PluginCfg{}
pb := PluginBroker{}
profiles := csconfig.NewDefaultConfig().API.Server.Profiles
profiles = append(profiles, &csconfig.ProfileCfg{
Notifications: []string{"dummy_default"},
})
err := pb.Init(&procCfg, profiles, &csconfig.ConfigurationPaths{
PluginDir: testPath,
NotificationDir: "./tests/notifications",
})
assert.NoError(t, err)
tomb := tomb.Tomb{}
go pb.Run(&tomb)
defer pb.Kill()
assert.NoFileExists(t, "./out")
defer os.Remove("./out")
pb.PluginChannel <- ProfileAlert{ProfileID: uint(0), Alert: &models.Alert{}}
pb.PluginChannel <- ProfileAlert{ProfileID: uint(0), Alert: &models.Alert{}}
time.Sleep(time.Second * 4)
assert.FileExists(t, "./out")
assert.Equal(t, types.GetLineCountForFile("./out"), 2)
}
func buildDummyPlugin() {
dir, err := os.MkdirTemp("./tests", "cs_plugin_test")
if err != nil {
log.Fatal(err)
}
cmd := exec.Command("go", "build", "-o", path.Join(dir, "notification-dummy"), "../../plugins/notifications/dummy/")
if err := cmd.Run(); err != nil {
log.Fatal(err)
}
testPath = dir
}
func setPluginPermTo(perm string) {
if err := exec.Command("chmod", perm, path.Join(testPath, "notification-dummy")).Run(); err != nil {
log.Fatal(errors.Wrapf(err, "chmod 744 %s", path.Join(testPath, "notification-dummy")))
}
}
func setUp() {
testMode = true
dir, err := ioutil.TempDir("./", "cs_plugin_test")
dir, err := os.MkdirTemp("./", "cs_plugin_test")
if err != nil {
log.Fatal(err)
}

22
pkg/csplugin/tests/notifications/dummy.yaml Normal file
View file

@ -0,0 +1,22 @@
type: dummy # Don't change
name: dummy_default # Must match the registered plugin in the profile
# One of "trace", "debug", "info", "warn", "error", "off"
log_level: info
# group_wait: # Time to wait collecting alerts before relaying a message to this plugin, eg "30s"
# group_threshold: # Amount of alerts that triggers a message before <group_wait> has expired, eg "10"
# max_retry: # Number of attempts to relay messages to plugins in case of error
# timeout: # Time to wait for response from the plugin before considering the attempt a failure, eg "10s"
#-------------------------
# plugin-specific options
# The following template receives a list of models.Alert objects
# The output goes in the logs and to a text file, if defined
format: |
{{.|toJson}}
#
output_file: ./out # notifications will be appended here. optional

107
pkg/csplugin/watcher_test.go Normal file
View file

@ -0,0 +1,107 @@
package csplugin
import (
"context"
"log"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/models"
"gopkg.in/tomb.v2"
"gotest.tools/v3/assert"
)
var ctx = context.Background()
func resetTestTomb(testTomb *tomb.Tomb) {
testTomb.Kill(nil)
if err := testTomb.Wait(); err != nil {
log.Fatal(err)
}
}
func resetWatcherAlertCounter(pw *PluginWatcher) {
for k := range pw.AlertCountByPluginName {
pw.AlertCountByPluginName[k] = 0
}
}
func insertNAlertsToPlugin(pw *PluginWatcher, n int, pluginName string) {
for i := 0; i < n; i++ {
pw.Inserts <- pluginName
}
}
func listenChannelWithTimeout(ctx context.Context, channel chan string) error {
select {
case <-channel:
case <-ctx.Done():
return ctx.Err()
}
return nil
}
func TestPluginWatcherInterval(t *testing.T) {
pw := PluginWatcher{}
alertsByPluginName := make(map[string][]*models.Alert)
testTomb := tomb.Tomb{}
configs := map[string]PluginConfig{
"testPlugin": {
GroupWait: time.Millisecond,
},
}
pw.Init(configs, alertsByPluginName)
pw.Start(&testTomb)
ct, cancel := context.WithTimeout(ctx, time.Microsecond)
defer cancel()
err := listenChannelWithTimeout(ct, pw.PluginEvents)
assert.ErrorContains(t, err, "context deadline exceeded")
resetTestTomb(&testTomb)
testTomb = tomb.Tomb{}
pw.Start(&testTomb)
ct, cancel = context.WithTimeout(ctx, time.Millisecond*5)
defer cancel()
err = listenChannelWithTimeout(ct, pw.PluginEvents)
assert.NilError(t, err)
resetTestTomb(&testTomb)
// This is to avoid the int complaining
}
func TestPluginAlertCountWatcher(t *testing.T) {
pw := PluginWatcher{}
alertsByPluginName := make(map[string][]*models.Alert)
configs := map[string]PluginConfig{
"testPlugin": {
GroupThreshold: 5,
},
}
testTomb := tomb.Tomb{}
pw.Init(configs, alertsByPluginName)
pw.Start(&testTomb)
// Channel won't contain any events since threshold is not crossed.
ct, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
err := listenChannelWithTimeout(ct, pw.PluginEvents)
assert.ErrorContains(t, err, "context deadline exceeded")
// Channel won't contain any events since threshold is not crossed.
resetWatcherAlertCounter(&pw)
insertNAlertsToPlugin(&pw, 4, "testPlugin")
ct, cancel = context.WithTimeout(ctx, time.Second)
defer cancel()
err = listenChannelWithTimeout(ct, pw.PluginEvents)
assert.ErrorContains(t, err, "context deadline exceeded")
// Channel will contain an event since threshold is crossed.
resetWatcherAlertCounter(&pw)
insertNAlertsToPlugin(&pw, 5, "testPlugin")
ct, cancel = context.WithTimeout(ctx, time.Second)
defer cancel()
err = listenChannelWithTimeout(ct, pw.PluginEvents)
assert.NilError(t, err)
resetTestTomb(&testTomb)
}

4
pkg/database/decisions.go
View file

@ -408,7 +408,7 @@ func (c *Client) DeleteDecisionsWithFilter(filter map[string][]string) (string,
return strconv.Itoa(nbDeleted), nil
}
// SoftDeleteDecisionsWithFilter udpate the expiration time to now() for the decisions matching the filter
// SoftDeleteDecisionsWithFilter updates the expiration time to now() for the decisions matching the filter
func (c *Client) SoftDeleteDecisionsWithFilter(filter map[string][]string) (string, error) {
var err error
var start_ip, start_sfx, end_ip, end_sfx int64
@ -426,6 +426,8 @@ func (c *Client) SoftDeleteDecisionsWithFilter(filter map[string][]string) (stri
}
case "scopes":
decisions = decisions.Where(decision.ScopeEQ(value[0]))
case "origin":
decisions = decisions.Where(decision.OriginEQ(value[0]))
case "value":
decisions = decisions.Where(decision.ValueEQ(value[0]))
case "type":

40
pkg/types/dataset_test.go Normal file
View file

@ -0,0 +1,40 @@
package types
import (
"io/ioutil"
"testing"
"github.com/stretchr/testify/assert"
"github.com/jarcoal/httpmock"
)
func TestDownladFile(t *testing.T) {
httpmock.Activate()
defer httpmock.DeactivateAndReset()
//OK
httpmock.RegisterResponder(
"GET",
"https://example.com/xx",
httpmock.NewStringResponder(200, "example content oneoneone"),
)
httpmock.RegisterResponder(
"GET",
"https://example.com/x",
httpmock.NewStringResponder(404, "not found"),
)
err := downloadFile("https://example.com/xx", "./example.txt")
assert.NoError(t, err)
content, err := ioutil.ReadFile("./example.txt")
assert.Equal(t, "example content oneoneone", string(content))
assert.NoError(t, err)
//bad uri
err = downloadFile("https://zz.com", "./example.txt")
assert.Error(t, err)
//404
err = downloadFile("https://example.com/x", "./example.txt")
assert.Error(t, err)
//bad target
err = downloadFile("https://example.com/xx", "")
assert.Error(t, err)
}

15
pkg/types/utils.go
View file

@ -1,6 +1,7 @@
package types
import (
"bufio"
"bytes"
"encoding/gob"
"fmt"
@ -243,3 +244,17 @@ func InSlice(str string, slice []string) bool {
func UtcNow() time.Time {
return time.Now().UTC()
}
func GetLineCountForFile(filepath string) int {
f, err := os.Open(filepath)
if err != nil {
log.Fatalf("unable to open log file %s", filepath)
}
defer f.Close()
lc := 0
fs := bufio.NewScanner(f)
for fs.Scan() {
lc++
}
return lc
}

4
plugins/notifications/dummy/main.go
View file

@ -46,7 +46,7 @@ func (s *DummyPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
if err != nil {
logger.Error(fmt.Sprintf("Cannot open notification file: %s", err))
}
if _, err := f.WriteString(notification.Text); err != nil {
if _, err := f.WriteString(notification.Text + "\n"); err != nil {
f.Close()
logger.Error(fmt.Sprintf("Cannot write notification to file: %s", err))
}
@ -55,7 +55,7 @@ func (s *DummyPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
logger.Error(fmt.Sprintf("Cannot close notification file: %s", err))
}
}
fmt.Print(notification.Text)
fmt.Println(notification.Text)
return &protobufs.Empty{}, nil
}