fix tests (#191)

* fix leakybucket test

docs/references/parsers.md

@@ -337,14 +337,15 @@ statics:
 data:
   - source_url: https://URL/TO/FILE
     dest_file: LOCAL_FILENAME
-    [type: regexp]
+    [type: (regexp|string)]
 ```
 
 `data` allows user to specify an external source of data.
 This section is only relevant when `cscli` is used to install parser from hub, as it will download the `source_url` and store it to `dest_file`. When the parser is not installed from the hub, {{crowdsec.name}} won't download the URL, but the file must exist for the parser to be loaded correctly.
 
-If `type` is set to `regexp`, the content of the file must be one valid (re2) regular expression per line.
-Those regexps will be compiled and kept in cache.
+The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line.
+The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory.
+Without specifying a `type`, the file will be downloaded and stored as file and not in memory.
 
 
 ```yaml
@@ -353,6 +354,7 @@ name: crowdsecurity/cdn-whitelist
 data:
   - source_url: https://www.cloudflare.com/ips-v4
     dest_file: cloudflare_ips.txt
+    type: string
 ```
 
 

docs/references/scenarios.md

@@ -362,14 +362,14 @@ If this expression is present and returns false, the overflow will be discarded.
 data:
   - source_url: https://URL/TO/FILE
     dest_file: LOCAL_FILENAME
-    [type: regexp]
+    [type: (regexp|string)]
 ```
 
 `data` allows user to specify an external source of data.
 This section is only relevant when `cscli` is used to install scenario from hub, as ill download the `source_url` and store it to `dest_file`. When the scenario is not installed from the hub, {{crowdsec.name}} won't download the URL, but the file must exist for the scenario to be loaded correctly.
-
-If `type` is set to `regexp`, the content of the file must be one valid (re2) regular expression per line.
-Those regexps will be compiled and kept in cache.
+The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line.
+The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory.
+Without specifying a `type`, the file will be downloaded and stored as file and not in memory.
 
 
 ```yaml
@@ -378,6 +378,7 @@ name: crowdsecurity/cdn-whitelist
 data:
   - source_url: https://www.cloudflare.com/ips-v4
     dest_file: cloudflare_ips.txt
+    type: string
 ```
 
 

pkg/exprhelpers/exprlib_test.go

@@ -160,6 +160,68 @@ func TestRegexpInFile(t *testing.T) {
 	}
 }
 
+func TestFileInit(t *testing.T) {
+	if err := Init(); err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	tests := []struct {
+		name     string
+		filename string
+		types    string
+		result   int
+		err      error
+	}{
+		{
+			name:     "file with type:string",
+			filename: "test_data.txt",
+			types:    "string",
+			result:   3,
+		},
+		{
+			name:     "file with type:re",
+			filename: "test_data_re.txt",
+			types:    "regex",
+			result:   2,
+		},
+		{
+			name:     "file without type",
+			filename: "test_data_no_type.txt",
+			types:    "",
+		},
+	}
+
+	for _, test := range tests {
+		err := FileInit(TestFolder, test.filename, test.types)
+		if err != nil {
+			log.Fatalf(err.Error())
+		}
+		if test.types == "string" {
+			if _, ok := dataFile[test.filename]; !ok {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+			if isOk := assert.Equal(t, test.result, len(dataFile[test.filename])); !isOk {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+		} else if test.types == "regex" {
+			if _, ok := dataFileRegex[test.filename]; !ok {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+			if isOk := assert.Equal(t, test.result, len(dataFileRegex[test.filename])); !isOk {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+		} else {
+			if _, ok := dataFileRegex[test.filename]; ok {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+			if _, ok := dataFile[test.filename]; ok {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+		}
+		log.Printf("test '%s' : OK", test.name)
+	}
+}
+
 func TestFile(t *testing.T) {
 	if err := Init(); err != nil {
 		log.Fatalf(err.Error())

pkg/exprhelpers/tests/test_data_no_type.txt

@@ -0,0 +1,3 @@
+Crowdsec
+Crowdsecurity
+CrowdSec