From 6624fce66a4bd53ef9ee5d6083e8715385d07f59 Mon Sep 17 00:00:00 2001
From: erenJag <64777133+erenJag@users.noreply.github.com>
Date: Mon, 24 Aug 2020 10:25:52 +0200
Subject: [PATCH] fix tests (#191)

* fix leakybucket test
---
 docs/references/parsers.md                  |  8 ++-
 docs/references/scenarios.md                |  9 +--
 pkg/exprhelpers/exprlib_test.go             | 62 +++++++++++++++++++++
 pkg/exprhelpers/tests/test_data_no_type.txt |  3 +
 4 files changed, 75 insertions(+), 7 deletions(-)
 create mode 100644 pkg/exprhelpers/tests/test_data_no_type.txt

diff --git a/docs/references/parsers.md b/docs/references/parsers.md
index d1a57f162..ba354b705 100644
--- a/docs/references/parsers.md
+++ b/docs/references/parsers.md
@@ -337,14 +337,15 @@ statics:
 data:
   - source_url: https://URL/TO/FILE
     dest_file: LOCAL_FILENAME
-    [type: regexp]
+    [type: (regexp|string)]
 ```
 
 `data` allows user to specify an external source of data.
 This section is only relevant when `cscli` is used to install parser from hub, as it will download the `source_url` and store it to `dest_file`. When the parser is not installed from the hub, {{crowdsec.name}} won't download the URL, but the file must exist for the parser to be loaded correctly.
 
-If `type` is set to `regexp`, the content of the file must be one valid (re2) regular expression per line.
-Those regexps will be compiled and kept in cache.
+The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line.
+The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory.
+Without specifying a `type`, the file will be downloaded and stored as file and not in memory.
 
 
 ```yaml
@@ -353,6 +354,7 @@ name: crowdsecurity/cdn-whitelist
 data:
   - source_url: https://www.cloudflare.com/ips-v4
     dest_file: cloudflare_ips.txt
+    type: string
 ```
 
 
diff --git a/docs/references/scenarios.md b/docs/references/scenarios.md
index 6232f6425..e0bb81693 100644
--- a/docs/references/scenarios.md
+++ b/docs/references/scenarios.md
@@ -362,14 +362,14 @@ If this expression is present and returns false, the overflow will be discarded.
 data:
   - source_url: https://URL/TO/FILE
     dest_file: LOCAL_FILENAME
-    [type: regexp]
+    [type: (regexp|string)]
 ```
 
 `data` allows user to specify an external source of data.
 This section is only relevant when `cscli` is used to install scenario from hub, as ill download the `source_url` and store it to `dest_file`. When the scenario is not installed from the hub, {{crowdsec.name}} won't download the URL, but the file must exist for the scenario to be loaded correctly.
-
-If `type` is set to `regexp`, the content of the file must be one valid (re2) regular expression per line.
-Those regexps will be compiled and kept in cache.
+The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line.
+The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory.
+Without specifying a `type`, the file will be downloaded and stored as file and not in memory.
 
 
 ```yaml
@@ -378,6 +378,7 @@ name: crowdsecurity/cdn-whitelist
 data:
   - source_url: https://www.cloudflare.com/ips-v4
     dest_file: cloudflare_ips.txt
+    type: string
 ```
 
 
diff --git a/pkg/exprhelpers/exprlib_test.go b/pkg/exprhelpers/exprlib_test.go
index b95f3f71a..7bd317bbe 100644
--- a/pkg/exprhelpers/exprlib_test.go
+++ b/pkg/exprhelpers/exprlib_test.go
@@ -160,6 +160,68 @@ func TestRegexpInFile(t *testing.T) {
 	}
 }
 
+func TestFileInit(t *testing.T) {
+	if err := Init(); err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	tests := []struct {
+		name     string
+		filename string
+		types    string
+		result   int
+		err      error
+	}{
+		{
+			name:     "file with type:string",
+			filename: "test_data.txt",
+			types:    "string",
+			result:   3,
+		},
+		{
+			name:     "file with type:re",
+			filename: "test_data_re.txt",
+			types:    "regex",
+			result:   2,
+		},
+		{
+			name:     "file without type",
+			filename: "test_data_no_type.txt",
+			types:    "",
+		},
+	}
+
+	for _, test := range tests {
+		err := FileInit(TestFolder, test.filename, test.types)
+		if err != nil {
+			log.Fatalf(err.Error())
+		}
+		if test.types == "string" {
+			if _, ok := dataFile[test.filename]; !ok {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+			if isOk := assert.Equal(t, test.result, len(dataFile[test.filename])); !isOk {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+		} else if test.types == "regex" {
+			if _, ok := dataFileRegex[test.filename]; !ok {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+			if isOk := assert.Equal(t, test.result, len(dataFileRegex[test.filename])); !isOk {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+		} else {
+			if _, ok := dataFileRegex[test.filename]; ok {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+			if _, ok := dataFile[test.filename]; ok {
+				t.Fatalf("test '%s' : NOK", test.name)
+			}
+		}
+		log.Printf("test '%s' : OK", test.name)
+	}
+}
+
 func TestFile(t *testing.T) {
 	if err := Init(); err != nil {
 		log.Fatalf(err.Error())
diff --git a/pkg/exprhelpers/tests/test_data_no_type.txt b/pkg/exprhelpers/tests/test_data_no_type.txt
new file mode 100644
index 000000000..a80d62137
--- /dev/null
+++ b/pkg/exprhelpers/tests/test_data_no_type.txt
@@ -0,0 +1,3 @@
+Crowdsec
+Crowdsecurity
+CrowdSec
\ No newline at end of file