parent
a6fabcf481
commit
6624fce66a
|
@ -337,14 +337,15 @@ statics:
|
||||||
data:
|
data:
|
||||||
- source_url: https://URL/TO/FILE
|
- source_url: https://URL/TO/FILE
|
||||||
dest_file: LOCAL_FILENAME
|
dest_file: LOCAL_FILENAME
|
||||||
[type: regexp]
|
[type: (regexp|string)]
|
||||||
```
|
```
|
||||||
|
|
||||||
`data` allows user to specify an external source of data.
|
`data` allows user to specify an external source of data.
|
||||||
This section is only relevant when `cscli` is used to install parser from hub, as it will download the `source_url` and store it to `dest_file`. When the parser is not installed from the hub, {{crowdsec.name}} won't download the URL, but the file must exist for the parser to be loaded correctly.
|
This section is only relevant when `cscli` is used to install parser from hub, as it will download the `source_url` and store it to `dest_file`. When the parser is not installed from the hub, {{crowdsec.name}} won't download the URL, but the file must exist for the parser to be loaded correctly.
|
||||||
|
|
||||||
If `type` is set to `regexp`, the content of the file must be one valid (re2) regular expression per line.
|
The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line.
|
||||||
Those regexps will be compiled and kept in cache.
|
The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory.
|
||||||
|
Without specifying a `type`, the file will be downloaded and stored as file and not in memory.
|
||||||
|
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
|
@ -353,6 +354,7 @@ name: crowdsecurity/cdn-whitelist
|
||||||
data:
|
data:
|
||||||
- source_url: https://www.cloudflare.com/ips-v4
|
- source_url: https://www.cloudflare.com/ips-v4
|
||||||
dest_file: cloudflare_ips.txt
|
dest_file: cloudflare_ips.txt
|
||||||
|
type: string
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -362,14 +362,14 @@ If this expression is present and returns false, the overflow will be discarded.
|
||||||
data:
|
data:
|
||||||
- source_url: https://URL/TO/FILE
|
- source_url: https://URL/TO/FILE
|
||||||
dest_file: LOCAL_FILENAME
|
dest_file: LOCAL_FILENAME
|
||||||
[type: regexp]
|
[type: (regexp|string)]
|
||||||
```
|
```
|
||||||
|
|
||||||
`data` allows user to specify an external source of data.
|
`data` allows user to specify an external source of data.
|
||||||
This section is only relevant when `cscli` is used to install scenario from hub, as ill download the `source_url` and store it to `dest_file`. When the scenario is not installed from the hub, {{crowdsec.name}} won't download the URL, but the file must exist for the scenario to be loaded correctly.
|
This section is only relevant when `cscli` is used to install scenario from hub, as ill download the `source_url` and store it to `dest_file`. When the scenario is not installed from the hub, {{crowdsec.name}} won't download the URL, but the file must exist for the scenario to be loaded correctly.
|
||||||
|
The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line.
|
||||||
If `type` is set to `regexp`, the content of the file must be one valid (re2) regular expression per line.
|
The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory.
|
||||||
Those regexps will be compiled and kept in cache.
|
Without specifying a `type`, the file will be downloaded and stored as file and not in memory.
|
||||||
|
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
|
@ -378,6 +378,7 @@ name: crowdsecurity/cdn-whitelist
|
||||||
data:
|
data:
|
||||||
- source_url: https://www.cloudflare.com/ips-v4
|
- source_url: https://www.cloudflare.com/ips-v4
|
||||||
dest_file: cloudflare_ips.txt
|
dest_file: cloudflare_ips.txt
|
||||||
|
type: string
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -160,6 +160,68 @@ func TestRegexpInFile(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFileInit(t *testing.T) {
|
||||||
|
if err := Init(); err != nil {
|
||||||
|
log.Fatalf(err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
filename string
|
||||||
|
types string
|
||||||
|
result int
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "file with type:string",
|
||||||
|
filename: "test_data.txt",
|
||||||
|
types: "string",
|
||||||
|
result: 3,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file with type:re",
|
||||||
|
filename: "test_data_re.txt",
|
||||||
|
types: "regex",
|
||||||
|
result: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file without type",
|
||||||
|
filename: "test_data_no_type.txt",
|
||||||
|
types: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
err := FileInit(TestFolder, test.filename, test.types)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf(err.Error())
|
||||||
|
}
|
||||||
|
if test.types == "string" {
|
||||||
|
if _, ok := dataFile[test.filename]; !ok {
|
||||||
|
t.Fatalf("test '%s' : NOK", test.name)
|
||||||
|
}
|
||||||
|
if isOk := assert.Equal(t, test.result, len(dataFile[test.filename])); !isOk {
|
||||||
|
t.Fatalf("test '%s' : NOK", test.name)
|
||||||
|
}
|
||||||
|
} else if test.types == "regex" {
|
||||||
|
if _, ok := dataFileRegex[test.filename]; !ok {
|
||||||
|
t.Fatalf("test '%s' : NOK", test.name)
|
||||||
|
}
|
||||||
|
if isOk := assert.Equal(t, test.result, len(dataFileRegex[test.filename])); !isOk {
|
||||||
|
t.Fatalf("test '%s' : NOK", test.name)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if _, ok := dataFileRegex[test.filename]; ok {
|
||||||
|
t.Fatalf("test '%s' : NOK", test.name)
|
||||||
|
}
|
||||||
|
if _, ok := dataFile[test.filename]; ok {
|
||||||
|
t.Fatalf("test '%s' : NOK", test.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.Printf("test '%s' : OK", test.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFile(t *testing.T) {
|
func TestFile(t *testing.T) {
|
||||||
if err := Init(); err != nil {
|
if err := Init(); err != nil {
|
||||||
log.Fatalf(err.Error())
|
log.Fatalf(err.Error())
|
||||||
|
|
3
pkg/exprhelpers/tests/test_data_no_type.txt
Normal file
3
pkg/exprhelpers/tests/test_data_no_type.txt
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
Crowdsec
|
||||||
|
Crowdsecurity
|
||||||
|
CrowdSec
|
Loading…
Reference in a new issue