2022-09-30 14:01:42 +00:00
package fileacquisition_test
2021-06-11 07:53:53 +00:00
import (
"fmt"
"os"
2022-05-17 10:14:59 +00:00
"runtime"
2021-06-11 07:53:53 +00:00
"testing"
"time"
2022-09-30 14:01:42 +00:00
fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file"
2022-02-15 11:50:33 +00:00
"github.com/crowdsecurity/crowdsec/pkg/cstest"
2021-06-11 07:53:53 +00:00
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
2022-09-30 14:01:42 +00:00
"github.com/stretchr/testify/require"
2021-06-11 07:53:53 +00:00
"gopkg.in/tomb.v2"
)
func TestBadConfiguration ( t * testing . T ) {
tests := [ ] struct {
2022-09-30 14:01:42 +00:00
name string
2021-06-11 07:53:53 +00:00
config string
expectedErr string
} {
{
2022-09-30 14:01:42 +00:00
name : "extra configuration key" ,
config : "foobar: asd.log" ,
2021-06-11 07:53:53 +00:00
expectedErr : "line 1: field foobar not found in type fileacquisition.FileConfiguration" ,
} ,
{
2022-09-30 14:01:42 +00:00
name : "missing filenames" ,
config : "mode: tail" ,
2021-06-11 07:53:53 +00:00
expectedErr : "no filename or filenames configuration provided" ,
} ,
{
2022-09-30 14:01:42 +00:00
name : "glob syntax error" ,
2021-06-11 07:53:53 +00:00
config : ` filename: "[asd-.log" ` ,
expectedErr : "Glob failure: syntax error in pattern" ,
} ,
2022-09-06 12:58:37 +00:00
{
2022-09-30 14:01:42 +00:00
name : "bad exclude regexp" ,
2022-09-06 12:58:37 +00:00
config : ` filenames : [ "asd.log" ]
exclude_regexps : [ "as[a-$d" ] ` ,
expectedErr : "Could not compile regexp as" ,
} ,
2021-06-11 07:53:53 +00:00
}
subLogger := log . WithFields ( log . Fields {
"type" : "file" ,
} )
2022-09-30 14:01:42 +00:00
for _ , tc := range tests {
tc := tc
t . Run ( tc . name , func ( t * testing . T ) {
f := fileacquisition . FileSource { }
err := f . Configure ( [ ] byte ( tc . config ) , subLogger )
cstest . RequireErrorContains ( t , err , tc . expectedErr )
} )
2021-06-11 07:53:53 +00:00
}
}
func TestConfigureDSN ( t * testing . T ) {
2022-09-30 14:01:42 +00:00
file := "/etc/passwd"
if runtime . GOOS == "windows" {
file = ` C:\Windows\System32\drivers\etc\hosts `
2022-05-17 10:14:59 +00:00
}
2022-09-30 14:01:42 +00:00
2021-06-11 07:53:53 +00:00
tests := [ ] struct {
dsn string
expectedErr string
} {
{
dsn : "asd://" ,
expectedErr : "invalid DSN asd:// for file source, must start with file://" ,
} ,
{
dsn : "file://" ,
expectedErr : "empty file:// DSN" ,
} ,
{
2022-09-30 14:01:42 +00:00
dsn : fmt . Sprintf ( "file://%s?log_level=warn" , file ) ,
2021-06-11 07:53:53 +00:00
} ,
{
2022-05-17 10:14:59 +00:00
dsn : fmt . Sprintf ( "file://%s?log_level=foobar" , file ) ,
2021-06-11 07:53:53 +00:00
expectedErr : "unknown level foobar: not a valid logrus Level:" ,
} ,
}
2022-09-30 14:01:42 +00:00
2021-06-11 07:53:53 +00:00
subLogger := log . WithFields ( log . Fields {
"type" : "file" ,
} )
2022-09-30 14:01:42 +00:00
for _ , tc := range tests {
tc := tc
t . Run ( tc . dsn , func ( t * testing . T ) {
f := fileacquisition . FileSource { }
err := f . ConfigureByDSN ( tc . dsn , map [ string ] string { "type" : "testtype" } , subLogger )
cstest . RequireErrorContains ( t , err , tc . expectedErr )
} )
2021-06-11 07:53:53 +00:00
}
}
func TestOneShot ( t * testing . T ) {
2022-09-30 14:01:42 +00:00
permDeniedFile := "/etc/shadow"
permDeniedError := "failed opening /etc/shadow: open /etc/shadow: permission denied"
if runtime . GOOS == "windows" {
// Technically, this is not a permission denied error, but we just want to test what happens
// if we do not have access to the file
permDeniedFile = ` C:\Windows\System32\config\SAM `
permDeniedError = ` failed opening C:\Windows\System32\config\SAM: open C:\Windows\System32\config\SAM: The process cannot access the file because it is being used by another process. `
2022-05-17 10:14:59 +00:00
}
2022-09-30 14:01:42 +00:00
2021-06-11 07:53:53 +00:00
tests := [ ] struct {
2022-09-30 14:01:42 +00:00
name string
2022-02-15 11:50:33 +00:00
config string
expectedConfigErr string
expectedErr string
expectedOutput string
expectedLines int
logLevel log . Level
setup func ( )
afterConfigure func ( )
teardown func ( )
2021-06-11 07:53:53 +00:00
} {
{
2022-09-30 14:01:42 +00:00
name : "permission denied" ,
2022-05-17 10:14:59 +00:00
config : fmt . Sprintf ( `
2021-06-11 07:53:53 +00:00
mode : cat
2022-05-17 10:14:59 +00:00
filename : % s ` , permDeniedFile ) ,
2022-09-30 14:01:42 +00:00
expectedErr : permDeniedError ,
logLevel : log . WarnLevel ,
expectedLines : 0 ,
2021-06-11 07:53:53 +00:00
} ,
{
2022-09-30 14:01:42 +00:00
name : "ignored directory" ,
2021-06-11 07:53:53 +00:00
config : `
mode : cat
filename : / ` ,
2022-09-30 14:01:42 +00:00
expectedOutput : "/ is a directory, ignoring it" ,
logLevel : log . WarnLevel ,
expectedLines : 0 ,
2021-06-11 07:53:53 +00:00
} ,
{
2022-09-30 14:01:42 +00:00
name : "glob syntax error" ,
2021-06-11 07:53:53 +00:00
config : `
mode : cat
filename : "[*-.log" ` ,
2022-02-15 11:50:33 +00:00
expectedConfigErr : "Glob failure: syntax error in pattern" ,
logLevel : log . WarnLevel ,
expectedLines : 0 ,
2021-06-11 07:53:53 +00:00
} ,
{
2022-09-30 14:01:42 +00:00
name : "no matching files" ,
2021-06-11 07:53:53 +00:00
config : `
mode : cat
filename : / do / not / exist ` ,
2022-09-30 14:01:42 +00:00
expectedOutput : "No matching files for pattern /do/not/exist" ,
logLevel : log . WarnLevel ,
expectedLines : 0 ,
2021-06-11 07:53:53 +00:00
} ,
{
2022-09-30 14:01:42 +00:00
name : "test.log" ,
2021-06-11 07:53:53 +00:00
config : `
mode : cat
filename : test_files / test . log ` ,
2022-09-30 14:01:42 +00:00
expectedLines : 5 ,
logLevel : log . WarnLevel ,
2021-06-11 07:53:53 +00:00
} ,
{
2022-09-30 14:01:42 +00:00
name : "test.log.gz" ,
2021-06-11 07:53:53 +00:00
config : `
mode : cat
filename : test_files / test . log . gz ` ,
2022-09-30 14:01:42 +00:00
expectedLines : 5 ,
logLevel : log . WarnLevel ,
2021-06-11 07:53:53 +00:00
} ,
{
2022-09-30 14:01:42 +00:00
name : "unexpected end of gzip stream" ,
2021-06-11 07:53:53 +00:00
config : `
mode : cat
filename : test_files / bad . gz ` ,
2022-09-30 14:01:42 +00:00
expectedErr : "failed to read gz test_files/bad.gz: unexpected EOF" ,
expectedLines : 0 ,
logLevel : log . WarnLevel ,
2021-06-11 07:53:53 +00:00
} ,
{
2022-09-30 14:01:42 +00:00
name : "deleted file" ,
2021-06-11 07:53:53 +00:00
config : `
mode : cat
filename : test_files / test_delete . log ` ,
setup : func ( ) {
2022-05-17 10:14:59 +00:00
f , _ := os . Create ( "test_files/test_delete.log" )
f . Close ( )
2021-06-11 07:53:53 +00:00
} ,
afterConfigure : func ( ) {
os . Remove ( "test_files/test_delete.log" )
} ,
2022-05-17 10:14:59 +00:00
expectedErr : "could not stat file test_files/test_delete.log" ,
2021-06-11 07:53:53 +00:00
} ,
}
2022-09-30 14:01:42 +00:00
for _ , tc := range tests {
tc := tc
t . Run ( tc . name , func ( t * testing . T ) {
logger , hook := test . NewNullLogger ( )
logger . SetLevel ( tc . logLevel )
subLogger := logger . WithFields ( log . Fields {
"type" : "file" ,
} )
tomb := tomb . Tomb { }
2022-10-05 14:40:09 +00:00
out := make ( chan types . Event , 100 )
2022-09-30 14:01:42 +00:00
f := fileacquisition . FileSource { }
if tc . setup != nil {
tc . setup ( )
}
err := f . Configure ( [ ] byte ( tc . config ) , subLogger )
cstest . RequireErrorContains ( t , err , tc . expectedConfigErr )
if tc . expectedConfigErr != "" {
return
}
if tc . afterConfigure != nil {
tc . afterConfigure ( )
}
err = f . OneShotAcquisition ( out , & tomb )
2022-10-05 14:40:09 +00:00
actualLines := len ( out )
2022-09-30 14:01:42 +00:00
cstest . RequireErrorContains ( t , err , tc . expectedErr )
if tc . expectedLines != 0 {
assert . Equal ( t , tc . expectedLines , actualLines )
}
if tc . expectedOutput != "" {
assert . Contains ( t , hook . LastEntry ( ) . Message , tc . expectedOutput )
hook . Reset ( )
}
if tc . teardown != nil {
tc . teardown ( )
}
} )
2021-06-11 07:53:53 +00:00
}
}
func TestLiveAcquisition ( t * testing . T ) {
2022-09-30 14:01:42 +00:00
permDeniedFile := "/etc/shadow"
permDeniedError := "unable to read /etc/shadow : open /etc/shadow: permission denied"
testPattern := "test_files/*.log"
if runtime . GOOS == "windows" {
// Technically, this is not a permission denied error, but we just want to test what happens
// if we do not have access to the file
permDeniedFile = ` C:\Windows\System32\config\SAM `
permDeniedError = ` unable to read C:\Windows\System32\config\SAM : open C:\Windows\System32\config\SAM: The process cannot access the file because it is being used by another process `
testPattern = ` test_files\\*.log ` // the \ must be escaped for the yaml config
2022-05-17 10:14:59 +00:00
}
2022-09-30 14:01:42 +00:00
2021-06-11 07:53:53 +00:00
tests := [ ] struct {
2022-09-30 14:01:42 +00:00
name string
2021-06-11 07:53:53 +00:00
config string
expectedErr string
expectedOutput string
expectedLines int
logLevel log . Level
setup func ( )
afterConfigure func ( )
teardown func ( )
} {
{
2022-05-17 10:14:59 +00:00
config : fmt . Sprintf ( `
2021-06-11 07:53:53 +00:00
mode : tail
2022-05-17 10:14:59 +00:00
filename : % s ` , permDeniedFile ) ,
expectedOutput : permDeniedError ,
2021-06-11 07:53:53 +00:00
logLevel : log . InfoLevel ,
expectedLines : 0 ,
2022-05-17 10:14:59 +00:00
name : "PermissionDenied" ,
2021-06-11 07:53:53 +00:00
} ,
{
config : `
mode : tail
filename : / ` ,
expectedOutput : "/ is a directory, ignoring it" ,
logLevel : log . WarnLevel ,
expectedLines : 0 ,
2022-05-17 10:14:59 +00:00
name : "Directory" ,
2021-06-11 07:53:53 +00:00
} ,
{
config : `
mode : tail
filename : / do / not / exist ` ,
expectedOutput : "No matching files for pattern /do/not/exist" ,
logLevel : log . WarnLevel ,
expectedLines : 0 ,
2022-05-17 10:14:59 +00:00
name : "badPattern" ,
2021-06-11 07:53:53 +00:00
} ,
{
2022-05-17 10:14:59 +00:00
config : fmt . Sprintf ( `
2021-06-11 07:53:53 +00:00
mode : tail
filenames :
2022-05-17 10:14:59 +00:00
- % s
force_inotify : true ` , testPattern ) ,
2022-09-30 14:01:42 +00:00
expectedLines : 5 ,
logLevel : log . DebugLevel ,
name : "basicGlob" ,
2021-06-11 07:53:53 +00:00
} ,
{
2022-05-17 10:14:59 +00:00
config : fmt . Sprintf ( `
2021-06-11 07:53:53 +00:00
mode : tail
filenames :
2022-05-17 10:14:59 +00:00
- % s
force_inotify : true ` , testPattern ) ,
2022-09-30 14:01:42 +00:00
expectedLines : 0 ,
logLevel : log . DebugLevel ,
name : "GlobInotify" ,
2021-06-11 07:53:53 +00:00
afterConfigure : func ( ) {
2022-05-17 10:14:59 +00:00
f , _ := os . Create ( "test_files/a.log" )
f . Close ( )
time . Sleep ( 1 * time . Second )
2021-06-11 07:53:53 +00:00
os . Remove ( "test_files/a.log" )
} ,
} ,
{
2022-05-17 10:14:59 +00:00
config : fmt . Sprintf ( `
2021-06-11 07:53:53 +00:00
mode : tail
filenames :
2022-05-17 10:14:59 +00:00
- % s
force_inotify : true ` , testPattern ) ,
2022-09-30 14:01:42 +00:00
expectedLines : 5 ,
logLevel : log . DebugLevel ,
name : "GlobInotifyChmod" ,
2021-06-11 07:53:53 +00:00
afterConfigure : func ( ) {
2022-05-17 10:14:59 +00:00
f , _ := os . Create ( "test_files/a.log" )
f . Close ( )
2021-06-11 07:53:53 +00:00
time . Sleep ( 1 * time . Second )
2022-09-30 14:01:42 +00:00
os . Chmod ( "test_files/a.log" , 0 o000 )
2021-06-11 07:53:53 +00:00
} ,
teardown : func ( ) {
2022-09-30 14:01:42 +00:00
os . Chmod ( "test_files/a.log" , 0 o644 )
2021-06-11 07:53:53 +00:00
os . Remove ( "test_files/a.log" )
} ,
} ,
{
2022-05-17 10:14:59 +00:00
config : fmt . Sprintf ( `
2021-06-11 07:53:53 +00:00
mode : tail
filenames :
2022-05-17 10:14:59 +00:00
- % s
force_inotify : true ` , testPattern ) ,
2022-09-30 14:01:42 +00:00
expectedLines : 5 ,
logLevel : log . DebugLevel ,
name : "InotifyMkDir" ,
2021-06-11 07:53:53 +00:00
afterConfigure : func ( ) {
2022-09-30 14:01:42 +00:00
os . Mkdir ( "test_files/pouet/" , 0 o700 )
2021-06-11 07:53:53 +00:00
} ,
teardown : func ( ) {
os . Remove ( "test_files/pouet/" )
} ,
} ,
}
2022-09-30 14:01:42 +00:00
for _ , tc := range tests {
tc := tc
t . Run ( tc . name , func ( t * testing . T ) {
logger , hook := test . NewNullLogger ( )
logger . SetLevel ( tc . logLevel )
subLogger := logger . WithFields ( log . Fields {
"type" : "file" ,
} )
tomb := tomb . Tomb { }
out := make ( chan types . Event )
f := fileacquisition . FileSource { }
if tc . setup != nil {
tc . setup ( )
}
err := f . Configure ( [ ] byte ( tc . config ) , subLogger )
require . NoError ( t , err )
if tc . afterConfigure != nil {
tc . afterConfigure ( )
}
actualLines := 0
if tc . expectedLines != 0 {
go func ( ) {
for {
select {
case <- out :
actualLines ++
case <- time . After ( 2 * time . Second ) :
return
}
2021-06-11 07:53:53 +00:00
}
2022-09-30 14:01:42 +00:00
} ( )
2021-06-11 07:53:53 +00:00
}
2022-09-30 14:01:42 +00:00
err = f . StreamingAcquisition ( out , & tomb )
cstest . RequireErrorContains ( t , err , tc . expectedErr )
if tc . expectedLines != 0 {
fd , err := os . Create ( "test_files/stream.log" )
2021-06-11 07:53:53 +00:00
if err != nil {
2022-09-30 14:01:42 +00:00
t . Fatalf ( "could not create test file : %s" , err )
}
for i := 0 ; i < 5 ; i ++ {
_ , err = fmt . Fprintf ( fd , "%d\n" , i )
if err != nil {
t . Fatalf ( "could not write test file : %s" , err )
os . Remove ( "test_files/stream.log" )
}
2021-06-11 07:53:53 +00:00
}
2022-09-30 14:01:42 +00:00
fd . Close ( )
// we sleep to make sure we detect the new file
time . Sleep ( 1 * time . Second )
os . Remove ( "test_files/stream.log" )
assert . Equal ( t , tc . expectedLines , actualLines )
2021-06-11 07:53:53 +00:00
}
2022-09-30 14:01:42 +00:00
if tc . expectedOutput != "" {
if hook . LastEntry ( ) == nil {
t . Fatalf ( "expected output %s, but got nothing" , tc . expectedOutput )
}
assert . Contains ( t , hook . LastEntry ( ) . Message , tc . expectedOutput )
hook . Reset ( )
2021-06-11 07:53:53 +00:00
}
2022-09-30 14:01:42 +00:00
if tc . teardown != nil {
tc . teardown ( )
}
2021-06-11 07:53:53 +00:00
2022-09-30 14:01:42 +00:00
tomb . Kill ( nil )
} )
2021-06-11 07:53:53 +00:00
}
}
2022-09-06 12:58:37 +00:00
func TestExclusion ( t * testing . T ) {
config := ` filenames : [ "test_files/*.log*" ]
exclude_regexps : [ "\\.gz$" ] `
logger , hook := test . NewNullLogger ( )
2022-09-30 14:01:42 +00:00
// logger.SetLevel(ts.logLevel)
2022-09-06 12:58:37 +00:00
subLogger := logger . WithFields ( log . Fields {
"type" : "file" ,
} )
2022-09-30 14:01:42 +00:00
f := fileacquisition . FileSource { }
if err := f . Configure ( [ ] byte ( config ) , subLogger ) ; err != nil {
2022-09-06 12:58:37 +00:00
subLogger . Fatalf ( "unexpected error: %s" , err )
}
2022-09-30 14:01:42 +00:00
expectedLogOutput := "Skipping file test_files/test.log.gz as it matches exclude pattern"
2022-09-28 14:18:00 +00:00
if runtime . GOOS == "windows" {
2022-09-30 14:01:42 +00:00
expectedLogOutput = ` Skipping file test_files\test.log.gz as it matches exclude pattern \.gz `
2022-09-28 14:18:00 +00:00
}
2022-09-30 14:01:42 +00:00
2022-09-06 12:58:37 +00:00
if hook . LastEntry ( ) == nil {
t . Fatalf ( "expected output %s, but got nothing" , expectedLogOutput )
}
2022-09-30 14:01:42 +00:00
2022-09-06 12:58:37 +00:00
assert . Contains ( t , hook . LastEntry ( ) . Message , expectedLogOutput )
hook . Reset ( )
}