* allow to specify a list of regular expressions to skip some specific files
This commit is contained in:
parent
414282a2c9
commit
9d199fd4a9
|
@ -9,6 +9,7 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -33,6 +34,7 @@ var linesRead = prometheus.NewCounterVec(
|
||||||
|
|
||||||
type FileConfiguration struct {
|
type FileConfiguration struct {
|
||||||
Filenames []string
|
Filenames []string
|
||||||
|
ExcludeRegexps []string `yaml:"exclude_regexps"`
|
||||||
Filename string
|
Filename string
|
||||||
ForceInotify bool `yaml:"force_inotify"`
|
ForceInotify bool `yaml:"force_inotify"`
|
||||||
configuration.DataSourceCommonCfg `yaml:",inline"`
|
configuration.DataSourceCommonCfg `yaml:",inline"`
|
||||||
|
@ -45,6 +47,7 @@ type FileSource struct {
|
||||||
tails map[string]bool
|
tails map[string]bool
|
||||||
logger *log.Entry
|
logger *log.Entry
|
||||||
files []string
|
files []string
|
||||||
|
exclude_regexps []*regexp.Regexp
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *FileSource) Configure(Config []byte, logger *log.Entry) error {
|
func (f *FileSource) Configure(Config []byte, logger *log.Entry) error {
|
||||||
|
@ -74,6 +77,13 @@ func (f *FileSource) Configure(Config []byte, logger *log.Entry) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrapf(err, "Could not create fsnotify watcher")
|
return errors.Wrapf(err, "Could not create fsnotify watcher")
|
||||||
}
|
}
|
||||||
|
for _, exclude := range f.config.ExcludeRegexps {
|
||||||
|
re, err := regexp.Compile(exclude)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Could not compile regexp %s", exclude)
|
||||||
|
}
|
||||||
|
f.exclude_regexps = append(f.exclude_regexps, re)
|
||||||
|
}
|
||||||
f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config)
|
f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config)
|
||||||
for _, pattern := range f.config.Filenames {
|
for _, pattern := range f.config.Filenames {
|
||||||
if f.config.ForceInotify {
|
if f.config.ForceInotify {
|
||||||
|
@ -97,6 +107,19 @@ func (f *FileSource) Configure(Config []byte, logger *log.Entry) error {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
|
|
||||||
|
//check if file is excluded
|
||||||
|
excluded := false
|
||||||
|
for _, pattern := range f.exclude_regexps {
|
||||||
|
if pattern.MatchString(file) {
|
||||||
|
excluded = true
|
||||||
|
f.logger.Infof("Skipping file %s as it matches exclude pattern %s", file, pattern)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if excluded {
|
||||||
|
continue
|
||||||
|
}
|
||||||
if files[0] != pattern && f.config.Mode == configuration.TAIL_MODE { //we have a glob pattern
|
if files[0] != pattern && f.config.Mode == configuration.TAIL_MODE { //we have a glob pattern
|
||||||
directory := filepath.Dir(file)
|
directory := filepath.Dir(file)
|
||||||
f.logger.Debugf("Will add watch to directory: %s", directory)
|
f.logger.Debugf("Will add watch to directory: %s", directory)
|
||||||
|
@ -232,6 +255,19 @@ func (f *FileSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) er
|
||||||
return f.monitorNewFiles(out, t)
|
return f.monitorNewFiles(out, t)
|
||||||
})
|
})
|
||||||
for _, file := range f.files {
|
for _, file := range f.files {
|
||||||
|
//before opening the file, check if we need to specifically avoid it. (XXX)
|
||||||
|
skip := false
|
||||||
|
for _, pattern := range f.exclude_regexps {
|
||||||
|
if pattern.MatchString(file) {
|
||||||
|
f.logger.Infof("file %s matches exclusion pattern %s, skipping", file, pattern.String())
|
||||||
|
skip = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if skip {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
//cf. https://github.com/crowdsecurity/crowdsec/issues/1168
|
//cf. https://github.com/crowdsecurity/crowdsec/issues/1168
|
||||||
//do not rely on stat, reclose file immediately as it's opened by Tail
|
//do not rely on stat, reclose file immediately as it's opened by Tail
|
||||||
fd, err := os.Open(file)
|
fd, err := os.Open(file)
|
||||||
|
@ -252,6 +288,7 @@ func (f *FileSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) er
|
||||||
f.logger.Warnf("%s is a directory, ignoring it.", file)
|
f.logger.Warnf("%s is a directory, ignoring it.", file)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: true, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}})
|
tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: true, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.logger.Errorf("Could not start tailing file %s : %s", file, err)
|
f.logger.Errorf("Could not start tailing file %s : %s", file, err)
|
||||||
|
@ -304,6 +341,20 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error {
|
||||||
if !matched {
|
if !matched {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//before opening the file, check if we need to specifically avoid it. (XXX)
|
||||||
|
skip := false
|
||||||
|
for _, pattern := range f.exclude_regexps {
|
||||||
|
if pattern.MatchString(event.Name) {
|
||||||
|
f.logger.Infof("file %s matches exclusion pattern %s, skipping", event.Name, pattern.String())
|
||||||
|
skip = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if skip {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
if f.tails[event.Name] {
|
if f.tails[event.Name] {
|
||||||
//we already have a tail on it, do not start a new one
|
//we already have a tail on it, do not start a new one
|
||||||
logger.Debugf("Already tailing file %s, not creating a new tail", event.Name)
|
logger.Debugf("Already tailing file %s, not creating a new tail", event.Name)
|
||||||
|
|
|
@ -32,6 +32,11 @@ func TestBadConfiguration(t *testing.T) {
|
||||||
config: `filename: "[asd-.log"`,
|
config: `filename: "[asd-.log"`,
|
||||||
expectedErr: "Glob failure: syntax error in pattern",
|
expectedErr: "Glob failure: syntax error in pattern",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
config: `filenames: ["asd.log"]
|
||||||
|
exclude_regexps: ["as[a-$d"]`,
|
||||||
|
expectedErr: "Could not compile regexp as",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
subLogger := log.WithFields(log.Fields{
|
subLogger := log.WithFields(log.Fields{
|
||||||
|
@ -440,3 +445,25 @@ force_inotify: true`, testPattern),
|
||||||
tomb.Kill(nil)
|
tomb.Kill(nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExclusion(t *testing.T) {
|
||||||
|
|
||||||
|
config := `filenames: ["test_files/*.log*"]
|
||||||
|
exclude_regexps: ["\\.gz$"]`
|
||||||
|
logger, hook := test.NewNullLogger()
|
||||||
|
//logger.SetLevel(ts.logLevel)
|
||||||
|
subLogger := logger.WithFields(log.Fields{
|
||||||
|
"type": "file",
|
||||||
|
})
|
||||||
|
f := FileSource{}
|
||||||
|
err := f.Configure([]byte(config), subLogger)
|
||||||
|
if err != nil {
|
||||||
|
subLogger.Fatalf("unexpected error: %s", err)
|
||||||
|
}
|
||||||
|
expectedLogOutput := "Skipping file test_files/test.log.gz as it matches exclude pattern"
|
||||||
|
if hook.LastEntry() == nil {
|
||||||
|
t.Fatalf("expected output %s, but got nothing", expectedLogOutput)
|
||||||
|
}
|
||||||
|
assert.Contains(t, hook.LastEntry().Message, expectedLogOutput)
|
||||||
|
hook.Reset()
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue