Sqlite : Support automatic db flushing (#91)
* add support for sqlite retention : max_records, max_records_age * reduce verbosity of cwhub
This commit is contained in:
parent
81ef26f406
commit
b9ae94b874
|
@ -157,8 +157,9 @@ cscli api credentials # Display your API credentials
|
||||||
|
|
||||||
outputConfig := outputs.OutputFactory{
|
outputConfig := outputs.OutputFactory{
|
||||||
BackendFolder: config.BackendPluginFolder,
|
BackendFolder: config.BackendPluginFolder,
|
||||||
|
Flush: false,
|
||||||
}
|
}
|
||||||
outputCTX, err = outputs.NewOutput(&outputConfig, false)
|
outputCTX, err = outputs.NewOutput(&outputConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -412,8 +412,9 @@ cscli backup restore ./my-backup`,
|
||||||
|
|
||||||
outputConfig := outputs.OutputFactory{
|
outputConfig := outputs.OutputFactory{
|
||||||
BackendFolder: config.BackendPluginFolder,
|
BackendFolder: config.BackendPluginFolder,
|
||||||
|
Flush: false,
|
||||||
}
|
}
|
||||||
outputCTX, err = outputs.NewOutput(&outputConfig, false)
|
outputCTX, err = outputs.NewOutput(&outputConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Failed to load output plugins : %v", err)
|
log.Fatalf("Failed to load output plugins : %v", err)
|
||||||
}
|
}
|
||||||
|
@ -453,8 +454,9 @@ cscli backup restore ./my-backup`,
|
||||||
|
|
||||||
outputConfig := outputs.OutputFactory{
|
outputConfig := outputs.OutputFactory{
|
||||||
BackendFolder: config.BackendPluginFolder,
|
BackendFolder: config.BackendPluginFolder,
|
||||||
|
Flush: false,
|
||||||
}
|
}
|
||||||
outputCTX, err = outputs.NewOutput(&outputConfig, false)
|
outputCTX, err = outputs.NewOutput(&outputConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Failed to load output plugins : %v", err)
|
log.Fatalf("Failed to load output plugins : %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -167,7 +167,7 @@ func BanAdd(target string, duration string, reason string, action string) error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
log.Infof("Wrote ban to database.")
|
log.Infof("%s %s for %s (%s)", action, target, duration, reason)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -188,9 +188,10 @@ You can add/delete/list or flush current bans in your local ban DB.`,
|
||||||
|
|
||||||
outputConfig := outputs.OutputFactory{
|
outputConfig := outputs.OutputFactory{
|
||||||
BackendFolder: config.BackendPluginFolder,
|
BackendFolder: config.BackendPluginFolder,
|
||||||
|
Flush: false,
|
||||||
}
|
}
|
||||||
|
|
||||||
outputCTX, err = outputs.NewOutput(&outputConfig, false)
|
outputCTX, err = outputs.NewOutput(&outputConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(err.Error())
|
return fmt.Errorf(err.Error())
|
||||||
}
|
}
|
||||||
|
@ -220,9 +221,10 @@ cscli ban add range 1.2.3.0/24 24h "the whole range"`,
|
||||||
Short: "Adds the specific ip to the ban db",
|
Short: "Adds the specific ip to the ban db",
|
||||||
Long: `Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration), expressed in s/m/h.`,
|
Long: `Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration), expressed in s/m/h.`,
|
||||||
Example: `cscli ban add ip 1.2.3.4 12h "the scan"`,
|
Example: `cscli ban add ip 1.2.3.4 12h "the scan"`,
|
||||||
Args: cobra.ExactArgs(3),
|
Args: cobra.MinimumNArgs(3),
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
if err := BanAdd(args[0], args[1], args[2], remediationType); err != nil {
|
reason := strings.Join(args[2:], " ")
|
||||||
|
if err := BanAdd(args[0], args[1], reason, remediationType); err != nil {
|
||||||
log.Fatalf("failed to add ban to sqlite : %v", err)
|
log.Fatalf("failed to add ban to sqlite : %v", err)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
@ -233,9 +235,10 @@ cscli ban add range 1.2.3.0/24 24h "the whole range"`,
|
||||||
Short: "Adds the specific ip to the ban db",
|
Short: "Adds the specific ip to the ban db",
|
||||||
Long: `Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) compatible, expressed in s/m/h.`,
|
Long: `Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) compatible, expressed in s/m/h.`,
|
||||||
Example: `cscli ban add range 1.2.3.0/24 12h "the whole range"`,
|
Example: `cscli ban add range 1.2.3.0/24 12h "the whole range"`,
|
||||||
Args: cobra.ExactArgs(3),
|
Args: cobra.MinimumNArgs(3),
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
if err := BanAdd(args[0], args[1], args[2], remediationType); err != nil {
|
reason := strings.Join(args[2:], " ")
|
||||||
|
if err := BanAdd(args[0], args[1], reason, remediationType); err != nil {
|
||||||
log.Fatalf("failed to add ban to sqlite : %v", err)
|
log.Fatalf("failed to add ban to sqlite : %v", err)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/outputs"
|
"github.com/crowdsecurity/crowdsec/pkg/outputs"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/parser"
|
"github.com/crowdsecurity/crowdsec/pkg/parser"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
"github.com/pkg/errors"
|
||||||
"github.com/sevlyar/go-daemon"
|
"github.com/sevlyar/go-daemon"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
@ -151,11 +152,22 @@ func LoadOutputs(cConfig *csconfig.CrowdSec) error {
|
||||||
return fmt.Errorf("Failed to load output profiles : %v", err)
|
return fmt.Errorf("Failed to load output profiles : %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
OutputRunner, err = outputs.NewOutput(cConfig.OutputConfig, cConfig.Daemonize)
|
//If the user is providing a single file (ie forensic mode), don't flush expired records
|
||||||
|
if cConfig.SingleFile != "" {
|
||||||
|
log.Infof("forensic mode, disable flush")
|
||||||
|
cConfig.OutputConfig.Flush = false
|
||||||
|
} else {
|
||||||
|
cConfig.OutputConfig.Flush = true
|
||||||
|
}
|
||||||
|
OutputRunner, err = outputs.NewOutput(cConfig.OutputConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("output plugins initialization error : %s", err.Error())
|
return fmt.Errorf("output plugins initialization error : %s", err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := OutputRunner.StartAutoCommit(); err != nil {
|
||||||
|
return errors.Wrap(err, "failed to start autocommit")
|
||||||
|
}
|
||||||
|
|
||||||
/* Init the API connector */
|
/* Init the API connector */
|
||||||
if cConfig.APIMode {
|
if cConfig.APIMode {
|
||||||
log.Infof("Loading API client")
|
log.Infof("Loading API client")
|
||||||
|
@ -277,7 +289,6 @@ func main() {
|
||||||
|
|
||||||
if err := LoadBuckets(cConfig); err != nil {
|
if err := LoadBuckets(cConfig); err != nil {
|
||||||
log.Fatalf("Failed to load scenarios: %s", err)
|
log.Fatalf("Failed to load scenarios: %s", err)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := LoadOutputs(cConfig); err != nil {
|
if err := LoadOutputs(cConfig); err != nil {
|
||||||
|
|
|
@ -129,8 +129,6 @@ func termHandler(sig os.Signal) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func serveOneTimeRun(outputRunner outputs.Output) error {
|
func serveOneTimeRun(outputRunner outputs.Output) error {
|
||||||
log.Infof("waiting for acquisition to finish")
|
|
||||||
|
|
||||||
if err := acquisTomb.Wait(); err != nil {
|
if err := acquisTomb.Wait(); err != nil {
|
||||||
log.Warningf("acquisition returned error : %s", err)
|
log.Warningf("acquisition returned error : %s", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,8 +6,12 @@ cscli_dir: "./config/crowdsec-cli"
|
||||||
log_dir: "./logs"
|
log_dir: "./logs"
|
||||||
log_mode: "stdout"
|
log_mode: "stdout"
|
||||||
log_level: info
|
log_level: info
|
||||||
|
prometheus: true
|
||||||
profiling: false
|
profiling: false
|
||||||
sqlite_path: "./test.db"
|
|
||||||
apimode: false
|
apimode: false
|
||||||
plugin:
|
plugin:
|
||||||
backend: "./config/plugins/backend"
|
backend: "./config/plugins/backend"
|
||||||
|
max_records: 10000
|
||||||
|
#30 days = 720 hours
|
||||||
|
max_records_age: 720h
|
||||||
|
|
|
@ -2,4 +2,3 @@ name: sqlite
|
||||||
path: /usr/local/lib/crowdsec/plugins/backend/sqlite.so
|
path: /usr/local/lib/crowdsec/plugins/backend/sqlite.so
|
||||||
config:
|
config:
|
||||||
db_path: /var/lib/crowdsec/data/crowdsec.db
|
db_path: /var/lib/crowdsec/data/crowdsec.db
|
||||||
flush: true
|
|
||||||
|
|
|
@ -7,7 +7,6 @@ cscli_dir: ${CFG}/cscli
|
||||||
log_mode: file
|
log_mode: file
|
||||||
log_level: info
|
log_level: info
|
||||||
profiling: false
|
profiling: false
|
||||||
sqlite_path: ${DATA}/crowdsec.db
|
|
||||||
apimode: true
|
apimode: true
|
||||||
daemon: true
|
daemon: true
|
||||||
prometheus: true
|
prometheus: true
|
||||||
|
@ -15,3 +14,4 @@ prometheus: true
|
||||||
http_listen: 127.0.0.1:6060
|
http_listen: 127.0.0.1:6060
|
||||||
plugin:
|
plugin:
|
||||||
backend: "/etc/crowdsec/plugins/backend"
|
backend: "/etc/crowdsec/plugins/backend"
|
||||||
|
max_records_age: 720h
|
||||||
|
|
|
@ -7,7 +7,6 @@ cscli_dir: ${CFG}/cscli
|
||||||
log_mode: stdout
|
log_mode: stdout
|
||||||
log_level: info
|
log_level: info
|
||||||
profiling: false
|
profiling: false
|
||||||
sqlite_path: ${DATA}/crowdsec.db
|
|
||||||
apimode: false
|
apimode: false
|
||||||
daemon: false
|
daemon: false
|
||||||
prometheus: false
|
prometheus: false
|
||||||
|
|
|
@ -25,14 +25,13 @@ type CrowdSec struct {
|
||||||
SingleFileLabel string //for forensic mode
|
SingleFileLabel string //for forensic mode
|
||||||
PIDFolder string `yaml:"pid_dir,omitempty"`
|
PIDFolder string `yaml:"pid_dir,omitempty"`
|
||||||
LogFolder string `yaml:"log_dir,omitempty"`
|
LogFolder string `yaml:"log_dir,omitempty"`
|
||||||
LogMode string `yaml:"log_mode,omitempty"` //like file, syslog or stdout ?
|
LogMode string `yaml:"log_mode,omitempty"` //like file, syslog or stdout ?
|
||||||
LogLevel log.Level `yaml:"log_level,omitempty"` //trace,debug,info,warning,error
|
LogLevel log.Level `yaml:"log_level,omitempty"` //trace,debug,info,warning,error
|
||||||
Daemonize bool `yaml:"daemon,omitempty"` //true -> go background
|
Daemonize bool `yaml:"daemon,omitempty"` //true -> go background
|
||||||
Profiling bool `yaml:"profiling,omitempty"` //true -> enable runtime profiling
|
Profiling bool `yaml:"profiling,omitempty"` //true -> enable runtime profiling
|
||||||
SQLiteFile string `yaml:"sqlite_path,omitempty"` //path to sqlite output
|
APIMode bool `yaml:"apimode,omitempty"` //true -> enable api push
|
||||||
APIMode bool `yaml:"apimode,omitempty"` //true -> enable api push
|
CsCliFolder string `yaml:"cscli_dir"` //cscli folder
|
||||||
CsCliFolder string `yaml:"cscli_dir"` //cscli folder
|
NbParsers int `yaml:"parser_routines"` //the number of go routines to start for parsing
|
||||||
NbParsers int `yaml:"parser_routines"` //the number of go routines to start for parsing
|
|
||||||
Linter bool
|
Linter bool
|
||||||
Prometheus bool
|
Prometheus bool
|
||||||
HTTPListen string `yaml:"http_listen,omitempty"`
|
HTTPListen string `yaml:"http_listen,omitempty"`
|
||||||
|
@ -53,7 +52,6 @@ func NewCrowdSecConfig() *CrowdSec {
|
||||||
PIDFolder: "/var/run/",
|
PIDFolder: "/var/run/",
|
||||||
LogFolder: "/var/log/",
|
LogFolder: "/var/log/",
|
||||||
LogMode: "stdout",
|
LogMode: "stdout",
|
||||||
SQLiteFile: "/var/lib/crowdsec/data/crowdsec.db",
|
|
||||||
APIMode: false,
|
APIMode: false,
|
||||||
NbParsers: 1,
|
NbParsers: 1,
|
||||||
Prometheus: false,
|
Prometheus: false,
|
||||||
|
@ -89,7 +87,6 @@ func (c *CrowdSec) GetOPT() error {
|
||||||
printInfo := flag.Bool("info", false, "print info-level on stdout")
|
printInfo := flag.Bool("info", false, "print info-level on stdout")
|
||||||
printVersion := flag.Bool("version", false, "display version")
|
printVersion := flag.Bool("version", false, "display version")
|
||||||
APIMode := flag.Bool("api", false, "perform pushes to api")
|
APIMode := flag.Bool("api", false, "perform pushes to api")
|
||||||
SQLiteMode := flag.Bool("sqlite", true, "write overflows to sqlite")
|
|
||||||
profileMode := flag.Bool("profile", false, "Enable performance profiling")
|
profileMode := flag.Bool("profile", false, "Enable performance profiling")
|
||||||
catFile := flag.String("file", "", "Process a single file in time-machine")
|
catFile := flag.String("file", "", "Process a single file in time-machine")
|
||||||
catFileType := flag.String("type", "", "Labels.type for file in time-machine")
|
catFileType := flag.String("type", "", "Labels.type for file in time-machine")
|
||||||
|
@ -156,9 +153,6 @@ func (c *CrowdSec) GetOPT() error {
|
||||||
if *printTrace {
|
if *printTrace {
|
||||||
c.LogLevel = log.TraceLevel
|
c.LogLevel = log.TraceLevel
|
||||||
}
|
}
|
||||||
if !*SQLiteMode {
|
|
||||||
c.SQLiteFile = ""
|
|
||||||
}
|
|
||||||
if *APIMode {
|
if *APIMode {
|
||||||
c.APIMode = true
|
c.APIMode = true
|
||||||
}
|
}
|
||||||
|
|
|
@ -123,7 +123,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
|
||||||
|
|
||||||
subs := strings.Split(path, "/")
|
subs := strings.Split(path, "/")
|
||||||
|
|
||||||
log.Debugf("path:%s, hubdir:%s, installdir:%s", path, Hubdir, Installdir)
|
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, Hubdir, Installdir)
|
||||||
/*we're in hub (~/.cscli/hub/)*/
|
/*we're in hub (~/.cscli/hub/)*/
|
||||||
if strings.HasPrefix(path, Hubdir) {
|
if strings.HasPrefix(path, Hubdir) {
|
||||||
inhub = true
|
inhub = true
|
||||||
|
@ -137,7 +137,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
|
||||||
fauthor = subs[len(subs)-2]
|
fauthor = subs[len(subs)-2]
|
||||||
stage = subs[len(subs)-3]
|
stage = subs[len(subs)-3]
|
||||||
ftype = subs[len(subs)-4]
|
ftype = subs[len(subs)-4]
|
||||||
log.Debugf("HUBB check [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype)
|
log.Tracef("HUBB check [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype)
|
||||||
|
|
||||||
} else if strings.HasPrefix(path, Installdir) { /*we're in install /etc/crowdsec/<type>/... */
|
} else if strings.HasPrefix(path, Installdir) { /*we're in install /etc/crowdsec/<type>/... */
|
||||||
if len(subs) < 3 {
|
if len(subs) < 3 {
|
||||||
|
@ -151,7 +151,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
|
||||||
stage = subs[len(subs)-2]
|
stage = subs[len(subs)-2]
|
||||||
ftype = subs[len(subs)-3]
|
ftype = subs[len(subs)-3]
|
||||||
fauthor = ""
|
fauthor = ""
|
||||||
log.Debugf("INSTALL check [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype)
|
log.Tracef("INSTALL check [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype)
|
||||||
}
|
}
|
||||||
|
|
||||||
//log.Printf("%s -> name:%s stage:%s", path, fname, stage)
|
//log.Printf("%s -> name:%s stage:%s", path, fname, stage)
|
||||||
|
@ -165,7 +165,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
|
||||||
return fmt.Errorf("unknown prefix in %s : fname:%s, fauthor:%s, stage:%s, ftype:%s", path, fname, fauthor, stage, ftype)
|
return fmt.Errorf("unknown prefix in %s : fname:%s, fauthor:%s, stage:%s, ftype:%s", path, fname, fauthor, stage, ftype)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debugf("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype)
|
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
we can encounter 'collections' in the form of a symlink :
|
we can encounter 'collections' in the form of a symlink :
|
||||||
|
@ -176,7 +176,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
|
||||||
if f.Mode()&os.ModeSymlink == 0 {
|
if f.Mode()&os.ModeSymlink == 0 {
|
||||||
local = true
|
local = true
|
||||||
skippedLocal++
|
skippedLocal++
|
||||||
log.Debugf("%s isn't a symlink", path)
|
log.Tracef("%s isn't a symlink", path)
|
||||||
} else {
|
} else {
|
||||||
hubpath, err = os.Readlink(path)
|
hubpath, err = os.Readlink(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -192,7 +192,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
log.Debugf("%s points to %s", path, hubpath)
|
log.Tracef("%s points to %s", path, hubpath)
|
||||||
}
|
}
|
||||||
|
|
||||||
//if it's not a symlink and not in hub, it's a local file, don't bother
|
//if it's not a symlink and not in hub, it's a local file, don't bother
|
||||||
|
@ -214,13 +214,13 @@ func parser_visit(path string, f os.FileInfo, err error) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
//try to find which configuration item it is
|
//try to find which configuration item it is
|
||||||
log.Debugf("check [%s] of %s", fname, ftype)
|
log.Tracef("check [%s] of %s", fname, ftype)
|
||||||
|
|
||||||
match := false
|
match := false
|
||||||
for k, v := range HubIdx[ftype] {
|
for k, v := range HubIdx[ftype] {
|
||||||
log.Debugf("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml")
|
log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml")
|
||||||
if fname != v.FileName {
|
if fname != v.FileName {
|
||||||
log.Debugf("%s != %s (filename)", fname, v.FileName)
|
log.Tracef("%s != %s (filename)", fname, v.FileName)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
//wrong stage
|
//wrong stage
|
||||||
|
@ -238,7 +238,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if path == Hubdir+"/"+v.RemotePath {
|
if path == Hubdir+"/"+v.RemotePath {
|
||||||
log.Debugf("marking %s as downloaded", v.Name)
|
log.Tracef("marking %s as downloaded", v.Name)
|
||||||
v.Downloaded = true
|
v.Downloaded = true
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
@ -274,10 +274,8 @@ func parser_visit(path string, f os.FileInfo, err error) error {
|
||||||
target.FileName = x[len(x)-1]
|
target.FileName = x[len(x)-1]
|
||||||
}
|
}
|
||||||
if version == v.Version {
|
if version == v.Version {
|
||||||
log.Debugf("%s is up-to-date", v.Name)
|
log.Tracef("%s is up-to-date", v.Name)
|
||||||
v.UpToDate = true
|
v.UpToDate = true
|
||||||
} else {
|
|
||||||
log.Debugf("%s is outdated", v.Name)
|
|
||||||
}
|
}
|
||||||
match = true
|
match = true
|
||||||
|
|
||||||
|
@ -310,18 +308,18 @@ func parser_visit(path string, f os.FileInfo, err error) error {
|
||||||
func CollecDepsCheck(v *Item) error {
|
func CollecDepsCheck(v *Item) error {
|
||||||
/*if it's a collection, ensure all the items are installed, or tag it as tainted*/
|
/*if it's a collection, ensure all the items are installed, or tag it as tainted*/
|
||||||
if v.Type == COLLECTIONS {
|
if v.Type == COLLECTIONS {
|
||||||
log.Debugf("checking submembers of %s installed:%t", v.Name, v.Installed)
|
log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
|
||||||
var tmp = [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections}
|
var tmp = [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections}
|
||||||
for idx, ptr := range tmp {
|
for idx, ptr := range tmp {
|
||||||
ptrtype := ItemTypes[idx]
|
ptrtype := ItemTypes[idx]
|
||||||
for _, p := range ptr {
|
for _, p := range ptr {
|
||||||
if val, ok := HubIdx[ptrtype][p]; ok {
|
if val, ok := HubIdx[ptrtype][p]; ok {
|
||||||
log.Debugf("check %s installed:%t", val.Name, val.Installed)
|
log.Tracef("check %s installed:%t", val.Name, val.Installed)
|
||||||
if !v.Installed {
|
if !v.Installed {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if val.Type == COLLECTIONS {
|
if val.Type == COLLECTIONS {
|
||||||
log.Debugf("collec, recurse.")
|
log.Tracef("collec, recurse.")
|
||||||
if err := CollecDepsCheck(&val); err != nil {
|
if err := CollecDepsCheck(&val); err != nil {
|
||||||
return fmt.Errorf("sub collection %s is broken : %s", val.Name, err)
|
return fmt.Errorf("sub collection %s is broken : %s", val.Name, err)
|
||||||
}
|
}
|
||||||
|
@ -341,7 +339,7 @@ func CollecDepsCheck(v *Item) error {
|
||||||
}
|
}
|
||||||
val.BelongsToCollections = append(val.BelongsToCollections, v.Name)
|
val.BelongsToCollections = append(val.BelongsToCollections, v.Name)
|
||||||
HubIdx[ptrtype][p] = val
|
HubIdx[ptrtype][p] = val
|
||||||
log.Debugf("checking for %s - tainted:%t uptodate:%t", p, v.Tainted, v.UpToDate)
|
log.Tracef("checking for %s - tainted:%t uptodate:%t", p, v.Tainted, v.UpToDate)
|
||||||
} else {
|
} else {
|
||||||
log.Fatalf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, v.Name)
|
log.Fatalf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, v.Name)
|
||||||
}
|
}
|
||||||
|
@ -653,7 +651,7 @@ func DownloadLatest(target Item, tdir string, overwrite bool, dataFolder string)
|
||||||
log.Debugf("Download %s sub-item : %s %s", target.Name, ptrtype, p)
|
log.Debugf("Download %s sub-item : %s %s", target.Name, ptrtype, p)
|
||||||
//recurse as it's a collection
|
//recurse as it's a collection
|
||||||
if ptrtype == COLLECTIONS {
|
if ptrtype == COLLECTIONS {
|
||||||
log.Debugf("collection, recurse")
|
log.Tracef("collection, recurse")
|
||||||
HubIdx[ptrtype][p], err = DownloadLatest(val, tdir, overwrite, dataFolder)
|
HubIdx[ptrtype][p], err = DownloadLatest(val, tdir, overwrite, dataFolder)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("Encountered error while downloading sub-item %s %s : %s.", ptrtype, p, err)
|
log.Errorf("Encountered error while downloading sub-item %s %s : %s.", ptrtype, p, err)
|
||||||
|
|
|
@ -22,24 +22,35 @@ type Backend interface {
|
||||||
Flush() error
|
Flush() error
|
||||||
Shutdown() error
|
Shutdown() error
|
||||||
DeleteAll() error
|
DeleteAll() error
|
||||||
|
StartAutoCommit() error
|
||||||
}
|
}
|
||||||
|
|
||||||
type BackendPlugin struct {
|
type BackendPlugin struct {
|
||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
Path string `yaml:"path"`
|
Path string `yaml:"path"`
|
||||||
ConfigFilePath string
|
ConfigFilePath string
|
||||||
Config map[string]string `yaml:"config"`
|
//Config is passed to the backend plugin.
|
||||||
ID string
|
//It contains specific plugin config + plugin config from main yaml file
|
||||||
funcs Backend
|
Config map[string]string `yaml:"config"`
|
||||||
|
ID string
|
||||||
|
funcs Backend
|
||||||
}
|
}
|
||||||
|
|
||||||
type BackendManager struct {
|
type BackendManager struct {
|
||||||
backendPlugins map[string]BackendPlugin
|
backendPlugins map[string]BackendPlugin
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewBackendPlugin(path string, isDaemon bool) (*BackendManager, error) {
|
func NewBackendPlugin(outputConfig map[string]string) (*BackendManager, error) {
|
||||||
var files []string
|
var files []string
|
||||||
var backendManager = &BackendManager{}
|
var backendManager = &BackendManager{}
|
||||||
|
var path string
|
||||||
|
|
||||||
|
if v, ok := outputConfig["backend"]; ok {
|
||||||
|
path = v
|
||||||
|
} else {
|
||||||
|
return nil, fmt.Errorf("missing 'backend' (path to backend plugins)")
|
||||||
|
}
|
||||||
|
//var path = output.BackendFolder
|
||||||
err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
|
err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
|
||||||
if filepath.Ext(path) == ".yaml" {
|
if filepath.Ext(path) == ".yaml" {
|
||||||
files = append(files, path)
|
files = append(files, path)
|
||||||
|
@ -88,17 +99,28 @@ func NewBackendPlugin(path string, isDaemon bool) (*BackendManager, error) {
|
||||||
|
|
||||||
// Add the interface and Init()
|
// Add the interface and Init()
|
||||||
newPlugin.funcs = bInterface
|
newPlugin.funcs = bInterface
|
||||||
if isDaemon {
|
// Merge backend config from main config file
|
||||||
newPlugin.Config["flush"] = "true"
|
if v, ok := outputConfig["debug"]; ok {
|
||||||
} else {
|
newPlugin.Config["debug"] = v
|
||||||
newPlugin.Config["flush"] = "false"
|
}
|
||||||
|
|
||||||
|
if v, ok := outputConfig["max_records"]; ok {
|
||||||
|
newPlugin.Config["max_records"] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
if v, ok := outputConfig["max_records_age"]; ok {
|
||||||
|
newPlugin.Config["max_records_age"] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
if v, ok := outputConfig["flush"]; ok {
|
||||||
|
newPlugin.Config["flush"] = v
|
||||||
}
|
}
|
||||||
|
|
||||||
err = newPlugin.funcs.Init(newPlugin.Config)
|
err = newPlugin.funcs.Init(newPlugin.Config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("plugin '%s' init error : %s", newPlugin.Name, err)
|
return nil, fmt.Errorf("plugin '%s' init error : %s", newPlugin.Name, err)
|
||||||
}
|
}
|
||||||
log.Infof("backend plugin '%s' loaded", newPlugin.Name)
|
log.Debugf("backend plugin '%s' loaded", newPlugin.Name)
|
||||||
backendManager.backendPlugins[newPlugin.Name] = newPlugin
|
backendManager.backendPlugins[newPlugin.Name] = newPlugin
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -175,6 +197,17 @@ func (b *BackendManager) IsBackendPlugin(plugin string) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *BackendManager) StartAutoCommit() error {
|
||||||
|
var err error
|
||||||
|
for _, plugin := range b.backendPlugins {
|
||||||
|
err = plugin.funcs.StartAutoCommit()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (b *BackendManager) ReadAT(timeAT time.Time) ([]map[string]string, error) {
|
func (b *BackendManager) ReadAT(timeAT time.Time) ([]map[string]string, error) {
|
||||||
var ret []map[string]string
|
var ret []map[string]string
|
||||||
var err error
|
var err error
|
||||||
|
|
|
@ -44,7 +44,6 @@ func GetExprEnv(ctx map[string]interface{}) map[string]interface{} {
|
||||||
}
|
}
|
||||||
|
|
||||||
func Init() error {
|
func Init() error {
|
||||||
log.Infof("Expr helper initiated")
|
|
||||||
dataFile = make(map[string][]string)
|
dataFile = make(map[string][]string)
|
||||||
dataFileRegex = make(map[string][]*regexp.Regexp)
|
dataFileRegex = make(map[string][]*regexp.Regexp)
|
||||||
return nil
|
return nil
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwplugin"
|
"github.com/crowdsecurity/crowdsec/pkg/cwplugin"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwapi"
|
"github.com/crowdsecurity/crowdsec/pkg/cwapi"
|
||||||
|
|
||||||
|
@ -18,10 +19,19 @@ import (
|
||||||
"gopkg.in/yaml.v2"
|
"gopkg.in/yaml.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
//OutputFactory is part of the main yaml configuration file, and holds generic backend config
|
||||||
type OutputFactory struct {
|
type OutputFactory struct {
|
||||||
BackendFolder string `yaml:"backend"`
|
BackendFolder string `yaml:"backend,omitempty"`
|
||||||
|
//For the db GC : how many records can we keep at most
|
||||||
|
MaxRecords string `yaml:"max_records,omitempty"`
|
||||||
|
//For the db GC what is the oldest records we tolerate
|
||||||
|
MaxRecordsAge string `yaml:"max_records_age,omitempty"`
|
||||||
|
//Should we automatically flush expired bans
|
||||||
|
Flush bool
|
||||||
|
Debug bool `yaml:"debug"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//Output holds the runtime objects of backend
|
||||||
type Output struct {
|
type Output struct {
|
||||||
API *cwapi.ApiCtx
|
API *cwapi.ApiCtx
|
||||||
bManager *cwplugin.BackendManager
|
bManager *cwplugin.BackendManager
|
||||||
|
@ -86,6 +96,10 @@ func OvflwToOrder(sig types.SignalOccurence, prof types.Profile) (*types.BanOrde
|
||||||
return &ordr, nil, warn
|
return &ordr, nil, warn
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (o *Output) StartAutoCommit() error {
|
||||||
|
return o.bManager.StartAutoCommit()
|
||||||
|
}
|
||||||
|
|
||||||
func (o *Output) Shutdown() error {
|
func (o *Output) Shutdown() error {
|
||||||
var reterr error
|
var reterr error
|
||||||
if o.API != nil {
|
if o.API != nil {
|
||||||
|
@ -100,8 +114,6 @@ func (o *Output) Shutdown() error {
|
||||||
reterr = err
|
reterr = err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
//bManager
|
|
||||||
//TBD : the backend(s) should be stopped in the same way
|
|
||||||
return reterr
|
return reterr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -286,19 +298,6 @@ func (o *Output) LoadAPIConfig(configFile string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o *Output) load(config *OutputFactory, isDaemon bool) error {
|
|
||||||
var err error
|
|
||||||
if config == nil {
|
|
||||||
return fmt.Errorf("missing output plugin configuration")
|
|
||||||
}
|
|
||||||
log.Debugf("loading backend plugins ...")
|
|
||||||
o.bManager, err = cwplugin.NewBackendPlugin(config.BackendFolder, isDaemon)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *Output) Delete(target string) (int, error) {
|
func (o *Output) Delete(target string) (int, error) {
|
||||||
nbDel, err := o.bManager.Delete(target)
|
nbDel, err := o.bManager.Delete(target)
|
||||||
return nbDel, err
|
return nbDel, err
|
||||||
|
@ -327,11 +326,30 @@ func (o *Output) ReadAT(timeAT time.Time) ([]map[string]string, error) {
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewOutput(config *OutputFactory, isDaemon bool) (*Output, error) {
|
func NewOutput(config *OutputFactory) (*Output, error) {
|
||||||
var output Output
|
var output Output
|
||||||
err := output.load(config, isDaemon)
|
var err error
|
||||||
|
|
||||||
|
if config == nil {
|
||||||
|
return nil, fmt.Errorf("missing output plugin configuration")
|
||||||
|
}
|
||||||
|
log.Debugf("loading backend plugins ...")
|
||||||
|
//turn the *OutputFactory into a map[string]string for less constraint
|
||||||
|
backendConfig := map[string]string{
|
||||||
|
"backend": config.BackendFolder,
|
||||||
|
"flush": strconv.FormatBool(config.Flush),
|
||||||
|
"debug": strconv.FormatBool(config.Debug)}
|
||||||
|
|
||||||
|
if config.MaxRecords != "" {
|
||||||
|
backendConfig["max_records"] = config.MaxRecords
|
||||||
|
}
|
||||||
|
if config.MaxRecordsAge != "" {
|
||||||
|
backendConfig["max_records_age"] = config.MaxRecordsAge
|
||||||
|
}
|
||||||
|
|
||||||
|
output.bManager, err = cwplugin.NewBackendPlugin(backendConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, errors.Wrap(err, "failed to load backend plugin")
|
||||||
}
|
}
|
||||||
return &output, nil
|
return &output, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -133,7 +133,8 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx) ([]Node, error) {
|
||||||
pctx.Stages = append(pctx.Stages, k)
|
pctx.Stages = append(pctx.Stages, k)
|
||||||
}
|
}
|
||||||
sort.Strings(pctx.Stages)
|
sort.Strings(pctx.Stages)
|
||||||
log.Infof("Stages loaded: %+v", pctx.Stages)
|
log.Infof("Loaded %d nodes, %d stages", len(nodes), len(pctx.Stages))
|
||||||
|
|
||||||
return nodes, nil
|
return nodes, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,9 +6,21 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
"github.com/pkg/errors"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func (c *Context) DeleteExpired() error {
|
||||||
|
//Delete the expired records
|
||||||
|
if c.flush {
|
||||||
|
retx := c.Db.Where(`strftime("%s", until) < strftime("%s", "now")`).Delete(types.BanApplication{})
|
||||||
|
if retx.RowsAffected > 0 {
|
||||||
|
log.Infof("Flushed %d expired entries from Ban Application", retx.RowsAffected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Context) Flush() error {
|
func (c *Context) Flush() error {
|
||||||
c.lock.Lock()
|
c.lock.Lock()
|
||||||
defer c.lock.Unlock()
|
defer c.lock.Unlock()
|
||||||
|
@ -21,18 +33,120 @@ func (c *Context) Flush() error {
|
||||||
}
|
}
|
||||||
c.tx = c.Db.Begin()
|
c.tx = c.Db.Begin()
|
||||||
c.lastCommit = time.Now()
|
c.lastCommit = time.Now()
|
||||||
//Delete the expired records
|
return nil
|
||||||
if c.flush {
|
}
|
||||||
retx := c.Db.Where(`strftime("%s", until) < strftime("%s", "now")`).Delete(types.BanApplication{})
|
|
||||||
if retx.RowsAffected > 0 {
|
func (c *Context) CleanUpRecordsByAge() error {
|
||||||
log.Infof("Flushed %d expired entries from Ban Application", retx.RowsAffected)
|
//let's fetch all expired records that are more than XX days olds
|
||||||
|
sos := []types.BanApplication{}
|
||||||
|
|
||||||
|
if c.maxDurationRetention == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
//look for soft-deleted events that are OLDER than maxDurationRetention
|
||||||
|
ret := c.Db.Unscoped().Table("ban_applications").Where("deleted_at is not NULL").
|
||||||
|
Where(fmt.Sprintf("deleted_at > date('now','-%d minutes')", int(c.maxDurationRetention.Minutes()))).
|
||||||
|
Order("updated_at desc").Find(&sos)
|
||||||
|
|
||||||
|
if ret.Error != nil {
|
||||||
|
return errors.Wrap(ret.Error, "failed to get count of old records")
|
||||||
|
}
|
||||||
|
|
||||||
|
//no events elligible
|
||||||
|
if len(sos) == 0 || ret.RowsAffected == 0 {
|
||||||
|
log.Debugf("no event older than %s", c.maxDurationRetention.String())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
//let's do it in a single transaction
|
||||||
|
delTx := c.Db.Unscoped().Begin()
|
||||||
|
delRecords := 0
|
||||||
|
|
||||||
|
for _, record := range sos {
|
||||||
|
copy := record
|
||||||
|
delTx.Unscoped().Table("signal_occurences").Where("ID = ?", copy.SignalOccurenceID).Delete(&types.SignalOccurence{})
|
||||||
|
delTx.Unscoped().Table("event_sequences").Where("signal_occurence_id = ?", copy.SignalOccurenceID).Delete(&types.EventSequence{})
|
||||||
|
delTx.Unscoped().Table("ban_applications").Delete(©)
|
||||||
|
//we need to delete associations : event_sequences, signal_occurences
|
||||||
|
delRecords++
|
||||||
|
}
|
||||||
|
ret = delTx.Unscoped().Commit()
|
||||||
|
if ret.Error != nil {
|
||||||
|
return errors.Wrap(ret.Error, "failed to delete records")
|
||||||
|
}
|
||||||
|
log.Printf("max_records_age: deleting %d events (max age:%s)", delRecords, c.maxDurationRetention)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Context) CleanUpRecordsByCount() error {
|
||||||
|
var count int
|
||||||
|
|
||||||
|
if c.maxEventRetention <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ret := c.Db.Unscoped().Table("ban_applications").Order("updated_at desc").Count(&count)
|
||||||
|
|
||||||
|
if ret.Error != nil {
|
||||||
|
return errors.Wrap(ret.Error, "failed to get bans count")
|
||||||
|
}
|
||||||
|
if count < c.maxEventRetention {
|
||||||
|
log.Debugf("%d < %d, don't cleanup", count, c.maxEventRetention)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sos := []types.BanApplication{}
|
||||||
|
/*get soft deleted records oldest to youngest*/
|
||||||
|
records := c.Db.Unscoped().Table("ban_applications").Where("deleted_at is not NULL").Where(`strftime("%s", deleted_at) < strftime("%s", "now")`).Find(&sos)
|
||||||
|
if records.Error != nil {
|
||||||
|
return errors.Wrap(records.Error, "failed to list expired bans for flush")
|
||||||
|
}
|
||||||
|
|
||||||
|
//let's do it in a single transaction
|
||||||
|
delTx := c.Db.Unscoped().Begin()
|
||||||
|
delRecords := 0
|
||||||
|
for _, ld := range sos {
|
||||||
|
copy := ld
|
||||||
|
delTx.Unscoped().Table("signal_occurences").Where("ID = ?", copy.SignalOccurenceID).Delete(&types.SignalOccurence{})
|
||||||
|
delTx.Unscoped().Table("event_sequences").Where("signal_occurence_id = ?", copy.SignalOccurenceID).Delete(&types.EventSequence{})
|
||||||
|
delTx.Unscoped().Table("ban_applications").Delete(©)
|
||||||
|
//we need to delete associations : event_sequences, signal_occurences
|
||||||
|
delRecords++
|
||||||
|
//let's delete as well the associated event_sequence
|
||||||
|
if count-delRecords <= c.maxEventRetention {
|
||||||
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if len(sos) > 0 {
|
||||||
|
//log.Printf("Deleting %d soft-deleted results out of %d total events (%d soft-deleted)", delRecords, count, len(sos))
|
||||||
|
log.Printf("max_records: deleting %d events. (%d soft-deleted)", delRecords, len(sos))
|
||||||
|
ret = delTx.Unscoped().Commit()
|
||||||
|
if ret.Error != nil {
|
||||||
|
return errors.Wrap(ret.Error, "failed to delete records")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.Debugf("didn't find any record to clean")
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Context) AutoCommit() {
|
func (c *Context) StartAutoCommit() error {
|
||||||
|
//TBD : we shouldn't start auto-commit if we are in cli mode ?
|
||||||
|
c.PusherTomb.Go(func() error {
|
||||||
|
c.autoCommit()
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Context) autoCommit() {
|
||||||
|
log.Debugf("starting autocommit")
|
||||||
ticker := time.NewTicker(200 * time.Millisecond)
|
ticker := time.NewTicker(200 * time.Millisecond)
|
||||||
|
cleanUpTicker := time.NewTicker(1 * time.Minute)
|
||||||
|
expireTicker := time.NewTicker(1 * time.Second)
|
||||||
|
if !c.flush {
|
||||||
|
log.Debugf("flush is disabled")
|
||||||
|
}
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-c.PusherTomb.Dying():
|
case <-c.PusherTomb.Dying():
|
||||||
|
@ -51,12 +165,25 @@ func (c *Context) AutoCommit() {
|
||||||
log.Errorf("error while closing db : %s", err)
|
log.Errorf("error while closing db : %s", err)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
|
case <-expireTicker.C:
|
||||||
|
if err := c.DeleteExpired(); err != nil {
|
||||||
|
log.Errorf("Error while deleting expired records: %s", err)
|
||||||
|
}
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
if atomic.LoadInt32(&c.count) != 0 &&
|
if atomic.LoadInt32(&c.count) != 0 &&
|
||||||
(atomic.LoadInt32(&c.count)%100 == 0 || time.Since(c.lastCommit) >= 500*time.Millisecond) {
|
(atomic.LoadInt32(&c.count)%100 == 0 || time.Since(c.lastCommit) >= 500*time.Millisecond) {
|
||||||
if err := c.Flush(); err != nil {
|
if err := c.Flush(); err != nil {
|
||||||
log.Errorf("failed to flush : %s", err)
|
log.Errorf("failed to flush : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
case <-cleanUpTicker.C:
|
||||||
|
if err := c.CleanUpRecordsByCount(); err != nil {
|
||||||
|
log.Errorf("error in max records cleanup : %s", err)
|
||||||
|
}
|
||||||
|
if err := c.CleanUpRecordsByAge(); err != nil {
|
||||||
|
log.Errorf("error in old records cleanup : %s", err)
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
"github.com/pkg/errors"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
"github.com/jinzhu/gorm"
|
"github.com/jinzhu/gorm"
|
||||||
|
@ -23,12 +24,27 @@ type Context struct {
|
||||||
count int32
|
count int32
|
||||||
lock sync.Mutex //booboo
|
lock sync.Mutex //booboo
|
||||||
PusherTomb tomb.Tomb
|
PusherTomb tomb.Tomb
|
||||||
|
//to manage auto cleanup : max number of records *or* oldest
|
||||||
|
maxEventRetention int
|
||||||
|
maxDurationRetention time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewSQLite(cfg map[string]string) (*Context, error) {
|
func NewSQLite(cfg map[string]string) (*Context, error) {
|
||||||
var err error
|
var err error
|
||||||
c := &Context{}
|
c := &Context{}
|
||||||
|
|
||||||
|
if v, ok := cfg["max_records"]; ok {
|
||||||
|
c.maxEventRetention, err = strconv.Atoi(v)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Ignoring invalid max_records '%s' : %s", v, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v, ok := cfg["max_records_age"]; ok {
|
||||||
|
c.maxDurationRetention, err = time.ParseDuration(v)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Ignoring invalid duration '%s' : %s", v, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
if _, ok := cfg["db_path"]; !ok {
|
if _, ok := cfg["db_path"]; !ok {
|
||||||
return nil, fmt.Errorf("please specify a 'db_path' to SQLite db in the configuration")
|
return nil, fmt.Errorf("please specify a 'db_path' to SQLite db in the configuration")
|
||||||
}
|
}
|
||||||
|
@ -36,6 +52,7 @@ func NewSQLite(cfg map[string]string) (*Context, error) {
|
||||||
if cfg["db_path"] == "" {
|
if cfg["db_path"] == "" {
|
||||||
return nil, fmt.Errorf("please specify a 'db_path' to SQLite db in the configuration")
|
return nil, fmt.Errorf("please specify a 'db_path' to SQLite db in the configuration")
|
||||||
}
|
}
|
||||||
|
log.Debugf("Starting SQLite backend, path:%s", cfg["db_path"])
|
||||||
|
|
||||||
c.Db, err = gorm.Open("sqlite3", cfg["db_path"]+"?_busy_timeout=1000")
|
c.Db, err = gorm.Open("sqlite3", cfg["db_path"]+"?_busy_timeout=1000")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -47,7 +64,10 @@ func NewSQLite(cfg map[string]string) (*Context, error) {
|
||||||
c.Db.LogMode(true)
|
c.Db.LogMode(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
c.flush, _ = strconv.ParseBool(cfg["flush"])
|
c.flush, err = strconv.ParseBool(cfg["flush"])
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "Unable to parse 'flush' flag")
|
||||||
|
}
|
||||||
// Migrate the schema
|
// Migrate the schema
|
||||||
c.Db.AutoMigrate(&types.EventSequence{}, &types.SignalOccurence{}, &types.BanApplication{})
|
c.Db.AutoMigrate(&types.EventSequence{}, &types.SignalOccurence{}, &types.BanApplication{})
|
||||||
c.Db.Model(&types.SignalOccurence{}).Related(&types.EventSequence{})
|
c.Db.Model(&types.SignalOccurence{}).Related(&types.EventSequence{})
|
||||||
|
@ -64,9 +84,5 @@ func NewSQLite(cfg map[string]string) (*Context, error) {
|
||||||
if c.tx == nil {
|
if c.tx == nil {
|
||||||
return nil, fmt.Errorf("failed to begin sqlite transac : %s", err)
|
return nil, fmt.Errorf("failed to begin sqlite transac : %s", err)
|
||||||
}
|
}
|
||||||
c.PusherTomb.Go(func() error {
|
|
||||||
c.AutoCommit()
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,10 @@ func (p *pluginDB) Shutdown() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *pluginDB) StartAutoCommit() error {
|
||||||
|
return p.CTX.StartAutoCommit()
|
||||||
|
}
|
||||||
|
|
||||||
func (p *pluginDB) Init(config map[string]string) error {
|
func (p *pluginDB) Init(config map[string]string) error {
|
||||||
var err error
|
var err error
|
||||||
log.Debugf("sqlite config : %+v \n", config)
|
log.Debugf("sqlite config : %+v \n", config)
|
||||||
|
|
Loading…
Reference in a new issue