beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

decouple bouncer dependencies: use go-cs-lib/pkg/* (#2216)

Browse source 
* decouple bouncer dependencies: use go-cs-lib/pkg/trace
* decouple bouncer dependencies: use go-cs-lib/pkg/version
* decouple bouncer dependencies: use go-cs-lib/pkg/yamlpatch
* decouple bouncer dependencies: use go-cs-lib/pkg/csstring
* unused import
This commit is contained in:
mmetc 2023-05-23 10:52:47 +02:00 committed by GitHub
parent eddb994c0b
commit 534328ca30
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
59 changed files with 191 additions and 1297 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Makefile
View file

@ -27,10 +27,10 @@ BUILD_CMD = build
GO_MODULE_NAME = github.com/crowdsecurity/crowdsec
LD_OPTS_VARS= \
-X '$(GO_MODULE_NAME)/pkg/cwversion.Version=$(BUILD_VERSION)' \
-X '$(GO_MODULE_NAME)/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP)' \
-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Version=$(BUILD_VERSION)' \
-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.BuildDate=$(BUILD_TIMESTAMP)' \
-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Tag=$(BUILD_TAG)' \
-X '$(GO_MODULE_NAME)/pkg/cwversion.Codename=$(BUILD_CODENAME)' \
-X '$(GO_MODULE_NAME)/pkg/cwversion.Tag=$(BUILD_TAG)' \
-X '$(GO_MODULE_NAME)/pkg/csconfig.defaultConfigDir=$(DEFAULT_CONFIGDIR)' \
-X '$(GO_MODULE_NAME)/pkg/csconfig.defaultDataDir=$(DEFAULT_DATADIR)'

5
cmd/crowdsec-cli/alerts.go
View file

@ -20,8 +20,9 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -219,7 +220,7 @@ func NewAlertsCmd() *cobra.Command {
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})

7
cmd/crowdsec-cli/capi.go
View file

@ -6,10 +6,11 @@ import (
"net/url"
"os"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -70,7 +71,7 @@ func NewCapiRegisterCmd() *cobra.Command {
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: capiUser,
Password: password,
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiurl,
VersionPrefix: CAPIURLPrefix,
}, nil)
@ -164,7 +165,7 @@ func NewCapiStatusCmd() *cobra.Command {
log.Fatalf("no scenarios installed, abort")
}
Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), nil)
Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
if err != nil {
log.Fatalf("init default client: %s", err)
}

5
cmd/crowdsec-cli/console.go
View file

@ -16,10 +16,11 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -101,7 +102,7 @@ After running this command your will need to validate the enrollment in the weba
MachineID: csConfig.API.Server.OnlineClient.Credentials.Login,
Password: password,
Scenarios: scenarios,
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v3",
})

5
cmd/crowdsec-cli/decisions.go
View file

@ -19,8 +19,9 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -120,7 +121,7 @@ func NewDecisionsCmd() *cobra.Command {
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: password,
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiurl,
VersionPrefix: "v1",
})

7
cmd/crowdsec-cli/lapi.go
View file

@ -14,11 +14,12 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/parser"
@ -51,7 +52,7 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
Client, err = apiclient.NewDefaultClient(apiurl,
LAPIURLPrefix,
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
fmt.Sprintf("crowdsec/%s", version.String()),
nil)
if err != nil {
log.Fatalf("init default client: %s", err)
@ -122,7 +123,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: lapiUser,
Password: password,
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiurl,
VersionPrefix: LAPIURLPrefix,
}, nil)

4
cmd/crowdsec-cli/metrics.go
View file

@ -17,7 +17,7 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
)
// FormatPrometheusMetrics is a complete rip from prom2json
@ -32,7 +32,7 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer types.CatchPanic("crowdsec/ShowPrometheus")
defer trace.CatchPanic("crowdsec/ShowPrometheus")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)

5
cmd/crowdsec-cli/notifications.go
View file

@ -20,11 +20,12 @@ import (
"github.com/spf13/cobra"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/csplugin"
"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
)
@ -273,7 +274,7 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})

4
cmd/crowdsec-cli/support.go
View file

@ -18,6 +18,8 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
@ -182,7 +184,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
Client, err = apiclient.NewDefaultClient(apiurl,
prefix,
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
fmt.Sprintf("crowdsec/%s", version.String()),
nil)
if err != nil {
return []byte(fmt.Sprintf("could not init client: %s", err))

4
cmd/crowdsec-cli/utils.go
View file

@ -21,6 +21,8 @@ import (
"github.com/texttheater/golang-levenshtein/levenshtein"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -515,7 +517,7 @@ func GetPrometheusMetric(url string) []*prom2json.Family {
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer types.CatchPanic("crowdsec/GetPrometheusMetric")
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)

7
cmd/crowdsec/api.go
View file

@ -7,9 +7,10 @@ import (
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/apiserver"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
@ -52,9 +53,9 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
func serveAPIServer(apiServer *apiserver.APIServer, apiReady chan bool) {
apiTomb.Go(func() error {
defer types.CatchPanic("crowdsec/serveAPIServer")
defer trace.CatchPanic("crowdsec/serveAPIServer")
go func() {
defer types.CatchPanic("crowdsec/runAPIServer")
defer trace.CatchPanic("crowdsec/runAPIServer")
log.Debugf("serving API after %s ms", time.Since(crowdsecT0))
if err := apiServer.Run(apiReady); err != nil {
log.Fatal(err)

12
cmd/crowdsec/crowdsec.go
View file

@ -8,6 +8,8 @@ import (
"path/filepath"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@ -53,7 +55,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
parserWg.Add(1)
for i := 0; i < cConfig.Crowdsec.ParserRoutinesCount; i++ {
parsersTomb.Go(func() error {
defer types.CatchPanic("crowdsec/runParse")
defer trace.CatchPanic("crowdsec/runParse")
if err := runParse(inputLineChan, inputEventChan, *parsers.Ctx, parsers.Nodes); err != nil { //this error will never happen as parser.Parse is not able to return errors
log.Fatalf("starting parse error : %s", err)
return err
@ -79,7 +81,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
for i := 0; i < cConfig.Crowdsec.BucketsRoutinesCount; i++ {
bucketsTomb.Go(func() error {
defer types.CatchPanic("crowdsec/runPour")
defer trace.CatchPanic("crowdsec/runPour")
if err := runPour(inputEventChan, holders, buckets, cConfig); err != nil {
log.Fatalf("starting pour error : %s", err)
return err
@ -97,7 +99,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
outputWg.Add(1)
for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
outputsTomb.Go(func() error {
defer types.CatchPanic("crowdsec/runOutput")
defer trace.CatchPanic("crowdsec/runOutput")
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil {
log.Fatalf("starting outputs error : %s", err)
return err
@ -132,9 +134,9 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) {
crowdsecTomb.Go(func() error {
defer types.CatchPanic("crowdsec/serveCrowdsec")
defer trace.CatchPanic("crowdsec/serveCrowdsec")
go func() {
defer types.CatchPanic("crowdsec/runCrowdsec")
defer trace.CatchPanic("crowdsec/runCrowdsec")
// this logs every time, even at config reload
log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
agentReady <- true

9
cmd/crowdsec/metrics.go
View file

@ -9,15 +9,16 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1"
"github.com/crowdsecurity/crowdsec/pkg/cache"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
/*prometheus*/
@ -61,7 +62,7 @@ var globalCsInfo = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "cs_info",
Help: "Information about Crowdsec.",
ConstLabels: prometheus.Labels{"version": cwversion.VersionStr()},
ConstLabels: prometheus.Labels{"version": version.String()},
},
)
@ -187,7 +188,7 @@ func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client,
return
}
defer types.CatchPanic("crowdsec/servePrometheus")
defer trace.CatchPanic("crowdsec/servePrometheus")
http.Handle("/metrics", computeDynamicMetrics(promhttp.Handler(), dbClient))
<-apiReady

5
cmd/crowdsec/output.go
View file

@ -7,10 +7,11 @@ import (
"sync"
"time"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/parser"
@ -88,7 +89,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
MachineID: apiConfig.Login,
Password: password,
Scenarios: scenarios,
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",

9
cmd/crowdsec/run_in_svc.go
View file

@ -10,10 +10,11 @@ import (
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/writer"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func StartRunSvc() error {
@ -22,7 +23,7 @@ func StartRunSvc() error {
err error
)
defer types.CatchPanic("crowdsec/StartRunSvc")
defer trace.CatchPanic("crowdsec/StartRunSvc")
// Set a default logger with level=fatal on stderr,
// in addition to the one we configure afterwards
@ -38,7 +39,7 @@ func StartRunSvc() error {
return err
}
log.Infof("Crowdsec %s", cwversion.VersionStr())
log.Infof("Crowdsec %s", version.String())
apiReady := make(chan bool, 1)
agentReady := make(chan bool, 1)

9
cmd/crowdsec/run_in_svc_windows.go
View file

@ -7,17 +7,18 @@ import (
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows/svc"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func StartRunSvc() error {
const svcName = "CrowdSec"
const svcDescription = "Crowdsec IPS/IDS"
defer types.CatchPanic("crowdsec/StartRunSvc")
defer trace.CatchPanic("crowdsec/StartRunSvc")
isRunninginService, err := svc.IsWindowsService()
if err != nil {
@ -66,7 +67,7 @@ func WindowsRun() error {
return err
}
// Configure logging
log.Infof("Crowdsec %s", cwversion.VersionStr())
log.Infof("Crowdsec %s", version.String())
apiReady := make(chan bool, 1)
agentReady := make(chan bool, 1)

4
cmd/crowdsec/serve.go
View file

@ -11,6 +11,8 @@ import (
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
@ -226,7 +228,7 @@ func HandleSignals(cConfig *csconfig.Config) error {
exitChan := make(chan error)
go func() {
defer types.CatchPanic("crowdsec/HandleSignals")
defer trace.CatchPanic("crowdsec/HandleSignals")
Loop:
for {
s := <-signalChan

6
debian/rules vendored
View file

@ -4,12 +4,6 @@ export DEB_VERSION=$(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d
export BUILD_VERSION=v${DEB_VERSION}-debian-pragmatic
export GO111MODULE=on
# LD_OPTS=-ldflags "-s -w -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Version=$(BUILD_VERSION) \
# -X github.com/crowdsecurity/crowdsec/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP) \
# -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Codename=$(BUILD_CODENAME) \
# -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Tag=$(BUILD_TAG) \
# -X github.com/crowdsecurity/crowdsec/pkg/cwversion.GoVersion=$(BUILD_GOVERSION)"
%:
dh $@

5
go.mod
View file

@ -48,9 +48,9 @@ require (
github.com/prometheus/client_model v0.3.0
github.com/prometheus/prom2json v1.3.0
github.com/r3labs/diff/v2 v2.14.1
github.com/sirupsen/logrus v1.9.0
github.com/sirupsen/logrus v1.9.2
github.com/spf13/cobra v1.7.0
github.com/stretchr/testify v1.8.2
github.com/stretchr/testify v1.8.3
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4
google.golang.org/grpc v1.47.0
@ -101,6 +101,7 @@ require (
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/crowdsecurity/go-cs-lib v0.0.0-20230522120244-fa545c12e7ee // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect

10
go.sum
View file

@ -172,6 +172,8 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
github.com/crowdsecurity/go-cs-lib v0.0.0-20230522120244-fa545c12e7ee h1:YD/SYJ0otjlyAxuLRzJNXTQyXcpccefTvaHextBT9mQ=
github.com/crowdsecurity/go-cs-lib v0.0.0-20230522120244-fa545c12e7ee/go.mod h1:9JJLSpGj1ZXnROV3xAcJvS/HTaUvuA8K3gGOpO4tfVc=
github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
@ -841,8 +843,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y=
github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@ -881,8 +883,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tetratelabs/wazero v1.0.0-rc.2 h1:OA3UUynnoqxrjCQ94mpAtdO4/oMxFQVNL2BXDMOc66Q=
github.com/tetratelabs/wazero v1.0.0-rc.2/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=

6
pkg/acquisition/acquisition.go
View file

@ -15,6 +15,8 @@ import (
tomb "gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch"
dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker"
@ -241,7 +243,7 @@ func GetMetrics(sources []DataSource, aggregated bool) error {
}
func transform(transformChan chan types.Event, output chan types.Event, AcquisTomb *tomb.Tomb, transformRuntime *vm.Program, logger *log.Entry) {
defer types.CatchPanic("crowdsec/acquis")
defer trace.CatchPanic("crowdsec/acquis")
logger.Infof("transformer started")
for {
select {
@ -296,7 +298,7 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb
log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc)
AcquisTomb.Go(func() error {
defer types.CatchPanic("crowdsec/acquis")
defer trace.CatchPanic("crowdsec/acquis")
var err error
outChan := output

6
pkg/acquisition/modules/file/file.go
View file

@ -14,6 +14,8 @@ import (
"strings"
"time"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/fsnotify/fsnotify"
"github.com/nxadm/tail"
"github.com/pkg/errors"
@ -336,7 +338,7 @@ func (f *FileSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) er
}
f.tails[file] = true
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/file/live/fsnotify")
defer trace.CatchPanic("crowdsec/acquis/file/live/fsnotify")
return f.tailFile(out, t, tail)
})
}
@ -419,7 +421,7 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error {
}
f.tails[event.Name] = true
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/tailfile")
defer trace.CatchPanic("crowdsec/acquis/tailfile")
return f.tailFile(out, t, tail)
})
}

6
pkg/acquisition/modules/journalctl/journalctl.go
View file

@ -15,6 +15,8 @@ import (
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -257,7 +259,7 @@ func (j *JournalCtlSource) GetName() string {
}
func (j *JournalCtlSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error {
defer types.CatchPanic("crowdsec/acquis/journalctl/oneshot")
defer trace.CatchPanic("crowdsec/acquis/journalctl/oneshot")
err := j.runJournalCtl(out, t)
j.logger.Debug("Oneshot journalctl acquisition is done")
return err
@ -266,7 +268,7 @@ func (j *JournalCtlSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb
func (j *JournalCtlSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/journalctl/streaming")
defer trace.CatchPanic("crowdsec/acquis/journalctl/streaming")
return j.runJournalCtl(out, t)
})
return nil

4
pkg/acquisition/modules/kafka/kafka.go
View file

@ -17,6 +17,8 @@ import (
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -190,7 +192,7 @@ func (k *KafkaSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) e
k.logger.Infof("start reader on topic '%s'", k.Config.Topic)
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/kafka/live")
defer trace.CatchPanic("crowdsec/acquis/kafka/live")
return k.RunReader(out, t)
})

6
pkg/acquisition/modules/kinesis/kinesis.go
View file

@ -19,6 +19,8 @@ import (
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -490,7 +492,7 @@ func (k *KinesisSource) ReadFromStream(out chan types.Event, t *tomb.Tomb) error
for _, shard := range shards.Shards {
shardId := *shard.ShardId
k.shardReaderTomb.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/kinesis/streaming/shard")
defer trace.CatchPanic("crowdsec/acquis/kinesis/streaming/shard")
return k.ReadFromShard(out, shardId)
})
}
@ -514,7 +516,7 @@ func (k *KinesisSource) ReadFromStream(out chan types.Event, t *tomb.Tomb) error
func (k *KinesisSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/kinesis/streaming")
defer trace.CatchPanic("crowdsec/acquis/kinesis/streaming")
if k.Config.UseEnhancedFanOut {
return k.EnhancedRead(out, t)
} else {

4
pkg/acquisition/modules/kubernetesaudit/k8s_audit.go
View file

@ -8,6 +8,8 @@ import (
"net/http"
"strings"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
@ -133,7 +135,7 @@ func (ka *KubernetesAuditSource) OneShotAcquisition(out chan types.Event, t *tom
func (ka *KubernetesAuditSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
ka.outChan = out
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/k8s-audit/live")
defer trace.CatchPanic("crowdsec/acquis/k8s-audit/live")
ka.logger.Infof("Starting k8s-audit server on %s:%d%s", ka.config.ListenAddr, ka.config.ListenPort, ka.config.WebhookPath)
t.Go(func() error {
err := ka.server.ListenAndServe()

4
pkg/acquisition/modules/syslog/syslog.go
View file

@ -12,6 +12,8 @@ import (
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/parser/rfc3164"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/parser/rfc5424"
@ -142,7 +144,7 @@ func (s *SyslogSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb)
}
s.serverTomb = s.server.StartServer()
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/syslog/live")
defer trace.CatchPanic("crowdsec/acquis/syslog/live")
return s.handleSyslogMsg(out, t, c)
})
return nil

4
pkg/acquisition/modules/wineventlog/wineventlog_windows.go
View file

@ -17,6 +17,8 @@ import (
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -321,7 +323,7 @@ func (w *WinEventLogSource) CanRun() error {
func (w *WinEventLogSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
t.Go(func() error {
defer types.CatchPanic("crowdsec/acquis/wineventlog/streaming")
defer trace.CatchPanic("crowdsec/acquis/wineventlog/streaming")
return w.getEvents(out, t)
})
return nil

11
pkg/apiclient/alerts_service_test.go
View file

@ -8,7 +8,8 @@ import (
"reflect"
"testing"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/models"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
@ -31,7 +32,7 @@ func TestAlertsListAsMachine(t *testing.T) {
client, err := NewClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
@ -233,7 +234,7 @@ func TestAlertsGetAsMachine(t *testing.T) {
client, err := NewClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
@ -423,7 +424,7 @@ func TestAlertsCreateAsMachine(t *testing.T) {
client, err := NewClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
@ -467,7 +468,7 @@ func TestAlertsDeleteAsMachine(t *testing.T) {
client, err := NewClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})

11
pkg/apiclient/auth_service_test.go
View file

@ -10,7 +10,8 @@ import (
"net/url"
"testing"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/models"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
@ -86,7 +87,7 @@ func TestWatcherRegister(t *testing.T) {
clientconfig := Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
}
@ -128,7 +129,7 @@ func TestWatcherAuth(t *testing.T) {
clientConfig := &Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
Scenarios: []string{"crowdsecurity/test"},
@ -216,7 +217,7 @@ func TestWatcherUnregister(t *testing.T) {
mycfg := &Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
Scenarios: []string{"crowdsecurity/test"},
@ -270,7 +271,7 @@ func TestWatcherEnroll(t *testing.T) {
mycfg := &Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
Scenarios: []string{"crowdsecurity/test"},

7
pkg/apiclient/client_http_test.go
View file

@ -8,7 +8,8 @@ import (
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/stretchr/testify/assert"
)
@ -23,7 +24,7 @@ func TestNewRequestInvalid(t *testing.T) {
client, err := NewClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
@ -56,7 +57,7 @@ func TestNewRequestTimeout(t *testing.T) {
client, err := NewClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})

13
pkg/apiclient/client_test.go
View file

@ -11,8 +11,9 @@ import (
"github.com/stretchr/testify/assert"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
)
/*this is a ripoff of google/go-github approach :
@ -55,7 +56,7 @@ func TestNewClientOk(t *testing.T) {
client, err := NewClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
@ -92,7 +93,7 @@ func TestNewClientKo(t *testing.T) {
client, err := NewClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
@ -143,7 +144,7 @@ func TestNewClientRegisterKO(t *testing.T) {
_, err = RegisterClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
}, &http.Client{})
@ -173,7 +174,7 @@ func TestNewClientRegisterOK(t *testing.T) {
client, err := RegisterClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
}, &http.Client{})
@ -201,7 +202,7 @@ func TestNewClientBadAnswer(t *testing.T) {
_, err = RegisterClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
}, &http.Client{})

5
pkg/apiclient/decisions_service_test.go
View file

@ -8,7 +8,8 @@ import (
"reflect"
"testing"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/modelscapi"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -495,7 +496,7 @@ func TestDeleteDecisions(t *testing.T) {
client, err := NewClient(&Config{
MachineID: "test_login",
Password: "test_password",
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})

5
pkg/apiclient/heartbeat.go
View file

@ -6,9 +6,10 @@ import (
"net/http"
"time"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
tomb "gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
)
type HeartBeatService service
@ -32,7 +33,7 @@ func (h *HeartBeatService) Ping(ctx context.Context) (bool, *Response, error) {
func (h *HeartBeatService) StartHeartBeat(ctx context.Context, t *tomb.Tomb) {
t.Go(func() error {
defer types.CatchPanic("crowdsec/apiClient/heartbeat")
defer trace.CatchPanic("crowdsec/apiClient/heartbeat")
hbTimer := time.NewTicker(1 * time.Minute)
for {
select {

14
pkg/apiserver/apic.go
View file

@ -17,9 +17,11 @@ import (
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/database/ent"
"github.com/crowdsecurity/crowdsec/pkg/database/ent/alert"
@ -193,7 +195,7 @@ func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, con
ret.apiClient, err = apiclient.NewClient(&apiclient.Config{
MachineID: config.Credentials.Login,
Password: password,
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v3",
@ -231,7 +233,7 @@ func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, con
// keep track of all alerts in cache and push it to CAPI every PushInterval.
func (a *apic) Push() error {
defer types.CatchPanic("lapi/pushToAPIC")
defer trace.CatchPanic("lapi/pushToAPIC")
var cache models.AddSignalsRequest
ticker := time.NewTicker(a.pushIntervalFirst)
@ -778,7 +780,7 @@ func setAlertScenario(add_counters map[string]map[string]int, delete_counters ma
}
func (a *apic) Pull() error {
defer types.CatchPanic("lapi/pullFromAPIC")
defer trace.CatchPanic("lapi/pullFromAPIC")
toldOnce := false
for {
@ -820,7 +822,7 @@ func (a *apic) Pull() error {
func (a *apic) GetMetrics() (*models.Metrics, error) {
metric := &models.Metrics{
ApilVersion: types.StrPtr(cwversion.VersionStr()),
ApilVersion: types.StrPtr(version.String()),
Machines: make([]*models.MetricsAgentInfo, 0),
Bouncers: make([]*models.MetricsBouncerInfo, 0),
}
@ -861,7 +863,7 @@ func (a *apic) GetMetrics() (*models.Metrics, error) {
}
func (a *apic) SendMetrics(stop chan (bool)) {
defer types.CatchPanic("lapi/metricsToAPIC")
defer trace.CatchPanic("lapi/metricsToAPIC")
ticker := time.NewTicker(a.metricsIntervalFirst)

21
pkg/apiserver/apic_test.go
View file

@ -20,10 +20,11 @@ import (
"github.com/stretchr/testify/require"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cstest"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/database/ent/decision"
"github.com/crowdsecurity/crowdsec/pkg/database/ent/machine"
@ -293,7 +294,7 @@ func TestAPICGetMetrics(t *testing.T) {
machineIDs: []string{},
bouncers: []string{},
expectedMetric: &models.Metrics{
ApilVersion: types.StrPtr(cwversion.VersionStr()),
ApilVersion: types.StrPtr(version.String()),
Bouncers: []*models.MetricsBouncerInfo{},
Machines: []*models.MetricsAgentInfo{},
},
@ -303,7 +304,7 @@ func TestAPICGetMetrics(t *testing.T) {
machineIDs: []string{"a", "b", "c"},
bouncers: []string{"1", "2", "3"},
expectedMetric: &models.Metrics{
ApilVersion: types.StrPtr(cwversion.VersionStr()),
ApilVersion: types.StrPtr(version.String()),
Bouncers: []*models.MetricsBouncerInfo{
{
CustomName: "1",
@ -661,7 +662,7 @@ func TestAPICWhitelists(t *testing.T) {
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)
@ -792,7 +793,7 @@ func TestAPICPullTop(t *testing.T) {
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)
@ -874,7 +875,7 @@ func TestAPICPullTopBLCacheFirstCall(t *testing.T) {
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)
@ -961,7 +962,7 @@ func TestAPICPullTopBLCacheForceCall(t *testing.T) {
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)
@ -1036,7 +1037,7 @@ func TestAPICPush(t *testing.T) {
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)
@ -1111,7 +1112,7 @@ func TestAPICSendMetrics(t *testing.T) {
apiClient, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)
@ -1179,7 +1180,7 @@ func TestAPICPull(t *testing.T) {
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)

6
pkg/apiserver/apiserver.go
View file

@ -12,6 +12,8 @@ import (
"strings"
"time"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers"
v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
@ -87,7 +89,7 @@ func CustomRecoveryWithWriter() gin.HandlerFunc {
log.Warningf("client %s disconnected : %s", c.ClientIP(), err)
c.Abort()
} else {
filename := types.WriteStackTrace(err)
filename := trace.WriteStackTrace(err)
log.Warningf("client %s error : %s", c.ClientIP(), err)
log.Warningf("stacktrace written to %s, please join to your issue", filename)
c.AbortWithStatus(http.StatusInternalServerError)
@ -325,7 +327,7 @@ func (s *APIServer) GetTLSConfig() (*tls.Config, error) {
}
func (s *APIServer) Run(apiReady chan bool) error {
defer types.CatchPanic("lapi/runServer")
defer trace.CatchPanic("lapi/runServer")
tlsCfg, err := s.GetTLSConfig()
if err != nil {
return errors.Wrap(err, "while creating TLS config")

5
pkg/apiserver/apiserver_test.go
View file

@ -11,8 +11,9 @@ import (
"testing"
"time"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/go-openapi/strfmt"
@ -33,7 +34,7 @@ var MachineTest = models.WatcherAuthRequest{
Password: &testPassword,
}
var UserAgent = fmt.Sprintf("crowdsec-test/%s", cwversion.Version)
var UserAgent = fmt.Sprintf("crowdsec-test/%s", version.Version)
var emptyBody = strings.NewReader("")
func LoadTestConfig() csconfig.Config {

7
pkg/apiserver/papi.go
View file

@ -8,6 +8,8 @@ import (
"sync"
"time"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/database"
@ -217,8 +219,7 @@ func (p *Papi) PullOnce(since time.Time, sync bool) error {
// PullPAPI is the long polling client for real-time decisions from PAPI
func (p *Papi) Pull() error {
defer types.CatchPanic("lapi/PullPAPI")
defer trace.CatchPanic("lapi/PullPAPI")
p.Logger.Infof("Starting Polling API Pull")
lastTimestamp := time.Time{}
@ -270,7 +271,7 @@ func (p *Papi) Pull() error {
}
func (p *Papi) SyncDecisions() error {
defer types.CatchPanic("lapi/syncDecisionsToCAPI")
defer trace.CatchPanic("lapi/syncDecisionsToCAPI")
var cache models.DecisionsDeleteRequest
ticker := time.NewTicker(p.SyncInterval)

3
pkg/csconfig/api.go
View file

@ -13,9 +13,10 @@ import (
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/yamlpatch"
)
type APICfg struct {

5
pkg/csconfig/config.go
View file

@ -9,9 +9,10 @@ import (
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/csstring"
"github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch"
"github.com/crowdsecurity/go-cs-lib/pkg/csstring"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/yamlpatch"
)
// defaultConfigDir is the base path to all configuration files, to be overridden in the Makefile */

3
pkg/csconfig/profiles.go
View file

@ -5,8 +5,9 @@ import (
"fmt"
"io"
"github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/yamlpatch"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
)

2
pkg/csconfig/simulation.go
View file

@ -4,7 +4,7 @@ import (
"fmt"
"path/filepath"
"github.com/crowdsecurity/crowdsec/pkg/yamlpatch"
"github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch"
"gopkg.in/yaml.v2"
)

3
pkg/csplugin/broker.go
View file

@ -19,8 +19,9 @@ import (
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/csstring"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/csstring"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
"github.com/crowdsecurity/crowdsec/pkg/types"

75
pkg/csstring/expand.go
View file

@ -1,75 +0,0 @@
package csstring
func seekClosingBracket(s string, i int) int {
for ; i < len(s); i++ {
if s[i] == '}' {
return i
}
}
return -1
}
func seekEndVarname(s string, i int) int {
// envvar names are more strict but this is good enough
for ; i < len(s); i++ {
if (s[i] < 'a' || s[i] > 'z') && (s[i] < 'A' || s[i] > 'Z') && (s[i] < '0' || s[i] > '9') && s[i] != '_' {
break
}
}
return i
}
func replaceVarBracket(s string, i int, mapping func(string) (string, bool)) string {
j := seekClosingBracket(s, i+2)
if j < 0 {
return s
}
if j < len(s) {
varName := s[i+2 : j]
if val, ok := mapping(varName); ok {
s = s[:i] + val + s[j+1:]
}
}
return s
}
func replaceVar(s string, i int, mapping func(string) (string, bool)) string {
if s[i+1] == '{' {
return replaceVarBracket(s, i, mapping)
}
j := seekEndVarname(s, i+1)
if j < 0 {
return s
}
if j > i+1 {
varName := s[i+1 : j]
if val, ok := mapping(varName); ok {
s = s[:i] + val + s[j:]
}
}
return s
}
// StrictExpand replaces ${var} or $var in the string according to the mapping
// function, like os.Expand. The difference is that the mapping function
// returns a boolean indicating whether the variable was found.
// If the variable was not found, the string is not modified.
//
// Whereas os.ExpandEnv uses os.Getenv, here we can use os.LookupEnv
// to distinguish between an empty variable and an undefined one.
func StrictExpand(s string, mapping func(string) (string, bool)) string {
for i := 0; i < len(s); i++ {
if s[i] == '$' {
s = replaceVar(s, i, mapping)
}
}
return s
}

98
pkg/csstring/expand_test.go
View file

@ -1,98 +0,0 @@
package csstring_test
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/crowdsecurity/crowdsec/pkg/csstring"
)
func TestStrictExpand(t *testing.T) {
t.Parallel()
testenv := func(key string) (string, bool) {
switch key {
case "USER":
return "testuser", true
case "HOME":
return "/home/testuser", true
case "empty":
return "", true
default:
return "", false
}
}
home, _ := testenv("HOME")
user, _ := testenv("USER")
tests := []struct {
input string
expected string
}{
{
input: "$HOME",
expected: home,
},
{
input: "${USER}",
expected: user,
},
{
input: "Hello, $USER!",
expected: fmt.Sprintf("Hello, %s!", user),
},
{
input: "My home directory is ${HOME}",
expected: fmt.Sprintf("My home directory is %s", home),
},
{
input: "This is a $SINGLE_VAR string with ${HOME}",
expected: fmt.Sprintf("This is a $SINGLE_VAR string with %s", home),
},
{
input: "This is a $SINGLE_VAR string with $HOME",
expected: fmt.Sprintf("This is a $SINGLE_VAR string with %s", home),
},
{
input: "This variable does not exist: $NON_EXISTENT_VAR",
expected: "This variable does not exist: $NON_EXISTENT_VAR",
},
{
input: "This is a $MULTI_VAR string with ${HOME} and ${USER}",
expected: fmt.Sprintf("This is a $MULTI_VAR string with %s and %s", home, user),
},
{
input: "This is a ${MULTI_VAR} string with $HOME and $USER",
expected: fmt.Sprintf("This is a ${MULTI_VAR} string with %s and %s", home, user),
},
{
input: "This is a plain string with no variables",
expected: "This is a plain string with no variables",
},
{
input: "$empty",
expected: "",
},
{
input: "",
expected: "",
},
{
input: "$USER:$empty:$HOME",
expected: fmt.Sprintf("%s::%s", user, home),
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.input, func(t *testing.T) {
t.Parallel()
output := csstring.StrictExpand(tc.input, testenv)
assert.Equal(t, tc.expected, output)
})
}
}

42
pkg/cwversion/version.go
View file

@ -8,27 +8,13 @@ import (
"runtime"
"strings"
version "github.com/hashicorp/go-version"
goversion "github.com/hashicorp/go-version"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
)
/*
Given a version number MAJOR.MINOR.PATCH, increment the:
MAJOR version when you make incompatible API changes,
MINOR version when you add functionality in a backwards compatible manner, and
PATCH version when you make backwards compatible bug fixes.
Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.
*/
var (
Version string // = "v0.0.0"
Codename string // = "SoumSoum"
BuildDate string // = "0000-00-00_00:00:00"
Tag string // = "dev"
GoVersion = runtime.Version()[2:] // = "1.13"
System = runtime.GOOS // = "linux"
Constraint_parser = ">= 1.0, <= 2.0"
Constraint_scenario = ">= 1.0, < 3.0"
@ -38,19 +24,19 @@ var (
func ShowStr() string {
ret := ""
ret += fmt.Sprintf("version: %s-%s\n", Version, Tag)
ret += fmt.Sprintf("version: %s-%s\n", version.Version, version.Tag)
ret += fmt.Sprintf("Codename: %s\n", Codename)
ret += fmt.Sprintf("BuildDate: %s\n", BuildDate)
ret += fmt.Sprintf("GoVersion: %s\n", GoVersion)
ret += fmt.Sprintf("BuildDate: %s\n", version.BuildDate)
ret += fmt.Sprintf("GoVersion: %s\n", version.GoVersion)
ret += fmt.Sprintf("Platform: %s\n", System)
return ret
}
func Show() {
log.Printf("version: %s-%s", Version, Tag)
log.Printf("version: %s-%s", version.Version, version.Tag)
log.Printf("Codename: %s", Codename)
log.Printf("BuildDate: %s", BuildDate)
log.Printf("GoVersion: %s", GoVersion)
log.Printf("BuildDate: %s", version.BuildDate)
log.Printf("GoVersion: %s", version.GoVersion)
log.Printf("Platform: %s\n", System)
log.Printf("Constraint_parser: %s", Constraint_parser)
log.Printf("Constraint_scenario: %s", Constraint_scenario)
@ -59,21 +45,21 @@ func Show() {
}
func VersionStr() string {
return fmt.Sprintf("%s-%s-%s", Version, System, Tag)
return fmt.Sprintf("%s-%s-%s", version.Version, System, version.Tag)
}
func VersionStrip() string {
version := strings.Split(Version, "~")
version := strings.Split(version.Version, "~")
version = strings.Split(version[0], "-")
return version[0]
}
func Statisfies(strvers string, constraint string) (bool, error) {
vers, err := version.NewVersion(strvers)
func Satisfies(strvers string, constraint string) (bool, error) {
vers, err := goversion.NewVersion(strvers)
if err != nil {
return false, fmt.Errorf("failed to parse '%s' : %v", strvers, err)
}
constraints, err := version.NewConstraint(constraint)
constraints, err := goversion.NewConstraint(constraint)
if err != nil {
return false, fmt.Errorf("failed to parse constraint '%s'", constraint)
}

4
pkg/leakybucket/bucket.go
View file

@ -6,6 +6,8 @@ import (
"sync/atomic"
"time"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/time/rate"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/davecgh/go-spew/spew"
@ -202,7 +204,7 @@ func LeakRoutine(leaky *Leaky) error {
firstEvent = true
)
defer types.CatchPanic(fmt.Sprintf("crowdsec/LeakRoutine/%s", leaky.Name))
defer trace.CatchPanic(fmt.Sprintf("crowdsec/LeakRoutine/%s", leaky.Name))
BucketsCurrentCount.With(prometheus.Labels{"name": leaky.Name}).Inc()
defer BucketsCurrentCount.With(prometheus.Labels{"name": leaky.Name}).Dec()

2
pkg/leakybucket/manager_load.go
View file

@ -200,7 +200,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
log.Tracef("no version in %s : %s, assuming '1.0'", bucketFactory.Name, f)
bucketFactory.FormatVersion = "1.0"
}
ok, err := cwversion.Statisfies(bucketFactory.FormatVersion, cwversion.Constraint_scenario)
ok, err := cwversion.Satisfies(bucketFactory.FormatVersion, cwversion.Constraint_scenario)
if err != nil {
log.Fatalf("Failed to check version : %s", err)
}

5
pkg/metabase/api.go
View file

@ -6,7 +6,8 @@ import (
"net/http"
"time"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/dghubble/sling"
log "github.com/sirupsen/logrus"
)
@ -37,7 +38,7 @@ var (
func NewAPIClient(url string) (*APIClient, error) {
httpClient := &http.Client{Timeout: 20 * time.Second}
return &APIClient{
CTX: sling.New().Client(httpClient).Base(url).Set("User-Agent", fmt.Sprintf("crowdsec/%s", cwversion.VersionStr())),
CTX: sling.New().Client(httpClient).Base(url).Set("User-Agent", fmt.Sprintf("crowdsec/%s", version.String())),
Client: httpClient,
}, nil
}

2
pkg/parser/stage.go
View file

@ -83,7 +83,7 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) (
log.Tracef("no version in %s, assuming '1.0'", node.Name)
node.FormatVersion = "1.0"
}
ok, err := cwversion.Statisfies(node.FormatVersion, cwversion.Constraint_parser)
ok, err := cwversion.Satisfies(node.FormatVersion, cwversion.Constraint_parser)
if err != nil {
log.Fatalf("Failed to check version : %s", err)
}

40
pkg/types/utils.go
View file

@ -9,15 +9,12 @@ import (
"os"
"path/filepath"
"regexp"
"runtime/debug"
"strconv"
"strings"
"time"
log "github.com/sirupsen/logrus"
"gopkg.in/natefinch/lumberjack.v2"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
)
var logFormatter log.Formatter
@ -25,7 +22,6 @@ var LogOutput *lumberjack.Logger //io.Writer
var logLevel log.Level
func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level, maxSize int, maxFiles int, maxAge int, compress *bool, forceColors bool) error {
/*Configure logs*/
if cfgMode == "file" {
_maxsize := 500
@ -87,7 +83,6 @@ func ConfigureLogger(clog *log.Logger) error {
}
func Clone(a, b interface{}) error {
buff := new(bytes.Buffer)
enc := gob.NewEncoder(buff)
dec := gob.NewDecoder(buff)
@ -100,40 +95,6 @@ func Clone(a, b interface{}) error {
return nil
}
func WriteStackTrace(iErr interface{}) string {
tmpfile, err := os.CreateTemp("", "crowdsec-crash.*.txt")
if err != nil {
log.Fatal(err)
}
if _, err := tmpfile.Write([]byte(fmt.Sprintf("error : %+v\n", iErr))); err != nil {
tmpfile.Close()
log.Fatal(err)
}
if _, err := tmpfile.Write([]byte(cwversion.ShowStr())); err != nil {
tmpfile.Close()
log.Fatal(err)
}
if _, err := tmpfile.Write(debug.Stack()); err != nil {
tmpfile.Close()
log.Fatal(err)
}
if err := tmpfile.Close(); err != nil {
log.Fatal(err)
}
return tmpfile.Name()
}
//CatchPanic is a util func that we should call from all go-routines to ensure proper stacktrace handling
func CatchPanic(component string) {
if r := recover(); r != nil {
log.Errorf("crowdsec - goroutine %s crashed : %s", component, r)
log.Errorf("please report this error to https://github.com/crowdsecurity/crowdsec/")
filename := WriteStackTrace(r)
log.Errorf("stacktrace/report is written to %s : please join it to your issue", filename)
log.Fatalf("crowdsec stopped")
}
}
func ParseDuration(d string) (time.Duration, error) {
durationStr := d
if strings.HasSuffix(d, "d") {
@ -181,7 +142,6 @@ func copyFileContents(src, dst string) (err error) {
/*copy the file, ioutile doesn't offer the feature*/
func CopyFile(sourceSymLink, destinationFile string) (err error) {
sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
if err != nil {
log.Infof("Not a symlink : %s", err)

168
pkg/yamlpatch/merge.go
View file

@ -1,168 +0,0 @@
//
// from https://github.com/uber-go/config/tree/master/internal/merge
//
// Copyright (c) 2019 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package yamlpatch
import (
"bytes"
"fmt"
"io"
"github.com/pkg/errors"
yaml "gopkg.in/yaml.v2"
)
type (
// YAML has three fundamental types. When unmarshaled into interface{},
// they're represented like this.
mapping = map[interface{}]interface{}
sequence = []interface{}
)
// YAML deep-merges any number of YAML sources, with later sources taking
// priority over earlier ones.
//
// Maps are deep-merged. For example,
// {"one": 1, "two": 2} + {"one": 42, "three": 3}
// == {"one": 42, "two": 2, "three": 3}
// Sequences are replaced. For example,
// {"foo": [1, 2, 3]} + {"foo": [4, 5, 6]}
// == {"foo": [4, 5, 6]}
//
// In non-strict mode, duplicate map keys are allowed within a single source,
// with later values overwriting previous ones. Attempting to merge
// mismatched types (e.g., merging a sequence into a map) replaces the old
// value with the new.
//
// Enabling strict mode returns errors in both of the above cases.
func YAML(sources [][]byte, strict bool) (*bytes.Buffer, error) {
var merged interface{}
var hasContent bool
for _, r := range sources {
d := yaml.NewDecoder(bytes.NewReader(r))
d.SetStrict(strict)
var contents interface{}
if err := d.Decode(&contents); err == io.EOF {
// Skip empty and comment-only sources, which we should handle
// differently from explicit nils.
continue
} else if err != nil {
return nil, fmt.Errorf("couldn't decode source: %v", err)
}
hasContent = true
pair, err := merge(merged, contents, strict)
if err != nil {
return nil, err // error is already descriptive enough
}
merged = pair
}
buf := &bytes.Buffer{}
if !hasContent {
// No sources had any content. To distinguish this from a source with just
// an explicit top-level null, return an empty buffer.
return buf, nil
}
enc := yaml.NewEncoder(buf)
if err := enc.Encode(merged); err != nil {
return nil, errors.Wrap(err, "couldn't re-serialize merged YAML")
}
return buf, nil
}
func merge(into, from interface{}, strict bool) (interface{}, error) {
// It's possible to handle this with a mass of reflection, but we only need
// to merge whole YAML files. Since we're always unmarshaling into
// interface{}, we only need to handle a few types. This ends up being
// cleaner if we just handle each case explicitly.
if into == nil {
return from, nil
}
if from == nil {
// Allow higher-priority YAML to explicitly nil out lower-priority entries.
return nil, nil
}
if IsScalar(into) && IsScalar(from) {
return from, nil
}
if IsSequence(into) && IsSequence(from) {
return from, nil
}
if IsMapping(into) && IsMapping(from) {
return mergeMapping(into.(mapping), from.(mapping), strict)
}
// YAML types don't match, so no merge is possible. For backward
// compatibility, ignore mismatches unless we're in strict mode and return
// the higher-priority value.
if !strict {
return from, nil
}
return nil, fmt.Errorf("can't merge a %s into a %s", describe(from), describe(into))
}
func mergeMapping(into, from mapping, strict bool) (mapping, error) {
merged := make(mapping, len(into))
for k, v := range into {
merged[k] = v
}
for k := range from {
m, err := merge(merged[k], from[k], strict)
if err != nil {
return nil, err
}
merged[k] = m
}
return merged, nil
}
// IsMapping reports whether a type is a mapping in YAML, represented as a
// map[interface{}]interface{}.
func IsMapping(i interface{}) bool {
_, is := i.(mapping)
return is
}
// IsSequence reports whether a type is a sequence in YAML, represented as an
// []interface{}.
func IsSequence(i interface{}) bool {
_, is := i.(sequence)
return is
}
// IsScalar reports whether a type is a scalar value in YAML.
func IsScalar(i interface{}) bool {
return !IsMapping(i) && !IsSequence(i)
}
func describe(i interface{}) string {
if IsMapping(i) {
return "mapping"
}
if IsSequence(i) {
return "sequence"
}
return "scalar"
}

238
pkg/yamlpatch/merge_test.go
View file

@ -1,238 +0,0 @@
// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package yamlpatch
import (
"bytes"
"os"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
yaml "gopkg.in/yaml.v2"
)
func trimcr(s string) string {
return strings.ReplaceAll(s, "\r\n", "\n")
}
func mustRead(t testing.TB, fname string) []byte {
contents, err := os.ReadFile(fname)
require.NoError(t, err, "failed to read file: %s", fname)
return contents
}
func dump(t testing.TB, actual, expected string) {
// It's impossible to debug YAML if the actual and expected values are
// printed on a single line.
t.Logf("Actual:\n\n%s\n\n", actual)
t.Logf("Expected:\n\n%s\n\n", expected)
}
func strip(s string) string {
// It's difficult to write string constants that are valid YAML. Normalize
// strings for ease of testing.
s = strings.TrimSpace(s)
s = strings.Replace(s, "\t", " ", -1)
return s
}
func canonicalize(t testing.TB, s string) string {
// round-trip to canonicalize formatting
var i interface{}
require.NoError(t,
yaml.Unmarshal([]byte(strip(s)), &i),
"canonicalize: couldn't unmarshal YAML",
)
formatted, err := yaml.Marshal(i)
require.NoError(t, err, "canonicalize: couldn't marshal YAML")
return string(bytes.TrimSpace(formatted))
}
func unmarshal(t testing.TB, s string) interface{} {
var i interface{}
require.NoError(t, yaml.Unmarshal([]byte(strip(s)), &i), "unmarshaling failed")
return i
}
func succeeds(t testing.TB, strict bool, left, right, expect string) {
l, r := unmarshal(t, left), unmarshal(t, right)
m, err := merge(l, r, strict)
require.NoError(t, err, "merge failed")
actualBytes, err := yaml.Marshal(m)
require.NoError(t, err, "couldn't marshal merged structure")
actual := canonicalize(t, string(actualBytes))
expect = canonicalize(t, expect)
if !assert.Equal(t, expect, actual) {
dump(t, actual, expect)
}
}
func fails(t testing.TB, strict bool, left, right string) {
_, err := merge(unmarshal(t, left), unmarshal(t, right), strict)
assert.Error(t, err, "merge succeeded")
}
func TestIntegration(t *testing.T) {
base := mustRead(t, "testdata/base.yaml")
prod := mustRead(t, "testdata/production.yaml")
expect := mustRead(t, "testdata/expect.yaml")
merged, err := YAML([][]byte{base, prod}, true /* strict */)
require.NoError(t, err, "merge failed")
if !assert.Equal(t, trimcr(string(expect)), merged.String(), "unexpected contents") {
dump(t, merged.String(), string(expect))
}
}
func TestEmpty(t *testing.T) {
full := []byte("foo: bar\n")
null := []byte("~")
tests := []struct {
desc string
sources [][]byte
expect string
}{
{"empty base", [][]byte{nil, full}, string(full)},
{"empty override", [][]byte{full, nil}, string(full)},
{"both empty", [][]byte{nil, nil}, ""},
{"null base", [][]byte{null, full}, string(full)},
{"null override", [][]byte{full, null}, "null\n"},
{"empty base and null override", [][]byte{nil, null}, "null\n"},
{"null base and empty override", [][]byte{null, nil}, "null\n"},
}
for _, tt := range tests {
tt := tt
t.Run(tt.desc, func(t *testing.T) {
merged, err := YAML(tt.sources, true /* strict */)
require.NoError(t, err, "merge failed")
assert.Equal(t, tt.expect, merged.String(), "wrong contents after merge")
})
}
}
func TestSuccess(t *testing.T) {
left := `
fun: [maserati, porsche]
practical: {toyota: camry, honda: accord}
occupants:
honda: {driver: jane, backseat: [nate]}
`
right := `
fun: [lamborghini, porsche]
practical: {honda: civic, nissan: altima}
occupants:
honda: {passenger: arthur, backseat: [nora]}
`
expect := `
fun: [lamborghini, porsche]
practical: {toyota: camry, honda: civic, nissan: altima}
occupants:
honda: {passenger: arthur, driver: jane, backseat: [nora]}
`
succeeds(t, true, left, right, expect)
succeeds(t, false, left, right, expect)
}
func TestErrors(t *testing.T) {
check := func(t testing.TB, strict bool, sources ...[]byte) error {
_, err := YAML(sources, strict)
return err
}
t.Run("tabs in source", func(t *testing.T) {
src := []byte("foo:\n\tbar:baz")
assert.Error(t, check(t, false, src), "expected error in permissive mode")
assert.Error(t, check(t, true, src), "expected error in strict mode")
})
t.Run("duplicated keys", func(t *testing.T) {
src := []byte("{foo: bar, foo: baz}")
assert.NoError(t, check(t, false, src), "expected success in permissive mode")
assert.Error(t, check(t, true, src), "expected error in permissive mode")
})
t.Run("merge error", func(t *testing.T) {
left := []byte("foo: [1, 2]")
right := []byte("foo: {bar: baz}")
assert.NoError(t, check(t, false, left, right), "expected success in permissive mode")
assert.Error(t, check(t, true, left, right), "expected error in strict mode")
})
}
func TestMismatchedTypes(t *testing.T) {
tests := []struct {
desc string
left, right string
}{
{"sequence and mapping", "[one, two]", "{foo: bar}"},
{"sequence and scalar", "[one, two]", "foo"},
{"mapping and scalar", "{foo: bar}", "foo"},
{"nested", "{foo: [one, two]}", "{foo: bar}"},
}
for _, tt := range tests {
tt := tt
t.Run(tt.desc+" strict", func(t *testing.T) {
fails(t, true, tt.left, tt.right)
})
t.Run(tt.desc+" permissive", func(t *testing.T) {
// prefer the higher-priority value
succeeds(t, false, tt.left, tt.right, tt.right)
})
}
}
func TestBooleans(t *testing.T) {
// YAML helpfully interprets many strings as Booleans.
tests := []struct {
in, out string
}{
{"yes", "true"},
{"YES", "true"},
{"on", "true"},
{"ON", "true"},
{"no", "false"},
{"NO", "false"},
{"off", "false"},
{"OFF", "false"},
}
for _, tt := range tests {
tt := tt
t.Run(tt.in, func(t *testing.T) {
succeeds(t, true, "", tt.in, tt.out)
succeeds(t, false, "", tt.in, tt.out)
})
}
}
func TestExplicitNil(t *testing.T) {
base := `foo: {one: two}`
override := `foo: ~`
expect := `foo: ~`
succeeds(t, true, base, override, expect)
succeeds(t, false, base, override, expect)
}

173
pkg/yamlpatch/patcher.go
View file

@ -1,173 +0,0 @@
package yamlpatch
import (
"bytes"
"io"
"os"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
type Patcher struct {
BaseFilePath string
PatchFilePath string
quiet bool
}
func NewPatcher(filePath string, suffix string) *Patcher {
return &Patcher{
BaseFilePath: filePath,
PatchFilePath: filePath + suffix,
quiet: false,
}
}
// SetQuiet sets the quiet flag, which will log as DEBUG_LEVEL instead of INFO
func (p *Patcher) SetQuiet(quiet bool) {
p.quiet = quiet
}
// read a single YAML file, check for errors (the merge package doesn't) then return the content as bytes.
func readYAML(filePath string) ([]byte, error) {
var content []byte
var err error
if content, err = os.ReadFile(filePath); err != nil {
return nil, errors.Wrap(err, "while reading yaml file")
}
var yamlMap map[interface{}]interface{}
if err = yaml.Unmarshal(content, &yamlMap); err != nil {
return nil, errors.Wrap(err, filePath)
}
return content, nil
}
// MergedPatchContent reads a YAML file and, if it exists, its patch file,
// then merges them and returns it serialized.
func (p *Patcher) MergedPatchContent() ([]byte, error) {
var err error
var base []byte
base, err = readYAML(p.BaseFilePath)
if err != nil {
return nil, err
}
var over []byte
over, err = readYAML(p.PatchFilePath)
if errors.Is(err, os.ErrNotExist) {
return base, nil
}
if err != nil {
return nil, err
}
logf := log.Infof
if p.quiet {
logf = log.Debugf
}
logf("Patching yaml: '%s' with '%s'", p.BaseFilePath, p.PatchFilePath)
var patched *bytes.Buffer
// strict mode true, will raise errors for duplicate map keys and
// overriding with a different type
patched, err = YAML([][]byte{base, over}, true)
if err != nil {
return nil, err
}
return patched.Bytes(), nil
}
// read multiple YAML documents inside a file, and writes them to a buffer
// separated by the appropriate '---' terminators.
func decodeDocuments(file *os.File, buf *bytes.Buffer, finalDashes bool) error {
var (
err error
docBytes []byte
)
dec := yaml.NewDecoder(file)
dec.SetStrict(true)
dashTerminator := false
for {
yml := make(map[interface{}]interface{})
err = dec.Decode(&yml)
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return errors.Wrapf(err, "while decoding %s", file.Name())
}
docBytes, err = yaml.Marshal(&yml)
if err != nil {
return errors.Wrapf(err, "while marshaling %s", file.Name())
}
if dashTerminator {
buf.Write([]byte("---\n"))
}
buf.Write(docBytes)
dashTerminator = true
}
if dashTerminator && finalDashes {
buf.Write([]byte("---\n"))
}
return nil
}
// PrependedPatchContent collates the base .yaml file with the .yaml.patch, by putting
// the content of the patch BEFORE the base document. The result is a multi-document
// YAML in all cases, even if the base and patch files are single documents.
func (p *Patcher) PrependedPatchContent() ([]byte, error) {
var (
result bytes.Buffer
patchFile *os.File
baseFile *os.File
err error
)
patchFile, err = os.Open(p.PatchFilePath)
// optional file, ignore if it does not exist
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, errors.Wrapf(err, "while opening %s", p.PatchFilePath)
}
if patchFile != nil {
if err = decodeDocuments(patchFile, &result, true); err != nil {
return nil, err
}
logf := log.Infof
if p.quiet {
logf = log.Debugf
}
logf("Prepending yaml: '%s' with '%s'", p.BaseFilePath, p.PatchFilePath)
}
baseFile, err = os.Open(p.BaseFilePath)
if err != nil {
return nil, errors.Wrapf(err, "while opening %s", p.BaseFilePath)
}
if err = decodeDocuments(baseFile, &result, false); err != nil {
return nil, err
}
return result.Bytes(), nil
}

313
pkg/yamlpatch/patcher_test.go
View file

@ -1,313 +0,0 @@
package yamlpatch_test
import (
"os"
"path/filepath"
"testing"
"github.com/crowdsecurity/crowdsec/pkg/yamlpatch"
"github.com/stretchr/testify/require"
)
// similar to the one in cstest, but with test number too. We cannot import
// cstest here because of circular dependency.
func requireErrorContains(t *testing.T, err error, expectedErr string) {
t.Helper()
if expectedErr != "" {
require.ErrorContains(t, err, expectedErr)
return
}
require.NoError(t, err)
}
func TestMergedPatchContent(t *testing.T) {
t.Parallel()
tests := []struct {
name string
base string
patch string
expected string
expectedErr string
}{
{
"invalid yaml in base",
"notayaml",
"",
"",
"config.yaml: yaml: unmarshal errors:",
},
{
"invalid yaml in base (detailed message)",
"notayaml",
"",
"",
"cannot unmarshal !!str `notayaml`",
},
{
"invalid yaml in patch",
"",
"notayaml",
"",
"config.yaml.local: yaml: unmarshal errors:",
},
{
"invalid yaml in patch (detailed message)",
"",
"notayaml",
"",
"cannot unmarshal !!str `notayaml`",
},
{
"basic merge",
"{'first':{'one':1,'two':2},'second':{'three':3}}",
"{'first':{'one':10,'dos':2}}",
"{'first':{'one':10,'dos':2,'two':2},'second':{'three':3}}",
"",
},
// bools and zero values; here the "mergo" package had issues
// so we used something simpler.
{
"bool merge - off if false",
"bool: on",
"bool: off",
"bool: false",
"",
},
{
"bool merge - on is true",
"bool: off",
"bool: on",
"bool: true",
"",
},
{
"string is not a bool - on to off",
"{'bool': 'on'}",
"{'bool': 'off'}",
"{'bool': 'off'}",
"",
},
{
"string is not a bool - off to on",
"{'bool': 'off'}",
"{'bool': 'on'}",
"{'bool': 'on'}",
"",
},
{
"bool merge - true to false",
"{'bool': true}",
"{'bool': false}",
"{'bool': false}",
"",
},
{
"bool merge - false to true",
"{'bool': false}",
"{'bool': true}",
"{'bool': true}",
"",
},
{
"string merge - value to value",
"{'string': 'value'}",
"{'string': ''}",
"{'string': ''}",
"",
},
{
"sequence merge - value to empty",
"{'sequence': [1, 2]}",
"{'sequence': []}",
"{'sequence': []}",
"",
},
{
"map merge - value to value",
"{'map': {'one': 1, 'two': 2}}",
"{'map': {}}",
"{'map': {'one': 1, 'two': 2}}",
"",
},
// mismatched types
{
"can't merge a sequence into a mapping",
"map: {'key': 'value'}",
"map: ['value1', 'value2']",
"",
"can't merge a sequence into a mapping",
},
{
"can't merge a scalar into a mapping",
"map: {'key': 'value'}",
"map: 3",
"",
"can't merge a scalar into a mapping",
},
{
"can't merge a mapping into a sequence",
"sequence: ['value1', 'value2']",
"sequence: {'key': 'value'}",
"",
"can't merge a mapping into a sequence",
},
{
"can't merge a scalar into a sequence",
"sequence: ['value1', 'value2']",
"sequence: 3",
"",
"can't merge a scalar into a sequence",
},
{
"can't merge a sequence into a scalar",
"scalar: true",
"scalar: ['value1', 'value2']",
"",
"can't merge a sequence into a scalar",
},
{
"can't merge a mapping into a scalar",
"scalar: true",
"scalar: {'key': 'value'}",
"",
"can't merge a mapping into a scalar",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
dirPath, err := os.MkdirTemp("", "yamlpatch")
require.NoError(t, err)
defer os.RemoveAll(dirPath)
configPath := filepath.Join(dirPath, "config.yaml")
patchPath := filepath.Join(dirPath, "config.yaml.local")
err = os.WriteFile(configPath, []byte(tc.base), 0o600)
require.NoError(t, err)
err = os.WriteFile(patchPath, []byte(tc.patch), 0o600)
require.NoError(t, err)
patcher := yamlpatch.NewPatcher(configPath, ".local")
patchedBytes, err := patcher.MergedPatchContent()
requireErrorContains(t, err, tc.expectedErr)
require.YAMLEq(t, tc.expected, string(patchedBytes))
})
}
}
func TestPrependedPatchContent(t *testing.T) {
t.Parallel()
tests := []struct {
name string
base string
patch string
expected string
expectedErr string
}{
// we test with scalars here, because YAMLeq does not work
// with multi-document files, so we need char-to-char comparison
// which is noisy with sequences and (unordered) mappings
{
"newlines are always appended, if missing, by yaml.Marshal()",
"foo: bar",
"",
"foo: bar\n",
"",
},
{
"prepend empty document",
"foo: bar\n",
"",
"foo: bar\n",
"",
},
{
"prepend a document to another",
"foo: bar",
"baz: qux",
"baz: qux\n---\nfoo: bar\n",
"",
},
{
"prepend document with same key",
"foo: true",
"foo: false",
"foo: false\n---\nfoo: true\n",
"",
},
{
"prepend multiple documents",
"one: 1\n---\ntwo: 2\n---\none: 3",
"four: 4\n---\none: 1.1",
"four: 4\n---\none: 1.1\n---\none: 1\n---\ntwo: 2\n---\none: 3\n",
"",
},
{
"invalid yaml in base",
"blablabla",
"",
"",
"config.yaml: yaml: unmarshal errors:",
},
{
"invalid yaml in base (detailed message)",
"blablabla",
"",
"",
"cannot unmarshal !!str `blablabla`",
},
{
"invalid yaml in patch",
"",
"blablabla",
"",
"config.yaml.local: yaml: unmarshal errors:",
},
{
"invalid yaml in patch (detailed message)",
"",
"blablabla",
"",
"cannot unmarshal !!str `blablabla`",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
dirPath, err := os.MkdirTemp("", "yamlpatch")
require.NoError(t, err)
defer os.RemoveAll(dirPath)
configPath := filepath.Join(dirPath, "config.yaml")
patchPath := filepath.Join(dirPath, "config.yaml.local")
err = os.WriteFile(configPath, []byte(tc.base), 0o600)
require.NoError(t, err)
err = os.WriteFile(patchPath, []byte(tc.patch), 0o600)
require.NoError(t, err)
patcher := yamlpatch.NewPatcher(configPath, ".local")
patchedBytes, err := patcher.PrependedPatchContent()
requireErrorContains(t, err, tc.expectedErr)
// YAMLeq does not handle multiple documents
require.Equal(t, tc.expected, string(patchedBytes))
})
}
}

13
pkg/yamlpatch/testdata/base.yaml vendored
View file

@ -1,13 +0,0 @@
fun:
- maserati
- porsche
practical:
toyota: camry
honda: accord
occupants:
honda:
driver: jane
backseat:
- nate

13
pkg/yamlpatch/testdata/expect.yaml vendored
View file

@ -1,13 +0,0 @@
fun:
- lamborghini
- porsche
occupants:
honda:
backseat:
- nora
driver: jane
passenger: arthur
practical:
honda: civic
nissan: altima
toyota: camry

13
pkg/yamlpatch/testdata/production.yaml vendored
View file

@ -1,13 +0,0 @@
fun:
- lamborghini
- porsche
practical:
honda: civic
nissan: altima
occupants:
honda:
passenger: arthur
backseat:
- nora